From 1726f58da22ee83abb9b04b86f5ee74b4ac2e0ef Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Fri, 9 Oct 2026 02:45:51 -0700 Subject: [PATCH 01/63] Add handled-error metadata and validate run view cursors --- src/firefly_weave/api/transport.py | 7 +- src/firefly_weave/contracts/run_views.py | 56 ++- src/firefly_weave/contracts/schema_export.py | 2 + src/firefly_weave/contracts/surface.py | 25 +- .../operations/fact_positions.py | 76 ++++ studio/src/app/operate/run-contracts.ts | 3 + studio/src/app/operate/run-views-fake.ts | 14 +- studio/src/app/operate/run-views.fixture.ts | 1 + studio/tests/run-views-contract.test.ts | 44 ++- studio/tests/run-views-fake.test.ts | 31 ++ studio/tests/run-views-oracle.ts | 31 +- tests/contracts/test_run_views_contract.py | 138 ++++++++ tests/contracts/test_run_views_snapshot.py | 15 +- .../contracts/run-views.snapshot.json | 326 ++++++++++++++++++ tests/unit/operations/test_fact_positions.py | 142 ++++++++ 15 files changed, 873 insertions(+), 38 deletions(-) create mode 100644 src/firefly_weave/operations/fact_positions.py create mode 100644 tests/unit/operations/test_fact_positions.py diff --git a/src/firefly_weave/api/transport.py b/src/firefly_weave/api/transport.py index 4c72b17e..fc07d1b4 100644 --- a/src/firefly_weave/api/transport.py +++ b/src/firefly_weave/api/transport.py @@ -117,7 +117,10 @@ def decode_sequence_cursor(value: str | None, scope: Scope, collection: str) -> def encode_cursor_v2(scope: Scope, collection: str, sort_value: JsonValue, identifier: str) -> str: """Time-ordered cursor: base64url of [2, scope, collection, sort value, identifier].""" raw = json.dumps([2, scope.model_dump(mode="json"), collection, sort_value, identifier], separators=(",", ":")) - return base64.urlsafe_b64encode(raw.encode()).decode().rstrip("=") + cursor = base64.urlsafe_b64encode(raw.encode()).decode().rstrip("=") + if len(cursor) > CURSOR_V2_LIMIT: + raise ValueError("Invalid scope-bound cursor") + return cursor def decode_cursor_v2(value: str | None, scope: Scope, collection: str) -> tuple[JsonValue, str] | None: @@ -138,7 +141,7 @@ def decode_cursor_v2(value: str | None, scope: Scope, collection: str) -> tuple[ if not isinstance(identifier, str) or encode_cursor_v2(scope, collection, sort_value, identifier) != value: raise ValueError() return sort_value, identifier - except (ValueError, TypeError, KeyError): + except (ValueError, TypeError, KeyError, RecursionError): raise ValueError("Invalid scope-bound cursor") from None diff --git a/src/firefly_weave/contracts/run_views.py b/src/firefly_weave/contracts/run_views.py index 63ebbfbf..20d76eff 100644 --- a/src/firefly_weave/contracts/run_views.py +++ b/src/firefly_weave/contracts/run_views.py @@ -26,6 +26,7 @@ import hashlib import json +import re from datetime import UTC, datetime, timedelta from typing import Annotated, Literal from uuid import UUID @@ -42,6 +43,9 @@ type RunStatus = Literal["queued", "running", "waiting", "suspended", "succeeded", "failed", "cancelled", "timed_out"] type RunOrigin = Literal["manual", "webhook", "schedule", "broker", "email", "provider", "retry", "call", "test"] type RunSummaryOrder = Literal["started_desc", "started_asc", "updated_desc"] +type RunListOrder = Literal["id", "started_desc", "started_asc", "updated_desc"] +type HandledErrorCount = Annotated[int, Field(ge=0)] +type StepHandledFailure = Literal["continue", "errorOutput"] type StepKind = Literal[ "action", "llm", @@ -78,6 +82,26 @@ type LogFieldValue = Annotated[str, Field(max_length=256)] | SafeInteger | FiniteFloat | bool | None +_TIMESTAMP = re.compile( + r"[0-9]{4}-[0-9]{2}-[0-9]{2}[Tt][0-9]{2}:[0-9]{2}:[0-9]{2}(?:\.[0-9]+)?(?:[Zz]|[+-][0-9]{2}:[0-9]{2})" +) + + +def utc_time(value: object) -> datetime | None: + """Parse a bounded RFC 3339 timestamp into a representable UTC instant.""" + if value is None: + return None + if type(value) is not str or len(value) > 64 or _TIMESTAMP.fullmatch(value) is None: + raise ValueError("Expected a timestamp with an offset") + if value[-1] not in "Zz" and (int(value[-5:-3]) > 23 or int(value[-2:]) > 59): + raise ValueError("Expected a valid timestamp") + try: + parsed = datetime.fromisoformat(value.upper().replace("Z", "+00:00")) + return parsed.astimezone(UTC) + except (ValueError, OverflowError): + raise ValueError("Expected a valid timestamp") from None + + def _same_step(node_id: str, instance_key: str) -> None: if instance_key and (instance_key == node_id or node_of(instance_key) != node_id): raise ValueError("instance_key must be empty or a longer key of the same step") @@ -147,6 +171,7 @@ class RunSummary(ContractModel): failed_step: FailedStep | None active_incidents: Count archived: bool + handled_errors: HandledErrorCount = Field(default=0, exclude_if=lambda value: value == 0) @model_validator(mode="after") def coherent(self) -> RunSummary: @@ -191,6 +216,7 @@ class StepFact(ContractModel): child_run_id: UUID | None log_entries: Count omissions: list[Omission] = Field(default_factory=list, max_length=8, exclude_if=lambda value: not value) + handled: StepHandledFailure | None = Field(default=None, exclude_if=lambda value: value is None) @model_validator(mode="after") def coherent(self) -> StepFact: @@ -202,6 +228,12 @@ def coherent(self) -> StepFact: raise ValueError("duration_ms is set exactly when the step started and ended") if (self.queue_wait_ms is not None) != (self.scheduled_at is not None and self.started_at is not None): raise ValueError("queue_wait_ms is set exactly when the step was scheduled and started") + if self.handled is not None and self.status != "failed": + raise ValueError("Only a failed step can have a handled failure") + if self.started_at and self.ended_at and self.ended_at < self.started_at: + raise ValueError("A step cannot end before it started") + if self.scheduled_at and self.started_at and self.started_at < self.scheduled_at: + raise ValueError("A step cannot start before it was scheduled") return self @@ -279,6 +311,7 @@ class RunSummaryQuery(ContractModel): business_key: BusinessKey | None = None correlation_key: BusinessKey | None = None has_active_incident: bool | None = None + has_handled_errors: bool | None = None include_archived: bool = False activation_id: UUID | None = None order: RunSummaryOrder = "started_desc" @@ -290,16 +323,10 @@ class RunSummaryQuery(ContractModel): def canonical_status(cls, value: list[RunStatus]) -> list[RunStatus]: return sorted(set(value)) - @field_validator("started_after", "started_before") + @field_validator("started_after", "started_before", mode="before") @classmethod - def utc(cls, value: datetime | None) -> datetime | None: - if value is None: - return None - try: - return value.astimezone(UTC) - except OverflowError: - # An offset can push the instant past year 9999 or before year 1; that is a bad value, not a server error. - raise ValueError("The time is outside the supported range") from None + def utc(cls, value: object) -> datetime | None: + return utc_time(value) @model_validator(mode="after") def coherent(self) -> RunSummaryQuery: @@ -321,6 +348,17 @@ def cursor_collection(self) -> str: return f"run_summaries:{_digest(filters)}:{self.order}" +class RunListQuery(RunSummaryQuery): + """`runs.list` filters, with ID order retained for existing callers.""" + + # The list schema also permits ID order; the summary schema keeps chronological orders only. + order: RunListOrder = "id" # type: ignore[assignment] + + def cursor_collection(self) -> str: + filters = self.model_dump(mode="json", exclude={"order", "limit", "cursor"}) + return f"runs:{_digest(filters)}:{self.order}" + + class RunStepQuery(ContractModel): """`runs.steps` query parameters; `step` is a static step ID and matches every instance.""" diff --git a/src/firefly_weave/contracts/schema_export.py b/src/firefly_weave/contracts/schema_export.py index b31c2e08..505836fe 100644 --- a/src/firefly_weave/contracts/schema_export.py +++ b/src/firefly_weave/contracts/schema_export.py @@ -192,6 +192,7 @@ def contract_models() -> dict[str, SchemaModel]: ) from firefly_weave.contracts.run_lifecycle import RunLifecycle, RunLifecycleRequest, RunPurgeRequest from firefly_weave.contracts.run_views import ( + RunListQuery, RunLogPage, RunLogQuery, RunStepQuery, @@ -296,6 +297,7 @@ def contract_models() -> dict[str, SchemaModel]: "run-purge-request": RunPurgeRequest, "run-summary-page": RunSummaryPage, "run-summary-query": RunSummaryQuery, + "run-list-query": RunListQuery, "step-fact-page": StepFactPage, "run-step-query": RunStepQuery, "run-log-page": RunLogPage, diff --git a/src/firefly_weave/contracts/surface.py b/src/firefly_weave/contracts/surface.py index 5520b71c..dbc39be6 100644 --- a/src/firefly_weave/contracts/surface.py +++ b/src/firefly_weave/contracts/surface.py @@ -157,6 +157,7 @@ ) from firefly_weave.contracts.run_lifecycle import RunLifecycle, RunLifecycleRequest, RunPurgeRequest from firefly_weave.contracts.run_views import ( + RunListQuery, RunLogPage, RunLogQuery, RunStepQuery, @@ -275,20 +276,6 @@ def native(self) -> OpenAPIOperation: {} if field.default is None or field.default_factory is not None else {"default": field.default} ) params.append(OpenAPIParameter(name, "query", _present(field.annotation), required=False, **documented)) - if self.id == "runs.list": - params += [ - OpenAPIParameter("business_key", "query", Annotated[str, Field(max_length=200)], required=False), - OpenAPIParameter("correlation_key", "query", Annotated[str, Field(max_length=200)], required=False), - OpenAPIParameter( - "status", - "query", - Literal[ - "queued", "running", "waiting", "suspended", "succeeded", "failed", "cancelled", "timed_out" - ], - required=False, - ), - OpenAPIParameter("include_archived", "query", bool, required=False, default=False), - ] if self.id == "human_tasks.list": params.append( OpenAPIParameter( @@ -315,7 +302,7 @@ def native(self) -> OpenAPIOperation: params.append(OpenAPIParameter("target_id", "query", UUID, required=False)) if self.id == "deployment_plans.list": params.append(OpenAPIParameter("deployment_id", "query", UUID, required=False)) - if self.page: + if self.page and self.query is None: params += [ OpenAPIParameter("limit", "query", PageLimit, required=False, default=50), OpenAPIParameter("cursor", "query", str, required=False), @@ -1380,7 +1367,13 @@ def native(self) -> OpenAPIOperation: "runs.start", ENVIRONMENT + "/runs", "POST", RunView, "run.start", StartRunRequest, (201,), idempotency=True ), Operation( - "runs.list", ENVIRONMENT + "/runs", "GET", Page[RunView | UnavailableResource], "run.read", page=True + "runs.list", + ENVIRONMENT + "/runs", + "GET", + Page[RunView | UnavailableResource], + "run.read", + page=True, + query=RunListQuery, ), Operation("runs.read", ENVIRONMENT + "/runs/{identifier}", "GET", RunView, "run.read"), Operation( diff --git a/src/firefly_weave/operations/fact_positions.py b/src/firefly_weave/operations/fact_positions.py new file mode 100644 index 00000000..babd211c --- /dev/null +++ b/src/firefly_weave/operations/fact_positions.py @@ -0,0 +1,76 @@ +# Copyright 2026 Firefly Software Foundation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# Author: Firefly Software Foundation +# SPDX-License-Identifier: Apache-2.0 + +"""Typed positions for chronological run and step fact queries.""" + +from dataclasses import dataclass +from datetime import datetime +from uuid import UUID + +from pydantic import TypeAdapter + +from firefly_weave.contracts.definitions import ResourceName +from firefly_weave.contracts.instance_keys import InstanceKeyTextOrEmpty, instance_view +from firefly_weave.contracts.run_views import utc_time +from firefly_weave.contracts.values import JsonValue + +type Position = tuple[JsonValue, str] | None + +_NODE: TypeAdapter[str] = TypeAdapter(ResourceName) +_INSTANCE: TypeAdapter[str] = TypeAdapter(InstanceKeyTextOrEmpty) + + +@dataclass(frozen=True) +class RunPosition: + """A nullable recorded time and the UUID tie breaker.""" + + at: datetime | None + id: UUID + + +@dataclass(frozen=True) +class StepPosition: + """A nullable scheduled time and the canonical step instance identity.""" + + scheduled_at: datetime | None + node_id: str + instance_key: str + + +def run_position(position: Position) -> RunPosition | None: + """Validate a decoded run cursor before using its values in SQL.""" + if position is None: + return None + value, identifier = position + if type(identifier) is not str: + raise ValueError("Invalid run position identifier") + return RunPosition(utc_time(value), UUID(identifier)) + + +def step_position(position: Position) -> StepPosition | None: + """Validate a decoded step cursor and its repeated instance identity.""" + if position is None: + return None + value, identifier = position + if type(value) is not list or len(value) != 3: + raise ValueError("Invalid step position") + at, node, key = value + node = _NODE.validate_python(node, strict=True) + key = _INSTANCE.validate_python(key, strict=True) + identity = instance_view(key or node) + if identity.node_id != node or identity.instance_key != key or identifier != (key or node): + raise ValueError("Invalid step position identity") + return StepPosition(utc_time(at), node, key) diff --git a/studio/src/app/operate/run-contracts.ts b/studio/src/app/operate/run-contracts.ts index 2461cc35..d93767ed 100644 --- a/studio/src/app/operate/run-contracts.ts +++ b/studio/src/app/operate/run-contracts.ts @@ -128,6 +128,7 @@ export interface RunSummary { failed_step: FailedStep | null; active_incidents: number; archived: boolean; + handled_errors?: number; } /** A run whose classification is unavailable: only its ID is shown. */ export interface UnavailableRun { @@ -157,6 +158,7 @@ export interface StepFact { error_code: string | null; child_run_id: string | null; log_entries: number; + handled?: "continue" | "errorOutput"; /** Only with include=output: why the output was not returned. */ omissions?: Omission[]; /** Only with include=output, when the recorded output may be shown (it can be null). */ @@ -206,6 +208,7 @@ export interface RunSummaryQuery { business_key?: string; correlation_key?: string; has_active_incident?: boolean; + has_handled_errors?: boolean; include_archived?: boolean; activation_id?: string; order?: RunSummaryOrder; diff --git a/studio/src/app/operate/run-views-fake.ts b/studio/src/app/operate/run-views-fake.ts index 382d2405..3fb0b06f 100644 --- a/studio/src/app/operate/run-views-fake.ts +++ b/studio/src/app/operate/run-views-fake.ts @@ -45,11 +45,9 @@ export const fakeEnvironment = type Kind = "string" | "boolean" | "integer" | "array"; type Values = Record; // Never looser than the server: whatever the server refuses, the fake refuses. -// It is stricter on some forms the server accepts: epoch-number timestamps, -// "_" or a space instead of "T", no seconds, a comma fraction, offsets without -// a colon, UUIDs written as urn:uuid:, without hyphens or in braces, and -// names, versions or keys over 200 characters (counted in UTF-16 units, so -// keys of 101 to 200 characters outside the BMP also fail here). +// It is stricter on UUIDs written as urn:uuid:, without hyphens or in braces, +// and names, versions or keys over 200 characters (counted in UTF-16 units, +// so keys of 101 to 200 characters outside the BMP also fail here). const name = /^[A-Za-z0-9][A-Za-z0-9_.-]*$/; const uuid = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; // Mirrors SEMVER_PATTERN in src/firefly_weave/contracts/definitions.py. @@ -103,6 +101,7 @@ const summaryParameters: Record = { business_key: "string", correlation_key: "string", has_active_incident: "boolean", + has_handled_errors: "boolean", include_archived: "boolean", activation_id: "string", order: "string", @@ -224,7 +223,7 @@ function instantMicros(value: string): bigint { return utc; } function timeOf(values: Values, key: string): bigint | undefined { - const value = text(values, key, instant); + const value = text(values, key, instant, 64); return value === undefined ? undefined : instantMicros(value); } /** Run timestamps come from the fixture, which has whole-millisecond precision. */ @@ -317,6 +316,7 @@ function summaries(params: URLSearchParams, data: RunViewsFixture): FakeAnswer { business_key: text(values, "business_key"), correlation_key: text(values, "correlation_key"), has_active_incident: values["has_active_incident"] as boolean | undefined, + has_handled_errors: values["has_handled_errors"] as boolean | undefined, include_archived: values["include_archived"] === true, activation_id: uuidOf(values, "activation_id"), }; @@ -373,6 +373,8 @@ function summaries(params: URLSearchParams, data: RunViewsFixture): FakeAnswer { run.correlation_key === filters.correlation_key) && (filters.has_active_incident === undefined || run.active_incidents > 0 === filters.has_active_incident) && + (filters.has_handled_errors === undefined || + (run.handled_errors ?? 0) > 0 === filters.has_handled_errors) && (filters.activation_id === undefined || run.activation_id === filters.activation_id); const field = order === "updated_desc" ? "updated_at" : "started_at"; diff --git a/studio/src/app/operate/run-views.fixture.ts b/studio/src/app/operate/run-views.fixture.ts index a0f0ece8..ca0178f8 100644 --- a/studio/src/app/operate/run-views.fixture.ts +++ b/studio/src/app/operate/run-views.fixture.ts @@ -191,6 +191,7 @@ export const runViewsFixture: RunViewsFixture = { status: "succeeded", origin: "retry", retried_from_run_id: fixtureIds.failed, + handled_errors: 1, started_at: "2026-10-07T11:00:00Z", ended_at: "2026-10-07T11:00:50Z", duration_ms: 50000, diff --git a/studio/tests/run-views-contract.test.ts b/studio/tests/run-views-contract.test.ts index ed6dac57..debe4ce6 100644 --- a/studio/tests/run-views-contract.test.ts +++ b/studio/tests/run-views-contract.test.ts @@ -36,7 +36,11 @@ import { fixtureIds, runViewsFixture, } from "../src/app/operate/run-views.fixture"; -import { pythonAvailable, validateWithPython } from "./run-views-oracle"; +import { + pythonAvailable, + roundTripWithPython, + validateWithPython, +} from "./run-views-oracle"; const summaries = runViewsFixture.runs.map((run) => run.summary); const placeholder = (id: string): RunSummaryItem => ({ @@ -160,3 +164,41 @@ describe("run view helpers", () => { ).toEqual(["invoice-approval"]); }); }); + +describe.skipIf(!pythonAvailable)("handled metadata compatibility", () => { + it("retains nonzero counts and handled failures through Python JSON serialization", () => { + const summary = summaries.find((item) => item.id === fixtureIds.retry)!; + const step = { + ...fact(fixtureIds.failed, "send[3]"), + handled: "errorOutput" as const, + }; + const result = roundTripWithPython({ + summaries: [{ items: [summary], next_cursor: null }], + steps: [{ items: [step], next_cursor: null, complete: true }], + logs: [], + }); + const item = result.summaries[0].items[0]; + if (isUnavailableRun(item)) throw new Error("Expected a run summary"); + expect(item.handled_errors).toBe(1); + expect(result.steps[0].items[0].handled).toBe("errorOutput"); + expect(item.status).toBe("succeeded"); + }); + it("keeps fields absent on older server records", () => { + const result = roundTripWithPython({ + summaries: [{ items: [summaries[0]], next_cursor: null }], + steps: [ + { + items: [fact(fixtureIds.failed, "send[3]")], + next_cursor: null, + complete: true, + }, + ], + logs: [], + }); + expect(result.summaries[0].items[0]).not.toHaveProperty("handled_errors"); + expect(result.steps[0].items[0]).not.toHaveProperty("handled"); + expect(runViewQuery({ has_handled_errors: false })).toBe( + "?has_handled_errors=false", + ); + }); +}); diff --git a/studio/tests/run-views-fake.test.ts b/studio/tests/run-views-fake.test.ts index dfa2ff4b..e4d541fb 100644 --- a/studio/tests/run-views-fake.test.ts +++ b/studio/tests/run-views-fake.test.ts @@ -76,6 +76,11 @@ const invalidQueries = [ "colour=red", "workflow=a&workflow=b", "include_test=1", + "started_after=1791374400", + "started_after=2026-10-07%2012:00:00Z", + "started_after=2026-10-07T12:00:00", + "started_after=2026-10-07T12:00:00." + "1".repeat(65) + "Z", + "has_handled_errors=1", "limit=101", "status=canceled", "started_after=2026-10-07T12:00:00+02:00", @@ -498,6 +503,8 @@ describe.skipIf(!pythonAvailable)("query decisions match parse_query", () => { }; const summaryQueries = [ "", + "has_handled_errors=true", + "has_handled_errors=false", "include_test=true&order=updated_desc&limit=100", "status=queued&status=failed&status=queued", "status=queued&status=queued&status=queued&status=queued&status=queued&status=queued&status=queued&status=queued", @@ -645,3 +652,27 @@ describe.skipIf(!pythonAvailable)("query decisions match parse_query", () => { ).toEqual([]); }); }); + +describe("handled failures", () => { + it("filters nonzero counts and treats absent counts as zero", async () => { + const yes = await listRunSummaries(api, { has_handled_errors: true }); + expect(ids(yes)).toEqual([fixtureIds.retry]); + expect(summary(yes, fixtureIds.retry).handled_errors).toBe(1); + const no = await listRunSummaries(api, { has_handled_errors: false }); + expect(ids(no)).not.toContain(fixtureIds.retry); + expect(ids(no)).toContain(fixtureIds.succeeded); + expect(summary(no, fixtureIds.succeeded)).not.toHaveProperty( + "handled_errors", + ); + }); + it("binds both handled filters to pagination cursors", async () => { + const first = await listRunSummaries(api, { limit: 1 }); + for (const has_handled_errors of [true, false]) + await expect( + listRunSummaries(api, { + cursor: first.next_cursor!, + has_handled_errors, + }), + ).rejects.toMatchObject({ status: 422, code: "WV-VALIDATION" }); + }); +}); diff --git a/studio/tests/run-views-oracle.ts b/studio/tests/run-views-oracle.ts index df73ffe0..52f83306 100644 --- a/studio/tests/run-views-oracle.ts +++ b/studio/tests/run-views-oracle.ts @@ -78,7 +78,11 @@ export function validateWithPython( } /** The query models that parse_query decodes in the run views. */ -export type QueryModel = "RunSummaryQuery" | "RunStepQuery" | "RunLogQuery"; +export type QueryModel = + | "RunSummaryQuery" + | "RunListQuery" + | "RunStepQuery" + | "RunLogQuery"; const queryScript = ` import json, sys from starlette.datastructures import QueryParams @@ -142,3 +146,28 @@ export function rejectionMessagesWithPython(queries: string[]): string[] { }), ) as string[]; } + +/** Serialize fixture pages through Python to check additive wire fields. */ +export function roundTripWithPython(pages: RunViewPages): { + summaries: import("../src/app/operate/run-contracts").RunSummaryPage[]; + steps: import("../src/app/operate/run-contracts").StepFactPage[]; + logs: import("../src/app/operate/run-contracts").RunLogPage[]; +} { + return JSON.parse( + execFileSync( + python, + [ + "-c", + ` +import json, sys +from firefly_weave.contracts import run_views as v +pages = json.load(sys.stdin) +for name, model in (("summaries", v.RunSummaryPage), ("steps", v.StepFactPage), ("logs", v.RunLogPage)): + pages[name] = [model.model_validate_json(json.dumps(page)).model_dump(mode="json") for page in pages[name]] +print(json.dumps(pages)) +`, + ], + { ...pythonOptions, input: JSON.stringify(pages) }, + ), + ); +} diff --git a/tests/contracts/test_run_views_contract.py b/tests/contracts/test_run_views_contract.py index bf768569..431f9ece 100644 --- a/tests/contracts/test_run_views_contract.py +++ b/tests/contracts/test_run_views_contract.py @@ -27,6 +27,7 @@ from firefly_weave.contracts import run_views from firefly_weave.contracts.run_views import ( FailedStep, + RunListQuery, RunLogEntry, RunLogPage, RunLogQuery, @@ -136,6 +137,7 @@ def test_field_names_and_order_are_frozen(): "failed_step", "active_incidents", "archived", + "handled_errors", ] assert list(StepFact.model_fields) == [ "node_id", @@ -154,6 +156,7 @@ def test_field_names_and_order_are_frozen(): "child_run_id", "log_entries", "omissions", + "handled", ] assert list(StepFactWithOutput.model_fields) == [*StepFact.model_fields, "output"] assert list(RunLogEntry.model_fields) == [ @@ -188,6 +191,7 @@ def test_field_names_and_order_are_frozen(): "business_key", "correlation_key", "has_active_incident", + "has_handled_errors", "include_archived", "activation_id", "order", @@ -507,3 +511,137 @@ def test_step_and_log_queries_bound_their_pages_and_take_static_step_ids(): assert logs != parse(RunLogQuery, {"level": "error"}).cursor_collection(run_id) assert logs != parse(RunLogQuery, {"level": "warning", "node_id": "send"}).cursor_collection(run_id) assert logs != parse(RunLogQuery, {"level": "warning", "source": "ai"}).cursor_collection(run_id) + + +@pytest.mark.parametrize("handled", ["continue", "errorOutput"]) +def test_handled_field_requires_failed_status(handled): + with pytest.raises(ValidationError): + parse(StepFact, {**STEP, "status": "succeeded", "handled": handled}) + assert parse(StepFact, {**STEP, "handled": handled}).handled == handled + assert "handled" not in parse(StepFact, STEP).model_dump(mode="json") + + +def test_handled_summary_default_is_omitted_and_nonzero_round_trips(): + assert "handled_errors" not in parse(RunSummary, SUMMARY).model_dump(mode="json") + value = {**SUMMARY, "handled_errors": 2} + assert parse(RunSummary, value).model_dump(mode="json") == value + with pytest.raises(ValidationError): + parse(RunSummary, {**SUMMARY, "handled_errors": -1}) + + +@pytest.mark.parametrize( + "change", + [ + {"duration_ms": -1}, + {"queue_wait_ms": -1}, + {"ended_at": "2026-10-07T12:00:01Z"}, + {"scheduled_at": "2026-10-07T12:00:03Z"}, + ], +) +def test_step_timings_cannot_be_negative_or_reversed(change): + with pytest.raises(ValidationError): + parse(StepFact, {**STEP, **change}) + + +QUERY_CHANGES = { + "workflow": "other", + "version": "1.3.0", + "status": ["failed"], + "origin": "webhook", + "started_after": "2026-10-01T00:00:00Z", + "started_before": "2026-10-08T00:00:00Z", + "include_test": True, + "caller_run_id": RUN, + "top_level_only": True, + "retried_from_run_id": RUN, + "business_key": "INV-1042", + "correlation_key": "thread", + "has_active_incident": True, + "has_handled_errors": True, + "include_archived": True, + "activation_id": RUN, + "order": "updated_desc", +} + + +@pytest.mark.parametrize("model", [RunSummaryQuery, RunListQuery]) +@pytest.mark.parametrize("field,value", QUERY_CHANGES.items()) +def test_every_run_query_field_binds_the_cursor(model, field, value): + base = parse(model, {"workflow": "invoice-approval"}) + changed = parse(model, {**base.model_dump(mode="json"), field: value}) + assert changed.cursor_collection() != base.cursor_collection() + + +def test_query_digest_coverage_includes_every_filter(): + assert set(QUERY_CHANGES) == set(RunSummaryQuery.model_fields) - {"limit", "cursor"} + plain = RunSummaryQuery() + assert plain.cursor_collection() != plain.model_copy(update={"has_handled_errors": True}).cursor_collection() + assert plain.cursor_collection() != plain.model_copy(update={"has_handled_errors": False}).cursor_collection() + with pytest.raises(ValidationError): + parse(RunSummaryQuery, {"order": "id"}) + assert RunListQuery().order == "id" + assert RunListQuery().cursor_collection().startswith("runs:") + assert RunListQuery().cursor_collection() != plain.cursor_collection() + assert ( + parse(RunListQuery, {"status": ["running", "failed", "running"]}).cursor_collection() + == parse(RunListQuery, {"status": ["failed", "running"], "limit": 1, "cursor": "abc"}).cursor_collection() + ) + + +@pytest.mark.parametrize( + "value", + [ + 1791374400, + True, + {}, + [], + "1791374400", + "2026-10-07 12:00:00Z", + "2026-10-07T12:00:00", + "2026-10-07T12:00Z", + "2026-10-07T12:00:00+00:60", + "2027-02-29T00:00:00Z", + "2026-10-07T12:00:00." + "1" * 64 + "Z", + ], +) +def test_query_timestamps_require_bounded_rfc3339(value): + with pytest.raises(ValidationError): + parse(RunSummaryQuery, {"started_after": value}) + + +@pytest.mark.parametrize( + "value,expected", + [ + ("2028-02-29T00:00:00Z", "2028-02-29T00:00:00+00:00"), + ("2026-10-07T12:00:00+23:59", "2026-10-06T12:01:00+00:00"), + ("2026-10-07t12:00:00z", "2026-10-07T12:00:00+00:00"), + ], +) +def test_query_timestamps_normalize_valid_leap_days_and_offsets(value, expected): + assert parse(RunSummaryQuery, {"started_after": value}).started_after.isoformat() == expected + + +def test_time_range_accepts_exactly_four_hundred_days(): + query = {"started_after": "2026-01-01T00:00:00Z", "started_before": "2027-02-05T00:00:00Z"} + assert parse(RunSummaryQuery, query).started_before is not None + for before in ["2027-02-05T00:00:00.000001Z", "2025-12-31T23:59:59Z"]: + with pytest.raises(ValidationError): + parse(RunSummaryQuery, {**query, "started_before": before}) + + +def test_run_list_surface_uses_one_typed_query(): + from firefly_weave.contracts.openapi import export_openapi + from firefly_weave.contracts.schema_export import export_schemas + from firefly_weave.contracts.surface import OPERATIONS + + operation = OPERATIONS["runs.list"] + assert operation.query is RunListQuery + parameters = export_openapi()["paths"][operation.canonical_path]["get"]["parameters"] + names = [item["name"] for item in parameters if item["in"] == "query"] + assert names == list(RunListQuery.model_fields) + assert export_schemas()["run-list-query"]["$defs"]["RunListOrder"]["enum"] == [ + "id", + "started_desc", + "started_asc", + "updated_desc", + ] diff --git a/tests/contracts/test_run_views_snapshot.py b/tests/contracts/test_run_views_snapshot.py index 57716396..9555989f 100644 --- a/tests/contracts/test_run_views_snapshot.py +++ b/tests/contracts/test_run_views_snapshot.py @@ -23,7 +23,15 @@ SNAPSHOT = Path(__file__).resolve().parents[1] / "fixtures/contracts/run-views.snapshot.json" OPERATION_IDS = ("run_summaries.list", "runs.steps", "runs.logs") # Pages embed their item schemas (RunSummary, StepFact, StepFactWithOutput, RunLogEntry) under $defs. -SCHEMAS = ("run-summary-page", "run-summary-query", "step-fact-page", "run-step-query", "run-log-page", "run-log-query") +SCHEMAS = ( + "run-summary-page", + "run-summary-query", + "run-list-query", + "step-fact-page", + "run-step-query", + "run-log-page", + "run-log-query", +) def render() -> str: @@ -79,6 +87,7 @@ def test_snapshot_documents_each_operation_exactly(): "business_key", "correlation_key", "has_active_incident", + "has_handled_errors", "include_archived", "activation_id", "order", @@ -109,9 +118,9 @@ def test_snapshot_documents_each_operation_exactly(): } assert sorted(document["schemas"]) == sorted(SCHEMAS) summary = document["schemas"]["run-summary-page"]["$defs"]["RunSummary"] - assert sorted(summary["required"]) == sorted(summary["properties"]) + assert set(summary["properties"]) - set(summary["required"]) == {"handled_errors"} steps = document["schemas"]["step-fact-page"]["$defs"] - assert set(steps["StepFact"]["properties"]) - set(steps["StepFact"]["required"]) == {"omissions"} + assert set(steps["StepFact"]["properties"]) - set(steps["StepFact"]["required"]) == {"omissions", "handled"} assert set(steps["StepFactWithOutput"]["required"]) - set(steps["StepFact"]["required"]) == {"output"} diff --git a/tests/fixtures/contracts/run-views.snapshot.json b/tests/fixtures/contracts/run-views.snapshot.json index 97026ef6..db185c12 100644 --- a/tests/fixtures/contracts/run-views.snapshot.json +++ b/tests/fixtures/contracts/run-views.snapshot.json @@ -18,6 +18,7 @@ "business_key", "correlation_key", "has_active_incident", + "has_handled_errors", "include_archived", "activation_id", "order", @@ -92,6 +93,282 @@ } }, "schemas": { + "run-list-query": { + "$defs": { + "BusinessKey": { + "maxLength": 200, + "type": "string" + }, + "Constraint_00404e686415370f": { + "type": "string" + }, + "Cursor": { + "maxLength": 2048, + "minLength": 1, + "pattern": "^[A-Za-z0-9_-]+$", + "type": "string" + }, + "ResourceName": { + "allOf": [ + { + "$ref": "#/$defs/Constraint_00404e686415370f" + }, + { + "pattern": "^[A-Za-z0-9][A-Za-z0-9_.-]*$", + "type": "string" + } + ] + }, + "RunListOrder": { + "enum": [ + "id", + "started_desc", + "started_asc", + "updated_desc" + ], + "type": "string" + }, + "RunOrigin": { + "enum": [ + "manual", + "webhook", + "schedule", + "broker", + "email", + "provider", + "retry", + "call", + "test" + ], + "type": "string" + }, + "RunStatus": { + "enum": [ + "queued", + "running", + "waiting", + "suspended", + "succeeded", + "failed", + "cancelled", + "timed_out" + ], + "type": "string" + }, + "SemVer": { + "allOf": [ + { + "$ref": "#/$defs/Constraint_00404e686415370f" + }, + { + "pattern": "^(?:0|[1-9][0-9]*)\\.(?:0|[1-9][0-9]*)\\.(?:0|[1-9][0-9]*)(?:-(?:(?:0|[1-9][0-9]*|[0-9]*[A-Za-z-][0-9A-Za-z-]*))(?:\\.(?:0|[1-9][0-9]*|[0-9]*[A-Za-z-][0-9A-Za-z-]*))*)?(?:\\+[0-9A-Za-z-]+(?:\\.[0-9A-Za-z-]+)*)?$", + "type": "string" + } + ] + }, + "StatusFilter": { + "items": { + "$ref": "#/$defs/RunStatus" + }, + "maxItems": 8, + "type": "array" + }, + "SummaryLimit": { + "maximum": 100, + "minimum": 1, + "type": "integer" + } + }, + "$id": "run-list-query.schema.json", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "description": "`runs.list` filters, with ID order retained for existing callers.", + "properties": { + "activation_id": { + "anyOf": [ + { + "format": "uuid", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Activation Id" + }, + "business_key": { + "anyOf": [ + { + "$ref": "#/$defs/BusinessKey" + }, + { + "type": "null" + } + ], + "default": null + }, + "caller_run_id": { + "anyOf": [ + { + "format": "uuid", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Caller Run Id" + }, + "correlation_key": { + "anyOf": [ + { + "$ref": "#/$defs/BusinessKey" + }, + { + "type": "null" + } + ], + "default": null + }, + "cursor": { + "anyOf": [ + { + "$ref": "#/$defs/Cursor" + }, + { + "type": "null" + } + ], + "default": null + }, + "has_active_incident": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Has Active Incident" + }, + "has_handled_errors": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Has Handled Errors" + }, + "include_archived": { + "default": false, + "title": "Include Archived", + "type": "boolean" + }, + "include_test": { + "default": false, + "title": "Include Test", + "type": "boolean" + }, + "limit": { + "$ref": "#/$defs/SummaryLimit", + "default": 50 + }, + "order": { + "$ref": "#/$defs/RunListOrder", + "default": "id" + }, + "origin": { + "anyOf": [ + { + "$ref": "#/$defs/RunOrigin" + }, + { + "type": "null" + } + ], + "default": null + }, + "retried_from_run_id": { + "anyOf": [ + { + "format": "uuid", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Retried From Run Id" + }, + "started_after": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Started After" + }, + "started_before": { + "anyOf": [ + { + "format": "date-time", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Started Before" + }, + "status": { + "$ref": "#/$defs/StatusFilter" + }, + "top_level_only": { + "default": false, + "title": "Top Level Only", + "type": "boolean" + }, + "version": { + "anyOf": [ + { + "$ref": "#/$defs/SemVer" + }, + { + "type": "null" + } + ], + "default": null + }, + "workflow": { + "anyOf": [ + { + "$ref": "#/$defs/ResourceName" + }, + { + "type": "null" + } + ], + "default": null + } + }, + "title": "RunListQuery", + "type": "object" + }, "run-log-page": { "$defs": { "Constraint_00404e686415370f": { @@ -553,6 +830,10 @@ "title": "FailedStep", "type": "object" }, + "HandledErrorCount": { + "minimum": 0, + "type": "integer" + }, "Milliseconds": { "minimum": 0, "type": "integer" @@ -702,6 +983,10 @@ } ] }, + "handled_errors": { + "$ref": "#/$defs/HandledErrorCount", + "default": 0 + }, "id": { "format": "uuid", "title": "Id", @@ -1085,6 +1370,18 @@ "default": null, "title": "Has Active Incident" }, + "has_handled_errors": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Has Handled Errors" + }, "include_archived": { "default": false, "title": "Include Archived", @@ -1344,6 +1641,17 @@ } ] }, + "handled": { + "anyOf": [ + { + "$ref": "#/$defs/StepHandledFailure" + }, + { + "type": "null" + } + ], + "default": null + }, "instance_key": { "maxLength": 512, "pattern": "^(?:@?[A-Za-z0-9][A-Za-z0-9_.:@-]*(?:\\[(?:0|[1-9][0-9]{0,14}|[1-8][0-9]{15}|900[0-6][0-9]{12}|90070[0-9]{11}|90071[0-8][0-9]{10}|900719[0-8][0-9]{9}|9007199[0-1][0-9]{8}|90071992[0-4][0-9]{7}|900719925[0-3][0-9]{6}|9007199254[0-6][0-9]{5}|90071992547[0-3][0-9]{4}|9007199254740[0-8][0-9]{2}|90071992547409[0-8][0-9]|9007199254740990|9007199254740991)\\])*(?:#[A-Za-z0-9]+(?:\\.[A-Za-z0-9]+)*)?(?:~(?:[1-9][0-9]{0,14}|[1-8][0-9]{15}|900[0-6][0-9]{12}|90070[0-9]{11}|90071[0-8][0-9]{10}|900719[0-8][0-9]{9}|9007199[0-1][0-9]{8}|90071992[0-4][0-9]{7}|900719925[0-3][0-9]{6}|9007199254[0-6][0-9]{5}|90071992547[0-3][0-9]{4}|9007199254740[0-8][0-9]{2}|90071992547409[0-8][0-9]|9007199254740990|9007199254740991))?)?$", @@ -1496,6 +1804,17 @@ } ] }, + "handled": { + "anyOf": [ + { + "$ref": "#/$defs/StepHandledFailure" + }, + { + "type": "null" + } + ], + "default": null + }, "instance_key": { "maxLength": 512, "pattern": "^(?:@?[A-Za-z0-9][A-Za-z0-9_.:@-]*(?:\\[(?:0|[1-9][0-9]{0,14}|[1-8][0-9]{15}|900[0-6][0-9]{12}|90070[0-9]{11}|90071[0-8][0-9]{10}|900719[0-8][0-9]{9}|9007199[0-1][0-9]{8}|90071992[0-4][0-9]{7}|900719925[0-3][0-9]{6}|9007199254[0-6][0-9]{5}|90071992547[0-3][0-9]{4}|9007199254740[0-8][0-9]{2}|90071992547409[0-8][0-9]|9007199254740990|9007199254740991)\\])*(?:#[A-Za-z0-9]+(?:\\.[A-Za-z0-9]+)*)?(?:~(?:[1-9][0-9]{0,14}|[1-8][0-9]{15}|900[0-6][0-9]{12}|90070[0-9]{11}|90071[0-8][0-9]{10}|900719[0-8][0-9]{9}|9007199[0-1][0-9]{8}|90071992[0-4][0-9]{7}|900719925[0-3][0-9]{6}|9007199254[0-6][0-9]{5}|90071992547[0-3][0-9]{4}|9007199254740[0-8][0-9]{2}|90071992547409[0-8][0-9]|9007199254740990|9007199254740991))?)?$", @@ -1601,6 +1920,13 @@ "title": "StepFactWithOutput", "type": "object" }, + "StepHandledFailure": { + "enum": [ + "continue", + "errorOutput" + ], + "type": "string" + }, "StepKind": { "enum": [ "action", diff --git a/tests/unit/operations/test_fact_positions.py b/tests/unit/operations/test_fact_positions.py new file mode 100644 index 00000000..c4ed64b6 --- /dev/null +++ b/tests/unit/operations/test_fact_positions.py @@ -0,0 +1,142 @@ +# Copyright 2026 Firefly Software Foundation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# Author: Firefly Software Foundation +# SPDX-License-Identifier: Apache-2.0 + +"""Typed run and step cursor boundaries and transport limits.""" + +import base64 +import json +import sys +from dataclasses import FrozenInstanceError +from datetime import UTC, datetime +from uuid import UUID + +import pytest + +from firefly_weave.api.transport import decode_cursor_v2, encode_cursor_v2 +from firefly_weave.contracts.access import Scope +from firefly_weave.contracts.run_views import RunSummaryQuery +from firefly_weave.operations.fact_positions import run_position, step_position + +RUN = "00000000-0000-0000-0000-000000000001" +SCOPE = Scope(tenant_id=UUID(RUN), project_id=UUID(RUN), environment_id=UUID(RUN)) + + +@pytest.mark.parametrize( + "value", + [ + 3, + True, + {}, + [], + ["time"], + "2026-10-09", + "2026-13-09T00:00:00Z", + "2026-10-09 00:00:00Z", + "2026-10-09T00:00:00", + "2026-10-09T00:00:00+00:60", + "9999-12-31T23:59:59-01:00", + "0001-01-01T00:00:00+00:01", + ], +) +def test_run_cursor_wrong_sql_type_is_validation(value): + with pytest.raises(ValueError): + run_position((value, RUN)) + + +@pytest.mark.parametrize("identifier", [None, 1, True, {}, [], "bad", ""]) +def test_run_cursor_uuid_must_be_a_string(identifier): + with pytest.raises(ValueError): + run_position((None, identifier)) + + +def test_positions_normalize_times_preserve_null_and_are_immutable(): + assert run_position(None) is None + assert step_position(None) is None + run = run_position(("2026-10-09T02:00:00+02:00", RUN)) + assert (run.at, run.id) == (datetime(2026, 10, 9, tzinfo=UTC), UUID(RUN)) + with pytest.raises(FrozenInstanceError): + run.at = None + assert run_position((None, RUN)).at is None + step = step_position(([None, "send", "send[3]#2~1"], "send[3]#2~1")) + assert (step.scheduled_at, step.node_id, step.instance_key) == (None, "send", "send[3]#2~1") + with pytest.raises(FrozenInstanceError): + step.node_id = "other" + plain = step_position((["2026-10-09T02:00:00+02:00", "send", ""], "send")) + assert (plain.scheduled_at, plain.instance_key) == (datetime(2026, 10, 9, tzinfo=UTC), "") + + +@pytest.mark.parametrize( + "value,identifier", + [ + ([None, "send", "other[3]"], "send[3]"), + ([None, "send", "send[3]"], "send"), + ([None, "send", "send"], "send"), + ([None, "@join:send", ""], "@join:send"), + ([None, "send", "send[03]"], "send[03]"), + ([None, 3, ""], "3"), + ([None, True, ""], "true"), + ([None, {}, ""], "x"), + ([None, [], ""], "x"), + ([None, "send", 3], "x"), + ([None, "send", True], "x"), + ([None, "send", {}], "x"), + ([None, "send", []], "x"), + ([True, "send", ""], "send"), + ([3, "send", ""], "send"), + ([{}, "send", ""], "send"), + ([[], "send", ""], "send"), + (None, "send"), + ({}, "send"), + ([], "send"), + ([None, "send"], "send"), + ([None, "send", "", "extra"], "send"), + ([None, "send", ""], None), + ], +) +def test_step_cursor_shape_and_identity_are_validated(value, identifier): + with pytest.raises(ValueError): + step_position((value, identifier)) + + +def test_cursor_encoded_length_boundary_is_enforced_both_ways(): + collection = "runs" + # A 1536-byte JSON envelope produces exactly 2048 base64url characters. + envelope = json.dumps([2, SCOPE.model_dump(mode="json"), collection, "", "x"], separators=(",", ":")) + value = "x" * (1536 - len(envelope.encode())) + cursor = encode_cursor_v2(SCOPE, collection, value, "x") + assert len(cursor) == 2048 + assert decode_cursor_v2(cursor, SCOPE, collection) == (value, "x") + with pytest.raises(ValueError): + encode_cursor_v2(SCOPE, collection, value + "x", "x") + with pytest.raises(ValueError): + decode_cursor_v2(cursor + "A", SCOPE, collection) + assert RunSummaryQuery(cursor="A" * 2048).cursor is not None + with pytest.raises(ValueError): + RunSummaryQuery(cursor="A" * 2049) + + +def test_deep_json_cursor_is_a_validation_error(): + # Stay below the encoded length cap even when the caller has a smaller recursion budget. + raw = "[2," + json.dumps(SCOPE.model_dump(mode="json")) + ',"runs",' + "[" * 600 + "0" + "]" * 600 + ',"x"]' + cursor = base64.urlsafe_b64encode(raw.encode()).decode().rstrip("=") + assert len(cursor) <= 2048 + previous = sys.getrecursionlimit() + try: + sys.setrecursionlimit(350) + with pytest.raises(ValueError, match="Invalid scope-bound cursor"): + decode_cursor_v2(cursor, SCOPE, "runs") + finally: + sys.setrecursionlimit(previous) From ee93fb2d48048a695fdb5bf341da373f67fed6ca Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Fri, 9 Oct 2026 03:37:43 -0700 Subject: [PATCH 02/63] Persist scoped operational fact tables with accounted backfill --- docs/operations/upgrades.md | 34 + migrations/versions/0031_operations_facts.py | 585 +++++++++++ src/firefly_weave/operations/facts.py | 59 ++ src/firefly_weave/persistence/migrations.py | 2 +- tests/integration/conftest.py | 53 + tests/integration/operations_support.py | 749 ++++++++++++++ .../integration/test_large_row_accounting.py | 19 + .../test_operations_facts_migration.py | 922 ++++++++++++++++++ 8 files changed, 2422 insertions(+), 1 deletion(-) create mode 100644 migrations/versions/0031_operations_facts.py create mode 100644 src/firefly_weave/operations/facts.py create mode 100644 tests/integration/operations_support.py create mode 100644 tests/integration/test_operations_facts_migration.py diff --git a/docs/operations/upgrades.md b/docs/operations/upgrades.md index db895ab4..8e5083d8 100644 --- a/docs/operations/upgrades.md +++ b/docs/operations/upgrades.md @@ -45,6 +45,7 @@ the desktop app never migrates a platform. | alpha13 | `0030_worker_presence` | No new schema migration. Workers retry explicit capacity rejections while they read task context or credentials, and an Agentic preparation timeout before provider execution counts as not started. Upgrade the Agentic and Files workers to 0.1.5 together with the server. | | alpha14 | `0030_worker_presence` | No new schema migration. Adds the detached Docker development platform (`weave platform up`); existing foreground installations keep working and are never converted. The Agentic and Files workers 0.1.6 pin this server version. | | alpha15 | `0030_worker_presence` | No new schema migration and no role change. Upgrade CLIs and SDKs together with the server, review native executor `capacity` and the legacy private-network settings before restarting, and finish runs that use text templates before any rollback to alpha14; see [what changes in alpha15](#what-changes-in-alpha15). The Agentic and Files workers 0.1.7 pin this server version. | +| Unreleased | `0031_operations_facts` | Adds [scoped operational fact storage](#operational-fact-storage), a bounded historical backfill, and reserved capacity for terminal fact updates. | ![Schema, compatibility and execution acceptance gates](../diagrams/operations-upgrade.svg) @@ -147,6 +148,39 @@ policy update, and this release has no in-place policy-change operation: a mismatch leaves the runtime restricted. Never rewrite the fingerprint or counters by hand to force readiness. +## Operational fact storage + +The `0031_operations_facts` migration adds five scoped operational projections +and two bounded traversal cursors. It does not add columns to stored runs or +events. Apply it explicitly with all writers stopped, using the procedure above. + +The migration holds schema locks in one transaction and reads retained runs in +UUID order, at most 10,000 IDs per batch. Its history work is proportional to the +number of runs and events. Classification fetches one bounded run at a time; +event timestamps are aggregated per batch. Allow temporary disk space for all +five projections, their indexes, and transaction/WAL overhead. Rehearse against a +restored copy to measure the maintenance window: measured 10,000-, 200,000- and +1,000,000-run deployment durations are not yet available. + +Historical run times come from accepted events. Missing, malformed or oversized +evidence remains unavailable, and malformed lifecycle values remain unknown +instead of being labeled as queued. A run requiring an unsupported IR version or +language feature keeps that separate classification. Historical origins remain +unknown unless an explicit start, retry or matching start receipt proves them. +Signal delivery receipts do not establish how a run started. Historical step +and task times are not invented, and older workers have an unknown registration +time and empty identity until observed. Each run projection records when its +coverage began. + +The new facts count toward ordinary storage usage. Active executions reserve +additional capacity for bounded terminal fact updates, including allocation +provenance. Backfill inventories retained data without granting extra admission +capacity: a platform already above a quota can still migrate, but new work must +satisfy the unchanged quotas. Purging an authorized archived run also removes +its run, step, task and incident facts; deleting a worker removes its facts. +The two cursor tables hold only one row per tenant or environment and have no +business payload or retained history. + ## Start and inspect compatibility Remove the migration credentials from the runtime environment, then start the diff --git a/migrations/versions/0031_operations_facts.py b/migrations/versions/0031_operations_facts.py new file mode 100644 index 00000000..a541d9f9 --- /dev/null +++ b/migrations/versions/0031_operations_facts.py @@ -0,0 +1,585 @@ +# Copyright 2026 Firefly Software Foundation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# Author: Firefly Software Foundation +# SPDX-License-Identifier: Apache-2.0 + +"""Scoped operational projections, bounded retained-history backfill and accounted control capacity.""" + +import json +from typing import Any +from uuid import UUID + +import sqlalchemy as sa +from alembic import op +from pydantic import TypeAdapter + +from firefly_weave.compiler.api import import_artifact +from firefly_weave.contracts.catalog import Activation +from firefly_weave.contracts.definitions import ResourceName +from firefly_weave.contracts.instance_keys import InstanceKeyText, instance_view +from firefly_weave.contracts.runtime import StartRunRequest +from firefly_weave.runtime.admission import POLICY, admission +from firefly_weave.runtime.models import RunState + +revision = "0031_operations_facts" +down_revision = "0030_worker_presence" + +SCOPE_COLUMNS = "tenant_id uuid NOT NULL,project_id uuid NOT NULL,environment_id uuid NOT NULL" +SCOPE_KEY = "tenant_id,project_id,environment_id" +SCOPE = "tenant_id=:tenant AND project_id=:project AND environment_id=:environment" +EXACT = " AND ".join( + f"{name}=nullif(current_setting('weave.{name}',true),'')::uuid" + for name in ("tenant_id", "project_id", "environment_id") +) +TABLE_KEYS = { + "run_facts": ("run_id",), + "step_facts": ("run_id", "node_id", "instance_key"), + "task_facts": ("task_id",), + "incident_facts": ("incident_id",), + "worker_facts": ("worker_id",), +} +STATUSES = "'queued','running','waiting','suspended','succeeded','failed','cancelled','timed_out'" +TERMINAL = "'succeeded','failed','cancelled','timed_out'" +ORIGINS = "'manual','webhook','schedule','broker','email','provider','retry','call','test'" +BATCH_SIZE = 10000 +FETCH_BYTES = 128 * 1024 * 1024 + + +def secure(table: str, key: tuple[str, ...]) -> None: + op.execute(f"ALTER TABLE {table} ENABLE ROW LEVEL SECURITY") + op.execute(f"ALTER TABLE {table} FORCE ROW LEVEL SECURITY") + op.execute(f"CREATE POLICY exact_scope ON {table} TO weave_app USING ({EXACT}) WITH CHECK ({EXACT})") + op.execute(f"GRANT SELECT,INSERT,UPDATE ON {table} TO weave_app") + op.execute(f"GRANT SELECT,DELETE ON {table} TO weave_retention_owner") + op.execute(f"CREATE POLICY operations_inventory ON {table} TO weave_retention_owner USING(true)") + arguments = ",".join("'" + name + "'" for name in key) + op.execute( + f"CREATE TRIGGER operation_usage AFTER INSERT OR UPDATE OR DELETE ON {table} " + f"FOR EACH ROW EXECUTE FUNCTION weave_usage_charge({arguments})" + ) + + +def tables() -> None: + run_parent = f"FOREIGN KEY({SCOPE_KEY},run_id) REFERENCES runs({SCOPE_KEY},id) ON DELETE CASCADE" + op.execute(f"""CREATE TABLE run_facts( + {SCOPE_COLUMNS},run_id uuid PRIMARY KEY,definition_version_id uuid NOT NULL, + workflow_name text NOT NULL,workflow_version text NOT NULL,activation_id uuid NOT NULL, + activation_name text NOT NULL,activation_revision integer NOT NULL CHECK(activation_revision>0), + status text,paused boolean NOT NULL DEFAULT false, + test boolean NOT NULL DEFAULT false,origin text CHECK(origin IN ({ORIGINS})), + caller_run_id uuid,caller_node_id text,caller_instance_key text,retried_from_run_id uuid, + business_key text,correlation_key text,started_at timestamptz,updated_at timestamptz,ended_at timestamptz, + failed_node_id text,failed_instance_key text,failed_error_code text, + active_incidents bigint NOT NULL DEFAULT 0 CHECK(active_incidents>=0), + handled_errors bigint NOT NULL DEFAULT 0 CHECK(handled_errors>=0), + classification_state text NOT NULL CHECK(classification_state IN ('available','unavailable','unsupported')), + classification_policy text NOT NULL,pinned_ir_version text, + CHECK((status IS NULL AND classification_state='unavailable') OR + (status IS NOT NULL AND status IN ({STATUSES}))), + pinned_features jsonb NOT NULL DEFAULT '[]',node_kinds jsonb NOT NULL DEFAULT '{{}}', + last_event_sequence bigint NOT NULL DEFAULT 0,facts_started_at timestamptz NOT NULL DEFAULT clock_timestamp(), + CHECK(CASE WHEN jsonb_typeof(pinned_features)='array' THEN + jsonb_array_length(pinned_features)<=256 AND octet_length(pinned_features::text)<=16384 AND + NOT jsonb_path_exists(pinned_features,'$[*] ? (@.type() != "string")') ELSE false END), + CHECK(CASE WHEN jsonb_typeof(node_kinds)='object' THEN octet_length(node_kinds::text)<=1048576 AND + jsonb_array_length(jsonb_path_query_array(node_kinds,'$.keyvalue()'))<=10000 AND + NOT jsonb_path_exists(node_kinds,'$.keyvalue().value ? (@.type() != "string")') ELSE false END), + {run_parent})""") + op.execute(f"""CREATE TABLE step_facts( + {SCOPE_COLUMNS},run_id uuid NOT NULL,node_id text NOT NULL,instance_key text NOT NULL DEFAULT '', + kind text NOT NULL,status text NOT NULL,scheduled_at timestamptz,started_at timestamptz,ended_at timestamptz, + attempts bigint NOT NULL DEFAULT 0 CHECK(attempts>=0),worker_id uuid,error_code text, + handled text CHECK(handled IN ('continue','errorOutput')),child_run_id uuid, + PRIMARY KEY(run_id,node_id,instance_key),CHECK(handled IS NULL OR status='failed'),{run_parent})""") + op.execute(f"""CREATE TABLE task_facts( + {SCOPE_COLUMNS},task_id uuid PRIMARY KEY,run_id uuid NOT NULL,node_id text NOT NULL, + instance_key text NOT NULL DEFAULT '',task_type text NOT NULL,status text NOT NULL, + created_at timestamptz NOT NULL,ready_since timestamptz,first_claimed_at timestamptz, + FOREIGN KEY({SCOPE_KEY},task_id) REFERENCES task_intents({SCOPE_KEY},id) ON DELETE CASCADE,{run_parent})""") + op.execute(f"""CREATE TABLE incident_facts( + {SCOPE_COLUMNS},incident_id uuid PRIMARY KEY,node_id text,instance_key text NOT NULL DEFAULT '', + opened_at timestamptz,updated_at timestamptz,workflow_name text NOT NULL,workflow_version text NOT NULL, + test boolean NOT NULL DEFAULT false,opened_at_estimated boolean NOT NULL DEFAULT false, + FOREIGN KEY({SCOPE_KEY},incident_id) REFERENCES incidents({SCOPE_KEY},id) ON DELETE CASCADE)""") + op.execute(f"""CREATE TABLE worker_facts( + {SCOPE_COLUMNS},worker_id uuid PRIMARY KEY,registered_at timestamptz,identity jsonb NOT NULL DEFAULT '{{}}', + drained_at timestamptz,CHECK(octet_length(identity::text)<=4096), + FOREIGN KEY({SCOPE_KEY},worker_id) REFERENCES worker_instances({SCOPE_KEY},id) ON DELETE CASCADE)""") + op.execute( + "CREATE TABLE operations_environment_cursors(tenant_id uuid PRIMARY KEY REFERENCES " + "tenants(id),environment_id uuid)" + ) + tenant = "tenant_id=nullif(current_setting('weave.tenant_id',true),'')::uuid" + op.execute("ALTER TABLE operations_environment_cursors ENABLE ROW LEVEL SECURITY") + op.execute("ALTER TABLE operations_environment_cursors FORCE ROW LEVEL SECURITY") + op.execute( + f"CREATE POLICY tenant_scope ON operations_environment_cursors TO weave_app USING " + f"({tenant}) WITH CHECK ({tenant})" + ) + op.execute("GRANT SELECT,INSERT,UPDATE ON operations_environment_cursors TO weave_app") + op.execute(f"""CREATE TABLE operations_fact_cursors({SCOPE_COLUMNS},after_run_id uuid, + PRIMARY KEY({SCOPE_KEY}),FOREIGN KEY({SCOPE_KEY}) REFERENCES environments(tenant_id,project_id,id) + ON DELETE CASCADE)""") + op.execute("ALTER TABLE operations_fact_cursors ENABLE ROW LEVEL SECURITY") + op.execute("ALTER TABLE operations_fact_cursors FORCE ROW LEVEL SECURITY") + op.execute( + f"CREATE POLICY exact_scope ON operations_fact_cursors TO weave_app USING ({EXACT}) WITH CHECK ({EXACT})" + ) + op.execute("GRANT SELECT,INSERT,UPDATE ON operations_fact_cursors TO weave_app") + + +def indexes() -> None: + for name, columns, condition in ( + ("started", "started_at DESC,run_id DESC", ""), + ("workflow", "workflow_name,started_at DESC,run_id DESC", ""), + ("status", "status,started_at DESC,run_id DESC", ""), + ("activation", "activation_id,started_at DESC,run_id DESC", ""), + ("updated", "updated_at DESC,run_id DESC", ""), + ("business_key", "business_key", ""), + ("correlation_key", "correlation_key", ""), + ("caller", "caller_run_id", "WHERE caller_run_id IS NOT NULL"), + ("retry", "retried_from_run_id", "WHERE retried_from_run_id IS NOT NULL"), + ("ended", "ended_at", "WHERE ended_at IS NOT NULL"), + ("active_test", "", "WHERE test AND ended_at IS NULL"), + ): + op.execute( + f"CREATE INDEX run_facts_{name} ON run_facts({SCOPE_KEY}{',' + columns if columns else ''}) {condition}" + ) + op.execute(f"CREATE INDEX step_facts_ended ON step_facts({SCOPE_KEY},ended_at)") + op.execute( + f"CREATE INDEX task_facts_ready ON task_facts({SCOPE_KEY},task_type,ready_since) WHERE ready_since IS NOT NULL" + ) + op.execute(f"CREATE INDEX incident_facts_node_opened ON incident_facts({SCOPE_KEY},node_id,opened_at,incident_id)") + op.execute(f"CREATE INDEX run_deadlines_operations_due ON run_deadlines({SCOPE_KEY},deadline) WHERE NOT consumed") + op.execute( + f"CREATE INDEX event_deliveries_operations_due ON event_deliveries({SCOPE_KEY},next_at) " + f"WHERE status IN ('pending','retry')" + ) + op.execute(f"CREATE INDEX incidents_operations_active ON incidents({SCOPE_KEY},status) WHERE status='active'") + + +def patch_function(signature: str, anchor: str, replacement: str) -> None: + connection = op.get_bind() + body = connection.scalar( + sa.text("SELECT pg_get_functiondef(cast(:signature AS regprocedure))"), {"signature": signature} + ) + if not isinstance(body, str) or body.count(anchor) != 1: + raise RuntimeError(f"Unexpected definition of {signature}; reconcile the control-accounting migration") + connection.exec_driver_sql(body.replace(anchor, replacement)) + + +def control_accounting() -> None: + anchor = "IF relation_name='runs' THEN RETURN (item->>'id')::uuid; END IF;" + patch_function( + "public.weave_control_target(text,jsonb)", + anchor, + """ + IF relation_name IN ('run_facts','step_facts','task_facts') THEN + RETURN (item->>'run_id')::uuid; + END IF; + IF relation_name='incident_facts' THEN + SELECT run_id INTO identifier FROM public.incidents WHERE id=(item->>'incident_id')::uuid; + RETURN identifier; + END IF; + """ + + anchor, + ) + anchor = "+(item->>'_weave_activation_bytes')::numeric+16777216+64*2048+65536+576*affected;" + predicate = "tenant_id=tenant AND project_id=project AND environment_id=(item->>'environment_id')::uuid" + patch_function( + "public.weave_reserve_refresh(text,uuid,boolean)", + anchor, + anchor[:-1] + + f""" + +2048+2048*((SELECT count(*) FROM public.step_facts WHERE {predicate} AND run_id=identifier) + +(SELECT count(*) FROM public.task_facts WHERE {predicate} AND run_id=identifier) + +(SELECT count(*) FROM public.incident_facts f JOIN public.incidents i ON + i.tenant_id=f.tenant_id AND i.project_id=f.project_id AND i.environment_id=f.environment_id + AND i.id=f.incident_id WHERE f.tenant_id=tenant AND f.project_id=project + AND f.environment_id=(item->>'environment_id')::uuid AND i.run_id=identifier));""", + ) + anchor = "ELSIF relation_name IN ('task_intents','task_leases','incidents','run_deadlines') THEN" + patch_function( + "public.weave_reservation_touch(text,jsonb)", + anchor, + "ELSIF relation_name IN ('task_intents','task_leases','incidents','run_deadlines'," + "'run_facts','step_facts','task_facts','incident_facts') THEN", + ) + + +def node_kinds(artifact: Any) -> dict[str, str]: + """Freeze author-step eligibility for the retained projection independently of live writers.""" + result = {} + for node in artifact.executable["graph"]["nodes"]: + key = node["id"] + if isinstance(key, str) and key.startswith("@"): + continue + TypeAdapter(InstanceKeyText).validate_python(key) + view = instance_view(key) + TypeAdapter(ResourceName).validate_python(view.node_id) + result[view.node_id] = node["kind"] + if len(result) > 10000 or len(json.dumps(result, ensure_ascii=False).encode()) > 1048576: + raise ValueError("Node-kind projection exceeds its bound") + return result + + +def classification(row: dict[str, Any]) -> dict[str, Any]: + result = { + "classification_state": "unavailable", + "classification_policy": POLICY, + "pinned_ir_version": None, + "pinned_features": "[]", + "node_kinds": "{}", + } + try: + RunState.model_validate_json(json.dumps(row["state"])) + Activation.model_validate_json(json.dumps(row["activation"])) + StartRunRequest.model_validate_json(json.dumps(row["request"])) + decision = admission(row) + executable = row["artifact"].get("executable", {}) + version, features = executable.get("irVersion"), executable.get("features", []) + if not isinstance(version, str) or len(version) > 256 or not isinstance(features, list): + return result + if len(features) > 256 or any(type(feature) is not str for feature in features): + return result + encoded = json.dumps(features, ensure_ascii=False) + if len(encoded.encode()) > 16384: + return result + result.update(classification_state=decision, pinned_ir_version=version, pinned_features=encoded) + if decision == "available": + result["node_kinds"] = json.dumps(node_kinds(import_artifact(row["artifact"])), ensure_ascii=False) + except (ValueError, TypeError, KeyError, AttributeError, RecursionError): + result.update(classification_state="unavailable", node_kinds="{}") + return result + + +EVENTS = f"""WITH ordered AS ( + SELECT e.*,lag(created_at) OVER(PARTITION BY run_id ORDER BY sequence) AS previous_at, + public.weave_run_events_bytes(e)<={FETCH_BYTES} AS bounded + FROM run_events e WHERE {SCOPE} AND run_id=ANY(:ids) +) SELECT run_id,min(created_at) AS started_at,max(created_at) AS updated_at, + max(created_at) FILTER(WHERE transition->'state'->>'status' IN ({TERMINAL})) AS ended_at, + min(sequence) AS first_sequence,max(sequence) AS last_event_sequence,count(*) AS event_count, + bool_and(coalesce(bounded AND jsonb_typeof(data)='object' AND jsonb_typeof(transition->'state')='object' + AND transition->'state'->>'status' IN ({STATUSES}) + AND transition->'state'->>'accepted_sequence'=sequence::text + AND (previous_at IS NULL OR created_at>=previous_at),false)) AS coherent, + min(created_at) FILTER(WHERE sequence=1 AND type='started') AS first_started, + min(data->>'origin') FILTER(WHERE sequence=1 AND type='started' AND data->>'origin' IN ({ORIGINS})) AS + explicit_origin + FROM ordered GROUP BY run_id""" + + +def receipt_origins(connection, params): + query = [] + for receipt, source, foreign, extra, origin in ( + ("trigger_receipts", "trigger_routes", "trigger_id", "", "webhook"), + ("broker_events", "broker_routes", "trigger_id", "", "broker"), + ("email_receipts", "email_sources", "source_id", "AND r.state='dispatched' AND r.signal_id IS NULL", "email"), + ): + query.append( + f"SELECT r.run_id,'{origin}' AS origin FROM {receipt} r JOIN {source} s " + "ON s.tenant_id=r.tenant_id AND s.project_id=r.project_id AND s.environment_id=r.environment_id " + f"AND s.id=r.{foreign} WHERE r.tenant_id=:tenant AND r.project_id=:project " + "AND r.environment_id=:environment AND r.run_id=ANY(:ids) AND s.activation_id IS NOT NULL " + extra + ) + query.append( + "SELECT run_id,'schedule' AS origin FROM schedule_occurrences WHERE " + + SCOPE + + " AND run_id=ANY(:ids) AND kind='started'" + ) + query.append( + "SELECT cast(r.receipt->>'run_id' AS uuid),'provider' AS origin FROM provider_receipts r " + "JOIN provider_sources s ON s.tenant_id=r.tenant_id AND s.project_id=r.project_id " + "AND s.environment_id=r.environment_id AND s.id=r.source_id " + "WHERE r.tenant_id=:tenant AND r.project_id=:project AND r.environment_id=:environment " + "AND r.receipt->>'run_id'=ANY(:text_ids) AND r.state='dispatched' " + "AND s.activation_id IS NOT NULL AND r.receipt->>'signal_id' IS NULL" + ) + result: dict[UUID, set[str]] = {} + for row in connection.execute( + sa.text(" UNION ALL ".join(query)), {**params, "text_ids": [str(v) for v in params["ids"]]} + ).mappings(): + result.setdefault(row["run_id"], set()).add(row["origin"]) + return result + + +def backfill_incidents(connection, params, events) -> None: + scoped = "i.tenant_id=:tenant AND i.project_id=:project AND i.environment_id=:environment" + rows = connection.execute( + sa.text(f"""WITH evidence AS ( + SELECT run_id,type,data->>'node_id' AS node_id,coalesce(data->>'generation','0') AS generation, + data->>'code' AS code,CASE WHEN type='incident_resolved' THEN data->>'incident_key' + ELSE (data->>'node_id')||':'||coalesce(data->>'generation','0') END AS incident_key, + min(created_at) AS first_at,max(created_at) AS last_at + FROM run_events e WHERE {SCOPE} AND run_id=ANY(:ids) + AND type IN ('incident_opened','incident_resolved') + AND public.weave_run_events_bytes(e)<={FETCH_BYTES} + GROUP BY run_id,type,data->>'node_id',coalesce(data->>'generation','0'),data->>'code', + CASE WHEN type='incident_resolved' THEN data->>'incident_key' + ELSE (data->>'node_id')||':'||coalesce(data->>'generation','0') END + ) SELECT i.*,f.workflow_name,f.workflow_version,f.test, + min(e.first_at) FILTER(WHERE e.type='incident_opened') AS exact_opened, + max(e.last_at) AS last_transition FROM incidents i JOIN run_facts f + ON f.tenant_id=i.tenant_id AND f.project_id=i.project_id + AND f.environment_id=i.environment_id AND f.run_id=i.run_id + LEFT JOIN evidence e ON e.run_id=i.run_id AND e.node_id=i.node_id + AND e.generation=coalesce(i.generation,0)::text AND e.incident_key=i.incident_key + AND (e.type='incident_resolved' OR e.code=i.origin_code) + WHERE {scoped} AND i.run_id=ANY(:ids) + GROUP BY i.id,f.workflow_name,f.workflow_version,f.test"""), + params, + ).mappings() + for row in rows: + node, instance = None, "" + try: + if row["node_id"] is not None and not row["node_id"].startswith("@"): + TypeAdapter(InstanceKeyText).validate_python(row["node_id"]) + view = instance_view(row["node_id"]) + TypeAdapter(ResourceName).validate_python(view.node_id) + node, instance = view.node_id, view.instance_key + except (ValueError, TypeError): + pass + event = events.get(row["run_id"], {}) + opened = row["exact_opened"] or event.get("updated_at") + times = [value for value in (opened, row["last_transition"], row["resolved_at"]) if value is not None] + connection.execute( + sa.text( + "INSERT INTO incident_facts(tenant_id,project_id,environment_id,incident_id," + "node_id,instance_key,opened_at,updated_at,workflow_name,workflow_version,test,opened_at_estimated) " + "VALUES(:tenant,:project,:environment,:incident,:node,:instance,:opened,:updated," + ":name,:version,:test,:estimated)" + ), + { + **params, + "incident": row["id"], + "node": node, + "instance": instance, + "opened": opened, + "updated": max(times) if times else None, + "name": row["workflow_name"], + "version": row["workflow_version"], + "test": row["test"], + "estimated": row["exact_opened"] is None, + }, + ) + + +def backfill_scope(connection, params) -> None: + after = None + while True: + ids = list( + connection.execute( + sa.text( + f"SELECT id FROM runs WHERE {SCOPE} " + "AND (cast(:after AS uuid) IS NULL OR id>:after) ORDER BY id LIMIT 10000" + ), + {**params, "after": after}, + ).scalars() + ) + if not ids: + break + batch = {**params, "ids": ids} + events = {row["run_id"]: dict(row) for row in connection.execute(sa.text(EVENTS), batch).mappings()} + origins = receipt_origins(connection, batch) + identities = connection.execute( + sa.text("""SELECT r.id,a.id AS activation_id,a.version_id AS definition_version_id, + a.name AS activation_name,a.revision AS activation_revision,v.name AS workflow_name,v.version AS + workflow_version, + CASE WHEN jsonb_typeof(r.state->'status')='string' THEN r.state->>'status' END AS status, + r.state->>'manual_paused' AS paused, + CASE WHEN jsonb_typeof(r.request->'business_key')='string' AND + char_length(r.request->>'business_key')<=200 THEN r.request->>'business_key' END AS business_key, + CASE WHEN jsonb_typeof(r.request->'correlation_key')='string' AND + char_length(r.request->>'correlation_key')<=200 THEN r.request->>'correlation_key' + END AS correlation_key, + l.parent_run_id,public.weave_runs_bytes(r) AS bytes, + (SELECT count(*) FROM incidents i WHERE i.tenant_id=r.tenant_id AND i.project_id=r.project_id + AND i.environment_id=r.environment_id AND i.run_id=r.id AND i.status='active') AS active_incidents + FROM runs r JOIN activation_revisions a ON a.tenant_id=r.tenant_id AND a.project_id=r.project_id + AND a.environment_id=r.environment_id AND a.id=r.activation_id + JOIN definition_versions v ON v.tenant_id=a.tenant_id AND v.project_id=a.project_id AND v.id=a.version_id + LEFT JOIN run_retry_links l ON l.tenant_id=r.tenant_id AND l.project_id=r.project_id + AND l.environment_id=r.environment_id AND l.run_id=r.id + WHERE r.tenant_id=:tenant AND r.project_id=:project AND r.environment_id=:environment AND r.id=ANY(:ids) + ORDER BY r.id"""), + batch, + ).mappings() + facts = [] + for identity in identities: + item = dict(identity) + event = events.get(item["id"], {}) + cached = classification({}) + state = None + if item["bytes"] <= FETCH_BYTES: + row = ( + connection.execute( + sa.text(f"SELECT artifact,state,activation,request FROM runs WHERE {SCOPE} AND id=:id"), + {**params, "id": item["id"]}, + ) + .mappings() + .one() + ) + cached = classification(dict(row)) + state = row["state"] + if not ( + event.get("coherent") + and event["first_sequence"] == 1 + and event["last_event_sequence"] == event["event_count"] + and event["first_started"] == event["started_at"] + and isinstance(state, dict) + and state.get("accepted_sequence") == event["last_event_sequence"] + and ( + item["status"] not in ("succeeded", "failed", "cancelled", "timed_out") + or event["ended_at"] is not None + ) + ): + cached.update(classification_state="unavailable", node_kinds="{}") + if item["status"] not in ( + "queued", + "running", + "waiting", + "suspended", + "succeeded", + "failed", + "cancelled", + "timed_out", + ): + item["status"] = None + cached.update(classification_state="unavailable", node_kinds="{}") + event = {**event, "started_at": None, "updated_at": None, "ended_at": None} + matching = origins.get(item["id"], set()) + origin = "retry" if item["parent_run_id"] else event.get("explicit_origin") + if origin is None and len(matching) == 1: + origin = next(iter(matching)) + # A call without its scoped parent identity is not supported historical evidence. + if origin == "call": + origin = None + facts.append( + { + **params, + **item, + **cached, + "origin": origin, + "test": origin == "test", + "paused": item["paused"] == "true", + "started_at": event.get("started_at"), + "updated_at": event.get("updated_at"), + "ended_at": event.get("ended_at") + if item["status"] in ("succeeded", "failed", "cancelled", "timed_out") + else None, + "last_event_sequence": event.get("last_event_sequence", 0), + } + ) + connection.execute( + sa.text("""INSERT INTO run_facts(tenant_id,project_id,environment_id,run_id,definition_version_id, + workflow_name,workflow_version,activation_id,activation_name,activation_revision,status,paused,test,origin, + retried_from_run_id,business_key,correlation_key,started_at,updated_at,ended_at,active_incidents, + classification_state,classification_policy,pinned_ir_version,pinned_features,node_kinds,last_event_sequence) + VALUES(:tenant,:project,:environment,:id,:definition_version_id,:workflow_name,:workflow_version, + :activation_id,:activation_name,:activation_revision,:status,:paused,:test,:origin,:parent_run_id, + :business_key,:correlation_key,:started_at,:updated_at,:ended_at,:active_incidents,:classification_state, + :classification_policy,:pinned_ir_version,cast(:pinned_features AS jsonb),cast(:node_kinds AS + jsonb),:last_event_sequence)"""), + facts, + ) + backfill_incidents(connection, batch, events) + after = ids[-1] + connection.execute( + sa.text( + f"INSERT INTO worker_facts(tenant_id,project_id,environment_id,worker_id) " + f"SELECT tenant_id,project_id,environment_id,id FROM worker_instances WHERE {SCOPE}" + ), + params, + ) + + +def scoped_work(connection, *, refresh: bool = False) -> None: + for tenant in connection.execute(sa.text("SELECT weave_tenant_ids()")).scalars(): + connection.execute(sa.text("SELECT set_config('weave.tenant_id',:value,true)"), {"value": str(tenant)}) + for project in connection.execute( + sa.text("SELECT id FROM projects WHERE tenant_id=:tenant ORDER BY id"), {"tenant": tenant} + ).scalars(): + connection.execute(sa.text("SELECT set_config('weave.project_id',:value,true)"), {"value": str(project)}) + for environment in connection.execute( + sa.text("SELECT id FROM environments WHERE tenant_id=:tenant AND project_id=:project ORDER BY id"), + {"tenant": tenant, "project": project}, + ).scalars(): + connection.execute( + sa.text("SELECT set_config('weave.environment_id',:value,true)"), {"value": str(environment)} + ) + params = {"tenant": tenant, "project": project, "environment": environment} + if not refresh: + backfill_scope(connection, params) + continue + after = None + while True: + ids = list( + connection.execute( + sa.text( + f"SELECT id FROM runs WHERE {SCOPE} AND state->>'status' NOT IN ({TERMINAL}) " + "AND (cast(:after AS uuid) IS NULL OR id>:after) ORDER BY id LIMIT 10000" + ), + {**params, "after": after}, + ).scalars() + ) + if not ids: + break + connection.execute( + sa.text("SELECT weave_reserve_refresh('run',id,false) FROM unnest(cast(:ids AS uuid[])) id"), + {"ids": ids}, + ) + after = ids[-1] + + +def upgrade() -> None: + connection = op.get_bind() + if not connection.scalar(sa.text("SELECT has_function_privilege(current_user,'weave_tenant_ids()','EXECUTE')")): + raise RuntimeError("Migration requires explicit EXECUTE on weave_tenant_ids() for tenant-scoped backfill") + if connection.scalar(sa.text("SELECT to_regclass('public.run_call_links')")) is not None or connection.scalar( + sa.text( + "SELECT EXISTS(SELECT 1 FROM information_schema.columns WHERE table_schema='public' AND " + "table_name='step_instances' AND column_name='handled')" + ) + ): + raise RuntimeError("Reconcile call and handled-failure history with the operational backfill before migrating") + previous = { + name: connection.scalar(sa.text("SELECT current_setting(:name,true)"), {"name": "weave." + name}) + for name in ("tenant_id", "project_id", "environment_id") + } + try: + with connection.begin_nested(): + tables() + indexes() + control_accounting() + scoped_work(connection) + for table, key in TABLE_KEYS.items(): + op.execute(f"""INSERT INTO operation_usage(tenant_id,project_id,environment_id,metric,value) + SELECT tenant_id,project_id,'00000000-0000-0000-0000-000000000000'::uuid,'ordinary_bytes', + sum((public.weave_row_metrics('{table}',to_jsonb(f))->>'ordinary_bytes')::bigint) + FROM {table} f GROUP BY tenant_id,project_id + ON CONFLICT(tenant_id,project_id,environment_id,metric) + DO UPDATE SET value=operation_usage.value+excluded.value""") + secure(table, key) + scoped_work(connection, refresh=True) + op.execute("UPDATE weave_schema_version SET version='0031_operations_facts'") + finally: + for name, value in previous.items(): + connection.execute( + sa.text("SELECT set_config(:name,:value,true)"), {"name": "weave." + name, "value": value or ""} + ) + + +def downgrade() -> None: + raise RuntimeError("Destructive downgrades are not supported") diff --git a/src/firefly_weave/operations/facts.py b/src/firefly_weave/operations/facts.py new file mode 100644 index 00000000..793944a4 --- /dev/null +++ b/src/firefly_weave/operations/facts.py @@ -0,0 +1,59 @@ +# Copyright 2026 Firefly Software Foundation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# Author: Firefly Software Foundation +# SPDX-License-Identifier: Apache-2.0 + +"""Transaction-bound SQL for operational projections.""" + +from dataclasses import dataclass +from typing import Any +from uuid import UUID + +from sqlalchemy import text + +from firefly_weave.contracts.run_views import RunOrigin, RunSummaryCaller +from firefly_weave.persistence.uow import Transaction + +FACT_TABLES: tuple[str, str, str, str, str] = ( + "run_facts", + "step_facts", + "task_facts", + "incident_facts", + "worker_facts", +) + + +@dataclass(frozen=True) +class RunStartFacts: + origin: RunOrigin = "manual" + test: bool = False + caller: RunSummaryCaller | None = None + retried_from_run_id: UUID | None = None + + +class FactRepository: + def __init__(self, tx: Transaction) -> None: + self.tx = tx + self.params = { + "tenant": tx.scope.tenant_id, + "project": tx.scope.project_id, + "environment": tx.scope.environment_id, + } + + async def execute(self, sql: str, **values: Any) -> None: + await self.tx.session.execute(text(sql), {**values, **self.params}) + + async def rows(self, sql: str, **values: Any) -> list[dict[str, Any]]: + result = await self.tx.session.execute(text(sql), {**values, **self.params}) + return [dict(row) for row in result.mappings()] diff --git a/src/firefly_weave/persistence/migrations.py b/src/firefly_weave/persistence/migrations.py index ca7f5bbc..39958643 100644 --- a/src/firefly_weave/persistence/migrations.py +++ b/src/firefly_weave/persistence/migrations.py @@ -29,7 +29,7 @@ from firefly_weave.settings import Settings, operations_from_env -SCHEMA_VERSION = "0030_worker_presence" +SCHEMA_VERSION = "0031_operations_facts" async def check_schema(connection: AsyncConnection) -> None: diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 394b1540..9699bdc6 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -505,3 +505,56 @@ async def replica_apps(authenticated_client): async with second.router.lifespan_context(second): assert first.state.pyfly.context is not second.state.pyfly.context yield first, second + + +@pytest.fixture +async def operations_case(worker_setup, access_db, provisioned): + from types import SimpleNamespace + + from firefly_weave.access.audit import AuditContext + from firefly_weave.contracts.runtime import StartRunRequest + from firefly_weave.persistence.uow import UnitOfWork + + tasks, runtime, workers, actor, scope, activation, instances = worker_setup + sessions, owner, access, url = access_db + uow = UnitOfWork(sessions) + + async def start(key="run"): + return await runtime.start( + actor, scope, StartRunRequest(activation_id=activation.id, input=3), key, context=AuditContext() + ) + + async def rows(sql, **params): + async with uow.open(scope, mutation=False) as tx: + return [ + dict(row) + for row in ( + await tx.session.execute( + text(sql), + { + "tenant": scope.tenant_id, + "project": scope.project_id, + "environment": scope.environment_id, + **params, + }, + ) + ).mappings() + ] + + return SimpleNamespace( + admin=provisioned[0], + tasks=tasks, + runtime=runtime, + workers=workers, + actor=actor, + scope=scope, + activation=activation, + instances=instances, + sessions=sessions, + owner=owner, + access=access, + url=url, + tx=lambda: uow.open(scope), + start=start, + rows=rows, + ) diff --git a/tests/integration/operations_support.py b/tests/integration/operations_support.py new file mode 100644 index 00000000..33d13a64 --- /dev/null +++ b/tests/integration/operations_support.py @@ -0,0 +1,749 @@ +# Copyright 2026 Firefly Software Foundation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# Author: Firefly Software Foundation +# SPDX-License-Identifier: Apache-2.0 + +"""Validated legacy fixtures and independent logical-usage inventory.""" + +import json +import runpy +from collections import defaultdict +from datetime import UTC, datetime, timedelta +from pathlib import Path +from uuid import UUID + +from sqlalchemy import text +from sqlalchemy.ext.asyncio import AsyncConnection + +from firefly_weave.contracts.access import Scope +from firefly_weave.contracts.catalog import Activation, ActivationRequest +from firefly_weave.contracts.runtime import RunView, StartRunRequest +from firefly_weave.runtime.models import RunState, RuntimeEvent, Transition + +FACT_TABLES = ("run_facts", "step_facts", "task_facts", "incident_facts", "worker_facts") +ZERO = UUID(int=0) +AT = datetime(2026, 1, 2, 3, 4, 5, tzinfo=UTC) + + +def identifier(value: int) -> UUID: + return UUID(int=value) + + +async def seed_prior_head(connection: AsyncConnection, *, count: int = 10001) -> dict[str, UUID]: + """Insert validated source models; no runtime service sees the previous schema.""" + fixture = runpy.run_path(str(Path(__file__).resolve().parents[1] / "conftest.py"))[ + "worker_runtime_fixture" + ].__wrapped__() + ids = { + name: identifier(index) + for index, name in enumerate( + ("tenant", "project", "environment", "other_environment", "principal", "version", "activation"), 1 + ) + } + ids.update( + first_run=identifier(1001), + last_run=identifier(1000 + count), + incident=identifier(101), + estimated_incident=identifier(102), + retry=identifier(1003), + active_run=identifier(1002), + no_events=identifier(1004), + malformed=identifier(1005), + unsupported=identifier(1006), + incoherent=identifier(1007), + explicit=identifier(1008), + ) + scope = Scope(tenant_id=ids["tenant"], project_id=ids["project"], environment_id=ids["environment"]) + artifact = fixture["artifact"] + activation = Activation( + id=ids["activation"], + revision=1, + name="worker-flow", + request=ActivationRequest( + version_id=ids["version"], + artifact_digest=artifact.digest, + scope=scope, + ), + ) + await connection.execute(text("INSERT INTO tenants VALUES(:tenant,'facts')"), ids) + await connection.execute(text("INSERT INTO projects VALUES(:project,:tenant,'facts')"), ids) + await connection.execute( + text( + "INSERT INTO environments VALUES(:environment,:tenant,:project,'first')," + "(:other_environment,:tenant,:project,'second')" + ), + ids, + ) + await connection.execute(text("INSERT INTO principals(id,kind) VALUES(:principal,'application')"), ids) + await connection.execute( + text( + "INSERT INTO definition_versions VALUES(:version,:tenant,:project,'Workflow'," + "'worker-flow','1.0.0',:digest,:definition_digest,cast(:document AS jsonb),cast(:artifact AS jsonb))" + ), + { + **ids, + "digest": artifact.digest, + "definition_digest": artifact.source_hash, + "document": fixture["source"], + "artifact": artifact.to_bytes().decode(), + }, + ) + await connection.execute( + text( + "INSERT INTO activation_revisions VALUES(:activation,:tenant,:project,:environment," + "'worker-flow',1,:version,cast(:payload AS jsonb))" + ), + {**ids, "payload": activation.model_dump_json()}, + ) + run_rows, event_rows = [], [] + for index in range(1, count + 1): + run = identifier(1000 + index) + state = RunState( + status="waiting" if index == 2 else "succeeded", + accepted_sequence=1, + admission_policy="classified-v1", + input=3, + ) + request = StartRunRequest( + activation_id=activation.id, input=3, business_key='España 雪 " \\', correlation_key="case-" + str(index) + ) + view = RunView( + id=run, + activation=activation, + artifact_digest=artifact.digest, + state=state, + business_key=request.business_key, + correlation_key=request.correlation_key, + ) + run_rows.append( + { + **ids, + "id": run, + "artifact": artifact.to_bytes().decode(), + "payload": activation.model_dump_json(), + "request": request.model_dump_json(), + "state": state.model_dump_json(), + } + ) + if index != 4: + event = RuntimeEvent( + id=identifier(100000 + index), + type="started", + sequence=1, + timestamp=AT, + data={"origin": "test"} if index == 8 else {"input": 3}, + ) + event_rows.append( + { + **ids, + "run": run, + "id": event.id, + "sequence": 1, + "type": event.type, + "data": json.dumps(event.data), + "at": event.timestamp, + "transition": Transition(state=state).model_dump_json(), + "response": view.model_dump_json(), + } + ) + for offset in range(0, len(run_rows), 1000): + await connection.execute( + text( + "INSERT INTO runs VALUES(:id,:tenant,:project,:environment,:activation,:principal," + "cast(:artifact AS jsonb),cast(:payload AS jsonb),cast(:request AS jsonb),cast(:state AS jsonb))" + ), + run_rows[offset : offset + 1000], + ) + for offset in range(0, len(event_rows), 1000): + await connection.execute( + text( + "INSERT INTO run_events VALUES(:tenant,:project,:environment,:run,:id,:sequence," + ":type,cast(:data AS jsonb),:at,'fixture',cast(:transition AS jsonb),cast(:response AS jsonb))" + ), + event_rows[offset : offset + 1000], + ) + await connection.execute( + text("INSERT INTO run_retry_links VALUES(:tenant,:project,:environment,:retry,:first_run)"), ids + ) + event = RuntimeEvent( + id=identifier(200001), + type="incident_opened", + sequence=2, + timestamp=AT + timedelta(seconds=5), + data={"node_id": "work[3]", "generation": 2, "code": "WV-TASK-AMBIGUOUS"}, + ) + state = RunState( + status="suspended", + accepted_sequence=2, + admission_policy="classified-v1", + input=3, + incidents={"work[3]:2": {"node_id": "work[3]", "generation": 2, "code": "WV-TASK-AMBIGUOUS"}}, + ) + await connection.execute( + text("UPDATE runs SET state=cast(:state AS jsonb) WHERE id=:active_run"), + {**ids, "state": state.model_dump_json()}, + ) + await connection.execute( + text( + "INSERT INTO run_events VALUES(:tenant,:project,:environment,:active_run,:id,2," + "'incident_opened',cast(:data AS jsonb),:at,'fixture',cast(:transition AS jsonb),cast(:response AS jsonb))" + ), + { + **ids, + "id": event.id, + "data": json.dumps(event.data), + "at": event.timestamp, + "transition": Transition(state=state).model_dump_json(), + "response": RunView( + id=ids["active_run"], activation=activation, artifact_digest=artifact.digest, state=state + ).model_dump_json(), + }, + ) + await connection.execute( + text( + "INSERT INTO incidents(id,tenant_id,project_id,environment_id,run_id,incident_key," + "node_id,generation,origin_code,code,status,revision) VALUES(:incident,:tenant,:project,:environment," + ":active_run,:incident_key,'work[3]',2,'WV-TASK-AMBIGUOUS','WV-TASK-AMBIGUOUS','active',1)," + "(:estimated_incident,:tenant,:project,:environment,:first_run,'@legacy',NULL,NULL," + "'WV-TASK-AMBIGUOUS','WV-TASK-AMBIGUOUS','closed',1)" + ), + {**ids, "incident_key": "work[3]:2"}, + ) + # Corruption trials begin with validated fixture models and change only the evidence under test. + await connection.execute(text("UPDATE runs SET artifact='{}' WHERE id=:malformed"), ids) + await connection.execute( + text( + "UPDATE runs SET artifact=jsonb_set(artifact,'{executable,irVersion}'," + "'\"weave/ir-future\"') WHERE id=:unsupported" + ), + ids, + ) + await connection.execute(text("UPDATE run_events SET transition='{}' WHERE run_id=:incoherent"), ids) + return ids + + +async def recompute_usage(connection: AsyncConnection, tenant: UUID, project: UUID) -> dict[tuple[UUID, str], int]: + """Inventory charged tables and retained control provenance independently of counters.""" + tables = ( + ( + await connection.execute( + text( + "SELECT c.relname FROM pg_trigger t JOIN pg_class c ON c.oid=t.tgrelid " + "JOIN pg_namespace n ON n.oid=c.relnamespace WHERE t.tgname='operation_usage' " + "AND n.nspname='public' " + "ORDER BY c.relname" + ) + ) + ) + .scalars() + .all() + ) + totals = defaultdict(int) + params = {"tenant": tenant, "project": project} + for table in tables: + projection = f"public.weave_{table}_account(r)" if table in ("runs", "run_events") else "to_jsonb(r)" + rows = ( + await connection.execute( + text( + f"SELECT item,public.weave_row_metrics('{table}',item) AS metrics FROM " + f"(SELECT {projection} AS item FROM {table} r) q WHERE " + "coalesce(item->>'tenant_id'," + "item->'event'->'scope'->>'tenant_id')=cast(cast(:tenant AS uuid) AS text) AND " + "coalesce(item->>'project_id'," + "item->'event'->'scope'->>'project_id')=cast(cast(:project AS uuid) AS text)" + ), + params, + ) + ).mappings() + for row in rows: + for metric, amount in row["metrics"].items(): + dimension = ( + ZERO + if metric in ("ordinary_bytes", "runs_retained", "debug_rows") + else UUID(row["item"].get("environment_id", str(ZERO))) + ) + if metric == "outbox_pending" and await connection.scalar( + text( + "SELECT EXISTS(SELECT 1 FROM operation_control_deliveries WHERE tenant_id=:tenant " + "AND project_id=:project AND id=:id)" + ), + {**params, "id": row["item"]["id"]}, + ): + metric, dimension = "control_outbox_pending", ZERO + totals[dimension, metric] += int(amount) + allocation = ( + ( + await connection.execute( + text( + "SELECT coalesce(sum(control_bytes),0) AS spent,count(*) AS rows " + "FROM operation_allocations WHERE tenant_id=:tenant AND project_id=:project" + ), + params, + ) + ) + .mappings() + .one() + ) + totals[ZERO, "ordinary_bytes"] -= int(allocation["spent"]) + totals[ZERO, "control_bytes"] += int(allocation["spent"]) + 512 * allocation["rows"] + reservation = ( + ( + await connection.execute( + text( + "SELECT coalesce(sum(amount),0) AS amount," + "coalesce(sum(outbox_slots),0) AS slots,count(*) AS rows FROM operation_reservations " + "WHERE tenant_id=:tenant AND project_id=:project" + ), + params, + ) + ) + .mappings() + .one() + ) + totals[ZERO, "ordinary_bytes"] += 512 * reservation["rows"] + totals[ZERO, "control_reserved"] += int(reservation["amount"]) + totals[ZERO, "control_outbox_reserved"] += int(reservation["slots"]) + return {key: value for key, value in totals.items() if value} + + +async def assert_usage(connection: AsyncConnection, ids: dict[str, UUID]) -> None: + expected = await recompute_usage(connection, ids["tenant"], ids["project"]) + actual = ( + await connection.execute( + text( + "SELECT environment_id,metric,value FROM operation_usage " + "WHERE tenant_id=:tenant AND project_id=:project AND value<>0" + ), + ids, + ) + ).all() + assert {(environment, metric): value for environment, metric, value in actual} == expected + + +async def seed_worker_source(connection, ids): + """Validated registration rows, including for databases without fact tables.""" + from firefly_weave.contracts.workers import WorkerInstance, WorkerRelease + + fixture = runpy.run_path(str(Path(__file__).resolve().parents[1] / "conftest.py"))[ + "worker_runtime_fixture" + ].__wrapped__() + contract = next( + value.definition.value for key, value in fixture["catalog"].resources.items() if key[0] == "TaskCapability" + ) + release = WorkerRelease(id=identifier(301), image_digest="sha256:" + "a" * 64, capabilities=[contract]) + worker = WorkerInstance( + id=identifier(302), release_id=release.id, principal_id=ids["principal"], task_types=["echo@1.2.0"], capacity=1 + ) + ids.update(worker=worker.id, release=release.id, task=identifier(400000)) + await connection.execute( + text( + "INSERT INTO worker_releases VALUES(:release,:tenant,:project,:environment,:digest,cast(:payload AS jsonb))" + ), + {**ids, "digest": release.image_digest, "payload": release.model_dump_json()}, + ) + await connection.execute( + text( + "INSERT INTO worker_instances(id,tenant_id,project_id,environment_id,release_id," + "principal_id,payload) VALUES(:worker,:tenant,:project,:environment,:release,:principal," + "cast(:payload AS jsonb))" + ), + {**ids, "payload": worker.model_dump_json()}, + ) + + +async def seed_fact_children(connection, ids, *, steps=1, tasks=1): + """Explicit source and projection inserts; runtime writers are tested separately.""" + from firefly_weave.runtime.models import TaskIntent + + await seed_worker_source(connection, ids) + await connection.execute( + text( + "INSERT INTO worker_facts(tenant_id,project_id,environment_id,worker_id) " + "VALUES(:tenant,:project,:environment,:worker)" + ), + ids, + ) + for index in range(steps): + node = "work" + str(index) + await connection.execute( + text("INSERT INTO step_instances VALUES(:tenant,:project,:environment,:active_run,:node,'waiting','null')"), + {**ids, "node": node}, + ) + await connection.execute( + text( + "INSERT INTO step_facts(tenant_id,project_id,environment_id,run_id,node_id," + "kind,status,scheduled_at) VALUES(:tenant,:project,:environment,:active_run,:node," + "'action','running',:at)" + ), + {**ids, "node": node, "at": AT}, + ) + for index in range(tasks): + task = TaskIntent( + node_id="work" + str(index), + action_digest="a" * 64, + action_reference="echo-action@2.0.0", + input=3, + connection_slot=None, + task_type="echo", + task_version="1.2.0", + deadline=AT + timedelta(hours=1), + ) + params = { + **ids, + "id": identifier(400000 + index), + "node": task.node_id, + "payload": task.model_dump_json(), + "key": "task-" + str(index), + "at": AT, + } + await connection.execute( + text( + "INSERT INTO task_intents VALUES(:id,:tenant,:project,:environment,:active_run," + ":node,cast(:payload AS jsonb),:release,:key,'ready')" + ), + params, + ) + await connection.execute( + text( + "INSERT INTO task_facts(tenant_id,project_id,environment_id,task_id,run_id," + "node_id,task_type,status,created_at,ready_since) VALUES(:tenant,:project,:environment,:id,:active_run," + ":node,'echo@1.2.0','ready',:at,:at)" + ), + params, + ) + + +async def scope_as_app(connection, ids, *, environment=None, tenant=None, project=None): + await connection.execute(text("SET LOCAL ROLE weave_app")) + for field, value in ( + ("tenant", tenant or ids["tenant"]), + ("project", project or ids["project"]), + ("environment", ids["environment"] if environment is None else environment), + ): + await connection.execute( + text("SELECT set_config(:name,:value,true)"), {"name": "weave." + field + "_id", "value": str(value)} + ) + await connection.execute( + text( + "SELECT pg_advisory_xact_lock(hashtextextended('weave.operations:' " + "|| cast(cast(:tenant AS uuid) AS text) || ':' || cast(cast(:project AS uuid) AS text),0))" + ), + ids, + ) + + +async def seed_ingress_receipts(connection, ids): + """Validated start and signal receipts for every durable ingress family.""" + from firefly_weave.compiler.api import compile_source + from firefly_weave.compiler.catalog import CatalogSnapshot + from firefly_weave.contracts.broker import AcceptedBrokerReceipt, BrokerTrigger, SourceBinding + from firefly_weave.contracts.connectors import ConnectionRevision + from firefly_weave.contracts.email import EmailReceipt, EmailSourceRequest + from firefly_weave.contracts.providers import ProviderEvent, ProviderReceipt, ProviderSource + from firefly_weave.contracts.schedules import ScheduleView + from firefly_weave.triggers.models import Trigger, TriggerReceipt + + source = json.dumps(json.loads(Path("tests/fixtures/e2-provider/connector.json").read_text())["manifest"]) + compiled = compile_source( + source, format="json", catalog=CatalogSnapshot.from_definitions([], adapters=["e2-inbox-fixture"]) + ) + assert compiled.ok + document = json.loads(source) + connector = identifier(501) + await connection.execute( + text( + "INSERT INTO definition_versions VALUES(:id,:tenant,:project,'Connector',:name," + ":version,:digest,:definition_digest,cast(:document AS jsonb),cast(:artifact AS jsonb))" + ), + { + **ids, + "id": connector, + "name": document["metadata"]["name"], + "version": document["metadata"]["version"], + "digest": compiled.artifact.digest, + "definition_digest": compiled.artifact.source_hash, + "document": source, + "artifact": compiled.artifact.to_bytes().decode(), + }, + ) + connection_model = ConnectionRevision( + id=identifier(502), + revision=1, + name="inbox", + connector_version_id=connector, + connector="inbox@1.0.0", + connector_digest=compiled.artifact.digest, + adapter="inbox", + ) + await connection.execute( + text( + "INSERT INTO connection_revisions VALUES(:id,:tenant,:project,:environment,'inbox',1," + ":connector,cast(:payload AS jsonb))" + ), + { + **ids, + "id": connection_model.id, + "connector": connector, + "payload": connection_model.model_dump_json(by_alias=True), + }, + ) + for signal in (False, True): + base = 600 if not signal else 700 + common = ( + {"kind": "signal", "run_id": identifier(1014), "signal": "continue"} + if signal + else {"kind": "run", "activation_id": ids["activation"]} + ) + webhook = Trigger( + id=identifier(base + 1), principal_id=ids["principal"], name="webhook", secret_ref="signing", **common + ) + await connection.execute( + text( + "INSERT INTO trigger_routes(id,tenant_id,project_id,environment_id,principal_id," + "activation_id,run_id,secret_ref,tolerance_seconds,max_body_bytes,payload) VALUES(:id,:tenant,:project," + ":environment,:principal,:activation_id,:run_id,'signing',300,1048576,cast(:payload AS jsonb))" + ), + { + **ids, + "id": webhook.id, + "activation_id": webhook.activation_id, + "run_id": webhook.run_id, + "payload": webhook.model_dump_json(), + }, + ) + for index in [14] if signal else [9, 15, 8, 3]: + receipt = TriggerReceipt( + id=identifier(base + 20 + index), + trigger_id=webhook.id, + event_id=str(index), + run_id=identifier(1000 + index), + signal_id=identifier(900) if signal else None, + ) + await connection.execute( + text( + "INSERT INTO trigger_receipts VALUES(:id,:tenant,:project,:environment,:trigger," + ":event,'fixture',:run,cast(:payload AS jsonb))" + ), + { + **ids, + "id": receipt.id, + "trigger": receipt.trigger_id, + "event": receipt.event_id, + "run": receipt.run_id, + "payload": receipt.model_dump_json(), + }, + ) + for family, offset in (("broker", 2), ("provider", 3)): + binding = SourceBinding( + id=identifier(base + offset + 50), + connection_revision_id=connection_model.id, + source_kind="kafka-trigger" if family == "broker" else "provider-source", + source_id=identifier(base + offset), + principal_id=ids["principal"], + source_fingerprint="fixture", + ) + await connection.execute( + text( + "INSERT INTO connection_source_bindings(id,tenant_id,project_id,environment_id," + "connection_revision_id,source_kind,source_id,principal_id,source_fingerprint) " + "VALUES(:id,:tenant,:project," + ":environment,:connection,:kind,:source,:principal,'fixture')" + ), + { + **ids, + "id": binding.id, + "connection": connection_model.id, + "kind": binding.source_kind, + "source": binding.source_id, + }, + ) + if family == "broker": + route = BrokerTrigger( + id=binding.source_id, + binding_id=binding.id, + principal_id=ids["principal"], + name="broker", + connection_revision_id=connection_model.id, + cluster_id="cluster", + topic="events", + dead_letter_policy="receipt", + **common, + ) + await connection.execute( + text( + "INSERT INTO broker_routes(id,tenant_id,project_id,environment_id,binding_id," + "principal_id,activation_id,run_id,payload) VALUES(:id,:tenant,:project," + ":environment,:binding,:principal," + ":activation_id,:run_id,cast(:payload AS jsonb))" + ), + { + **ids, + "id": route.id, + "binding": binding.id, + "activation_id": route.activation_id, + "run_id": route.run_id, + "payload": route.model_dump_json(), + }, + ) + for index in [14] if signal else [11, 15]: + receipt = AcceptedBrokerReceipt( + id=identifier(base + 100 + index), + trigger_id=route.id, + event_id=identifier(base + 200 + index), + run_id=identifier(1000 + index), + signal_id=identifier(900) if signal else None, + ) + await connection.execute( + text( + "INSERT INTO broker_events VALUES(:id,:tenant,:project,:environment," + ":trigger,'cluster',:event,'fixture',:run,cast(:payload AS jsonb))" + ), + { + **ids, + "id": receipt.id, + "trigger": route.id, + "event": receipt.event_id, + "run": receipt.run_id, + "payload": receipt.model_dump_json(), + }, + ) + else: + provider = ProviderSource( + id=binding.source_id, + scope=Scope(tenant_id=ids["tenant"], project_id=ids["project"], environment_id=ids["environment"]), + binding_id=binding.id, + principal_id=ids["principal"], + name="provider", + provider="inbox", + package="e2-inbox-fixture", + package_version="1.0.0", + adapter_version="1.0.0", + schema_digest="a" * 64, + connection_revision_id=connection_model.id, + policy={}, + **common, + ) + await connection.execute( + text( + "INSERT INTO provider_sources(id,tenant_id,project_id,environment_id," + "binding_id,principal_id,activation_id,run_id,payload) VALUES(:id,:tenant," + ":project,:environment," + ":binding,:principal,:activation_id,:run_id,cast(:payload AS jsonb))" + ), + { + **ids, + "id": provider.id, + "binding": binding.id, + "activation_id": provider.activation_id, + "run_id": provider.run_id, + "payload": provider.model_dump_json(), + }, + ) + for index in [14] if signal else [13, 17]: + event = ProviderEvent(event_id=str(index), kind="message", payload={}) + receipt = ProviderReceipt( + id=identifier(base + 300 + index), + source_id=provider.id, + provider="inbox", + event_id=event.event_id, + kind=event.kind, + fingerprint=event.fingerprint, + received_at=AT, + state="pending" if index == 17 else "dispatched", + run_id=identifier(1000 + index), + signal_id=identifier(900) if signal else None, + ) + await connection.execute( + text( + "INSERT INTO provider_receipts VALUES(:id,:tenant,:project,:environment," + ":source,:event,'message',:fingerprint,cast(:payload AS jsonb),'{}'," + "cast(:receipt AS jsonb),:state,:at)" + ), + { + **ids, + "id": receipt.id, + "source": provider.id, + "event": event.event_id, + "fingerprint": event.fingerprint, + "payload": event.model_dump_json(), + "receipt": receipt.model_dump_json(), + "state": receipt.state, + "at": AT, + }, + ) + email = EmailSourceRequest( + connection_revision_id=connection_model.id, activation_id=None if signal else ids["activation"] + ) + email_id = identifier(base + 4) + await connection.execute( + text( + "INSERT INTO email_sources(id,tenant_id,project_id,environment_id,principal_id," + "connection_revision_id,activation_id,mailbox,policy) VALUES(:id,:tenant,:project," + ":environment,:principal," + ":connection,:activation_id,'INBOX',cast(:policy AS jsonb))" + ), + { + **ids, + "id": email_id, + "connection": connection_model.id, + "activation_id": email.activation_id, + "policy": email.model_dump_json(), + }, + ) + receipt = EmailReceipt( + id=identifier(base + 400), + source_id=email_id, + transport_key="mail", + message_id=None, + state="dispatched", + run_id=identifier(1014 if signal else 1012), + signal_id=identifier(900) if signal else None, + ) + await connection.execute( + text( + "INSERT INTO email_receipts(id,tenant_id,project_id,environment_id,source_id," + "transport_key,state,run_id,signal_id) VALUES(:id,:tenant,:project,:environment," + ":source,'mail','dispatched'," + ":run,:signal)" + ), + {**ids, "id": receipt.id, "source": email_id, "run": receipt.run_id, "signal": receipt.signal_id}, + ) + schedule = ScheduleView( + id=identifier(550), + cron="0 * * * *", + activation_id=ids["activation"], + revision=1, + principal_id=ids["principal"], + status="disabled", + next_due_at=AT, + ) + await connection.execute( + text("INSERT INTO schedules VALUES(:id,:tenant,:project,:environment,1,'disabled',:at,NULL)"), + {**ids, "id": schedule.id, "at": AT}, + ) + await connection.execute( + text( + "INSERT INTO schedule_revisions VALUES(:tenant,:project,:environment,:id,1,:principal," + ":activation,cast(:payload AS jsonb),:at)" + ), + {**ids, "id": schedule.id, "payload": schedule.model_dump_json(), "at": AT}, + ) + await connection.execute( + text( + "INSERT INTO schedule_occurrences(tenant_id,project_id,environment_id,schedule_id," + "revision,instant,through,observed_at,kind,run_id) VALUES(:tenant,:project,:environment," + ":id,1,:at,:at,:at,'started',:run)" + ), + {**ids, "id": schedule.id, "at": AT, "run": identifier(1010)}, + ) diff --git a/tests/integration/test_large_row_accounting.py b/tests/integration/test_large_row_accounting.py index 8a27b0fb..e84c7aeb 100644 --- a/tests/integration/test_large_row_accounting.py +++ b/tests/integration/test_large_row_accounting.py @@ -75,3 +75,22 @@ async def test_all_column_measurements_equal_jsonb_and_reject_new_column_drift(w raise RuntimeError("rollback owned schema trial") async with access_db[1]() as owner: assert await owner.scalar(text("SELECT public.weave_runs_bytes(r)=octet_length(to_jsonb(r)::text) FROM runs r")) + + +async def test_operational_facts_are_in_the_charged_inventory(migration_db): + from firefly_weave.operations.facts import FACT_TABLES + + async with migration_db() as owner: + rows = ( + await owner.execute( + text( + "SELECT c.relname,t.tgargs FROM pg_trigger t " + "JOIN pg_class c ON c.oid=t.tgrelid WHERE t.tgname='operation_usage' AND c.relname=ANY(:tables)" + ), + {"tables": list(FACT_TABLES)}, + ) + ).all() + assert {row.relname for row in rows} == set(FACT_TABLES) + assert ( + next(row.tgargs for row in rows if row.relname == "step_facts") == b"run_id\x00node_id\x00instance_key\x00" + ) diff --git a/tests/integration/test_operations_facts_migration.py b/tests/integration/test_operations_facts_migration.py new file mode 100644 index 00000000..1ba95816 --- /dev/null +++ b/tests/integration/test_operations_facts_migration.py @@ -0,0 +1,922 @@ +# Copyright 2026 Firefly Software Foundation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# Author: Firefly Software Foundation +# SPDX-License-Identifier: Apache-2.0 + +"""Real PostgreSQL upgrade, isolation and allocation checks for operational facts.""" + +import pytest +from operations_support import FACT_TABLES, assert_usage, seed_prior_head +from sqlalchemy import text +from sqlalchemy.ext.asyncio import create_async_engine +from test_incident_migration import upgrade + +pytestmark = pytest.mark.integration + + +async def test_all_fact_tables_use_exact_environment_policy(migration_db): + async with migration_db() as session: + rows = ( + ( + await session.execute( + text( + "SELECT tablename,qual,with_check FROM pg_policies " + "WHERE tablename=ANY(:tables) AND policyname='exact_scope'" + ), + {"tables": list(FACT_TABLES)}, + ) + ) + .mappings() + .all() + ) + assert len(rows) == 5 + assert all("environment_id" in row["qual"] and row["qual"] == row["with_check"] for row in rows) + + +async def test_forward_backfill_preserves_source_columns_and_counts_usage(empty_settings): + engine = create_async_engine(empty_settings.database_url.get_secret_value(), hide_parameters=True) + try: + async with engine.begin() as connection: + await upgrade(connection, "0030_worker_presence") + ids = await seed_prior_head(connection, count=20) + await assert_usage(connection, ids) + snapshots = {} + for table in ("runs", "run_events"): + snapshots[table] = ( + (await connection.execute(text(f"SELECT to_jsonb(r)::text FROM {table} r ORDER BY id"))) + .scalars() + .all() + ) + before = ( + await connection.execute( + text( + "SELECT table_name,column_name,data_type,ordinal_position " + "FROM information_schema.columns WHERE table_name IN ('runs','run_events') " + "ORDER BY table_name,ordinal_position" + ) + ) + ).all() + await upgrade(connection, "0031_operations_facts") + after = ( + await connection.execute( + text( + "SELECT table_name,column_name,data_type,ordinal_position " + "FROM information_schema.columns WHERE table_name IN ('runs','run_events') " + "ORDER BY table_name,ordinal_position" + ) + ) + ).all() + assert before == after + for table in ("runs", "run_events"): + assert ( + await connection.execute(text(f"SELECT to_jsonb(r)::text FROM {table} r ORDER BY id")) + ).scalars().all() == snapshots[table] + assert await connection.scalar(text("SELECT version FROM weave_schema_version")) == "0031_operations_facts" + assert await connection.scalar(text("SELECT version_num FROM alembic_version")) == "0031_operations_facts" + assert await connection.scalar(text("SELECT count(*) FROM run_facts")) == 20 + await assert_usage(connection, ids) + finally: + await engine.dispose() + + +@pytest.fixture +async def retained(empty_settings): + engine = create_async_engine(empty_settings.database_url.get_secret_value(), hide_parameters=True) + try: + async with engine.begin() as connection: + await upgrade(connection, "0030_worker_presence") + ids = await seed_prior_head(connection, count=20) + await upgrade(connection, "0031_operations_facts") + yield engine, ids + finally: + await engine.dispose() + + +async def test_legacy_inventory_crosses_ten_thousand_boundary_without_relaxing_admission(empty_settings): + from sqlalchemy.exc import DBAPIError + + engine = create_async_engine(empty_settings.database_url.get_secret_value(), hide_parameters=True) + try: + async with engine.begin() as connection: + # The retained-run quota did not exist when this legacy inventory was accepted. + await upgrade(connection, "0020_whatsapp_status") + ids = await seed_prior_head(connection) + await upgrade(connection, "0030_worker_presence") + assert await connection.scalar(text("SELECT count(*) FROM runs")) == 10001 + assert await connection.scalar(text("SELECT version_num FROM alembic_version")) == "0030_worker_presence" + await assert_usage(connection, ids) + with pytest.raises(DBAPIError) as caught: + async with connection.begin_nested(): + await connection.execute( + text( + "INSERT INTO runs SELECT 'eeeeeeee-0000-0000-0000-000000000001'::uuid," + "tenant_id,project_id,environment_id,activation_id,principal_id," + "artifact,activation,request,state " + "FROM runs WHERE id=:first_run" + ), + ids, + ) + assert caught.value.orig.sqlstate == "WQ001" + before = await connection.scalar(text("SELECT value FROM operation_usage WHERE metric='control_reserved'")) + await upgrade(connection, "0031_operations_facts") + assert await connection.scalar(text("SELECT count(*) FROM run_facts")) == 10001 + assert await connection.scalar(text("SELECT count(*) FROM run_facts WHERE run_id=:last_run"), ids) == 1 + assert ( + await connection.scalar(text("SELECT value FROM operation_usage WHERE metric='control_reserved'")) + == before + 4096 + ) + await assert_usage(connection, ids) + finally: + await engine.dispose() + + +async def test_backfilled_evidence_is_honest_and_metadata_is_bounded(retained): + from datetime import timedelta + + from operations_support import AT + + engine, ids = retained + async with engine.connect() as connection: + rows = {row["run_id"]: row for row in (await connection.execute(text("SELECT * FROM run_facts"))).mappings()} + first = rows[ids["first_run"]] + assert first["started_at"] == first["updated_at"] == first["ended_at"] == AT + assert first["origin"] is None and first["classification_state"] == "available" + assert first["business_key"] == 'España 雪 " \\' + assert first["node_kinds"] == {"work": "action"} + assert first["workflow_name"] == first["activation_name"] == "worker-flow" + assert first["workflow_version"] == "1.0.0" and first["activation_revision"] == 1 + assert first["facts_started_at"] > AT + assert rows[ids["retry"]]["origin"] == "retry" + assert rows[ids["retry"]]["retried_from_run_id"] == ids["first_run"] + assert rows[ids["retry"]]["caller_run_id"] is None + assert rows[ids["explicit"]]["origin"] == "test" and rows[ids["explicit"]]["test"] + for name in ("no_events", "malformed", "incoherent"): + assert rows[ids[name]]["classification_state"] == "unavailable" + assert rows[ids["no_events"]]["started_at"] is None + unsupported = rows[ids["unsupported"]] + assert unsupported["classification_state"] == "unsupported" + assert unsupported["pinned_ir_version"] == "weave/ir-future" + assert unsupported["classification_policy"] == "classified-v1" + assert unsupported["node_kinds"] == {} + incidents = { + row["incident_id"]: row + for row in (await connection.execute(text("SELECT * FROM incident_facts"))).mappings() + } + exact = incidents[ids["incident"]] + assert (exact["node_id"], exact["instance_key"]) == ("work", "work[3]") + assert exact["opened_at"] == exact["updated_at"] == AT + timedelta(seconds=5) + assert not exact["opened_at_estimated"] + assert incidents[ids["estimated_incident"]]["opened_at"] == AT + assert incidents[ids["estimated_incident"]]["opened_at_estimated"] + assert rows[ids["active_run"]]["active_incidents"] == 1 + assert await connection.scalar(text("SELECT count(*) FROM step_facts")) == 0 + assert await connection.scalar(text("SELECT count(*) FROM task_facts")) == 0 + + +async def test_migration_restores_all_scope_settings(empty_settings): + engine = create_async_engine(empty_settings.database_url.get_secret_value(), hide_parameters=True) + try: + async with engine.begin() as connection: + await upgrade(connection, "0030_worker_presence") + ids = await seed_prior_head(connection, count=20) + for name in ("tenant", "project", "environment"): + await connection.execute( + text("SELECT set_config(:name,:value,true)"), + {"name": "weave." + name + "_id", "value": str(ids[name])}, + ) + await upgrade(connection, "0031_operations_facts") + for name in ("tenant", "project", "environment"): + assert await connection.scalar( + text("SELECT current_setting(:name)"), {"name": "weave." + name + "_id"} + ) == str(ids[name]) + finally: + await engine.dispose() + + +async def test_each_fact_table_enforces_scope_parent_and_delete_authority(retained): + import json + from uuid import uuid4 + + from operations_support import scope_as_app, seed_fact_children + from sqlalchemy.exc import DBAPIError + + engine, ids = retained + async with engine.begin() as connection: + await seed_fact_children(connection, ids) + assert ( + await connection.execute( + text( + "SELECT bool_and(relrowsecurity AND relforcerowsecurity) FROM pg_class WHERE relname=ANY(:tables)" + ), + {"tables": list(FACT_TABLES)}, + ) + ).scalar() + samples = { + table: (await connection.execute(text(f"SELECT to_jsonb(f) FROM {table} f LIMIT 1"))).scalar() + for table in FACT_TABLES + } + for table in FACT_TABLES: + sample = samples[table] + for environment in (ids["other_environment"], ""): + async with engine.begin() as connection: + await scope_as_app(connection, ids, environment=environment) + assert await connection.scalar(text(f"SELECT count(*) FROM {table}")) == 0 + result = await connection.execute(text(f"UPDATE {table} SET tenant_id=tenant_id")) + assert result.rowcount == 0 + with pytest.raises(DBAPIError) as caught: + async with connection.begin_nested(): + await connection.execute( + text( + f"INSERT INTO {table} SELECT " + f"(jsonb_populate_record(NULL::{table},cast(:row AS jsonb))).*" + ), + {"row": json.dumps(sample)}, + ) + assert caught.value.orig.sqlstate == "42501" + async with engine.begin() as connection: + await scope_as_app(connection, ids) + assert await connection.scalar(text(f"SELECT count(*) FROM {table}")) > 0 + with pytest.raises(DBAPIError) as caught: + async with connection.begin_nested(): + await connection.execute(text(f"DELETE FROM {table}")) + assert caught.value.orig.sqlstate == "42501" + async with engine.begin() as connection: + await scope_as_app(connection, ids, environment=ids["other_environment"]) + forged = {**sample, "environment_id": str(ids["other_environment"])} + # Retain the actual parent's ID; a different scope cannot reference it. + if table == "step_facts": + forged["node_id"] = "foreign" + else: + key = { + "run_facts": "run_id", + "task_facts": "task_id", + "incident_facts": "incident_id", + "worker_facts": "worker_id", + }[table] + await connection.execute(text("RESET ROLE")) + await connection.execute(text(f"DELETE FROM {table} WHERE {key}=:id"), {"id": sample[key]}) + await scope_as_app(connection, ids, environment=ids["other_environment"]) + with pytest.raises(DBAPIError) as caught: + async with connection.begin_nested(): + await connection.execute( + text( + f"INSERT INTO {table} SELECT (jsonb_populate_record(NULL::{table},cast(:row AS jsonb))).*" + ), + {"row": json.dumps(forged)}, + ) + assert caught.value.orig.sqlstate == "23503" + async with engine.begin() as connection: + await scope_as_app(connection, ids, project=uuid4()) + assert await connection.scalar(text(f"SELECT count(*) FROM {table}")) == 0 + + +async def test_bookkeeping_has_bounded_keys_and_scope_enforcement(retained): + from uuid import uuid4 + + from operations_support import scope_as_app + from sqlalchemy.exc import DBAPIError + + engine, ids = retained + async with engine.begin() as connection: + await scope_as_app(connection, ids) + await connection.execute(text("INSERT INTO operations_environment_cursors VALUES(:tenant,:environment)"), ids) + await connection.execute( + text("INSERT INTO operations_fact_cursors VALUES(:tenant,:project,:environment,:last_run)"), ids + ) + for table in ("operations_environment_cursors", "operations_fact_cursors"): + with pytest.raises(DBAPIError) as caught: + async with connection.begin_nested(): + await connection.execute(text(f"INSERT INTO {table} SELECT * FROM {table}")) + assert caught.value.orig.sqlstate == "23505" + with pytest.raises(DBAPIError) as caught: + async with connection.begin_nested(): + await connection.execute(text(f"DELETE FROM {table}")) + assert caught.value.orig.sqlstate == "42501" + await scope_as_app(connection, ids, environment=ids["other_environment"]) + assert await connection.scalar(text("SELECT count(*) FROM operations_fact_cursors")) == 0 + with pytest.raises(DBAPIError) as caught: + async with connection.begin_nested(): + await connection.execute( + text("INSERT INTO operations_fact_cursors VALUES(:tenant,:project,:environment,NULL)"), ids + ) + assert caught.value.orig.sqlstate == "42501" + await scope_as_app(connection, ids, tenant=uuid4()) + assert await connection.scalar(text("SELECT count(*) FROM operations_environment_cursors")) == 0 + with pytest.raises(DBAPIError) as caught: + async with connection.begin_nested(): + await connection.execute(text("INSERT INTO operations_environment_cursors VALUES(:tenant,NULL)"), ids) + assert caught.value.orig.sqlstate == "42501" + await connection.execute(text("RESET ROLE")) + assert not await connection.scalar( + text( + "SELECT EXISTS(SELECT 1 FROM pg_trigger WHERE tgname='operation_usage' " + "AND tgrelid IN ('operations_environment_cursors'::regclass,'operations_fact_cursors'::regclass))" + ) + ) + + +async def test_fact_payload_bounds_and_status_checks(retained): + import json + + from operations_support import seed_fact_children + from sqlalchemy.exc import DBAPIError + + engine, ids = retained + async with engine.begin() as connection: + await seed_fact_children(connection, ids) + changes = [ + ("run_facts", "status", "unknown"), + ("run_facts", "origin", "unknown"), + ("run_facts", "pinned_features", {}), + ("run_facts", "pinned_features", [1]), + ("run_facts", "pinned_features", ["a"] * 257), + ("run_facts", "pinned_features", ["a" * 16384]), + ("run_facts", "node_kinds", []), + ("run_facts", "node_kinds", {str(i): "action" for i in range(10001)}), + ("run_facts", "node_kinds", {"work": "a" * 1048576}), + ("worker_facts", "identity", {"a": "x" * 4090}), + ("step_facts", "handled", "continue"), + ] + for table, column, value in changes: + is_json = column in ("pinned_features", "node_kinds", "identity") + with pytest.raises(DBAPIError) as caught: + async with connection.begin_nested(): + await connection.execute( + text(f"UPDATE {table} SET {column}=" + ("cast(:value AS jsonb)" if is_json else ":value")), + {"value": json.dumps(value) if is_json else value}, + ) + assert caught.value.orig.sqlstate == "23514", (table, column) + await connection.execute( + text("UPDATE worker_facts SET identity=cast(:value AS jsonb)"), {"value": json.dumps({"a": "x" * 4087})} + ) + assert await connection.scalar(text("SELECT octet_length(identity::text) FROM worker_facts")) == 4096 + await assert_usage(connection, ids) + + +async def test_seeded_source_and_facts_roll_back_together_when_quota_is_full(retained): + from operations_support import scope_as_app, seed_fact_children + from sqlalchemy.exc import DBAPIError + + engine, ids = retained + async with engine.begin() as connection: + await seed_fact_children(connection, ids) + await connection.execute( + text( + "UPDATE operation_policy SET limits=jsonb_set(limits,'{ordinary_bytes}'," + "to_jsonb((SELECT value FROM operation_usage WHERE metric='ordinary_bytes')))" + ) + ) + before = await connection.scalar(text("SELECT state FROM runs WHERE id=:active_run"), ids) + await scope_as_app(connection, ids) + with pytest.raises(DBAPIError) as caught: + async with connection.begin_nested(): + await connection.execute( + text("UPDATE runs SET state=jsonb_set(state,'{status}','\"waiting\"') WHERE id=:active_run"), ids + ) + await connection.execute( + text("UPDATE run_facts SET failed_error_code=repeat('uncommitted',100) WHERE run_id=:active_run"), + ids, + ) + assert caught.value.orig.sqlstate == "WQ001" + await connection.execute(text("RESET ROLE")) + assert await connection.scalar(text("SELECT state FROM runs WHERE id=:active_run"), ids) == before + assert not await connection.scalar(text("SELECT paused FROM run_facts WHERE run_id=:active_run"), ids) + await assert_usage(connection, ids) + + +@pytest.mark.parametrize("steps,tasks", [(1, 1), (10000, 1000)]) +async def test_seeded_fact_growth_uses_terminal_control_reserve(retained, steps, tasks): + from operations_support import AT, scope_as_app, seed_fact_children + + engine, ids = retained + async with engine.begin() as connection: + await seed_fact_children(connection, ids, steps=steps, tasks=tasks) + await assert_usage(connection, ids) + await connection.execute( + text( + "UPDATE operation_policy SET limits=jsonb_set(limits,'{ordinary_bytes}'," + "to_jsonb((SELECT value FROM operation_usage WHERE metric='ordinary_bytes')))" + ) + ) + original = await connection.scalar(text("SELECT value FROM operation_usage WHERE metric='ordinary_bytes'")) + async with engine.begin() as connection: + await scope_as_app(connection, ids) + await connection.execute(text("SELECT weave_control_begin('run',:active_run)"), ids) + await connection.execute( + text("UPDATE run_facts SET status='cancelled',updated_at=:at,ended_at=:at WHERE run_id=:active_run"), + {**ids, "at": AT}, + ) + await connection.execute( + text("UPDATE step_facts SET status='cancelled',ended_at=:at,worker_id=:worker WHERE run_id=:active_run"), + {**ids, "at": AT}, + ) + await connection.execute( + text("UPDATE task_facts SET status='cancelled',first_claimed_at=:at WHERE run_id=:active_run"), + {**ids, "at": AT}, + ) + await connection.execute( + text("UPDATE incident_facts SET updated_at=:at WHERE incident_id=:incident"), {**ids, "at": AT} + ) + await connection.execute(text("UPDATE task_intents SET status='cancelled' WHERE run_id=:active_run"), ids) + await connection.execute( + text("UPDATE runs SET state=jsonb_set(state,'{status}','\"cancelled\"') WHERE id=:active_run"), ids + ) + await connection.execute(text("SELECT weave_control_finish()")) + async with engine.begin() as connection: + await assert_usage(connection, ids) + assert ( + await connection.scalar(text("SELECT value FROM operation_usage WHERE metric='ordinary_bytes'")) <= original + ) + assert await connection.scalar(text("SELECT value FROM operation_usage WHERE metric='control_reserved'")) == 0 + assert ( + await connection.scalar( + text( + "SELECT count(*) FROM operation_allocations " + "WHERE relation IN ('run_facts','step_facts','task_facts')" + ) + ) + == steps + tasks + 1 + ) + # Archive admission is separate from the saturated terminal control. + await connection.execute( + text("UPDATE operation_policy SET limits=jsonb_set(limits,'{ordinary_bytes}','4294967296')") + ) + await connection.execute( + text( + "INSERT INTO run_archives(tenant_id,project_id,environment_id,run_id,archived,revision) " + "VALUES(:tenant,:project,:environment,:active_run,true,1)" + ), + ids, + ) + async with engine.begin() as connection: + await scope_as_app(connection, ids) + await connection.execute(text("SELECT weave_purge_run(:active_run,1)"), ids) + async with engine.begin() as connection: + for table in ("run_facts", "step_facts", "task_facts"): + assert await connection.scalar(text(f"SELECT count(*) FROM {table} WHERE run_id=:active_run"), ids) == 0 + assert ( + await connection.scalar(text("SELECT count(*) FROM incident_facts WHERE incident_id=:incident"), ids) == 0 + ) + await connection.execute(text("DELETE FROM worker_instances WHERE id=:worker"), ids) + assert await connection.scalar(text("SELECT count(*) FROM worker_facts")) == 0 + await assert_usage(connection, ids) + + +async def test_ingress_origin_uses_start_receipts_and_preserves_precedence(empty_settings): + from operations_support import identifier, seed_ingress_receipts + + engine = create_async_engine(empty_settings.database_url.get_secret_value(), hide_parameters=True) + try: + async with engine.begin() as connection: + await upgrade(connection, "0030_worker_presence") + ids = await seed_prior_head(connection, count=20) + await seed_ingress_receipts(connection, ids) + await assert_usage(connection, ids) + await upgrade(connection, "0031_operations_facts") + origins = dict((await connection.execute(text("SELECT run_id,origin FROM run_facts"))).all()) + for index, origin in ( + (9, "webhook"), + (10, "schedule"), + (11, "broker"), + (12, "email"), + (13, "provider"), + (14, None), + (15, None), + (8, "test"), + (3, "retry"), + (17, None), + ): + assert origins[identifier(1000 + index)] == origin + await assert_usage(connection, ids) + finally: + await engine.dispose() + + +@pytest.mark.parametrize( + "change", ["CREATE TABLE run_call_links(id uuid)", "ALTER TABLE step_instances ADD COLUMN handled text"] +) +async def test_new_history_contract_requires_reconciled_backfill(empty_settings, change): + engine = create_async_engine(empty_settings.database_url.get_secret_value(), hide_parameters=True) + try: + async with engine.begin() as connection: + await upgrade(connection, "0030_worker_presence") + await connection.execute(text(change)) + with pytest.raises(RuntimeError, match="Reconcile call and handled-failure history"): + await upgrade(connection, "0031_operations_facts") + assert await connection.scalar(text("SELECT to_regclass('run_facts')")) is None + assert await connection.scalar(text("SELECT version_num FROM alembic_version")) == "0030_worker_presence" + finally: + await engine.dispose() + + +@pytest.mark.parametrize( + "signature,anchor", + [ + ("weave_control_target(text,jsonb)", "IF relation_name='runs' THEN RETURN (item->>'id')::uuid; END IF;"), + ("weave_reserve_refresh(text,uuid,boolean)", "+576*affected;"), + ("weave_reservation_touch(text,jsonb)", "'task_intents','task_leases','incidents','run_deadlines'"), + ], +) +@pytest.mark.parametrize("duplicate", [False, True]) +async def test_control_function_patch_aborts_on_unknown_or_duplicate_anchor( + empty_settings, signature, anchor, duplicate +): + engine = create_async_engine(empty_settings.database_url.get_secret_value(), hide_parameters=True) + try: + async with engine.begin() as connection: + await upgrade(connection, "0030_worker_presence") + body = await connection.scalar( + text("SELECT pg_get_functiondef(cast(:signature AS regprocedure))"), {"signature": signature} + ) + if duplicate: + changed = body.replace(anchor, "/* " + anchor + " */ " + anchor) + else: + changed = body.replace(anchor, anchor.replace("'", " '", 1) if "'" in anchor else "+576 * affected;") + await connection.exec_driver_sql(changed) + with pytest.raises(RuntimeError, match="Unexpected definition"): + async with engine.begin() as connection: + await upgrade(connection, "0031_operations_facts") + async with engine.connect() as connection: + assert await connection.scalar(text("SELECT to_regclass('run_facts')")) is None + assert await connection.scalar(text("SELECT version_num FROM alembic_version")) == "0030_worker_presence" + finally: + await engine.dispose() + + +async def test_fact_repository_uses_transaction_scope_and_rolls_back_with_source(retained): + from uuid import uuid4 + + from sqlalchemy.ext.asyncio import async_sessionmaker + + from firefly_weave.contracts.access import Scope + from firefly_weave.operations.facts import FactRepository, RunStartFacts + from firefly_weave.persistence.uow import UnitOfWork + + engine, ids = retained + scope = Scope(tenant_id=ids["tenant"], project_id=ids["project"], environment_id=ids["environment"]) + uow = UnitOfWork(async_sessionmaker(engine, expire_on_commit=False)) + defaults = RunStartFacts() + assert defaults.origin == "manual" and not defaults.test + assert defaults.caller is defaults.retried_from_run_id is None + with pytest.raises(RuntimeError, match="fixture rollback"): + async with uow.open(scope) as tx: + await tx.session.execute(text("SET LOCAL ROLE weave_app")) + facts = FactRepository(tx) + await facts.execute( + "UPDATE run_facts SET paused=true WHERE tenant_id=:tenant AND project_id=:project " + "AND environment_id=:environment AND run_id=:run", + run=ids["active_run"], + tenant=uuid4(), + ) + rows = await facts.rows( + "SELECT run_id,paused FROM run_facts WHERE tenant_id=:tenant AND project_id=:project " + "AND environment_id=:environment AND run_id=:run", + run=ids["active_run"], + environment=uuid4(), + ) + assert rows == [{"run_id": ids["active_run"], "paused": True}] + await facts.execute( + "UPDATE runs SET state=jsonb_set(state,'{manual_paused}','true') WHERE " + "tenant_id=:tenant AND project_id=:project AND environment_id=:environment AND id=:run", + run=ids["active_run"], + ) + raise RuntimeError("fixture rollback") + async with engine.connect() as connection: + assert not await connection.scalar(text("SELECT paused FROM run_facts WHERE run_id=:active_run"), ids) + assert not (await connection.scalar(text("SELECT state FROM runs WHERE id=:active_run"), ids)).get( + "manual_paused" + ) + await assert_usage(connection, ids) + + +@pytest.mark.parametrize("key", ["work[03]", "work:private", "work@other", "work[0]" + "a" * 512, 3]) +async def test_backfill_node_map_rejects_non_author_keys(key): + import runpy + from types import SimpleNamespace + + migration = runpy.run_path("migrations/versions/0031_operations_facts.py") + artifact = SimpleNamespace(executable={"graph": {"nodes": [{"id": key, "kind": "action"}]}}) + with pytest.raises(ValueError): + migration["node_kinds"](artifact) + artifact.executable["graph"]["nodes"] = [{"id": "@join:work", "kind": "join"}, {"id": "work[3]", "kind": "action"}] + assert migration["node_kinds"](artifact) == {"work": "action"} + + +async def test_missing_and_mismatched_incident_history_stays_estimated(empty_settings): + from operations_support import AT, identifier + + engine = create_async_engine(empty_settings.database_url.get_secret_value(), hide_parameters=True) + try: + async with engine.begin() as connection: + await upgrade(connection, "0030_worker_presence") + ids = await seed_prior_head(connection, count=20) + await connection.execute(text("UPDATE incidents SET incident_key='different' WHERE id=:incident"), ids) + await connection.execute( + text( + "INSERT INTO incidents(id,tenant_id,project_id,environment_id,run_id," + "incident_key,origin_code,code,status,revision) VALUES(:id,:tenant,:project," + ":environment,:no_events," + "'@legacy','WV-TASK-AMBIGUOUS','WV-TASK-AMBIGUOUS','closed',1)" + ), + {**ids, "id": identifier(103)}, + ) + await upgrade(connection, "0031_operations_facts") + first = ( + (await connection.execute(text("SELECT * FROM incident_facts WHERE incident_id=:incident"), ids)) + .mappings() + .one() + ) + assert first["opened_at_estimated"] and first["opened_at"] > AT + absent = ( + ( + await connection.execute( + text("SELECT * FROM incident_facts WHERE incident_id=:id"), {"id": identifier(103)} + ) + ) + .mappings() + .one() + ) + assert absent["opened_at"] is absent["updated_at"] is None + assert absent["opened_at_estimated"] + finally: + await engine.dispose() + + +async def test_malformed_event_in_otherwise_valid_history_is_unavailable(empty_settings): + from datetime import timedelta + + from operations_support import AT, identifier + + engine = create_async_engine(empty_settings.database_url.get_secret_value(), hide_parameters=True) + try: + async with engine.begin() as connection: + await upgrade(connection, "0030_worker_presence") + ids = await seed_prior_head(connection, count=20) + await connection.execute( + text("UPDATE runs SET state=jsonb_set(state,'{accepted_sequence}','2') WHERE id=:first_run"), ids + ) + await connection.execute( + text( + "INSERT INTO run_events SELECT tenant_id,project_id,environment_id,run_id," + ":event,2,'task_completed',data,:at,request_hash,'{}',response FROM run_events " + "WHERE run_id=:first_run" + ), + {**ids, "event": identifier(300001), "at": AT + timedelta(seconds=1)}, + ) + await upgrade(connection, "0031_operations_facts") + assert ( + await connection.scalar(text("SELECT classification_state FROM run_facts WHERE run_id=:first_run"), ids) + == "unavailable" + ) + finally: + await engine.dispose() + + +async def test_oversized_source_and_event_are_not_admitted_by_backfill(empty_settings): + from operations_support import identifier + from sqlalchemy import event + + engine = create_async_engine(empty_settings.database_url.get_secret_value(), hide_parameters=True) + fetched = [] + try: + async with engine.begin() as connection: + await upgrade(connection, "0030_worker_presence") + ids = await seed_prior_head(connection, count=20) + await connection.execute( + text( + "UPDATE runs SET request=jsonb_set(request,'{input}',to_jsonb(repeat('x',134217728))) WHERE id=:id" + ), + {"id": identifier(1018)}, + ) + await connection.execute( + text("UPDATE run_events SET data=jsonb_build_object('input',repeat('x',134217728)) WHERE run_id=:id"), + {"id": identifier(1019)}, + ) + + def observe(conn, cursor, statement, parameters, context, executemany): + if statement.startswith("SELECT artifact,state,activation,request FROM runs"): + fetched.append(parameters[-1]) + + event.listen(engine.sync_engine, "before_cursor_execute", observe) + await upgrade(connection, "0031_operations_facts") + event.remove(engine.sync_engine, "before_cursor_execute", observe) + assert identifier(1018) not in fetched and identifier(1019) in fetched + rows = ( + await connection.execute( + text("SELECT classification_state,node_kinds FROM run_facts WHERE run_id=ANY(:ids)"), + {"ids": [identifier(1018), identifier(1019)]}, + ) + ).all() + assert rows == [("unavailable", {}), ("unavailable", {})] + await assert_usage(connection, ids) + finally: + await engine.dispose() + + +async def test_migration_requires_catalog_only_execute_authority(empty_settings): + from uuid import uuid4 + + engine = create_async_engine(empty_settings.database_url.get_secret_value(), hide_parameters=True) + role = "weave_facts_migration_" + uuid4().hex + try: + async with engine.begin() as connection: + await upgrade(connection, "0030_worker_presence") + await connection.execute(text(f"CREATE ROLE {role} NOLOGIN NOSUPERUSER NOBYPASSRLS")) + await connection.execute(text(f"GRANT USAGE,CREATE ON SCHEMA public TO {role}")) + await connection.execute(text(f"GRANT SELECT,UPDATE ON weave_schema_version,alembic_version TO {role}")) + await connection.execute(text(f"SET LOCAL ROLE {role}")) + with pytest.raises(RuntimeError, match="explicit EXECUTE"): + await upgrade(connection, "0031_operations_facts") + assert not await connection.scalar(text("SELECT pg_has_role(current_user,'weave_catalog_reader','MEMBER')")) + assert await connection.scalar(text("SELECT to_regclass('run_facts')")) is None + finally: + await engine.dispose() + + +async def test_reserved_fact_growth_rolls_back_allocations_and_source(retained): + from operations_support import AT, scope_as_app, seed_fact_children + + engine, ids = retained + async with engine.begin() as connection: + await seed_fact_children(connection, ids) + await connection.execute(text("UPDATE incident_facts SET updated_at=NULL WHERE incident_id=:incident"), ids) + original = ( + await connection.execute(text("SELECT metric,environment_id,value FROM operation_usage ORDER BY 1,2")) + ).all() + await connection.execute( + text( + "UPDATE operation_policy SET limits=jsonb_set(limits,'{ordinary_bytes}'," + "to_jsonb((SELECT value FROM operation_usage WHERE metric='ordinary_bytes')))" + ) + ) + with pytest.raises(RuntimeError, match="rollback terminal facts"): + async with engine.begin() as connection: + await scope_as_app(connection, ids) + await connection.execute(text("SELECT weave_control_begin('run',:active_run)"), ids) + await connection.execute( + text("UPDATE incident_facts SET updated_at=:at WHERE incident_id=:incident"), {**ids, "at": AT} + ) + await connection.execute( + text("UPDATE step_facts SET ended_at=:at WHERE run_id=:active_run"), {**ids, "at": AT} + ) + await connection.execute( + text("UPDATE runs SET state=jsonb_set(state,'{status}','\"cancelled\"') WHERE id=:active_run"), ids + ) + await connection.execute(text("SELECT weave_control_finish()")) + assert ( + await connection.scalar( + text("SELECT count(*) FROM operation_allocations WHERE relation='incident_facts'") + ) + == 1 + ) + raise RuntimeError("rollback terminal facts") + async with engine.connect() as connection: + assert ( + await connection.execute(text("SELECT metric,environment_id,value FROM operation_usage ORDER BY 1,2")) + ).all() == original + assert ( + await connection.scalar(text("SELECT state->>'status' FROM runs WHERE id=:active_run"), ids) == "suspended" + ) + assert ( + await connection.scalar(text("SELECT updated_at FROM incident_facts WHERE incident_id=:incident"), ids) + is None + ) + assert await connection.scalar(text("SELECT count(*) FROM operation_allocations")) == 0 + await assert_usage(connection, ids) + + +async def test_legacy_worker_registration_and_step_task_times_remain_unknown(empty_settings): + from operations_support import seed_worker_source + + engine = create_async_engine(empty_settings.database_url.get_secret_value(), hide_parameters=True) + try: + async with engine.begin() as connection: + await upgrade(connection, "0030_worker_presence") + ids = await seed_prior_head(connection, count=20) + await seed_worker_source(connection, ids) + await upgrade(connection, "0031_operations_facts") + row = ( + await connection.execute(text("SELECT worker_id,registered_at,identity,drained_at FROM worker_facts")) + ).one() + assert row == (ids["worker"], None, {}, None) + assert await connection.scalar(text("SELECT count(*) FROM step_facts")) == 0 + assert await connection.scalar(text("SELECT count(*) FROM task_facts")) == 0 + assert ( + await connection.scalar( + text("SELECT weave_control_target('worker_facts',to_jsonb(f)) FROM worker_facts f") + ) + is None + ) + await assert_usage(connection, ids) + finally: + await engine.dispose() + + +async def test_fact_query_indexes_cover_scope_and_ordering(migration_db): + async with migration_db() as session: + definitions = dict( + (await session.execute(text("SELECT indexname,indexdef FROM pg_indexes WHERE schemaname='public'"))).all() + ) + scope = "tenant_id, project_id, environment_id" + for name, suffix in ( + ("started", "started_at DESC, run_id DESC"), + ("workflow", "workflow_name, started_at DESC, run_id DESC"), + ("status", "status, started_at DESC, run_id DESC"), + ("activation", "activation_id, started_at DESC, run_id DESC"), + ("updated", "updated_at DESC, run_id DESC"), + ): + assert "(" + scope + ", " + suffix + ")" in definitions["run_facts_" + name] + for name in ("business_key", "correlation_key", "caller", "retry", "ended", "active_test"): + assert scope in definitions["run_facts_" + name] + assert "WHERE (test AND (ended_at IS NULL))" in definitions["run_facts_active_test"] + for name in ( + "step_facts_ended", + "task_facts_ready", + "incident_facts_node_opened", + "run_deadlines_operations_due", + "event_deliveries_operations_due", + "incidents_operations_active", + ): + assert scope in definitions[name] + + +async def test_failed_upgrade_restores_scope_and_keeps_prior_schema(empty_settings): + from sqlalchemy.exc import DBAPIError + + engine = create_async_engine(empty_settings.database_url.get_secret_value(), hide_parameters=True) + try: + async with engine.begin() as connection: + await upgrade(connection, "0030_worker_presence") + ids = await seed_prior_head(connection, count=20) + for name in ("tenant", "project", "environment"): + await connection.execute( + text("SELECT set_config(:name,:value,true)"), + {"name": "weave." + name + "_id", "value": str(ids[name])}, + ) + await connection.execute(text("ALTER TABLE runs ADD COLUMN fixture_schema_drift text")) + with pytest.raises(DBAPIError) as caught: + await upgrade(connection, "0031_operations_facts") + assert caught.value.orig.sqlstate == "WQ003" + assert await connection.scalar(text("SELECT to_regclass('run_facts')")) is None + assert await connection.scalar(text("SELECT version_num FROM alembic_version")) == "0030_worker_presence" + for name in ("tenant", "project", "environment"): + assert await connection.scalar( + text("SELECT current_setting(:name)"), {"name": "weave." + name + "_id"} + ) == str(ids[name]) + finally: + await engine.dispose() + + +@pytest.mark.parametrize( + "mutation", ["state-'status'", "jsonb_set(state,'{status}','\"unknown\"')", "jsonb_set(state,'{status}','3')"] +) +async def test_malformed_legacy_status_has_no_invented_lifecycle(empty_settings, mutation): + from sqlalchemy.exc import DBAPIError + + engine = create_async_engine(empty_settings.database_url.get_secret_value(), hide_parameters=True) + try: + async with engine.begin() as connection: + await upgrade(connection, "0030_worker_presence") + ids = await seed_prior_head(connection, count=20) + await connection.execute(text(f"UPDATE runs SET state={mutation} WHERE id=:first_run"), ids) + before = await connection.scalar(text("SELECT state FROM runs WHERE id=:first_run"), ids) + await upgrade(connection, "0031_operations_facts") + row = ( + await connection.execute( + text( + "SELECT status,classification_state,node_kinds,started_at,updated_at,ended_at " + "FROM run_facts WHERE run_id=:first_run" + ), + ids, + ) + ).one() + assert row == (None, "unavailable", {}, None, None, None) + assert ( + await connection.scalar( + text("SELECT count(*) FROM run_facts WHERE run_id=:first_run AND status='queued'"), ids + ) + == 0 + ) + assert await connection.scalar(text("SELECT state FROM runs WHERE id=:first_run"), ids) == before + for classification in ("available", "unsupported"): + with pytest.raises(DBAPIError) as caught: + async with connection.begin_nested(): + await connection.execute( + text("UPDATE run_facts SET classification_state=:classification WHERE run_id=:first_run"), + {**ids, "classification": classification}, + ) + assert caught.value.orig.sqlstate == "23514" + await assert_usage(connection, ids) + finally: + await engine.dispose() From 5e3ca6d8c91046cb4282be481cf317877ba31674 Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Fri, 9 Oct 2026 04:03:23 -0700 Subject: [PATCH 03/63] Withhold operational facts with inconsistent lifecycle history --- migrations/versions/0031_operations_facts.py | 3 + .../test_operations_facts_migration.py | 102 ++++++++++++++++++ 2 files changed, 105 insertions(+) diff --git a/migrations/versions/0031_operations_facts.py b/migrations/versions/0031_operations_facts.py index a541d9f9..fb09b2a5 100644 --- a/migrations/versions/0031_operations_facts.py +++ b/migrations/versions/0031_operations_facts.py @@ -266,11 +266,13 @@ def classification(row: dict[str, Any]) -> dict[str, Any]: EVENTS = f"""WITH ordered AS ( SELECT e.*,lag(created_at) OVER(PARTITION BY run_id ORDER BY sequence) AS previous_at, + max(sequence) OVER(PARTITION BY run_id) AS final_sequence, public.weave_run_events_bytes(e)<={FETCH_BYTES} AS bounded FROM run_events e WHERE {SCOPE} AND run_id=ANY(:ids) ) SELECT run_id,min(created_at) AS started_at,max(created_at) AS updated_at, max(created_at) FILTER(WHERE transition->'state'->>'status' IN ({TERMINAL})) AS ended_at, min(sequence) AS first_sequence,max(sequence) AS last_event_sequence,count(*) AS event_count, + max(transition->'state'->>'status') FILTER(WHERE sequence=final_sequence) AS last_status, bool_and(coalesce(bounded AND jsonb_typeof(data)='object' AND jsonb_typeof(transition->'state')='object' AND transition->'state'->>'status' IN ({STATUSES}) AND transition->'state'->>'accepted_sequence'=sequence::text @@ -440,6 +442,7 @@ def backfill_scope(connection, params) -> None: and event["first_started"] == event["started_at"] and isinstance(state, dict) and state.get("accepted_sequence") == event["last_event_sequence"] + and item["status"] == event["last_status"] and ( item["status"] not in ("succeeded", "failed", "cancelled", "timed_out") or event["ended_at"] is not None diff --git a/tests/integration/test_operations_facts_migration.py b/tests/integration/test_operations_facts_migration.py index 1ba95816..9a654fd2 100644 --- a/tests/integration/test_operations_facts_migration.py +++ b/tests/integration/test_operations_facts_migration.py @@ -920,3 +920,105 @@ async def test_malformed_legacy_status_has_no_invented_lifecycle(empty_settings, await assert_usage(connection, ids) finally: await engine.dispose() + + +@pytest.mark.parametrize( + "source_status,event_status,initial_status,expected,unsupported", + [ + ("waiting", "succeeded", None, "unavailable", False), + ("failed", "succeeded", None, "unavailable", False), + ("succeeded", "waiting", None, "unavailable", False), + ("failed", "succeeded", "waiting", "unavailable", False), + ("succeeded", "succeeded", None, "available", False), + ("waiting", "waiting", "running", "available", False), + ("failed", "failed", "waiting", "available", False), + ("succeeded", "succeeded", "waiting", "unsupported", True), + ], +) +async def test_latest_event_lifecycle_must_match_retained_source( + empty_settings, source_status, event_status, initial_status, expected, unsupported +): + import json + from datetime import timedelta + + from operations_support import AT, identifier + + from firefly_weave.runtime.models import RunState, Transition + + engine = create_async_engine(empty_settings.database_url.get_secret_value(), hide_parameters=True) + try: + async with engine.begin() as connection: + await upgrade(connection, "0030_worker_presence") + ids = await seed_prior_head(connection, count=20) + run = ids["unsupported" if unsupported else "first_run"] + sequence = 2 if initial_status else 1 + state = RunState( + status=source_status, accepted_sequence=sequence, admission_policy="classified-v1", input=3 + ) + await connection.execute( + text("UPDATE runs SET state=cast(:state AS jsonb) WHERE id=:run"), + {"run": run, "state": state.model_dump_json()}, + ) + first_state = RunState( + status=initial_status or event_status, accepted_sequence=1, admission_policy="classified-v1", input=3 + ) + await connection.execute( + text( + "UPDATE run_events SET transition=cast(:transition AS jsonb)," + "response=jsonb_set(response,'{state}',cast(:state AS jsonb)) WHERE run_id=:run" + ), + { + "run": run, + "transition": Transition(state=first_state).model_dump_json(), + "state": first_state.model_dump_json(), + }, + ) + if initial_status: + latest = RunState(status=event_status, accepted_sequence=2, admission_policy="classified-v1", input=3) + await connection.execute( + text( + "INSERT INTO run_events SELECT tenant_id,project_id,environment_id,run_id," + ":event,2,'task_completed',data,:at,request_hash,cast(:transition AS jsonb)," + "jsonb_set(response,'{state}',cast(:state AS jsonb)) FROM run_events WHERE run_id=:run" + ), + { + "run": run, + "event": identifier(300002), + "at": AT + timedelta(seconds=1), + "transition": Transition(state=latest).model_dump_json(), + "state": latest.model_dump_json(), + }, + ) + sources = {} + for table in ("runs", "run_events"): + sources[table] = ( + (await connection.execute(text(f"SELECT to_jsonb(r)::text FROM {table} r ORDER BY id"))) + .scalars() + .all() + ) + assert ( + RunState.model_validate_json( + json.dumps(await connection.scalar(text("SELECT state FROM runs WHERE id=:run"), {"run": run})) + ).status + == source_status + ) + await upgrade(connection, "0031_operations_facts") + fact = ( + (await connection.execute(text("SELECT * FROM run_facts WHERE run_id=:run"), {"run": run})) + .mappings() + .one() + ) + assert fact["status"] == source_status + assert fact["classification_state"] == expected + assert fact["node_kinds"] == ({"work": "action"} if expected == "available" else {}) + if expected != "unavailable": + assert fact["started_at"] == AT + assert fact["updated_at"] == AT + timedelta(seconds=sequence - 1) + assert fact["ended_at"] == (fact["updated_at"] if source_status in {"succeeded", "failed"} else None) + for table in sources: + assert ( + await connection.execute(text(f"SELECT to_jsonb(r)::text FROM {table} r ORDER BY id")) + ).scalars().all() == sources[table] + await assert_usage(connection, ids) + finally: + await engine.dispose() From c1c905e9d70132cc3fd5be9062adc230d480a3ac Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Fri, 9 Oct 2026 05:20:09 -0700 Subject: [PATCH 04/63] Project run and task facts with accepted runtime changes --- src/firefly_weave/email/source.py | 2 + src/firefly_weave/operations/facts.py | 358 +++++++++- src/firefly_weave/operations/incidents.py | 13 +- src/firefly_weave/providers/dispatcher.py | 2 + src/firefly_weave/runtime/recovery.py | 10 +- src/firefly_weave/runtime/repository.py | 74 +- src/firefly_weave/runtime/service.py | 33 +- .../runtime/terminal_capacity.py | 14 +- src/firefly_weave/triggers/kafka.py | 2 + src/firefly_weave/triggers/scheduler.py | 2 + src/firefly_weave/triggers/service.py | 2 + src/firefly_weave/workers/leases.py | 7 +- tests/integration/providers/test_inbox.py | 1 + tests/integration/test_human_tasks.py | 11 + tests/integration/test_leases.py | 16 +- .../test_operations_fact_writes.py | 668 ++++++++++++++++++ tests/integration/test_operations_hot_path.py | 183 +++++ tests/integration/test_run_lifecycle.py | 15 +- tests/integration/test_runs.py | 16 +- tests/integration/test_schedules.py | 1 + tests/integration/test_terminal_capacity.py | 134 ++++ tests/integration/test_webhooks.py | 1 + .../email/test_email_dispatch_capacity.py | 1 + tests/unit/operations/test_fact_projection.py | 167 +++++ .../providers/test_dispatcher_capacity.py | 1 + tests/unit/runtime/test_recovery_admission.py | 2 +- .../unit/triggers/test_scheduler_capacity.py | 1 + tests/unit/triggers/test_start_origin.py | 82 +++ 28 files changed, 1772 insertions(+), 47 deletions(-) create mode 100644 tests/integration/test_operations_fact_writes.py create mode 100644 tests/integration/test_operations_hot_path.py create mode 100644 tests/unit/operations/test_fact_projection.py create mode 100644 tests/unit/triggers/test_start_origin.py diff --git a/src/firefly_weave/email/source.py b/src/firefly_weave/email/source.py index 9df2c3e6..c76aae2b 100644 --- a/src/firefly_weave/email/source.py +++ b/src/firefly_weave/email/source.py @@ -36,6 +36,7 @@ from firefly_weave.email.mime import parse_message from firefly_weave.email.service import EmailService, unavailable from firefly_weave.email.transport import IMAPTransport +from firefly_weave.operations.facts import RunStartFacts from firefly_weave.persistence.uow import Transaction from firefly_weave.runtime.repository import SCOPE, RuntimeRepository from firefly_weave.runtime.service import RuntimeService @@ -493,6 +494,7 @@ async def dispatch(self, actor: Principal, scope: Scope, receipt_id: UUID) -> di key, context=AuditContext(), tx=tx, + start_facts=RunStartFacts(origin="email"), ) run_id, signal_id = outcome_run.id, None await db.execute( diff --git a/src/firefly_weave/operations/facts.py b/src/firefly_weave/operations/facts.py index 793944a4..a13da6b1 100644 --- a/src/firefly_weave/operations/facts.py +++ b/src/firefly_weave/operations/facts.py @@ -16,14 +16,31 @@ """Transaction-bound SQL for operational projections.""" +import json from dataclasses import dataclass -from typing import Any +from datetime import datetime +from typing import Any, Literal from uuid import UUID +from pydantic import TypeAdapter from sqlalchemy import text -from firefly_weave.contracts.run_views import RunOrigin, RunSummaryCaller +from firefly_weave.contracts.catalog import Activation +from firefly_weave.contracts.definitions import ResourceName +from firefly_weave.contracts.instance_keys import InstanceKeyText, InstanceView, instance_view +from firefly_weave.contracts.run_views import RunOrigin, RunSummaryCaller, StepKind +from firefly_weave.contracts.runtime import RunView, StartRunRequest from firefly_weave.persistence.uow import Transaction +from firefly_weave.runtime.models import ( + ControlCommand, + Deadline, + HumanTaskIntent, + RunState, + RuntimeEvent, + TaskIntent, + TerminalControl, + Transition, +) FACT_TABLES: tuple[str, str, str, str, str] = ( "run_facts", @@ -34,6 +51,78 @@ ) +SCOPE = "tenant_id=:tenant AND project_id=:project AND environment_id=:environment" +TERMINAL = {"succeeded", "failed", "cancelled", "timed_out"} + + +def author_instance(key: str) -> InstanceView | None: + if type(key) is not str: + raise ValueError("Instance key must be text") + if key.startswith("@"): + return None + TypeAdapter(InstanceKeyText).validate_python(key, strict=True) + value = instance_view(key) + TypeAdapter(ResourceName).validate_python(value.node_id, strict=True) + return value + + +def listing_metadata(row: dict[str, Any]) -> dict[str, Any]: + """Bounded cache metadata; execution admission remains the runtime's authority.""" + from firefly_weave.compiler.api import import_artifact + from firefly_weave.runtime import admission as retained + from firefly_weave.runtime.kernel import workflow + + result: dict[str, Any] = dict( + classification_state="unavailable", + classification_policy=retained.POLICY, + pinned_ir_version=None, + pinned_features=[], + node_kinds={}, + ) + try: + RunState.model_validate_json(json.dumps(row["state"])) + Activation.model_validate_json(json.dumps(row["activation"])) + StartRunRequest.model_validate_json(json.dumps(row["request"])) + decision = retained.admission(row) + executable = row["artifact"].get("executable", {}) + version, features = executable.get("irVersion"), executable.get("features", []) + if not isinstance(version, str) or len(version) > 256 or not isinstance(features, list): + return result + if len(features) > 256 or any(type(value) is not str for value in features): + return result + if len(json.dumps(features, ensure_ascii=False).encode()) > 16384: + return result + result.update(classification_state=decision, pinned_ir_version=version, pinned_features=features) + if decision == "available": + kinds: dict[str, StepKind] = {} + for node in workflow(import_artifact(row["artifact"])).graph.nodes: + identity = author_instance(node.id) + if identity is not None: + kinds[identity.node_id] = TypeAdapter(StepKind).validate_python(node.kind, strict=True) + if len(kinds) > 10000 or len(json.dumps(kinds, ensure_ascii=False).encode()) > 1048576: + raise ValueError("Node-kind projection exceeds its bound") + result["node_kinds"] = kinds + except (ValueError, TypeError, KeyError, AttributeError, RecursionError): + result.update(classification_state="unavailable", node_kinds={}) + state = row.get("state") + status = state.get("status") if isinstance(state, dict) else None + if type(status) is not str or status not in {"queued", "running", "waiting", "suspended", *TERMINAL}: + result.update(classification_state="unavailable", node_kinds={}) + return result + + +def task_fact_update(source: str) -> str: + """Enlist task metadata in the existing authoritative source statement.""" + return ( + "WITH changed AS (" + source + " AND status<>'handled' RETURNING id,status,next_attempt_at) " + "UPDATE task_facts f SET status=c.status,ready_since=CASE WHEN c.status='ready' " + "THEN greatest(:at,coalesce(c.next_attempt_at,:at)) ELSE NULL END," + "first_claimed_at=CASE WHEN c.status='leased' THEN coalesce(f.first_claimed_at,:at) " + "ELSE f.first_claimed_at END FROM changed c WHERE f.task_id=c.id " + "AND f.tenant_id=:tenant AND f.project_id=:project AND f.environment_id=:environment" + ) + + @dataclass(frozen=True) class RunStartFacts: origin: RunOrigin = "manual" @@ -41,6 +130,21 @@ class RunStartFacts: caller: RunSummaryCaller | None = None retried_from_run_id: UUID | None = None + def __post_init__(self) -> None: + TypeAdapter(RunOrigin).validate_python(self.origin, strict=True) + if type(self.test) is not bool or self.test != (self.origin == "test"): + raise ValueError("Test origin and test flag must agree") + if (self.origin == "call") != (self.caller is not None): + raise ValueError("Call origin requires a caller") + if self.caller is not None: + if not isinstance(self.caller, RunSummaryCaller): + raise ValueError("Call origin requires typed caller metadata") + RunSummaryCaller.model_validate_json(self.caller.model_dump_json()) + if (self.origin == "retry") != (self.retried_from_run_id is not None): + raise ValueError("Retry origin requires a source run") + if self.retried_from_run_id is not None and not isinstance(self.retried_from_run_id, UUID): + raise ValueError("Retry source must be a UUID") + class FactRepository: def __init__(self, tx: Transaction) -> None: @@ -57,3 +161,253 @@ async def execute(self, sql: str, **values: Any) -> None: async def rows(self, sql: str, **values: Any) -> list[dict[str, Any]]: result = await self.tx.session.execute(text(sql), {**values, **self.params}) return [dict(row) for row in result.mappings()] + + async def insert_run( + self, view: RunView, artifact: dict[str, Any], request: StartRunRequest, facts: RunStartFacts, at: datetime + ) -> None: + metadata = listing_metadata( + dict( + state=view.state.model_dump(mode="json"), + artifact=artifact, + activation=view.activation.model_dump(mode="json"), + request=request.model_dump(mode="json"), + ) + ) + await self.execute( + "INSERT INTO run_facts(tenant_id,project_id,environment_id,run_id,definition_version_id," + "workflow_name,workflow_version,activation_id,activation_name,activation_revision,status,paused," + "test,origin,caller_run_id,caller_node_id,caller_instance_key,retried_from_run_id,business_key," + "correlation_key,started_at,updated_at,classification_state,classification_policy,pinned_ir_version," + "pinned_features,node_kinds,facts_started_at) SELECT :tenant,:project,:environment,:run,:version," + "name,version,:activation,:name,:revision,:status,:paused,:test,:origin,:caller_run,:caller_node," + ":caller_instance,:retried,:business,:correlation,:at,:at,:classification_state,:classification_policy," + ":pinned_ir_version,cast(:pinned_features AS jsonb),cast(:node_kinds AS jsonb),:at " + "FROM definition_versions WHERE tenant_id=:tenant AND project_id=:project AND id=:version", + run=view.id, + version=view.activation.request.version_id, + activation=view.activation.id, + name=view.activation.name, + revision=view.activation.revision, + status=view.state.status, + paused=view.state.manual_paused, + test=facts.test, + origin=facts.origin, + caller_run=facts.caller.run_id if facts.caller else None, + caller_node=facts.caller.node_id if facts.caller else None, + caller_instance=facts.caller.instance_key if facts.caller else None, + retried=facts.retried_from_run_id, + business=request.business_key, + correlation=request.correlation_key, + at=at, + **{ + **metadata, + "pinned_features": json.dumps(metadata["pinned_features"], ensure_ascii=False), + "node_kinds": json.dumps(metadata["node_kinds"], ensure_ascii=False), + }, + ) + + def task_values( + self, task_id: UUID, run_id: UUID, command: TaskIntent, task_type: str, at: datetime + ) -> dict[str, Any]: + identity = author_instance(command.node_id) + if identity is None: + raise ValueError("Task requires an author step") + return dict( + id=task_id, + run=run_id, + fact_node=identity.node_id, + instance=identity.instance_key, + task_type=task_type, + at=at, + ) + + async def project_run(self, view: RunView | TerminalControl, event: RuntimeEvent, result: Transition) -> None: + failed = None + if result.state.status == "failed": + key = next( + (c.node_id for c in result.commands if isinstance(c, ControlCommand) and c.kind == "terminate"), + event.data.get("node_id"), + ) + failed = author_instance(key) if isinstance(key, str) else None + rows = await self.rows( + f"UPDATE run_facts SET status=:status,paused=:paused,updated_at=greatest(updated_at,:at)," + "ended_at=CASE WHEN :terminal THEN :at ELSE ended_at END,failed_node_id=:failed_node," + "failed_instance_key=:failed_instance,failed_error_code=:code,last_event_sequence=:sequence," + "classification_state=CASE WHEN :unavailable THEN 'unavailable' ELSE classification_state END," + f"active_incidents=(SELECT count(*) FROM incidents WHERE {SCOPE} AND run_id=:run AND status='active') " + f"WHERE {SCOPE} AND run_id=:run AND last_event_sequence<:sequence RETURNING node_kinds", + run=view.id, + status=result.state.status, + paused=result.state.manual_paused, + at=event.timestamp, + terminal=result.state.status in TERMINAL, + failed_node=failed.node_id if failed else None, + failed_instance=failed.instance_key if failed else None, + code=result.state.incident if result.state.status == "failed" else None, + sequence=event.sequence, + unavailable=result.state.unavailable, + ) + if not rows: + return + kinds = rows[0]["node_kinds"] + steps: dict[tuple[str, str], dict[str, Any]] = {} + + def add(key: str, status: str, *, error: str | None = None) -> None: + identity = author_instance(key) + if identity is None: + return + kind: StepKind = TypeAdapter(StepKind).validate_python(kinds[identity.node_id], strict=True) + steps[identity.node_id, identity.instance_key] = dict( + node_id=identity.node_id, + instance_key=identity.instance_key, + kind=kind, + status=status, + started_at=None if kind in {"action", "llm", "agent"} else event.timestamp.isoformat(), + ended_at=event.timestamp.isoformat() if status in TERMINAL else None, + error_code=error, + ) + + if not isinstance(view, TerminalControl): + for command in result.commands: + if isinstance(command, (TaskIntent, Deadline, HumanTaskIntent)): + add(command.node_id, "scheduled" if isinstance(command, TaskIntent) else "waiting") + elif command.kind == "spawn_branch": + add(command.node_id, "running") + for change in result.steps: + identity = author_instance(change.node_id) + if identity is None: + continue + kind = kinds[identity.node_id] + status = ( + "succeeded" + if change.status == "completed" + else ("scheduled" if kind in {"action", "llm", "agent"} else "waiting") + ) + add(change.node_id, status) + for owner, join in result.state.joins.items(): + if join.status == "waiting" and (owner, "") not in steps: + add(owner, "running") + node = event.data.get("node_id") + if isinstance(node, str) and any(value.id == event.id for value in result.state.deferred_results): + add(node, "succeeded") + if isinstance(node, str) and event.type in {"task_failed", "incident_opened", "recovery_scheduled"}: + output = event.data.get("output") + code = output.get("code") if isinstance(output, dict) else event.data.get("code") + add( + node, + "scheduled" if event.type == "recovery_scheduled" else "failed", + error=code if isinstance(code, str) else None, + ) + if ( + isinstance(node, str) + and event.type == "incident_resolved" + and event.data.get("kind") == "retry_safe" + and result.state.status not in TERMINAL + ): + add(node, "scheduled") + if failed: + add(failed.instance_key or failed.node_id, "failed", error=result.state.incident) + if not steps and result.state.status not in TERMINAL: + return + closed = result.state.status if result.state.status in {"cancelled", "timed_out"} else "cancelled" + if result.state.status in TERMINAL: + for step in steps.values(): + if step["status"] in {"scheduled", "running", "waiting"}: + step.update(status=closed, ended_at=event.timestamp.isoformat()) + await self.execute( + "WITH projected AS (INSERT INTO step_facts(tenant_id,project_id,environment_id,run_id,node_id," + "instance_key,kind,status,scheduled_at,started_at,ended_at,error_code) " + "SELECT :tenant,:project,:environment,:run,s.node_id,s.instance_key,s.kind,s.status,:at," + "s.started_at,s.ended_at,s.error_code FROM jsonb_to_recordset(cast(:steps AS jsonb)) " + "AS s(node_id text,instance_key text,kind text,status text,started_at timestamptz," + "ended_at timestamptz,error_code text) ON CONFLICT(run_id,node_id,instance_key) DO UPDATE SET " + "status=excluded.status,started_at=coalesce(step_facts.started_at,excluded.started_at)," + "ended_at=CASE WHEN step_facts.status=excluded.status THEN coalesce(step_facts.ended_at,excluded.ended_at) " + "ELSE excluded.ended_at END,error_code=excluded.error_code WHERE step_facts.handled IS NULL " + "RETURNING node_id,instance_key) UPDATE step_facts SET status=:closed,ended_at=:at " + f"WHERE {SCOPE} AND run_id=:run AND :terminal AND status IN ('scheduled','running','waiting') " + "AND NOT EXISTS(SELECT 1 FROM projected p WHERE p.node_id=step_facts.node_id " + "AND p.instance_key=step_facts.instance_key)", + run=view.id, + at=event.timestamp, + steps=json.dumps(list(steps.values())), + terminal=result.state.status in TERMINAL, + closed=closed, + ) + + async def step_claimed(self, task_id: UUID, generation: int, worker_id: UUID, at: datetime) -> None: + await self.execute( + "UPDATE step_facts s SET started_at=coalesce(s.started_at,:at),status='running'," + "ended_at=NULL,error_code=NULL,attempts=greatest(s.attempts,:generation),worker_id=:worker " + "FROM task_facts f WHERE f.task_id=:task AND f.status='leased' " + "AND f.tenant_id=:tenant AND f.project_id=:project AND f.environment_id=:environment " + "AND s.tenant_id=f.tenant_id AND s.project_id=f.project_id AND s.environment_id=f.environment_id " + "AND s.run_id=f.run_id AND s.node_id=f.node_id AND s.instance_key=f.instance_key AND s.handled IS NULL", + task=task_id, + generation=generation, + worker=worker_id, + at=at, + ) + + async def project_terminal( + self, run_id: UUID, status: Literal["cancelled", "timed_out"], at: datetime, sequence: int + ) -> None: + if status not in {"cancelled", "timed_out"}: + raise ValueError("Terminal control status required") + await self.execute( + "WITH changed AS (UPDATE run_facts SET status=:status,updated_at=greatest(updated_at,:at),ended_at=:at," + "failed_node_id=NULL,failed_instance_key=NULL,failed_error_code=NULL,active_incidents=0," + f"last_event_sequence=:sequence WHERE {SCOPE} AND run_id=:run AND last_event_sequence<:sequence " + "RETURNING run_id), tasks AS (UPDATE task_facts f SET status=t.status,ready_since=NULL FROM " + "task_intents t WHERE f.task_id=t.id AND f.run_id IN (SELECT run_id FROM changed) " + "AND f.tenant_id=:tenant AND f.project_id=:project AND f.environment_id=:environment " + "AND t.tenant_id=f.tenant_id AND t.project_id=f.project_id AND t.environment_id=f.environment_id " + f"RETURNING f.task_id) UPDATE step_facts SET status=:status,ended_at=:at WHERE {SCOPE} " + "AND run_id IN (SELECT run_id FROM changed) AND status IN ('scheduled','running','waiting')", + run=run_id, + status=status, + at=at, + sequence=sequence, + ) + + async def handled_failure( + self, + run_id: UUID, + key: str, + kind: StepKind, + disposition: Literal["continue", "errorOutput"], + error_code: str, + at: datetime, + *, + task_id: UUID | None = None, + ) -> None: + identity = author_instance(key) + if identity is None or disposition not in {"continue", "errorOutput"}: + raise ValueError("Handled failure requires an author step and disposition") + TypeAdapter(StepKind).validate_python(kind, strict=True) + if task_id is not None: + tasks = await self.rows( + f"SELECT run_id,node_id,status FROM task_intents WHERE {SCOPE} AND id=:task", task=task_id + ) + if len(tasks) != 1 or tasks[0] != dict(run_id=run_id, node_id=key, status="handled"): + raise ValueError("Handled task authority does not match the projection") + await self.execute( + "WITH changed AS (INSERT INTO step_facts(tenant_id,project_id,environment_id,run_id,node_id," + "instance_key,kind,status,ended_at,error_code,handled) VALUES(:tenant,:project,:environment,:run," + ":node,:instance,:kind,'failed',:at,:code,:handled) ON CONFLICT(run_id,node_id,instance_key) " + "DO UPDATE SET status='failed',ended_at=excluded.ended_at,error_code=excluded.error_code," + "handled=excluded.handled WHERE step_facts.handled IS NULL RETURNING run_id) " + f"UPDATE run_facts SET handled_errors=handled_errors+1 WHERE {SCOPE} " + "AND run_id IN (SELECT run_id FROM changed)", + run=run_id, + node=identity.node_id, + instance=identity.instance_key, + kind=kind, + at=at, + code=error_code, + handled=disposition, + ) + if task_id is not None: + await self.execute( + f"UPDATE task_facts SET status='handled',ready_since=NULL WHERE {SCOPE} AND task_id=:task", task=task_id + ) diff --git a/src/firefly_weave/operations/incidents.py b/src/firefly_weave/operations/incidents.py index d7065863..33d75c79 100644 --- a/src/firefly_weave/operations/incidents.py +++ b/src/firefly_weave/operations/incidents.py @@ -34,6 +34,7 @@ from firefly_weave.contracts.definitions import ActionDefinition, load_definition from firefly_weave.contracts.operations import IncidentResolution, IncidentView from firefly_weave.definitions.models import CatalogError +from firefly_weave.operations.facts import task_fact_update from firefly_weave.persistence.uow import Transaction from firefly_weave.runtime.kernel import transition_async as transition from firefly_weave.runtime.kernel import validate_action, workflow @@ -209,16 +210,20 @@ async def resolve( await repository.persist(view, event, result, event_hash(event)) if task is not None and result.state.status not in TERMINAL: await repository.execute( - f"UPDATE task_intents SET status=:status,next_attempt_at=NULL WHERE {SCOPE} AND id=:id", + task_fact_update( + f"UPDATE task_intents SET status=:status,next_attempt_at=NULL WHERE {SCOPE} AND id=:id" + ), + at=event.timestamp, id=task["id"], status="ready" if request.kind == "retry_safe" else "completed", ) elif task is not None and request.kind == "accept_reconciled_result": - await repository.task_status(task["id"], "completed") + await repository.task_status(task["id"], "completed", at=event.timestamp) await repository.execute( - "UPDATE incidents SET status='resolved',actor_id=:actor,resolved_at=:now," + "WITH changed AS (UPDATE incidents SET status='resolved',actor_id=:actor,resolved_at=:now," "resolution=cast(:resolution AS jsonb) " - f"WHERE {SCOPE} AND id=:id", + f"WHERE {SCOPE} AND id=:id RETURNING id) UPDATE incident_facts SET updated_at=:now " + f"WHERE {SCOPE} AND incident_id IN (SELECT id FROM changed)", id=incident_id, actor=actor.id, now=event.timestamp, diff --git a/src/firefly_weave/providers/dispatcher.py b/src/firefly_weave/providers/dispatcher.py index 6421ba6d..524f5350 100644 --- a/src/firefly_weave/providers/dispatcher.py +++ b/src/firefly_weave/providers/dispatcher.py @@ -33,6 +33,7 @@ from firefly_weave.contracts.providers import ProviderReceipt from firefly_weave.contracts.runtime import StartRunRequest from firefly_weave.definitions.models import CatalogError, capacity_rejected +from firefly_weave.operations.facts import RunStartFacts from firefly_weave.providers.repository import ProviderRepository from firefly_weave.providers.service import ProviderIngressService from firefly_weave.runtime.repository import SCOPE @@ -71,6 +72,7 @@ async def dispatch_one(self, scope: Scope, receipt_id: UUID) -> ProviderReceipt: key, context=AuditContext(), tx=tx, + start_facts=RunStartFacts(origin="provider"), ) changes.update(state="dispatched", run_id=run.id, reason=None) else: diff --git a/src/firefly_weave/runtime/recovery.py b/src/firefly_weave/runtime/recovery.py index 00699bce..6d49313e 100644 --- a/src/firefly_weave/runtime/recovery.py +++ b/src/firefly_weave/runtime/recovery.py @@ -33,6 +33,7 @@ from firefly_weave.contracts.operations import TaskTiming from firefly_weave.contracts.runtime import RecoveryReport from firefly_weave.definitions.models import CatalogError +from firefly_weave.operations.facts import task_fact_update from firefly_weave.persistence.uow import Transaction from firefly_weave.runtime.deadlines import DeadlineService from firefly_weave.runtime.kernel import transition_async as transition @@ -139,13 +140,15 @@ async def _recover(self, tx: Transaction, kind: str) -> RecoveryReport: tx.session.info["weave_runtime_candidate"] = row["id"] result = await transition(view.state, event, import_artifact(row["artifact"])) await repository.persist(view.model_copy(update={"state": result.state}), event, result, event_hash(event)) - await repository.task_status(identifier, "incident") + await repository.task_status(identifier, "incident", at=event.timestamp) incidents += 1 for identifier in await self.tasks.recovery_candidates(tx, limit) if kind == "retry" else []: locked = await self.runtime.lock_task(tx, identifier, skip_locked=True) if locked is None: continue row, task = locked + if task["status"] == "handled": + continue if await self.runtime.observe_unavailable(tx, row): continue attempt = await self.tasks.recovery_attempt(tx, identifier) @@ -231,7 +234,10 @@ async def _recover(self, tx: Transaction, kind: str) -> RecoveryReport: if result.state.status in TERMINAL: continue await repository.execute( - f"UPDATE task_intents SET status=:status,next_attempt_at=:next WHERE {SCOPE} AND id=:id", + task_fact_update( + f"UPDATE task_intents SET status=:status,next_attempt_at=:next WHERE {SCOPE} AND id=:id" + ), + at=now, id=identifier, status="ready" if allowed else "incident", next=next_attempt if allowed else None, diff --git a/src/firefly_weave/runtime/repository.py b/src/firefly_weave/runtime/repository.py index 7693f872..c950edd4 100644 --- a/src/firefly_weave/runtime/repository.py +++ b/src/firefly_weave/runtime/repository.py @@ -28,6 +28,7 @@ from firefly_weave.contracts.operations import TaskTiming from firefly_weave.contracts.runtime import RunView, StartRunRequest, UnavailableRunAcknowledgment from firefly_weave.definitions.models import CatalogError +from firefly_weave.operations.facts import FactRepository, RunStartFacts, author_instance, task_fact_update from firefly_weave.operations.outbox import OutboxService from firefly_weave.persistence.uow import Transaction from firefly_weave.runtime.models import ( @@ -111,7 +112,16 @@ async def require_work(self, identifier: UUID, state: dict[str, Any]) -> None: self.tx.session.info["weave_runtime_candidate"] = identifier await execute_pure(partial(logical_size, state, STATE_BYTES)) - async def insert(self, view: RunView, artifact: dict[str, Any], request: StartRunRequest, principal: UUID) -> None: + async def insert( + self, + view: RunView, + artifact: dict[str, Any], + request: StartRunRequest, + principal: UUID, + *, + start_facts: RunStartFacts, + at: datetime, + ) -> None: await self.execute( "INSERT INTO runs VALUES(:id,:tenant,:project,:environment,:activation_id,:principal," "cast(:artifact AS jsonb),cast(:activation AS jsonb),cast(:request AS jsonb),cast(:state AS jsonb))", @@ -124,6 +134,8 @@ async def insert(self, view: RunView, artifact: dict[str, Any], request: StartRu state=view.state.model_dump_json(), ) + await FactRepository(self.tx).insert_run(view, artifact, request, start_facts, at) + async def event(self, run_id: UUID, event_id: UUID) -> dict[str, Any] | None: row = ( ( @@ -180,7 +192,7 @@ async def persist( state=result.state.model_dump_json(), run=view.id, ) - await self.project_incidents(view) + await self.project_incidents(view, at=event.timestamp) for change in result.steps: await self.execute( "INSERT INTO step_instances " @@ -205,13 +217,19 @@ async def persist( task_version=pin.task_version, connector_target=pin.model_dump(mode="json"), ) + identifier = uuid4() + values = FactRepository(self.tx).task_values( + identifier, view.id, command, f"{payload['task_type']}@{payload['task_version']}", event.timestamp + ) await self.execute( - "INSERT INTO task_intents(id,tenant_id,project_id,environment_id,run_id,node_id,payloa" - "d,worker_release_id,operation_key,status) " + "WITH inserted AS (INSERT INTO task_intents(id,tenant_id,project_id,environment_id,run_id,node_id," + "payload,worker_release_id,operation_key,status) " "VALUES(:id,:tenant,:project,:environment,:run,:node," - "cast(:payload AS jsonb),:release,:operation,'ready')", - id=uuid4(), - run=view.id, + "cast(:payload AS jsonb),:release,:operation,'ready') RETURNING id) " + "INSERT INTO task_facts(tenant_id,project_id,environment_id,task_id,run_id,node_id,instance_key," + "task_type,status,created_at,ready_since) SELECT :tenant,:project,:environment,id,:run," + ":fact_node,:instance,:task_type,'ready',:at,:at FROM inserted", + **values, node=command.node_id, payload=json.dumps(payload), release=release, @@ -234,7 +252,10 @@ async def persist( await self.record_evidence(view, event, result, task_timing) if event.type == "task_completed": await self.execute( - f"UPDATE task_intents SET status='completed' WHERE {SCOPE} AND run_id=:run AND node_id=:node", + task_fact_update( + f"UPDATE task_intents SET status='completed' WHERE {SCOPE} AND run_id=:run AND node_id=:node" + ), + at=event.timestamp, run=view.id, node=event.data["node_id"], ) @@ -266,12 +287,16 @@ async def persist( ), ) await self.execute( - f"UPDATE task_intents SET status='cancelled' WHERE {SCOPE} AND run_id=:run " - "AND status IN ('ready','leased','retry_pending','incident','failed')", + task_fact_update( + f"UPDATE task_intents SET status='cancelled' WHERE {SCOPE} AND run_id=:run " + "AND status IN ('ready','leased','retry_pending','incident','failed')" + ), + at=event.timestamp, run=view.id, ) await self.execute(f"UPDATE run_deadlines SET consumed=true WHERE {SCOPE} AND run_id=:run", run=view.id) + await FactRepository(self.tx).project_run(view, event, result) await self.outbox.append( self.tx, IntegrationEvent( @@ -376,9 +401,12 @@ async def locked_task( ) return (dict(run), dict(task)) if task else None - async def task_status(self, identifier: UUID, status: str) -> None: + async def task_status(self, identifier: UUID, status: str, *, at: datetime | None = None) -> None: await self.execute( - f"UPDATE task_intents SET status=:status WHERE {SCOPE} AND id=:id", id=identifier, status=status + task_fact_update(f"UPDATE task_intents SET status=:status WHERE {SCOPE} AND id=:id"), + id=identifier, + status=status, + at=at if at is not None else await self.now(), ) async def task_deadline(self, run_id: UUID, task_deadline: datetime) -> datetime: @@ -443,19 +471,29 @@ async def expired_ready(self, limit: int) -> list[UUID]: ) return [row["id"] for row in rows] - async def project_incidents(self, view: RunView | TerminalControl) -> None: + async def project_incidents(self, view: RunView | TerminalControl, *, at: datetime | None = None) -> None: """Projection only: never changes or repairs authoritative accepted state.""" + at = at if at is not None else await self.now() active = {key: value.model_dump() for key, value in view.state.incidents.items()} if view.state.status == "suspended" and not active and view.state.incident: active["@legacy"] = {"node_id": None, "generation": None, "code": view.state.incident} for key, value in active.items(): + identity = author_instance(value["node_id"]) if value["node_id"] is not None else None await self.execute( - "INSERT INTO incidents(id,tenant_id,project_id,environment_id,run_id,incident_key," + "WITH changed AS (INSERT INTO incidents(id,tenant_id,project_id,environment_id,run_id,incident_key," "node_id,generation,origin_code,code,status,revision) VALUES(" "md5(cast(cast(:run AS uuid) AS text) || ':' || :key)::uuid,:tenant,:project,:environment,:run,:key," ":node,:generation,:code,:code,'active',1) ON CONFLICT(run_id,incident_key) DO UPDATE " "SET code=excluded.code,revision=incidents.revision+1 " - "WHERE incidents.status='active' AND incidents.code<>excluded.code", + "WHERE incidents.status='active' AND incidents.code<>excluded.code RETURNING id) " + "INSERT INTO incident_facts(tenant_id,project_id,environment_id,incident_id,node_id,instance_key," + "opened_at,updated_at,workflow_name,workflow_version,test) SELECT :tenant,:project,:environment," + "c.id,:fact_node,:instance,:at,:at,f.workflow_name,f.workflow_version,f.test FROM changed c " + f"JOIN run_facts f ON f.run_id=:run AND f.{SCOPE.replace(' AND ', ' AND f.')} " + "ON CONFLICT(incident_id) DO UPDATE SET updated_at=excluded.updated_at", + fact_node=identity.node_id if identity else None, + instance=identity.instance_key if identity else "", + at=at, run=view.id, key=key, node=value["node_id"], @@ -463,8 +501,10 @@ async def project_incidents(self, view: RunView | TerminalControl) -> None: code=value["code"], ) await self.execute( - f"UPDATE incidents SET status='closed',revision=revision+1 WHERE {SCOPE} AND run_id=:run " - "AND status='active' AND NOT (incident_key=ANY(cast(:keys AS text[])))", + f"WITH closed AS (UPDATE incidents SET status='closed',revision=revision+1 WHERE {SCOPE} AND run_id=:run " + "AND status='active' AND NOT (incident_key=ANY(cast(:keys AS text[]))) RETURNING id) " + f"UPDATE incident_facts SET updated_at=:at WHERE {SCOPE} AND incident_id IN (SELECT id FROM closed)", + at=at, run=view.id, keys=list(active), ) diff --git a/src/firefly_weave/runtime/service.py b/src/firefly_weave/runtime/service.py index 09e174c5..bb64d056 100644 --- a/src/firefly_weave/runtime/service.py +++ b/src/firefly_weave/runtime/service.py @@ -44,6 +44,7 @@ ) from firefly_weave.definitions.models import CatalogError from firefly_weave.definitions.service import DefinitionService +from firefly_weave.operations.facts import RunStartFacts from firefly_weave.persistence.idempotency import Idempotency from firefly_weave.persistence.uow import Transaction from firefly_weave.runtime.kernel import KernelError, action_schemas, validate, validate_action, workflow @@ -165,8 +166,12 @@ async def start( context: AuditContext, tx: Transaction | None = None, not_after: datetime | None = None, + start_facts: RunStartFacts | None = None, ) -> RunView: self.require(actor, scope, "run.start", context) + start_facts = start_facts if start_facts is not None else RunStartFacts() + if not isinstance(start_facts, RunStartFacts): + raise ValueError("Trusted start facts required") async with self.definitions.transaction(scope, tx) as enlisted: replay = Idempotency( enlisted, @@ -203,7 +208,11 @@ async def start( if not_after is not None and now >= not_after: raise CatalogError(409, "WV-SCHEDULE-STALE", "Occurrence grace expired during admission") event = RuntimeEvent( - id=uuid4(), type="started", timestamp=now, sequence=1, data={"admission_policy": "classified-v1"} + id=uuid4(), + type="started", + timestamp=now, + sequence=1, + data={"admission_policy": "classified-v1", "origin": start_facts.origin}, ) result = await transition(RunState(input=request.input), event, artifact) self.require(await load_principal(enlisted.session, actor.id), scope, "run.start", context) @@ -218,7 +227,7 @@ async def start( correlation_key=request.correlation_key, business_key=request.business_key, ) - await repository.insert(view, envelope, request, actor.id) + await repository.insert(view, envelope, request, actor.id, start_facts=start_facts, at=now) from firefly_weave.files.authority import admit_files await admit_files(enlisted, view.id, request.input, actor, context=context) @@ -305,8 +314,10 @@ async def observe_unavailable(self, tx: Transaction, row: dict[str, Any]) -> boo await RuntimeRepository(tx, self.definitions.outbox).observe_policy_block(row) return decision != "available" - async def set_task_status(self, tx: Transaction, identifier: UUID, status: str) -> None: - await RuntimeRepository(tx, self.definitions.outbox).task_status(identifier, status) + async def set_task_status( + self, tx: Transaction, identifier: UUID, status: str, *, at: datetime | None = None + ) -> None: + await RuntimeRepository(tx, self.definitions.outbox).task_status(identifier, status, at=at) async def finish_verified_task( self, @@ -387,9 +398,9 @@ async def finish_verified_task( ), ) if rejection_code: - await repository.task_status(verified.task["id"], "incident") + await repository.task_status(verified.task["id"], "incident", at=event.timestamp) elif failed: - await repository.task_status(verified.task["id"], "failed") + await repository.task_status(verified.task["id"], "failed", at=event.timestamp) else: from firefly_weave.runtime.waits import settle @@ -560,7 +571,15 @@ async def retry_run( raise CatalogError(409, "WV-RUNTIME-STATE", "Only a terminal run can create a linked retry") # Retry replay is owned by the outer receipt; the nested start gets a fresh identity. start_key = str(uuid4()) - view = await self.start(current, scope, request, start_key, context=context, tx=tx) + view = await self.start( + current, + scope, + request, + start_key, + context=context, + tx=tx, + start_facts=RunStartFacts(origin="retry", retried_from_run_id=run_id), + ) await repository.execute( "INSERT INTO run_retry_links VALUES(:tenant,:project,:environment,:run,:parent)", run=view.id, diff --git a/src/firefly_weave/runtime/terminal_capacity.py b/src/firefly_weave/runtime/terminal_capacity.py index 3ea10b11..e942531c 100644 --- a/src/firefly_weave/runtime/terminal_capacity.py +++ b/src/firefly_weave/runtime/terminal_capacity.py @@ -28,6 +28,7 @@ from firefly_weave.definitions.models import CatalogError from firefly_weave.operations.execution import execute_pure from firefly_weave.operations.exports import lock_digest +from firefly_weave.operations.facts import FactRepository, task_fact_update from firefly_weave.operations.redaction import Omission from firefly_weave.runtime.admission import terminal_signals from firefly_weave.runtime.models import RuntimeEvent @@ -155,7 +156,10 @@ async def persist_terminal( break await revoke_attempts(repository.tx, row["id"], tasks, control_failure=event.type == "cancelled") await repository.execute( - f"UPDATE task_intents SET status='cancelled' WHERE {SCOPE} AND id=ANY(cast(:ids AS uuid[]))", + task_fact_update( + f"UPDATE task_intents SET status='cancelled' WHERE {SCOPE} AND id=ANY(cast(:ids AS uuid[]))" + ), + at=event.timestamp, ids=[task["id"] for task in tasks], ) await repository.execute( @@ -163,7 +167,10 @@ async def persist_terminal( run=row["id"], ) await repository.execute( - f"UPDATE incidents SET status='closed',revision=revision+1 WHERE {SCOPE} AND run_id=:run AND status='active'", + f"WITH closed AS (UPDATE incidents SET status='closed',revision=revision+1 WHERE {SCOPE} AND run_id=:run " + "AND status='active' RETURNING id) UPDATE incident_facts SET updated_at=:at " + f"WHERE {SCOPE} AND incident_id IN (SELECT id FROM closed)", + at=event.timestamp, run=row["id"], ) if row.get("artifact"): @@ -203,6 +210,9 @@ async def persist_terminal( sequence=event.sequence, evidence=evidence.model_dump_json(), ) + await FactRepository(repository.tx).project_terminal( + row["id"], cast(Literal["cancelled", "timed_out"], event.type), event.timestamp, event.sequence + ) await repository.outbox.append( repository.tx, IntegrationEvent( diff --git a/src/firefly_weave/triggers/kafka.py b/src/firefly_weave/triggers/kafka.py index b9be6bd8..536aede9 100644 --- a/src/firefly_weave/triggers/kafka.py +++ b/src/firefly_weave/triggers/kafka.py @@ -50,6 +50,7 @@ from firefly_weave.contracts.runtime import StartRunRequest from firefly_weave.contracts.values import JsonObject, JsonValue from firefly_weave.definitions.models import CatalogError +from firefly_weave.operations.facts import RunStartFacts from firefly_weave.operations.redaction import project from firefly_weave.persistence.idempotency import lock from firefly_weave.persistence.uow import Transaction @@ -416,6 +417,7 @@ async def consume(self, record: BrokerRecord, *, authority: ConsumerAuthority) - f"kafka:{route.id}:{event}", context=AuditContext(), tx=tx, + start_facts=RunStartFacts(origin="broker"), ) run_id = run.id else: diff --git a/src/firefly_weave/triggers/scheduler.py b/src/firefly_weave/triggers/scheduler.py index 78c5de3f..1ab3ccfd 100644 --- a/src/firefly_weave/triggers/scheduler.py +++ b/src/firefly_weave/triggers/scheduler.py @@ -31,6 +31,7 @@ from firefly_weave.contracts.access import Scope from firefly_weave.contracts.runtime import StartRunRequest from firefly_weave.definitions.models import CatalogError, capacity_rejected +from firefly_weave.operations.facts import RunStartFacts from firefly_weave.persistence.uow import Transaction from firefly_weave.runtime.repository import SCOPE, RuntimeRepository from firefly_weave.runtime.service import RuntimeService @@ -127,6 +128,7 @@ async def _fire(self, tx: Transaction, row: dict[str, Any]) -> int: occurrence_key(row["id"], row["revision"], instant), context=AuditContext(), tx=tx, + start_facts=RunStartFacts(origin="schedule"), not_after=instant + timedelta(seconds=60), ) await repository.execute( diff --git a/src/firefly_weave/triggers/service.py b/src/firefly_weave/triggers/service.py index d147c49c..aed2b4c4 100644 --- a/src/firefly_weave/triggers/service.py +++ b/src/firefly_weave/triggers/service.py @@ -37,6 +37,7 @@ from firefly_weave.contracts.access import Scope from firefly_weave.contracts.runtime import StartRunRequest from firefly_weave.definitions.models import CatalogError +from firefly_weave.operations.facts import RunStartFacts from firefly_weave.runtime.repository import SCOPE, RuntimeRepository from firefly_weave.runtime.service import RuntimeService from firefly_weave.runtime.signals import SignalService @@ -209,6 +210,7 @@ async def receive(self, trigger_id: UUID, raw_body: bytes, headers: dict[str, st f"webhook:{trigger.id}:{event}", context=context, tx=tx, + start_facts=RunStartFacts(origin="webhook"), ) run_id = run.id else: diff --git a/src/firefly_weave/workers/leases.py b/src/firefly_weave/workers/leases.py index 75591631..cbbd2cbb 100644 --- a/src/firefly_weave/workers/leases.py +++ b/src/firefly_weave/workers/leases.py @@ -51,6 +51,7 @@ UnavailableCompletionReceipt, ) from firefly_weave.definitions.models import CatalogError +from firefly_weave.operations.facts import FactRepository from firefly_weave.operations.redaction import Omission from firefly_weave.persistence.uow import Transaction from firefly_weave.runtime.repository import RuntimeRepository @@ -174,7 +175,8 @@ async def claim(self, tx: Transaction, worker_id: UUID, limit: int) -> list[Task capability=capability, policy=json.dumps({"side_effect": action["sideEffect"], "retry": action["retry"]}), ) - await self.runtime.set_task_status(enlisted, identifier, "leased") + await self.runtime.set_task_status(enlisted, identifier, "leased", at=now) + await FactRepository(enlisted).step_claimed(identifier, generation, worker_id, now) lease = TaskLease( proof=LeaseProof(task_id=identifier, generation=generation, owner=worker_id, token=token), input=payload["input"], @@ -577,6 +579,9 @@ async def recovery_candidates(self, tx: Transaction, limit: int) -> list[UUID]: rows = await WorkerRepository(tx).rows( f"SELECT task_id FROM task_leases WHERE {SCOPE} AND " "((status='active' AND expires_at<=clock_timestamp()) OR status='failed') " + "AND EXISTS (SELECT 1 FROM task_intents t WHERE t.id=task_leases.task_id " + "AND t.tenant_id=:tenant AND t.project_id=:project AND t.environment_id=:environment " + "AND t.status<>'handled') " "AND task_id NOT IN (SELECT t.id FROM task_intents t JOIN runtime_capacity_blocks b ON " "b.run_id=t.run_id WHERE b.active) " f"AND (SELECT {runnable('r')} FROM runs r WHERE r.id=" diff --git a/tests/integration/providers/test_inbox.py b/tests/integration/providers/test_inbox.py index 50f96de2..4468c0f5 100644 --- a/tests/integration/providers/test_inbox.py +++ b/tests/integration/providers/test_inbox.py @@ -157,6 +157,7 @@ async def test_duplicate_dispatch_creates_one_run(inbox, access_db): assert results[0] == results[1] assert results[0].state == "dispatched" assert len(await rows(inbox, access_db, "runs")) == 1 + assert [row["origin"] for row in await rows(inbox, access_db, "run_facts")] == ["provider"] assert len(await rows(inbox, access_db, "provider_fixture_effects")) == 1 diff --git a/tests/integration/test_human_tasks.py b/tests/integration/test_human_tasks.py index b28eb481..93d06fa7 100644 --- a/tests/integration/test_human_tasks.py +++ b/tests/integration/test_human_tasks.py @@ -201,6 +201,17 @@ async def test_claim_complete_retry_and_conflicting_payload(client, headers, env assert [entry["action"] for entry in detail.json()["history"]] == ["created", "claim", "complete"] async with access_db[1]() as session: assert await session.scalar(text("SELECT count(*) FROM human_task_decisions")) == 1 + fact = ( + ( + await session.execute( + text("SELECT kind,status,scheduled_at,started_at,ended_at,attempts FROM step_facts") + ) + ) + .mappings() + .one() + ) + assert fact["kind"] == "humanTask" and fact["status"] == "succeeded" + assert fact["scheduled_at"] == fact["started_at"] <= fact["ended_at"] and fact["attempts"] == 0 assert await session.scalar(text("SELECT count(*) FROM task_intents")) == 0 assert await session.scalar(text("SELECT count(*) FROM run_deadlines WHERE NOT consumed")) == 0 diff --git a/tests/integration/test_leases.py b/tests/integration/test_leases.py index 284ab737..ccb9d23b 100644 --- a/tests/integration/test_leases.py +++ b/tests/integration/test_leases.py @@ -556,7 +556,9 @@ async def test_internal_verified_port_rejects_mismatched_context( await runtime.finish_verified_task(tx, replace(verified, task=task), uuid4(), 4) -async def test_compatible_task_not_starved_by_incompatible_page(worker_setup, worker_runtime_fixture, access_db): +async def test_compatible_task_not_starved_by_incompatible_page( + worker_setup, worker_runtime_fixture, access_db, monkeypatch +): import json from uuid import UUID @@ -607,6 +609,10 @@ async def test_compatible_task_not_starved_by_incompatible_page(worker_setup, wo "mixed-other", context=AuditContext(), ) + from firefly_weave.runtime import repository + + identifiers = iter([UUID(int=index) for index in range(1, 102)] + [UUID(int=2**127)]) + monkeypatch.setattr(repository, "uuid4", lambda: next(identifiers)) for index in range(101): await runtime.start( actor, @@ -615,15 +621,9 @@ async def test_compatible_task_not_starved_by_incompatible_page(worker_setup, wo f"other-{index}", context=AuditContext(), ) - target = await runtime.start( + await runtime.start( actor, scope, StartRunRequest(activation_id=echo_activation.id, input=999), "eligible", context=AuditContext() ) - # Arrange deterministic UUID ordering only; every intent was created by authorized runtime services. - async with access_db[1].begin() as tx: - rows = (await tx.execute(text("SELECT id,run_id FROM task_intents ORDER BY id"))).all() - for index, row in enumerate(rows, 1): - identifier = UUID(int=2**127) if row.run_id == target.id else UUID(int=index) - await tx.execute(text("UPDATE task_intents SET id=:new WHERE id=:old"), {"new": identifier, "old": row.id}) instance = await workers.register_instance( actor, scope, diff --git a/tests/integration/test_operations_fact_writes.py b/tests/integration/test_operations_fact_writes.py new file mode 100644 index 00000000..d36fe759 --- /dev/null +++ b/tests/integration/test_operations_fact_writes.py @@ -0,0 +1,668 @@ +# Copyright 2026 Firefly Software Foundation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# Author: Firefly Software Foundation +# SPDX-License-Identifier: Apache-2.0 + +"""Facts share real accepted runtime, lease and incident transactions.""" + +from uuid import uuid4 + +import pytest +from sqlalchemy import text + +from firefly_weave.access.audit import AuditContext +from firefly_weave.operations.facts import FactRepository + +pytestmark = pytest.mark.integration + + +@pytest.fixture +def worker_runtime_fixture(worker_runtime_fixture): + from firefly_weave.compiler.catalog import CatalogSnapshot + from firefly_weave.contracts.definitions import load_definition + + action = worker_runtime_fixture["action"].model_dump(by_alias=True) + action["spec"]["retry"] = {"maxAttempts": 3, "initialDelaySeconds": 1, "maxDelaySeconds": 2} + action = load_definition(action) + contract = worker_runtime_fixture["catalog"].resolve("TaskCapability", "echo@1.2.0").definition.value + return { + **worker_runtime_fixture, + "action": action, + "catalog": CatalogSnapshot.from_definitions([action], tasks=[contract]), + } + + +async def test_start_replay_does_not_add_facts(operations_case): + case = operations_case + first, second = await case.start("same"), await case.start("same") + assert first.id == second.id + rows = await case.rows( + "SELECT origin,test,handled_errors,last_event_sequence FROM run_facts WHERE run_id=:id", id=first.id + ) + assert rows == [ + {"origin": "manual", "test": False, "handled_errors": 0, "last_event_sequence": first.state.accepted_sequence} + ] + steps = await case.rows( + "SELECT node_id,instance_key,kind,status,scheduled_at,started_at,attempts FROM step_facts WHERE run_id=:id", + id=first.id, + ) + assert len(steps) == 1 + assert {k: steps[0][k] for k in ("node_id", "instance_key", "kind", "status", "started_at", "attempts")} == dict( + node_id="work", instance_key="", kind="action", status="scheduled", started_at=None, attempts=0 + ) + assert steps[0]["scheduled_at"] is not None + + +async def test_claim_complete_replay_preserves_first_times(operations_case, task_service, worker_ids): + case = operations_case + run = await case.start() + before = (await case.rows("SELECT * FROM task_facts WHERE run_id=:id", id=run.id))[0] + assert before["status"] == "ready" and before["ready_since"] == before["created_at"] + async with case.tx() as tx: + lease = (await task_service.claim(tx, worker_ids[0], 1))[0] + async with case.tx() as tx: + await task_service.heartbeat(tx, lease.proof) + claimed = (await case.rows("SELECT * FROM step_facts WHERE run_id=:id", id=run.id))[0] + assert claimed["status"] == "running" and claimed["attempts"] == 1 + assert claimed["started_at"] >= claimed["scheduled_at"] + completion = uuid4() + async with case.tx() as tx: + await task_service.complete(tx, lease.proof, completion, 42) + snapshot = await case.rows("SELECT * FROM step_facts WHERE run_id=:id", id=run.id) + async with case.tx() as tx: + await task_service.complete(tx, lease.proof, completion, 42) + assert await case.rows("SELECT * FROM step_facts WHERE run_id=:id", id=run.id) == snapshot + assert snapshot[0]["status"] == "succeeded" and snapshot[0]["started_at"] == claimed["started_at"] + task = (await case.rows("SELECT * FROM task_facts WHERE run_id=:id", id=run.id))[0] + assert task["status"] == "completed" and task["ready_since"] is None + assert task["first_claimed_at"] == claimed["started_at"] + fact = (await case.rows("SELECT * FROM run_facts WHERE run_id=:id", id=run.id))[0] + assert fact["status"] == "succeeded" and fact["ended_at"] == snapshot[0]["ended_at"] + + +@pytest.mark.parametrize("point", ["before_projection", "before_outbox"]) +async def test_facts_rollback_with_source(point, operations_case, monkeypatch): + from firefly_weave.operations.outbox import OutboxService + + case = operations_case + run = await case.start() + tables = ( + "runs", + "run_events", + "run_facts", + "step_facts", + "task_facts", + "task_intents", + "operation_usage", + "operation_reservations", + "operation_allocations", + "event_deliveries", + "integration_events", + "operation_control_deliveries", + ) + before = { + table: await case.rows(f"SELECT to_jsonb(t) AS row FROM {table} t ORDER BY to_jsonb(t)::text") + for table in tables + } + + async def fail(*args, **kwargs): + raise RuntimeError("projection rollback") + + if point == "before_projection": + monkeypatch.setattr(FactRepository, "project_run", fail) + else: + monkeypatch.setattr(OutboxService, "append", fail) + with pytest.raises(RuntimeError, match="projection rollback"): + async with case.tx() as tx: + await case.runtime.cancel(tx, run.id, "stop", actor=case.actor, scope=case.scope, context=AuditContext()) + after = { + table: await case.rows(f"SELECT to_jsonb(t) AS row FROM {table} t ORDER BY to_jsonb(t)::text") + for table in tables + } + assert after == before + + +async def test_handled_hook_is_idempotent_and_does_not_change_authority(operations_case): + case = operations_case + run = await case.start() + async with case.tx() as tx: + at = await tx.session.scalar(text("SELECT clock_timestamp()")) + facts = FactRepository(tx) + for _ in range(2): + await facts.handled_failure(run.id, "work[3]", "action", "continue", "ECHO_FAILED", at) + rows = await case.rows( + "SELECT handled_errors,status,failed_node_id,active_incidents FROM run_facts WHERE run_id=:id", id=run.id + ) + assert rows == [{"handled_errors": 1, "status": run.state.status, "failed_node_id": None, "active_incidents": 0}] + assert (await case.rows("SELECT state FROM runs WHERE id=:id", id=run.id))[0]["state"] == run.state.model_dump( + mode="json" + ) + task = (await case.rows("SELECT task_id FROM task_facts WHERE run_id=:id", id=run.id))[0]["task_id"] + async with case.tx() as tx: + with pytest.raises(ValueError): + await FactRepository(tx).handled_failure( + run.id, "work", "action", "continue", "ECHO_FAILED", at, task_id=task + ) + assert (await case.rows("SELECT status FROM task_intents WHERE id=:id", id=task))[0]["status"] == "ready" + + +async def test_pause_resume_cancel_retry_preserves_trusted_origin(operations_case): + from firefly_weave.contracts.human_tasks import ManualControlRequest + from firefly_weave.contracts.runtime import StartRunRequest + + case = operations_case + run = await case.start() + for paused in (True, False): + view = await case.runtime.manual_control( + case.actor, + case.scope, + run.id, + ManualControlRequest(reason="control", expected_revision=int(not paused)), + str(uuid4()), + paused=paused, + context=AuditContext(), + ) + fact = ( + await case.rows("SELECT paused,updated_at,last_event_sequence FROM run_facts WHERE run_id=:id", id=run.id) + )[0] + assert fact["paused"] is paused + assert fact["last_event_sequence"] == view.state.accepted_sequence + async with case.tx() as tx: + await case.runtime.cancel(tx, run.id, "stop", actor=case.actor, scope=case.scope, context=AuditContext()) + async with case.tx() as tx: + retry = await case.runtime.retry_run( + tx, + run.id, + StartRunRequest(activation_id=case.activation.id, input=3), + "retry", + actor=case.actor, + scope=case.scope, + context=AuditContext(), + ) + async with case.tx() as tx: + duplicate = await case.runtime.retry_run( + tx, + run.id, + StartRunRequest(activation_id=case.activation.id, input=3), + "retry", + actor=case.actor, + scope=case.scope, + context=AuditContext(), + ) + assert duplicate.id == retry.id + assert await case.rows("SELECT origin,retried_from_run_id FROM run_facts WHERE run_id=:id", id=retry.id) == [ + {"origin": "retry", "retried_from_run_id": run.id} + ] + + +@pytest.mark.parametrize("resolution", ["retry_safe", "accept_reconciled_result"]) +async def test_incident_resolution_updates_fact_times_and_task_status( + operations_case, task_service, worker_ids, services, resolution +): + from firefly_weave.contracts.operations import IncidentResolution + from firefly_weave.contracts.workers import TaskError + from firefly_weave.operations.incidents import IncidentService + + case = operations_case + run = await case.start() + async with case.tx() as tx: + lease = (await task_service.claim(tx, worker_ids[0], 1))[0] + async with case.tx() as tx: + await task_service.fail(tx, lease.proof, TaskError(completion_id=uuid4(), code="CONNECTION_LOST")) + before = ( + await case.rows( + "SELECT i.id,i.revision,f.opened_at,f.updated_at FROM incidents i " + "JOIN incident_facts f ON f.incident_id=i.id WHERE i.run_id=:id", + id=run.id, + ) + )[0] + assert before["opened_at"] == before["updated_at"] + assert (await case.rows("SELECT active_incidents FROM run_facts WHERE run_id=:id", id=run.id))[0][ + "active_incidents" + ] == 1 + request = IncidentResolution( + kind=resolution, + receipt_id=uuid4(), + reason="reviewed", + **({"output": 42, "evidence_reference": "case-42"} if resolution == "accept_reconciled_result" else {}), + ) + async with case.tx() as tx: + await ( + services(case.sessions) + .resolve(IncidentService) + .resolve( + tx, + before["id"], + request, + before["revision"], + actor=case.actor, + scope=case.scope, + context=AuditContext(), + ) + ) + after = ( + await case.rows( + "SELECT f.opened_at,f.updated_at,i.resolved_at,i.status FROM incident_facts f " + "JOIN incidents i ON i.id=f.incident_id WHERE i.id=:id", + id=before["id"], + ) + )[0] + assert after["opened_at"] == before["opened_at"] + assert after["updated_at"] == after["resolved_at"] >= before["updated_at"] + assert after["status"] == "resolved" + task = ( + await case.rows( + "SELECT t.status,f.status AS fact_status,f.ready_since FROM task_intents t " + "JOIN task_facts f ON f.task_id=t.id WHERE t.id=:id", + id=lease.proof.task_id, + ) + )[0] + assert task["status"] == task["fact_status"] == ("ready" if resolution == "retry_safe" else "completed") + assert (task["ready_since"] is not None) is (resolution == "retry_safe") + assert (await case.rows("SELECT status FROM step_facts WHERE run_id=:id", id=run.id))[0]["status"] == ( + "scheduled" if resolution == "retry_safe" else "succeeded" + ) + assert (await case.rows("SELECT active_incidents FROM run_facts WHERE run_id=:id", id=run.id))[0][ + "active_incidents" + ] == 0 + + +async def test_expired_generation_requeues_at_effective_due_instant( + operations_case, task_service, worker_ids, services +): + from firefly_weave.runtime.recovery import RecoveryService + + case = operations_case + run = await case.start() + async with case.tx() as tx: + old = (await task_service.claim(tx, worker_ids[0], 1))[0] + first = (await case.rows("SELECT first_claimed_at FROM task_facts WHERE task_id=:id", id=old.proof.task_id))[0][ + "first_claimed_at" + ] + async with case.owner.begin() as tx: + await tx.execute(text("UPDATE task_leases SET expires_at=clock_timestamp()-interval '1 second'")) + report = ( + await services(case.sessions) + .resolve(RecoveryService) + .scan(case.scope, 100, actor=case.actor, context=AuditContext()) + ) + assert report.resumed == 1 + row = ( + await case.rows( + "SELECT t.status,t.next_attempt_at,f.status AS fact_status,f.ready_since,f.first_claimed_at " + "FROM task_intents t " + "JOIN task_facts f ON f.task_id=t.id WHERE t.id=:id", + id=old.proof.task_id, + ) + )[0] + assert row["status"] == row["fact_status"] == "ready" + assert row["ready_since"] == row["next_attempt_at"] and row["first_claimed_at"] == first + async with case.owner.begin() as tx: + await tx.execute( + text("UPDATE task_intents SET next_attempt_at=clock_timestamp()-interval '1 second' WHERE id=:id"), + {"id": old.proof.task_id}, + ) + async with case.tx() as tx: + new = (await task_service.claim(tx, worker_ids[1], 1))[0] + assert new.proof.generation == 2 + assert (await case.rows("SELECT attempts,started_at FROM step_facts WHERE run_id=:id", id=run.id))[0] == { + "attempts": 2, + "started_at": first, + } + from firefly_weave.access.authorization import AccessDenied + from firefly_weave.definitions.models import CatalogError + + with pytest.raises((AccessDenied, CatalogError)): + async with case.tx() as tx: + await task_service.complete(tx, old.proof, uuid4(), 7) + async with case.tx() as tx: + await task_service.complete(tx, new.proof, uuid4(), 42) + assert (await case.rows("SELECT attempts FROM step_facts WHERE run_id=:id", id=run.id))[0]["attempts"] == 2 + + +async def start_inline(case, steps, output, *, worker_pins=None): + import json + + from firefly_weave.contracts.catalog import ActivationRequest + from firefly_weave.contracts.runtime import StartRunRequest + + definitions = case.runtime.definitions + document = { + "apiVersion": "weave/v1alpha1", + "kind": "Workflow", + "metadata": {"name": "inline-facts", "version": "1.0.0"}, + "spec": {"inputSchema": {}, "outputSchema": {}, "timeoutSeconds": 600, "steps": steps, "output": output}, + } + published = await definitions.publish( + case.actor, case.scope, "Workflow", json.dumps(document), "json", str(uuid4()), context=AuditContext() + ) + activation = await definitions.activate( + case.actor, + case.scope, + ActivationRequest( + scope=case.scope, + version_id=published.id, + artifact_digest=published.digest, + worker_release_ids=worker_pins or {}, + ), + str(uuid4()), + context=AuditContext(), + ) + return await case.runtime.start( + case.actor, + case.scope, + StartRunRequest(activation_id=activation.id, input={"origin": "test"}), + str(uuid4()), + context=AuditContext(), + ) + + +@pytest.mark.parametrize("mode", ["null", "fail", "wait", "signal", "switch"]) +async def test_inline_wait_and_terminal_steps_have_honest_times(operations_case, services, mode): + case = operations_case + step = {"id": "item", "kind": "transform", "value": {"literal": None}} + if mode == "fail": + step = {"id": "item", "kind": "fail", "code": "DECLINED", "message": "Declined"} + elif mode == "wait": + step = {"id": "item", "kind": "wait", "durationSeconds": 1} + elif mode == "signal": + step = {"id": "item", "kind": "signal", "name": "approved", "timeoutSeconds": 300, "payloadSchema": {}} + if mode == "switch": + step = { + "id": "item", + "kind": "switch", + "cases": [{"when": {"literal": True}, "steps": [], "output": {"literal": None}}], + "default": {"steps": [], "output": {"literal": 1}}, + } + run = await start_inline(case, [step], {"literal": None}) + row = (await case.rows("SELECT * FROM step_facts WHERE run_id=:id", id=run.id))[0] + assert row["scheduled_at"] == row["started_at"] + assert row["attempts"] == 0 + if mode in {"wait", "signal"}: + assert row["status"] == "waiting" and row["ended_at"] is None + if mode == "signal": + from firefly_weave.runtime.signals import SignalService + + async with case.tx() as tx: + await ( + services(case.sessions) + .resolve(SignalService) + .deliver( + tx, + run.id, + "signal", + "approved", + None, + actor=case.actor, + scope=case.scope, + context=AuditContext(), + ) + ) + else: + import asyncio + + from firefly_weave.runtime.deadlines import DeadlineService + + await asyncio.sleep(1.05) + async with case.tx() as tx: + assert await services(case.sessions).resolve(DeadlineService).tick(tx, 100) == 1 + ended = (await case.rows("SELECT * FROM step_facts WHERE run_id=:id", id=run.id))[0] + assert ended["status"] == "succeeded" and ended["ended_at"] >= row["started_at"] + assert ended["started_at"] == row["started_at"] + elif mode == "fail": + assert row["status"] == "failed" and row["error_code"] == "DECLINED" + assert row["ended_at"] == row["started_at"] + assert await case.rows( + "SELECT status,failed_node_id,failed_error_code FROM run_facts WHERE run_id=:id", id=run.id + ) == [{"status": "failed", "failed_node_id": "item", "failed_error_code": "DECLINED"}] + assert not await case.rows("SELECT output FROM step_instances WHERE run_id=:id", id=run.id) + else: + assert row["status"] == "succeeded" and row["ended_at"] == row["started_at"] + assert await case.rows("SELECT output FROM step_instances WHERE run_id=:id AND node_id='item'", id=run.id) == [ + {"output": None} + ] + assert (await case.rows("SELECT origin,test FROM run_facts WHERE run_id=:id", id=run.id))[0] == { + "origin": "manual", + "test": False, + } + + +async def test_completion_behind_pause_keeps_actual_end_time(operations_case, task_service, worker_ids): + from firefly_weave.contracts.human_tasks import ManualControlRequest + + case = operations_case + run = await case.start() + async with case.tx() as tx: + lease = (await task_service.claim(tx, worker_ids[0], 1))[0] + await case.runtime.manual_control( + case.actor, + case.scope, + run.id, + ManualControlRequest(reason="pause", expected_revision=0), + "pause", + paused=True, + context=AuditContext(), + ) + async with case.tx() as tx: + await task_service.complete(tx, lease.proof, uuid4(), 42) + before = (await case.rows("SELECT status,started_at,ended_at FROM step_facts WHERE run_id=:id", id=run.id))[0] + assert before["status"] == "succeeded" and before["ended_at"] is not None + await case.runtime.manual_control( + case.actor, + case.scope, + run.id, + ManualControlRequest(reason="resume", expected_revision=1), + "resume", + paused=False, + context=AuditContext(), + ) + assert (await case.rows("SELECT status,started_at,ended_at FROM step_facts WHERE run_id=:id", id=run.id))[ + 0 + ] == before + + +async def test_handled_task_is_not_resurrected_by_recovery(operations_case, task_service, worker_ids, services): + from firefly_weave.runtime.recovery import RecoveryService + + case = operations_case + run = await case.start() + async with case.tx() as tx: + lease = (await task_service.claim(tx, worker_ids[0], 1))[0] + # A future language settlement owns this source state; the hook only mirrors it. + await tx.session.execute( + text("UPDATE task_intents SET status='handled' WHERE id=:id"), {"id": lease.proof.task_id} + ) + at = await tx.session.scalar(text("SELECT clock_timestamp()")) + await FactRepository(tx).handled_failure( + run.id, "work", "action", "errorOutput", "HANDLED", at, task_id=lease.proof.task_id + ) + from firefly_weave.runtime.repository import RuntimeRepository + + async with case.tx() as tx: + for status in ("ready", "failed", "incident", "completed", "cancelled", "leased"): + await RuntimeRepository(tx).task_status(lease.proof.task_id, status, at=at) + assert ( + await tx.session.scalar( + text("SELECT status FROM task_intents WHERE id=:id"), {"id": lease.proof.task_id} + ) + == "handled" + ) + async with case.owner.begin() as tx: + await tx.execute(text("UPDATE task_leases SET expires_at=clock_timestamp()-interval '1 second'")) + async with case.tx() as tx: + assert await case.tasks.recovery_candidates(tx, 100) == [] + await ( + services(case.sessions).resolve(RecoveryService).scan(case.scope, 100, actor=case.actor, context=AuditContext()) + ) + assert await case.rows("SELECT status,ready_since FROM task_facts WHERE task_id=:id", id=lease.proof.task_id) == [ + {"status": "handled", "ready_since": None} + ] + async with case.tx() as tx: + assert await task_service.claim(tx, worker_ids[1], 1) == [] + + +@pytest.mark.parametrize("origin", ["call", "test"]) +async def test_reserved_internal_start_metadata_is_projected(operations_case, origin): + from firefly_weave.contracts.run_views import RunSummaryCaller + from firefly_weave.contracts.runtime import StartRunRequest + from firefly_weave.operations.facts import RunStartFacts + + case = operations_case + caller = RunSummaryCaller(run_id=uuid4(), node_id="invoke", instance_key="invoke[3]") if origin == "call" else None + run = await case.runtime.start( + case.actor, + case.scope, + StartRunRequest(activation_id=case.activation.id, input=3), + "internal", + context=AuditContext(), + start_facts=RunStartFacts(origin=origin, test=origin == "test", caller=caller), + ) + assert await case.rows( + "SELECT origin,test,caller_run_id,caller_node_id,caller_instance_key FROM run_facts WHERE run_id=:id", id=run.id + ) == [ + { + "origin": origin, + "test": origin == "test", + "caller_run_id": caller.run_id if caller else None, + "caller_node_id": "invoke" if caller else None, + "caller_instance_key": "invoke[3]" if caller else None, + } + ] + + +async def test_policy_unavailable_control_withholds_metadata_and_closes_steps(operations_case): + case = operations_case + run = await case.start() + async with case.owner.begin() as owner: + await owner.execute(text("UPDATE runs SET artifact='{}' WHERE id=:id"), {"id": run.id}) + async with case.tx() as tx: + acknowledgment = await case.runtime.cancel( + tx, run.id, "stop", actor=case.actor, scope=case.scope, context=AuditContext() + ) + assert acknowledgment.unavailable + fact = ( + await case.rows( + "SELECT status,classification_state,failed_node_id,active_incidents FROM run_facts WHERE run_id=:id", + id=run.id, + ) + )[0] + assert fact == { + "status": "cancelled", + "classification_state": "unavailable", + "failed_node_id": None, + "active_incidents": 0, + } + assert await case.rows("SELECT status FROM step_facts WHERE run_id=:id", id=run.id) == [{"status": "cancelled"}] + + +async def test_recovery_rechecks_handled_status_after_candidate_selection( + operations_case, task_service, worker_ids, services, monkeypatch +): + from firefly_weave.runtime.recovery import RecoveryService + from firefly_weave.workers.leases import TaskService + + case = operations_case + run = await case.start() + async with case.tx() as tx: + lease = (await task_service.claim(tx, worker_ids[0], 1))[0] + async with case.owner.begin() as tx: + await tx.execute(text("UPDATE task_leases SET expires_at=clock_timestamp()-interval '1 second'")) + original = TaskService.recovery_candidates + + async def settle_after_selection(self, tx, limit): + candidates = await original(self, tx, limit) + assert candidates == [lease.proof.task_id] + await tx.session.execute( + text("UPDATE task_intents SET status='handled' WHERE id=:id"), {"id": lease.proof.task_id} + ) + at = await tx.session.scalar(text("SELECT clock_timestamp()")) + await FactRepository(tx).handled_failure( + run.id, "work", "action", "continue", "HANDLED", at, task_id=lease.proof.task_id + ) + return candidates + + monkeypatch.setattr(TaskService, "recovery_candidates", settle_after_selection) + report = ( + await services(case.sessions) + .resolve(RecoveryService) + .scan(case.scope, 100, actor=case.actor, context=AuditContext()) + ) + assert report.resumed == report.incidents == 0 + assert await case.rows("SELECT status FROM task_intents WHERE id=:id", id=lease.proof.task_id) == [ + {"status": "handled"} + ] + assert len(await case.rows("SELECT id FROM run_events WHERE run_id=:id", id=run.id)) == 1 + from operations_support import assert_usage + + async with case.owner() as owner: + await assert_usage(owner, {"tenant": case.scope.tenant_id, "project": case.scope.project_id}) + + +@pytest.mark.parametrize("after", [False, True]) +async def test_failed_start_fact_insert_rolls_back_source_and_usage(operations_case, monkeypatch, after): + case = operations_case + before = await case.rows("SELECT to_jsonb(u) AS value FROM operation_usage u ORDER BY to_jsonb(u)::text") + original = FactRepository.insert_run + + async def fail(self, *args, **kwargs): + if after: + await original(self, *args, **kwargs) + raise RuntimeError("start fact rollback") + + monkeypatch.setattr(FactRepository, "insert_run", fail) + with pytest.raises(RuntimeError, match="start fact rollback"): + await case.start() + for table in ("runs", "run_events", "run_facts", "task_intents", "task_facts", "step_facts"): + assert await case.rows(f"SELECT count(*) AS count FROM {table}") == [{"count": 0}] + assert await case.rows("SELECT to_jsonb(u) AS value FROM operation_usage u ORDER BY to_jsonb(u)::text") == before + + +async def test_claim_does_not_fabricate_missing_historical_step(operations_case, task_service, worker_ids): + case = operations_case + run = await case.start() + async with case.owner.begin() as owner: + await owner.execute(text("DELETE FROM step_facts WHERE run_id=:id"), {"id": run.id}) + async with case.tx() as tx: + assert len(await task_service.claim(tx, worker_ids[0], 1)) == 1 + assert await case.rows("SELECT node_id FROM step_facts WHERE run_id=:id", id=run.id) == [] + + +async def test_inline_parallel_failure_closes_tasks_scheduled_in_the_same_transition(operations_case): + case = operations_case + branches = { + "a": { + "steps": [{"id": "work", "kind": "action", "uses": "echo-action@2.0.0", "with": {"literal": 3}}], + "output": {"literal": None}, + }, + "b": { + "steps": [{"id": "fail", "kind": "fail", "code": "DECLINED", "message": "Declined"}], + "output": {"literal": None}, + }, + } + run = await start_inline( + case, + [{"id": "fork", "kind": "parallel", "concurrency": 2, "branches": branches}], + {"literal": None}, + worker_pins=case.activation.request.worker_release_ids, + ) + assert run.state.status == "failed" + facts = await case.rows("SELECT node_id,status FROM step_facts WHERE run_id=:id ORDER BY node_id", id=run.id) + assert facts == [ + {"node_id": "fail", "status": "failed"}, + {"node_id": "fork", "status": "cancelled"}, + {"node_id": "work", "status": "cancelled"}, + ] + assert await case.rows("SELECT status,ready_since FROM task_facts WHERE run_id=:id", id=run.id) == [ + {"status": "cancelled", "ready_since": None} + ] diff --git a/tests/integration/test_operations_hot_path.py b/tests/integration/test_operations_hot_path.py new file mode 100644 index 00000000..7af10a10 --- /dev/null +++ b/tests/integration/test_operations_hot_path.py @@ -0,0 +1,183 @@ +# Copyright 2026 Firefly Software Foundation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# Author: Firefly Software Foundation +# SPDX-License-Identifier: Apache-2.0 + +"""Measure emitted SQL for projection work in real service transactions.""" + +import json +from collections import Counter +from uuid import uuid4 + +import pytest +from sqlalchemy import event + +from firefly_weave.access.audit import AuditContext +from firefly_weave.contracts.workers import TaskError +from firefly_weave.runtime.repository import RuntimeRepository + +pytestmark = pytest.mark.integration + + +@pytest.fixture +def worker_runtime_fixture(worker_runtime_fixture, request): + mode = getattr(request, "param", "sequential") + document = json.loads(worker_runtime_fixture["source"]) + if isinstance(mode, int): + document["spec"]["steps"] = [ + { + "id": "fork", + "kind": "parallel", + "concurrency": mode, + "branches": { + f"branch{i}": { + "steps": [ + {"id": f"work{i}", "kind": "action", "uses": "echo-action@2.0.0", "with": {"ref": "/input"}} + ], + "output": {"ref": f"/steps/work{i}/output"}, + } + for i in range(mode) + }, + } + ] + document["spec"]["outputSchema"] = {} + document["spec"]["output"] = {"ref": "/steps/fork/output"} + elif mode == "inline": + document["spec"]["steps"] = [ + {"id": "copy", "kind": "transform", "value": {"ref": "/input"}}, + *document["spec"]["steps"], + ] + document["spec"]["output"] = {"ref": "/steps/copy/output"} + return {**worker_runtime_fixture, "source": json.dumps(document)} + + +@pytest.fixture +async def captured(operations_case): + statements = [] + engine = operations_case.sessions.kw["bind"].sync_engine + + def capture(connection, cursor, statement, parameters, context, executemany): + statements.append(" ".join(statement.split()).lower()) + + event.listen(engine, "before_cursor_execute", capture) + try: + yield statements + finally: + event.remove(engine, "before_cursor_execute", capture) + + +def assert_batch(statements, tasks): + projection = [ + s + for s in statements + if s.startswith("update run_facts ") or s.startswith("with projected as (insert into step_facts") + ] + assert 1 <= len(projection) <= 2 + step = [s for s in projection if "insert into step_facts" in s] + assert len(step) == 1 and "jsonb_to_recordset" in step[0] + sources = [s for s in statements if "insert into task_intents" in s] + assert len(sources) == tasks + assert all("insert into task_facts" in s and "returning id" in s for s in sources) + assert not any( + s.startswith("insert into task_facts") or s.startswith("insert into incident_facts") for s in statements + ) + assert not any(s.startswith("select") and ("from step_facts" in s or "from run_facts" in s) for s in statements) + print( + json.dumps( + { + "phase": "persist", + "tasks": tasks, + "normalized_counts": dict(Counter(statements)), + "statements": len(statements), + "projection_statements": len(projection), + "task_source_ctes": len(sources), + } + ) + ) + + +@pytest.mark.parametrize("worker_runtime_fixture", ["sequential", "inline", 1, 10, 100], indirect=True) +async def test_persist_projection_uses_bounded_sql(operations_case, captured, monkeypatch, request): + original = RuntimeRepository.persist + batches = [] + + async def capture_persist(self, *args, **kwargs): + captured.clear() + await original(self, *args, **kwargs) + batches.append(list(captured)) + + monkeypatch.setattr(RuntimeRepository, "persist", capture_persist) + run = await operations_case.start() + mode = request.node.callspec.params["worker_runtime_fixture"] + count = mode if isinstance(mode, int) else 1 + assert len(batches) == 1 + assert_batch(batches[0], count) + case = operations_case + assert len(await case.rows("SELECT task_id FROM task_facts WHERE run_id=:id", id=run.id)) == count + assert len(await case.rows("SELECT node_id FROM step_facts WHERE run_id=:id", id=run.id)) == ( + count + 1 if isinstance(mode, int) or mode == "inline" else 1 + ) + from operations_support import assert_usage + + async with case.owner() as owner: + await assert_usage(owner, {"tenant": case.scope.tenant_id, "project": case.scope.project_id}) + + +async def test_successful_claim_adds_one_step_update_repeated_claim_adds_none( + operations_case, captured, task_service, worker_ids +): + case = operations_case + await case.start() + async with case.tx() as tx: + captured.clear() + lease = (await task_service.claim(tx, worker_ids[0], 1))[0] + accepted = list(captured) + assert sum(s.startswith("update step_facts s ") for s in accepted) == 1 + assert sum("update task_intents set status=" in s and "update task_facts" in s for s in accepted) == 1 + async with case.tx() as tx: + captured.clear() + assert await task_service.claim(tx, worker_ids[1], 1) == [] + rejected = list(captured) + assert not any("update step_facts" in s for s in rejected) + print( + json.dumps( + {"phase": "claim", "accepted_counts": dict(Counter(accepted)), "rejected_counts": dict(Counter(rejected))} + ) + ) + async with case.tx() as tx: + captured.clear() + await task_service.heartbeat(tx, lease.proof) + assert not any("update step_facts" in s for s in captured) + + +async def test_incident_source_and_fact_share_statements(operations_case, captured, task_service, worker_ids): + case = operations_case + run = await case.start() + async with case.tx() as tx: + lease = (await task_service.claim(tx, worker_ids[0], 1))[0] + async with case.tx() as tx: + captured.clear() + await task_service.fail(tx, lease.proof, TaskError(completion_id=uuid4(), code="CONNECTION_LOST")) + created = list(captured) + inserts = [s for s in created if "insert into incidents" in s] + assert len(inserts) == 1 and "insert into incident_facts" in inserts[0] + print(json.dumps({"phase": "incident_open", "normalized_counts": dict(Counter(created))})) + async with case.tx() as tx: + captured.clear() + await case.runtime.cancel(tx, run.id, "stop", actor=case.actor, scope=case.scope, context=AuditContext()) + closed = [s for s in captured if "update incidents set status='closed'" in s] + assert len(closed) == 1 and "update incident_facts" in closed[0] + print(json.dumps({"phase": "incident_close", "normalized_counts": dict(Counter(closed))})) + fact = (await case.rows("SELECT active_incidents FROM run_facts WHERE run_id=:id", id=run.id))[0] + assert fact["active_incidents"] == 0 diff --git a/tests/integration/test_run_lifecycle.py b/tests/integration/test_run_lifecycle.py index e74fc899..4459cbc9 100644 --- a/tests/integration/test_run_lifecycle.py +++ b/tests/integration/test_run_lifecycle.py @@ -87,10 +87,23 @@ async def test_archive_restore_purge_and_no_resurrection( ) assert replay.status_code == 410, replay.text async with access_db[1]() as observer: - for table in ("runs", "run_events", "run_event_evidence", "step_instances", "runtime_capacity_blocks"): + for table in ( + "runs", + "run_events", + "run_event_evidence", + "step_instances", + "runtime_capacity_blocks", + "run_facts", + "step_facts", + "task_facts", + "incident_facts", + ): assert await observer.scalar(text(f"SELECT count(*) FROM {table}")) == 0 assert await observer.scalar(text("SELECT count(*) FROM access_audit WHERE action='run.purge'")) == 1 assert not await observer.scalar(text("SELECT has_table_privilege('weave_app','runs','DELETE')")) + from operations_support import assert_usage + + await assert_usage(observer, {"tenant": author[2].tenant_id, "project": author[2].project_id}) async def test_purge_reference_failure_is_atomic( diff --git a/tests/integration/test_runs.py b/tests/integration/test_runs.py index c78fd457..2ce742d5 100644 --- a/tests/integration/test_runs.py +++ b/tests/integration/test_runs.py @@ -311,7 +311,16 @@ async def test_durable_task_intents_rollback_and_restart_scan( try: async with UnitOfWork(sessions).open(author[2]) as tx: repository = RuntimeRepository(tx, services(sessions).resolve(DefinitionService).outbox) - await repository.insert(view, json.loads(flow.to_bytes()), request, actor.id) + from firefly_weave.operations.facts import RunStartFacts + + await repository.insert( + view, + json.loads(flow.to_bytes()), + request, + actor.id, + start_facts=RunStartFacts(), + at=accepted.timestamp, + ) await repository.persist(view, accepted, transition_result, event_hash(accepted)) assert len(await repository.ready()) == 1 async with UnitOfWork(sessions).open(author[2], mutation=False) as observer: @@ -359,6 +368,7 @@ async def test_rollback_between_event_and_task_insert( from firefly_weave.contracts.runtime import RunView, StartRunRequest from firefly_weave.definitions.service import DefinitionService + from firefly_weave.operations.facts import RunStartFacts from firefly_weave.persistence.uow import UnitOfWork from firefly_weave.runtime.kernel import transition from firefly_weave.runtime.models import RunState, RuntimeEvent @@ -389,7 +399,7 @@ async def test_rollback_between_event_and_task_insert( original = repository.execute async def injected(sql, **values): - if sql.startswith("INSERT INTO task_intents"): + if "INSERT INTO task_intents" in sql: assert await tx.session.scalar(text("SELECT count(*) FROM run_events")) == 1 raise RuntimeError("injected") await original(sql, **values) @@ -400,6 +410,8 @@ async def injected(sql, **values): json.loads(artifact.to_bytes()), StartRunRequest.model_validate_json(json.dumps(start_request)), actor.id, + start_facts=RunStartFacts(), + at=event.timestamp, ) await repository.persist(view, event, result, event_hash(event)) async with owner() as session: diff --git a/tests/integration/test_schedules.py b/tests/integration/test_schedules.py index 22a2abde..6d3d8609 100644 --- a/tests/integration/test_schedules.py +++ b/tests/integration/test_schedules.py @@ -79,6 +79,7 @@ async def test_two_schedulers_create_one_run_for_same_occurrence( == 1 ) assert await session.scalar(text("SELECT count(*) FROM runs")) == 1 + assert await session.scalar(text("SELECT origin FROM run_facts")) == "schedule" @pytest.fixture diff --git a/tests/integration/test_terminal_capacity.py b/tests/integration/test_terminal_capacity.py index 6e2b8a7c..1f3626e7 100644 --- a/tests/integration/test_terminal_capacity.py +++ b/tests/integration/test_terminal_capacity.py @@ -100,6 +100,24 @@ async def assert_equivalent(owner, before, kind): ) assert await owner.scalar(text("SELECT sum(value) FROM operation_usage WHERE metric='control_bytes'")) > 0 + fact = ( + (await owner.execute(text("SELECT * FROM run_facts WHERE run_id=:id"), {"id": before["id"]})).mappings().one() + ) + assert fact["status"] == kind and fact["ended_at"] == event["created_at"] + assert fact["updated_at"] == event["created_at"] and fact["last_event_sequence"] == event["sequence"] + assert fact["failed_node_id"] is None and fact["failed_error_code"] is None and fact["active_incidents"] == 0 + assert not await owner.scalar( + text("SELECT count(*) FROM task_facts WHERE run_id=:id AND (status<>'cancelled' OR ready_since IS NOT NULL)"), + {"id": before["id"]}, + ) + assert not await owner.scalar( + text("SELECT count(*) FROM step_facts WHERE run_id=:id AND status IN ('scheduled','running','waiting')"), + {"id": before["id"]}, + ) + from operations_support import assert_usage + + await assert_usage(owner, {"tenant": before["tenant_id"], "project": before["project_id"]}) + @pytest.mark.parametrize("kind", ["cancelled", "timed_out"]) async def test_sql_terminal_matches_sequential_kernel_at_full_pools( @@ -116,6 +134,11 @@ async def test_sql_terminal_matches_sequential_kernel_at_full_pools( await owner.execute(text("UPDATE run_deadlines SET deadline=clock_timestamp()-interval '1 second'")) await exhaust_pools(owner, author[2].project_id) monkeypatch.setattr(repository, "RUN_FETCH_BYTES", 1) + + async def reject_payload_fetch(*args, **kwargs): + raise AssertionError("Oversized terminal control fetched run payload") + + monkeypatch.setattr(repository.RuntimeRepository, "run", reject_payload_fetch) if kind == "cancelled": response = await client.post(f"{env_url}/runs/{waiting_run}/cancel", headers=headers, json={"reason": "stop"}) assert response.status_code == 200, response.text @@ -147,6 +170,11 @@ async def test_sql_terminal_matches_parallel_kernel_and_fences_leases( await owner.execute(text("UPDATE run_deadlines SET deadline=clock_timestamp()-interval '1 second'")) await exhaust_pools(owner, worker_setup[4].project_id) monkeypatch.setattr(repository, "RUN_FETCH_BYTES", 1) + + async def reject_payload_fetch(*args, **kwargs): + raise AssertionError("Oversized terminal control fetched run payload") + + monkeypatch.setattr(repository.RuntimeRepository, "run", reject_payload_fetch) async with transaction_factory() as tx: if kind == "cancelled": result = ( @@ -272,3 +300,109 @@ async def test_capacity_rejection_still_allows_issued_signal_timeout( if mode != "timeout": assert not await owner.scalar(text("SELECT count(*) FROM wait_wakeups")) assert not await owner.scalar(text("SELECT count(*) FROM signal_receipts WHERE consumed")) + + +@pytest.mark.parametrize("point", ["before_projection", "after_projection"]) +async def test_oversized_terminal_fact_failure_rolls_back_control_reserve( + point, queued_task, task_service, transaction_factory, worker_ids, worker_setup, access_db, services, monkeypatch +): + import firefly_weave.runtime.repository as repository + from firefly_weave.access.audit import AuditContext + from firefly_weave.operations.facts import FactRepository + from firefly_weave.operations.outbox import OutboxService + from firefly_weave.runtime.service import RuntimeService + + async with transaction_factory() as tx: + assert await task_service.claim(tx, worker_ids[0], 1) + async with access_db[1].begin() as owner: + await exhaust_pools(owner, worker_setup[4].project_id) + tables = ( + "runs", + "run_events", + "task_intents", + "task_leases", + "run_facts", + "step_facts", + "task_facts", + "incident_facts", + "operation_usage", + "operation_reservations", + "operation_allocations", + "event_deliveries", + "integration_events", + "operation_control_deliveries", + ) + + async def snapshot(): + async with access_db[1]() as owner: + return { + table: list( + ( + await owner.execute(text(f"SELECT to_jsonb(t) FROM {table} t ORDER BY to_jsonb(t)::text")) + ).scalars() + ) + for table in tables + } + + before = await snapshot() + monkeypatch.setattr(repository, "RUN_FETCH_BYTES", 1) + + async def fail(*args, **kwargs): + raise RuntimeError("terminal projection rollback") + + if point == "before_projection": + monkeypatch.setattr(FactRepository, "project_terminal", fail) + else: + monkeypatch.setattr(OutboxService, "append", fail) + with pytest.raises(RuntimeError, match="terminal projection rollback"): + async with transaction_factory() as tx: + await ( + services(access_db[0]) + .resolve(RuntimeService) + .cancel( + tx, queued_task.id, "stop", actor=worker_setup[3], scope=worker_setup[4], context=AuditContext() + ) + ) + assert await snapshot() == before + + +async def test_cancel_at_full_ordinary_quota_updates_facts_from_control_reserve( + queued_task, task_service, transaction_factory, worker_ids, worker_setup, access_db, services +): + from firefly_weave.access.audit import AuditContext + from firefly_weave.contracts.workers import TaskError + from firefly_weave.runtime.service import RuntimeService + + async with transaction_factory() as tx: + lease = (await task_service.claim(tx, worker_ids[0], 1))[0] + async with transaction_factory() as tx: + await task_service.fail(tx, lease.proof, TaskError(completion_id=UUID(int=59), code="CONNECTION_LOST")) + async with access_db[1].begin() as owner: + before = (await owner.execute(text("SELECT * FROM runs WHERE id=:id"), {"id": queued_task.id})).mappings().one() + opened = (await owner.execute(text("SELECT opened_at,updated_at FROM incident_facts"))).one() + await exhaust_pools(owner, worker_setup[4].project_id) + async with transaction_factory() as tx: + result = ( + await services(access_db[0]) + .resolve(RuntimeService) + .cancel(tx, queued_task.id, "stop", actor=worker_setup[3], scope=worker_setup[4], context=AuditContext()) + ) + assert result.state.status == "cancelled" + async with access_db[1]() as owner: + incident = (await owner.execute(text("SELECT opened_at,updated_at FROM incident_facts"))).one() + assert incident.opened_at == opened.opened_at and incident.updated_at > opened.updated_at + fact = ( + ( + await owner.execute( + text("SELECT status,active_incidents,ended_at,last_event_sequence FROM run_facts WHERE run_id=:id"), + {"id": queued_task.id}, + ) + ) + .mappings() + .one() + ) + assert fact["status"] == "cancelled" and fact["active_incidents"] == 0 and fact["ended_at"] is not None + assert fact["last_event_sequence"] == result.state.accepted_sequence + from operations_support import assert_usage + + await assert_usage(owner, {"tenant": before["tenant_id"], "project": before["project_id"]}) diff --git a/tests/integration/test_webhooks.py b/tests/integration/test_webhooks.py index 7edba90a..f536776e 100644 --- a/tests/integration/test_webhooks.py +++ b/tests/integration/test_webhooks.py @@ -124,6 +124,7 @@ async def test_atomic_receipt_duplicate_and_conflict(webhook_setup, access_db): assert error.value.status == 409 async with access_db[1]() as session: assert await session.scalar(text("SELECT count(*) FROM trigger_receipts")) == 1 + assert await session.scalar(text("SELECT origin FROM run_facts")) == "webhook" assert await session.scalar(text("SELECT count(*) FROM runs")) == 1 diff --git a/tests/unit/email/test_email_dispatch_capacity.py b/tests/unit/email/test_email_dispatch_capacity.py index f2a1227c..c90374c6 100644 --- a/tests/unit/email/test_email_dispatch_capacity.py +++ b/tests/unit/email/test_email_dispatch_capacity.py @@ -114,6 +114,7 @@ def admit(key): return SimpleNamespace(id=uuid4()) async def start(actor, scope, request, key, **options): + assert options["start_facts"].origin == "email" return admit(key) async def deliver(tx, run_id, key, signal, payload, **options): diff --git a/tests/unit/operations/test_fact_projection.py b/tests/unit/operations/test_fact_projection.py new file mode 100644 index 00000000..35f2fdd3 --- /dev/null +++ b/tests/unit/operations/test_fact_projection.py @@ -0,0 +1,167 @@ +# Copyright 2026 Firefly Software Foundation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# Author: Firefly Software Foundation +# SPDX-License-Identifier: Apache-2.0 + +"""Projection classification and trusted start metadata boundaries.""" + +import json +import runpy +from pathlib import Path +from uuid import uuid4 + +import pytest + +from firefly_weave.operations import facts + + +@pytest.mark.parametrize( + "key,want", + [ + ("work", ("work", "")), + ("work[3]", ("work", "work[3]")), + ("work[3]#retry~2", ("work", "work[3]#retry~2")), + ("@join:x", None), + ("@any malformed synthetic", None), + ], +) +def test_author_identity(key, want): + project = getattr(facts, "author_instance", None) + assert callable(project), "Author identity projection is missing" + actual = project(key) + assert (None if actual is None else (actual.node_id, actual.instance_key)) == want + + +@pytest.mark.parametrize("key", ["", "work[03]", "work:x", "work@x", "work[1]\n", 3, None, "x" * 513]) +def test_invalid_author_identity_is_not_hidden(key): + project = getattr(facts, "author_instance", None) + assert callable(project), "Author identity projection is missing" + with pytest.raises(ValueError): + project(key) + + +@pytest.mark.parametrize( + "values", + [ + {"origin": "user"}, + {"origin": "retry"}, + {"origin": "manual", "retried_from_run_id": uuid4()}, + {"origin": "call"}, + {"origin": "test"}, + {"test": True}, + {"test": "yes"}, + ], +) +def test_rejects_incoherent_trusted_origin(values): + with pytest.raises(ValueError): + facts.RunStartFacts(**values) + + +def test_classification_matches_migration_with_raw_status_normalization(worker_runtime_fixture): + from firefly_weave.contracts.access import Scope + from firefly_weave.contracts.catalog import Activation, ActivationRequest + from firefly_weave.contracts.runtime import StartRunRequest + from firefly_weave.runtime.models import RunState + + project = getattr(facts, "listing_metadata", None) + assert callable(project), "Listing classification projection is missing" + frozen = runpy.run_path(str(Path(__file__).resolve().parents[3] / "migrations/versions/0031_operations_facts.py"))[ + "classification" + ] + artifact = json.loads(worker_runtime_fixture["artifact"].to_bytes()) + activation = Activation( + id=uuid4(), + name="worker-flow", + revision=1, + request=ActivationRequest( + version_id=uuid4(), + artifact_digest=artifact["digest"], + scope=Scope(tenant_id=uuid4(), project_id=uuid4(), environment_id=uuid4()), + ), + ) + row = dict( + state=RunState(admission_policy="classified-v1").model_dump(mode="json"), + activation=activation.model_dump(mode="json"), + request=StartRunRequest(activation_id=activation.id, input=3).model_dump(mode="json"), + artifact=artifact, + ) + cases = [ + row, + *[{**row, "artifact": bad} for bad in (None, [], {}, {"executable": []}, {"executable": {"features": 4}})], + *[{**row, "state": bad} for bad in (None, [], {}, {"status": "alien"}, {"status": 1})], + ] + cases.extend( + {**row, "state": {**row["state"], "status": status}} + for status in ( + None, + 7, + [], + "unknown", + "queued", + "running", + "waiting", + "suspended", + "succeeded", + "failed", + "cancelled", + "timed_out", + ) + ) + for changes in ( + {"irVersion": "x" * 256}, + {"irVersion": "x" * 257}, + {"features": ["future-" + str(index) for index in range(256)]}, + {"features": ["future-" + str(index) for index in range(257)]}, + {"features": ["é" * 100 for _ in range(100)]}, + {"features": [None]}, + ): + changed = {**artifact, "executable": {**artifact["executable"], **changes}} + cases.append({**row, "artifact": changed}) + for key, value in (("id", "bad:author"), ("kind", "unknownAuthorKind")): + changed = json.loads(json.dumps(artifact)) + node = next(node for node in changed["executable"]["graph"]["nodes"] if node["id"] == "work") + node[key] = value + cases.append({**row, "artifact": changed}) + for case in cases: + expected = frozen(case) + # The retained migration also normalizes raw status after classification. + state = case.get("state") + status = state.get("status") if isinstance(state, dict) else None + if type(status) is not str or status not in { + "queued", + "running", + "waiting", + "suspended", + "succeeded", + "failed", + "cancelled", + "timed_out", + }: + expected.update(classification_state="unavailable", node_kinds="{}") + expected["node_kinds"] = json.loads(expected["node_kinds"]) + expected["pinned_features"] = json.loads(expected["pinned_features"]) + assert project(case) == expected + assert project(row)["node_kinds"] == {"work": "action"} + + +def test_call_origin_requires_a_typed_caller(): + with pytest.raises(ValueError): + facts.RunStartFacts(origin="call", caller={"run_id": uuid4(), "node_id": "caller", "instance_key": ""}) + + +def test_origin_is_not_a_request_body_field(): + from firefly_weave.contracts.runtime import StartRunRequest + + with pytest.raises(ValueError): + StartRunRequest.model_validate({"activation_id": str(uuid4()), "input": 3, "origin": "provider"}) diff --git a/tests/unit/providers/test_dispatcher_capacity.py b/tests/unit/providers/test_dispatcher_capacity.py index 07e9e712..6ebcd7af 100644 --- a/tests/unit/providers/test_dispatcher_capacity.py +++ b/tests/unit/providers/test_dispatcher_capacity.py @@ -97,6 +97,7 @@ async def transaction(scope, principal, *, mutation=True): raise async def start(actor, scope, request, key, **options): + assert options["start_facts"].origin == "provider" if harness.errors: raise harness.errors.pop(0) return SimpleNamespace(id=uuid4()) diff --git a/tests/unit/runtime/test_recovery_admission.py b/tests/unit/runtime/test_recovery_admission.py index 84090d40..dd8d450e 100644 --- a/tests/unit/runtime/test_recovery_admission.py +++ b/tests/unit/runtime/test_recovery_admission.py @@ -90,7 +90,7 @@ async def now(self): async def persist(self, view, event, result, digest, **timing): harness.persisted.append(event.type) - async def task_status(self, identifier, status): + async def task_status(self, identifier, status, *, at=None): harness.expired.remove(identifier) def transition(state, event, artifact): diff --git a/tests/unit/triggers/test_scheduler_capacity.py b/tests/unit/triggers/test_scheduler_capacity.py index 4039c894..7f9aeda1 100644 --- a/tests/unit/triggers/test_scheduler_capacity.py +++ b/tests/unit/triggers/test_scheduler_capacity.py @@ -121,6 +121,7 @@ async def load_principal(session, identifier): monkeypatch.setattr("firefly_weave.triggers.scheduler.load_principal", load_principal) async def start(actor, scope, request, key, **options): + assert options["start_facts"].origin == "schedule" # Run admission reduces the first event through the pure kernel, like `transition_async`. harness.keys.append(key) if harness.errors: diff --git a/tests/unit/triggers/test_start_origin.py b/tests/unit/triggers/test_start_origin.py new file mode 100644 index 00000000..b09d0f8f --- /dev/null +++ b/tests/unit/triggers/test_start_origin.py @@ -0,0 +1,82 @@ +# Copyright 2026 Firefly Software Foundation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# Author: Firefly Software Foundation +# SPDX-License-Identifier: Apache-2.0 + +"""Broker admission supplies trusted origin separately from the event body.""" + +from contextlib import asynccontextmanager +from types import SimpleNamespace +from uuid import uuid4 + +import pytest + +from firefly_weave.contracts.access import Scope +from firefly_weave.contracts.broker import BrokerRecord +from firefly_weave.triggers.kafka import ConsumerAuthority, KafkaTrigger + + +async def test_broker_start_origin_cannot_come_from_payload(monkeypatch): + from firefly_weave.triggers import kafka + + scope = Scope(tenant_id=uuid4(), project_id=uuid4(), environment_id=uuid4()) + authority = ConsumerAuthority(scope, uuid4(), uuid4(), 1) + route = SimpleNamespace( + id=authority.trigger_id, cluster_id="cluster", topic="events", payload_schema={}, activation_id=uuid4() + ) + calls = [] + + class ReachedStart(Exception): + pass + + @asynccontextmanager + async def transaction(*args): + yield SimpleNamespace(scope=scope) + + async def start(actor, received_scope, request, key, **options): + calls.append((request.input, options["start_facts"])) + raise ReachedStart + + async def checked(*args, **kwargs): + return route, SimpleNamespace(id=uuid4()) + + async def target(*args): + return {}, {} + + class Repository: + def __init__(self, tx): + pass + + async def rows(self, *args, **kwargs): + return [] + + monkeypatch.setattr(kafka, "RuntimeRepository", Repository) + trigger = KafkaTrigger( + SimpleNamespace(definitions=SimpleNamespace(transaction=transaction), start=start), None, None + ) + monkeypatch.setattr(trigger, "checked", checked) + monkeypatch.setattr(trigger, "target", target) + record = BrokerRecord( + cluster_id="cluster", + topic="events", + partition=0, + offset=1, + event_id=str(uuid4()), + raw_value=b'{"origin":"manual","test":true}', + ) + with pytest.raises(ReachedStart): + await trigger.consume(record, authority=authority) + assert len(calls) == 1 + assert calls[0][0] == {"origin": "manual", "test": True} + assert calls[0][1].origin == "broker" and calls[0][1].test is False From af18354125bb7f9aced3cd5f9cf675c6a73e1d9e Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Fri, 9 Oct 2026 06:08:12 -0700 Subject: [PATCH 05/63] Preserve runtime authority and unknown fact timing Keep incomplete listing metadata from vetoing admitted transitions, retain unknown historical schedule and start times, and allow typed called children to inherit test status. --- src/firefly_weave/operations/facts.py | 55 +++-- tests/integration/test_human_tasks.py | 17 +- .../test_operations_fact_writes.py | 194 +++++++++++++++++- tests/integration/test_operations_hot_path.py | 3 + tests/unit/operations/test_fact_projection.py | 26 +++ 5 files changed, 275 insertions(+), 20 deletions(-) diff --git a/src/firefly_weave/operations/facts.py b/src/firefly_weave/operations/facts.py index a13da6b1..79a254db 100644 --- a/src/firefly_weave/operations/facts.py +++ b/src/firefly_weave/operations/facts.py @@ -132,8 +132,12 @@ class RunStartFacts: def __post_init__(self) -> None: TypeAdapter(RunOrigin).validate_python(self.origin, strict=True) - if type(self.test) is not bool or self.test != (self.origin == "test"): - raise ValueError("Test origin and test flag must agree") + if ( + type(self.test) is not bool + or (self.origin == "test" and not self.test) + or (self.test and self.origin not in {"test", "call"}) + ): + raise ValueError("Test flag requires a direct test or inherited call origin") if (self.origin == "call") != (self.caller is not None): raise ValueError("Call origin requires a caller") if self.caller is not None: @@ -252,38 +256,54 @@ async def project_run(self, view: RunView | TerminalControl, event: RuntimeEvent kinds = rows[0]["node_kinds"] steps: dict[tuple[str, str], dict[str, Any]] = {} - def add(key: str, status: str, *, error: str | None = None) -> None: + def kind_of(node: str) -> StepKind | None: + # Retained runs can be admitted even when their listing cache is incomplete. + try: + return TypeAdapter(StepKind).validate_python(kinds.get(node), strict=True) + except ValueError: + return None + + def add(key: str, status: str, *, scheduled: bool = False, error: str | None = None) -> None: identity = author_instance(key) if identity is None: return - kind: StepKind = TypeAdapter(StepKind).validate_python(kinds[identity.node_id], strict=True) + kind = kind_of(identity.node_id) + observed = scheduled or kind in {"transform", "decisionTable", "fail"} + previous = steps.get((identity.node_id, identity.instance_key), {}) + at = event.timestamp.isoformat() steps[identity.node_id, identity.instance_key] = dict( node_id=identity.node_id, instance_key=identity.instance_key, kind=kind, status=status, - started_at=None if kind in {"action", "llm", "agent"} else event.timestamp.isoformat(), - ended_at=event.timestamp.isoformat() if status in TERMINAL else None, + scheduled_at=previous.get("scheduled_at") or (at if observed else None), + started_at=previous.get("started_at") + or (at if observed and status != "scheduled" and kind not in {"action", "llm", "agent"} else None), + ended_at=at if status in TERMINAL else None, error_code=error, ) if not isinstance(view, TerminalControl): + tasks = {command.node_id for command in result.commands if isinstance(command, TaskIntent)} for command in result.commands: if isinstance(command, (TaskIntent, Deadline, HumanTaskIntent)): - add(command.node_id, "scheduled" if isinstance(command, TaskIntent) else "waiting") + add(command.node_id, "scheduled" if isinstance(command, TaskIntent) else "waiting", scheduled=True) elif command.kind == "spawn_branch": - add(command.node_id, "running") + join = result.state.joins.get(command.node_id) + first = result.state.branches.get(join.branches[0]) if join and join.branches else None + # Later branch dispatches do not witness the join's original activation. + add(command.node_id, "running", scheduled=first is not None and command.branch == first.name) for change in result.steps: identity = author_instance(change.node_id) if identity is None: continue - kind = kinds[identity.node_id] + kind = kind_of(identity.node_id) status = ( "succeeded" if change.status == "completed" - else ("scheduled" if kind in {"action", "llm", "agent"} else "waiting") + else ("scheduled" if kind in {"action", "llm", "agent"} or change.node_id in tasks else "waiting") ) - add(change.node_id, status) + add(change.node_id, status, scheduled=change.status == "waiting") for owner, join in result.state.joins.items(): if join.status == "waiting" and (owner, "") not in steps: add(owner, "running") @@ -317,11 +337,16 @@ def add(key: str, status: str, *, error: str | None = None) -> None: await self.execute( "WITH projected AS (INSERT INTO step_facts(tenant_id,project_id,environment_id,run_id,node_id," "instance_key,kind,status,scheduled_at,started_at,ended_at,error_code) " - "SELECT :tenant,:project,:environment,:run,s.node_id,s.instance_key,s.kind,s.status,:at," + "SELECT :tenant,:project,:environment,:run,s.node_id,s.instance_key,coalesce(s.kind,known.kind)," + "s.status,s.scheduled_at," "s.started_at,s.ended_at,s.error_code FROM jsonb_to_recordset(cast(:steps AS jsonb)) " - "AS s(node_id text,instance_key text,kind text,status text,started_at timestamptz," - "ended_at timestamptz,error_code text) ON CONFLICT(run_id,node_id,instance_key) DO UPDATE SET " - "status=excluded.status,started_at=coalesce(step_facts.started_at,excluded.started_at)," + "AS s(node_id text,instance_key text,kind text,status text,scheduled_at timestamptz,started_at timestamptz," + "ended_at timestamptz,error_code text) LEFT JOIN step_facts known ON known.run_id=:run " + "AND known.tenant_id=:tenant AND known.project_id=:project AND known.environment_id=:environment " + "AND known.node_id=s.node_id AND known.instance_key=s.instance_key " + "WHERE coalesce(s.kind,known.kind) IS NOT NULL ON CONFLICT(run_id,node_id,instance_key) DO UPDATE SET " + "status=excluded.status,scheduled_at=coalesce(step_facts.scheduled_at,excluded.scheduled_at)," + "started_at=coalesce(step_facts.started_at,excluded.started_at)," "ended_at=CASE WHEN step_facts.status=excluded.status THEN coalesce(step_facts.ended_at,excluded.ended_at) " "ELSE excluded.ended_at END,error_code=excluded.error_code WHERE step_facts.handled IS NULL " "RETURNING node_id,instance_key) UPDATE step_facts SET status=:closed,ended_at=:at " diff --git a/tests/integration/test_human_tasks.py b/tests/integration/test_human_tasks.py index 93d06fa7..2acf0041 100644 --- a/tests/integration/test_human_tasks.py +++ b/tests/integration/test_human_tasks.py @@ -183,8 +183,14 @@ async def mutate(client, headers, env_url, task, command, payload, key): ) -async def test_claim_complete_retry_and_conflicting_payload(client, headers, env_url, human_run, access_db, author): +@pytest.mark.parametrize("historical", [False, True]) +async def test_claim_complete_retry_and_conflicting_payload( + client, headers, env_url, human_run, access_db, author, historical +): run, task = human_run + if historical: + async with access_db[1]() as session, session.begin(): + await session.execute(text("DELETE FROM step_facts WHERE run_id=:id"), {"id": run}) claimed = await mutate(client, headers, env_url, task, "claim", {"expected_revision": 1}, "claim") assert claimed.status_code == 200, claimed.text payload = {"expected_revision": 2, "decision": "reject", "data": {"note": "No"}} @@ -211,9 +217,16 @@ async def test_claim_complete_retry_and_conflicting_payload(client, headers, env .one() ) assert fact["kind"] == "humanTask" and fact["status"] == "succeeded" - assert fact["scheduled_at"] == fact["started_at"] <= fact["ended_at"] and fact["attempts"] == 0 + assert fact["ended_at"] is not None and fact["attempts"] == 0 + if historical: + assert fact["scheduled_at"] is None and fact["started_at"] is None + else: + assert fact["scheduled_at"] == fact["started_at"] <= fact["ended_at"] assert await session.scalar(text("SELECT count(*) FROM task_intents")) == 0 assert await session.scalar(text("SELECT count(*) FROM run_deadlines WHERE NOT consumed")) == 0 + from operations_support import assert_usage + + await assert_usage(session, {"tenant": author[2].tenant_id, "project": author[2].project_id}) async def test_pause_allows_fact_then_resume_continues(client, headers, env_url, human_run): diff --git a/tests/integration/test_operations_fact_writes.py b/tests/integration/test_operations_fact_writes.py index d36fe759..291e95a8 100644 --- a/tests/integration/test_operations_fact_writes.py +++ b/tests/integration/test_operations_fact_writes.py @@ -368,8 +368,11 @@ async def start_inline(case, steps, output, *, worker_pins=None): ) -@pytest.mark.parametrize("mode", ["null", "fail", "wait", "signal", "switch"]) -async def test_inline_wait_and_terminal_steps_have_honest_times(operations_case, services, mode): +@pytest.mark.parametrize( + "mode,historical", + [(mode, False) for mode in ("null", "fail", "wait", "signal", "switch")] + [("wait", True), ("signal", True)], +) +async def test_inline_wait_and_terminal_steps_have_honest_times(operations_case, services, mode, historical): case = operations_case step = {"id": "item", "kind": "transform", "value": {"literal": None}} if mode == "fail": @@ -391,6 +394,9 @@ async def test_inline_wait_and_terminal_steps_have_honest_times(operations_case, assert row["attempts"] == 0 if mode in {"wait", "signal"}: assert row["status"] == "waiting" and row["ended_at"] is None + if historical: + async with case.owner.begin() as owner: + await owner.execute(text("DELETE FROM step_facts WHERE run_id=:id"), {"id": run.id}) if mode == "signal": from firefly_weave.runtime.signals import SignalService @@ -419,7 +425,10 @@ async def test_inline_wait_and_terminal_steps_have_honest_times(operations_case, assert await services(case.sessions).resolve(DeadlineService).tick(tx, 100) == 1 ended = (await case.rows("SELECT * FROM step_facts WHERE run_id=:id", id=run.id))[0] assert ended["status"] == "succeeded" and ended["ended_at"] >= row["started_at"] - assert ended["started_at"] == row["started_at"] + if historical: + assert ended["scheduled_at"] is None and ended["started_at"] is None + else: + assert ended["scheduled_at"] == row["scheduled_at"] and ended["started_at"] == row["started_at"] elif mode == "fail": assert row["status"] == "failed" and row["error_code"] == "DECLINED" assert row["ended_at"] == row["started_at"] @@ -436,6 +445,10 @@ async def test_inline_wait_and_terminal_steps_have_honest_times(operations_case, "origin": "manual", "test": False, } + from operations_support import assert_usage + + async with case.owner() as owner: + await assert_usage(owner, {"tenant": case.scope.tenant_id, "project": case.scope.project_id}) async def test_completion_behind_pause_keeps_actual_end_time(operations_case, task_service, worker_ids): @@ -666,3 +679,178 @@ async def test_inline_parallel_failure_closes_tasks_scheduled_in_the_same_transi assert await case.rows("SELECT status,ready_since FROM task_facts WHERE run_id=:id", id=run.id) == [ {"status": "cancelled", "ready_since": None} ] + + +@pytest.mark.parametrize("cache", ["backfilled", "incomplete", "invalid"]) +@pytest.mark.parametrize("missing_step", [False, True]) +async def test_listing_cache_cannot_veto_admitted_completion( + operations_case, task_service, worker_ids, cache, missing_step +): + import json + import runpy + from pathlib import Path + + from operations_support import assert_usage + + from firefly_weave.runtime.admission import admission + + case = operations_case + run = await case.start() + params = { + "tenant": case.scope.tenant_id, + "project": case.scope.project_id, + "environment": case.scope.environment_id, + } + async with case.owner.begin() as owner: + if cache == "backfilled": + for table in ("run_event_evidence", "run_events", "run_facts"): + await owner.execute(text(f"DELETE FROM {table} WHERE run_id=:id"), {"id": run.id}) + frozen = runpy.run_path( + str(Path(__file__).resolve().parents[2] / "migrations/versions/0031_operations_facts.py") + ) + await owner.run_sync(lambda session: frozen["backfill_scope"](session.connection(), params)) + else: + await owner.execute( + text("UPDATE run_facts SET node_kinds=cast(:kinds AS jsonb) WHERE run_id=:id"), + { + "id": run.id, + "kinds": json.dumps({"other": "transform"} if cache == "incomplete" else {"work": "future"}), + }, + ) + if missing_step: + await owner.execute(text("DELETE FROM step_facts WHERE run_id=:id"), {"id": run.id}) + before = (await case.rows("SELECT classification_state,node_kinds FROM run_facts WHERE run_id=:id", id=run.id))[0] + if cache == "backfilled": + assert before == {"classification_state": "unavailable", "node_kinds": {}} + source = (await case.rows("SELECT artifact,state,activation,request FROM runs WHERE id=:id", id=run.id))[0] + assert admission(source) == "available" + async with case.tx() as tx: + lease = (await task_service.claim(tx, worker_ids[0], 1))[0] + async with case.tx() as tx: + await task_service.complete(tx, lease.proof, uuid4(), 42) + assert (await case.rows("SELECT state->>'status' AS status FROM runs WHERE id=:id", id=run.id)) == [ + {"status": "succeeded"} + ] + assert (await case.rows("SELECT status FROM run_facts WHERE run_id=:id", id=run.id)) == [{"status": "succeeded"}] + assert (await case.rows("SELECT classification_state,node_kinds FROM run_facts WHERE run_id=:id", id=run.id))[ + 0 + ] == before + assert await case.rows("SELECT status FROM step_facts WHERE run_id=:id", id=run.id) == ( + [] if missing_step else [{"status": "succeeded"}] + ) + async with case.owner() as owner: + await assert_usage(owner, params) + + +async def test_historical_worker_completion_preserves_unknown_times_and_atomicity( + operations_case, task_service, worker_ids, monkeypatch +): + from operations_support import assert_usage + + from firefly_weave.operations.outbox import OutboxService + + case = operations_case + run = await case.start() + async with case.tx() as tx: + lease = (await task_service.claim(tx, worker_ids[0], 1))[0] + async with case.owner.begin() as owner: + await owner.execute(text("DELETE FROM step_facts WHERE run_id=:id"), {"id": run.id}) + tables = ( + "runs", + "run_events", + "run_facts", + "step_facts", + "task_facts", + "task_intents", + "task_leases", + "operation_usage", + "operation_reservations", + "operation_allocations", + "event_deliveries", + "integration_events", + "operation_control_deliveries", + ) + before = { + table: await case.rows(f"SELECT to_jsonb(t) AS row FROM {table} t ORDER BY to_jsonb(t)::text") + for table in tables + } + + async def fail(*args, **kwargs): + raise RuntimeError("completion rollback") + + completion = uuid4() + with monkeypatch.context() as patch: + patch.setattr(OutboxService, "append", fail) + with pytest.raises(RuntimeError, match="completion rollback"): + async with case.tx() as tx: + await task_service.complete(tx, lease.proof, completion, 42) + assert { + table: await case.rows(f"SELECT to_jsonb(t) AS row FROM {table} t ORDER BY to_jsonb(t)::text") + for table in tables + } == before + async with case.tx() as tx: + await task_service.complete(tx, lease.proof, completion, 42) + fact = (await case.rows("SELECT * FROM step_facts WHERE run_id=:id", id=run.id))[0] + assert fact["status"] == "succeeded" and fact["ended_at"] is not None + assert fact["scheduled_at"] is None and fact["started_at"] is None and fact["attempts"] == 0 + async with case.tx() as tx: + await task_service.complete(tx, lease.proof, completion, 42) + assert (await case.rows("SELECT * FROM step_facts WHERE run_id=:id", id=run.id))[0] == fact + async with case.owner() as owner: + await assert_usage(owner, {"tenant": case.scope.tenant_id, "project": case.scope.project_id}) + + +@pytest.mark.parametrize("historical", [False, True]) +@pytest.mark.parametrize("late_start", [False, True]) +async def test_join_progress_preserves_first_observed_times( + operations_case, task_service, worker_ids, historical, late_start +): + from operations_support import assert_usage + + case = operations_case + branches = { + name: { + "steps": [{"id": name, "kind": "action", "uses": "echo-action@2.0.0", "with": {"literal": 3}}], + "output": {"literal": None}, + } + for name in ("first", "second") + } + steps = [{"id": "fork", "kind": "parallel", "concurrency": 1, "branches": branches}] + if late_start: + steps.insert(0, {"id": "lead", "kind": "action", "uses": "echo-action@2.0.0", "with": {"literal": 3}}) + run = await start_inline( + case, + steps, + {"literal": None}, + worker_pins=case.activation.request.worker_release_ids, + ) + if late_start: + async with case.tx() as tx: + lease = (await task_service.claim(tx, worker_ids[0], 1))[0] + async with case.tx() as tx: + await task_service.complete(tx, lease.proof, uuid4(), 42) + original = ( + await case.rows("SELECT scheduled_at,started_at FROM step_facts WHERE run_id=:id AND node_id='fork'", id=run.id) + )[0] + assert original["scheduled_at"] is not None and original["started_at"] == original["scheduled_at"] + if historical: + async with case.owner.begin() as owner: + await owner.execute(text("DELETE FROM step_facts WHERE run_id=:id AND node_id='fork'"), {"id": run.id}) + for status in ("running", "succeeded"): + async with case.tx() as tx: + lease = (await task_service.claim(tx, worker_ids[0], 1))[0] + async with case.tx() as tx: + await task_service.complete(tx, lease.proof, uuid4(), 42) + fact = ( + await case.rows( + "SELECT status,scheduled_at,started_at,ended_at FROM step_facts WHERE run_id=:id AND node_id='fork'", + id=run.id, + ) + )[0] + assert fact["status"] == status + assert {key: fact[key] for key in ("scheduled_at", "started_at")} == ( + {"scheduled_at": None, "started_at": None} if historical else original + ) + assert (fact["ended_at"] is not None) is (status == "succeeded") + async with case.owner() as owner: + await assert_usage(owner, {"tenant": case.scope.tenant_id, "project": case.scope.project_id}) diff --git a/tests/integration/test_operations_hot_path.py b/tests/integration/test_operations_hot_path.py index 7af10a10..3d176bb2 100644 --- a/tests/integration/test_operations_hot_path.py +++ b/tests/integration/test_operations_hot_path.py @@ -128,6 +128,9 @@ async def capture_persist(self, *args, **kwargs): assert len(await case.rows("SELECT node_id FROM step_facts WHERE run_id=:id", id=run.id)) == ( count + 1 if isinstance(mode, int) or mode == "inline" else 1 ) + for row in await case.rows("SELECT kind,scheduled_at,started_at FROM step_facts WHERE run_id=:id", id=run.id): + assert row["scheduled_at"] is not None + assert row["started_at"] == (None if row["kind"] == "action" else row["scheduled_at"]) from operations_support import assert_usage async with case.owner() as owner: diff --git a/tests/unit/operations/test_fact_projection.py b/tests/unit/operations/test_fact_projection.py index 35f2fdd3..3b2c1aaa 100644 --- a/tests/unit/operations/test_fact_projection.py +++ b/tests/unit/operations/test_fact_projection.py @@ -160,6 +160,32 @@ def test_call_origin_requires_a_typed_caller(): facts.RunStartFacts(origin="call", caller={"run_id": uuid4(), "node_id": "caller", "instance_key": ""}) +@pytest.mark.parametrize("inherited", [False, True]) +def test_call_origin_preserves_inherited_test_flag(inherited): + from firefly_weave.contracts.run_views import RunSummaryCaller + + caller = RunSummaryCaller(run_id=uuid4(), node_id="invoke", instance_key="invoke[3]") + value = facts.RunStartFacts(origin="call", test=inherited, caller=caller) + assert value.origin == "call" and value.test is inherited and value.caller == caller + + +def test_inherited_test_flag_does_not_relax_caller_or_retry_validation(): + from firefly_weave.contracts.run_views import RunSummaryCaller + + caller = RunSummaryCaller(run_id=uuid4(), node_id="invoke", instance_key="") + for values in ( + {"origin": "call", "test": True}, + {"origin": "call", "test": True, "caller": caller.model_dump(mode="json")}, + {"origin": "call", "test": True, "caller": caller, "retried_from_run_id": uuid4()}, + {"origin": "call", "test": "true", "caller": caller}, + {"origin": "test", "test": False}, + {"origin": "test", "test": True, "caller": caller}, + ): + with pytest.raises(ValueError): + facts.RunStartFacts(**values) + assert facts.RunStartFacts(origin="test", test=True).test is True + + def test_origin_is_not_a_request_body_field(): from firefly_weave.contracts.runtime import StartRunRequest From 812152b00f0a1b3976c35cc8160028e60c27bf94 Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Fri, 9 Oct 2026 07:02:27 -0700 Subject: [PATCH 06/63] Serve chronological run summaries and classified step timelines --- CHANGELOG.md | 8 + docs/operations/upgrades.md | 20 + docs/reference/api.md | 31 +- src/firefly_weave/api/runs.py | 39 +- src/firefly_weave/contracts/run_views.py | 25 +- src/firefly_weave/contracts/surface.py | 2 - src/firefly_weave/operations/fact_reads.py | 465 ++++++++++ src/firefly_weave/operations/facts.py | 6 +- src/firefly_weave/operations/run_views.py | 65 +- src/firefly_weave/operations/telemetry.py | 2 +- src/firefly_weave/operations/transport.py | 20 +- src/firefly_weave/runtime/service.py | 49 +- src/firefly_weave/sdk/client.py | 43 +- tests/contracts/test_run_views_contract.py | 3 +- tests/contracts/test_run_views_snapshot.py | 8 +- tests/contracts/test_run_views_surface.py | 15 +- .../contracts/run-views.snapshot.json | 14 +- tests/integration/test_run_fact_reads.py | 853 ++++++++++++++++++ tests/unit/operations/test_run_views.py | 255 +++++- tests/unit/operations/test_transport.py | 41 + tests/unit/runtime/test_run_filters.py | 13 + 21 files changed, 1882 insertions(+), 95 deletions(-) create mode 100644 src/firefly_weave/operations/fact_reads.py create mode 100644 tests/integration/test_run_fact_reads.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 25eb8cf5..799ff9ce 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,14 @@ SPDX-License-Identifier: Apache-2.0 ## Unreleased +- Serve bounded run summaries and classified step timelines with fresh scoped + authorization. Timeline output remains optional; legacy timings stay unknown. + Logs remain unavailable with HTTP 501. +- Change `runs.list` to newest-started-first order. Existing v1 cursors remain + supported with explicit `order=id`; chronological orders use v2 cursors. + Update-time pagination is a live view and may require a refresh after changes. + Restored executions reappear in every order. + - Keep typing in the selected form field when deferred row autofocus completes. Rapid row additions focus the newest item without overriding a later choice. - Update Studio's YAML parser to 2.8.3 so deeply nested documents produce a diff --git a/docs/operations/upgrades.md b/docs/operations/upgrades.md index 8e5083d8..3ca5b030 100644 --- a/docs/operations/upgrades.md +++ b/docs/operations/upgrades.md @@ -148,6 +148,26 @@ policy update, and this release has no in-place policy-change operation: a mismatch leaves the runtime restricted. Never rewrite the fingerprint or counters by hand to force readiness. +## Run list and timeline compatibility + +The server now serves run summaries and step timelines. `runs.list` changes its +default to `started_desc`, with `started_asc` and `updated_desc` also available. +Clients that continue a legacy v1 cursor must send `order=id` and retain its +original filters. New filters bind ID cursors to the normalized query; time +orders use v2 cursors. Restart pagination after changing filters or order. + +The update-time order is a live view: a later update can move an item between +pages. Refresh from the first page to see the latest ordering. Unknown historical +times sort last and are never replaced by the fact projection's creation date. +Restored runs reappear even though their archive lifecycle record is retained. + +Old runs may have incomplete timelines. The server reads their missing author +step metadata in batches of 500, retaining only the requested page candidates. +A request stops after 100,000 keys or two seconds with `429 WV-RUNTIME-LIMIT`; +the time budget can be reached first. Synthetic control instances do not count +toward completeness or the key cap. Step outputs remain optional and classified. +`runs.logs` still answers authorized requests with `501 WV-UNAVAILABLE`. + ## Operational fact storage The `0031_operations_facts` migration adds five scoped operational projections diff --git a/docs/reference/api.md b/docs/reference/api.md index f7b3130c..ada3db09 100644 --- a/docs/reference/api.md +++ b/docs/reference/api.md @@ -412,13 +412,20 @@ Replay reports `consistent`, `inconsistent`, or `incomplete`: a history prefix, or a history with unavailable or redacted parts, is never reported as success. See [history and replay](history-and-replay.md). -**Run summaries, steps, and logs.** `run_summaries.list`, `runs.steps`, and -`runs.logs` are published so that clients can build against them. Until this -server serves them, an authorized request answers `501` with `WV-UNAVAILABLE`. - -- These lists are ordered by time, not by ID: run summaries by start time - (newest first by default) or by last update, steps by scheduled time, and log - entries by time. +**Run summaries, steps, and logs.** `run_summaries.list` and `runs.steps` serve +scoped operational metadata after checking both token and current permissions. +`runs.logs` remains a published contract: an authorized request answers `501` +with `WV-UNAVAILABLE`. + +- Run summaries and `runs.list` default to `order=started_desc`. They also accept + `started_asc` and `updated_desc`; `runs.list` additionally accepts `order=id`. + Use explicit `order=id` with legacy v1 cursors. The original business-key, + correlation-key, single-status and archive filters retain their v1 collection; + additional filters bind the cursor to the normalized query. +- Chronological lists use v2 cursors with UUID tie breakers and unknown times + last. `updated_desc` is a live view: updates can move a run between pages, + so refresh from the first page to see its latest position. Steps sort by + scheduled time, static step ID and full instance key; unknown times sort last. - `runs.steps` and `runs.logs` accept a `limit` of 1 to 500 (200 by default). - Each cursor is bound to every filter and to the order. After you change a filter, start again from the first page. @@ -429,6 +436,16 @@ server serves them, an authorized request answers `501` with `WV-UNAVAILABLE`. - Send times as RFC 3339 with an offset. Use `Z`, or encode `+` as `%2B` in the query string. +Step timelines do not read payloads unless `include=output` is requested. Only +the displayed outputs are fetched, after checking the bounded run artifact and +classifying each output. An omitted output carries an omission reason; a +returned JSON `null` is distinct from an omitted output. Synthetic control +instances do not appear. `complete=false` means the run has author step +instances without facts, even when they fall outside the current page or filter. +Their unknown timestamps remain null. Legacy metadata is streamed in batches +of 500, up to 100,000 keys and two seconds per request; exceeding either budget +answers `429 WV-RUNTIME-LIMIT`. + **Answers that say "unavailable".** Cancellation and task completion can return an unavailable acknowledgment when the platform cannot classify the historical data or compare payloads; task completion then includes diff --git a/src/firefly_weave/api/runs.py b/src/firefly_weave/api/runs.py index 8384b0e9..dee83422 100644 --- a/src/firefly_weave/api/runs.py +++ b/src/firefly_weave/api/runs.py @@ -25,9 +25,11 @@ from firefly_weave.api.access import request_scope from firefly_weave.api.surface import operation -from firefly_weave.api.transport import page_request, page_response +from firefly_weave.api.transport import canonical_path, decode_cursor, decode_cursor_v2, parse_query from firefly_weave.contracts.human_tasks import ManualControlRequest -from firefly_weave.contracts.runtime import RunListFilters, SignalRequest, StartRunRequest +from firefly_weave.contracts.run_views import RunListQuery +from firefly_weave.contracts.runtime import SignalRequest, StartRunRequest +from firefly_weave.operations.fact_positions import RunPosition, run_position from firefly_weave.runtime.service import RuntimeService from firefly_weave.runtime.signals import SignalService @@ -82,25 +84,24 @@ async def signal(self, request: Request) -> JSONResponse: @operation("runs.list") async def discover(self, request: Request) -> JSONResponse: scope = request_scope(request, environment=True) - values: dict[str, str | bool] = { - key: request.query_params[key] for key in RunListFilters.model_fields if key in request.query_params - } - if "include_archived" in values: - if values["include_archived"] not in {"true", "false"}: - raise ValueError("include_archived must be true or false") - values["include_archived"] = values["include_archived"] == "true" - filters = RunListFilters.model_validate(values) - collection = filters.cursor_collection() - limit, cursor = page_request(request, scope, collection) + query = parse_query(request.query_params, RunListQuery) + position: RunPosition | UUID | None = None + if query.order == "id": + position = ( + decode_cursor(query.cursor, scope, query.cursor_collection()) + if canonical_path(request.url.path) + else UUID(query.cursor) + if query.cursor + else None + ) + else: + position = run_position(decode_cursor_v2(query.cursor, scope, query.cursor_collection())) result = await self.service.list( - request.state.principal, - scope, - limit=limit, - cursor=cursor, - filters=filters, - context=request.state.audit_context, + request.state.principal, scope, query=query, position=position, context=request.state.audit_context ) - return JSONResponse(page_response(result, scope, collection, request.url.path)) + if query.order == "id" and not canonical_path(request.url.path) and result["next_cursor"] is not None: + result["next_cursor"] = str(decode_cursor(result["next_cursor"], scope, query.cursor_collection())) + return JSONResponse(result) @operation("runs.pause") async def pause(self, request: Request) -> JSONResponse: diff --git a/src/firefly_weave/contracts/run_views.py b/src/firefly_weave/contracts/run_views.py index 20d76eff..d6e92072 100644 --- a/src/firefly_weave/contracts/run_views.py +++ b/src/firefly_weave/contracts/run_views.py @@ -28,7 +28,7 @@ import json import re from datetime import UTC, datetime, timedelta -from typing import Annotated, Literal +from typing import TYPE_CHECKING, Annotated, Literal from uuid import UUID from pydantic import AwareDatetime, Field, field_validator, model_validator @@ -40,6 +40,9 @@ from firefly_weave.contracts.values import FiniteFloat, JsonData, SafeInteger from firefly_weave.operations.redaction import Omission +if TYPE_CHECKING: + from firefly_weave.contracts.runtime import RunListFilters + type RunStatus = Literal["queued", "running", "waiting", "suspended", "succeeded", "failed", "cancelled", "timed_out"] type RunOrigin = Literal["manual", "webhook", "schedule", "broker", "email", "provider", "retry", "call", "test"] type RunSummaryOrder = Literal["started_desc", "started_asc", "updated_desc"] @@ -349,15 +352,31 @@ def cursor_collection(self) -> str: class RunListQuery(RunSummaryQuery): - """`runs.list` filters, with ID order retained for existing callers.""" + """Chronological `runs.list` filters, with explicit ID order for legacy cursors.""" # The list schema also permits ID order; the summary schema keeps chronological orders only. - order: RunListOrder = "id" # type: ignore[assignment] + order: RunListOrder = "started_desc" # type: ignore[assignment] def cursor_collection(self) -> str: + legacy = self.legacy_filters() + if legacy is not None: + return legacy.cursor_collection() filters = self.model_dump(mode="json", exclude={"order", "limit", "cursor"}) return f"runs:{_digest(filters)}:{self.order}" + def legacy_filters(self) -> RunListFilters | None: + from firefly_weave.contracts.runtime import RunListFilters + + excluded = {"order", "limit", "cursor", "business_key", "correlation_key", "status", "include_archived"} + if self.order != "id" or len(self.status) > 1 or self.model_dump(exclude=excluded, exclude_defaults=True): + return None + return RunListFilters( + business_key=self.business_key, + correlation_key=self.correlation_key, + status=self.status[0] if self.status else None, + include_archived=self.include_archived, + ) + class RunStepQuery(ContractModel): """`runs.steps` query parameters; `step` is a static step ID and matches every instance.""" diff --git a/src/firefly_weave/contracts/surface.py b/src/firefly_weave/contracts/surface.py index dbc39be6..595af133 100644 --- a/src/firefly_weave/contracts/surface.py +++ b/src/firefly_weave/contracts/surface.py @@ -1413,7 +1413,6 @@ def native(self) -> OpenAPIOperation: RunSummaryPage, "run.read", query=RunSummaryQuery, - served=False, ), Operation( "runs.steps", @@ -1422,7 +1421,6 @@ def native(self) -> OpenAPIOperation: StepFactPage, "run.read", query=RunStepQuery, - served=False, ), Operation( "runs.logs", diff --git a/src/firefly_weave/operations/fact_reads.py b/src/firefly_weave/operations/fact_reads.py new file mode 100644 index 00000000..b3dcfe6c --- /dev/null +++ b/src/firefly_weave/operations/fact_reads.py @@ -0,0 +1,465 @@ +# Copyright 2026 Firefly Software Foundation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# Author: Firefly Software Foundation +# SPDX-License-Identifier: Apache-2.0 + +"""Scoped, bounded operational metadata reads; execution remains runtime-owned.""" + +import asyncio +import heapq +import json +from collections.abc import Mapping +from dataclasses import dataclass, field +from datetime import UTC, datetime +from typing import Any, cast +from uuid import UUID + +from pydantic import TypeAdapter +from sqlalchemy import text + +from firefly_weave.compiler.ir import accepted_ir_versions +from firefly_weave.contracts import language_features +from firefly_weave.contracts.instance_keys import split_instance +from firefly_weave.contracts.public import UnavailableResource +from firefly_weave.contracts.run_views import ( + RunListQuery, + RunStepQuery, + RunSummary, + RunSummaryQuery, + StepFact, + StepFactWithOutput, + StepKind, +) +from firefly_weave.definitions.models import CatalogError +from firefly_weave.operations.fact_positions import RunPosition, StepPosition +from firefly_weave.operations.facts import FactRepository, author_instance +from firefly_weave.operations.redaction import Omission, SafeProjection +from firefly_weave.runtime import admission + +_RUN_SCOPE = "f.tenant_id=:tenant AND f.project_id=:project AND f.environment_id=:environment" +_RUN_SELECT = ( + "SELECT f.*,coalesce(a.archived,false) AS archived,EXISTS(SELECT 1 FROM run_policy_blocks b " + "WHERE b.tenant_id=f.tenant_id AND b.project_id=f.project_id AND b.environment_id=f.environment_id " + "AND b.run_id=f.run_id) AS policy_blocked FROM run_facts f LEFT JOIN run_archives a ON " + "a.tenant_id=f.tenant_id AND a.project_id=f.project_id AND a.environment_id=f.environment_id " + "AND a.run_id=f.run_id WHERE " + _RUN_SCOPE +) +_MISSING = ( + "s.tenant_id=:tenant AND s.project_id=:project AND s.environment_id=:environment AND s.run_id=:run " + "AND left(s.node_id,1)<>'@' AND NOT EXISTS(SELECT 1 FROM step_facts f " + "WHERE f.tenant_id=s.tenant_id AND f.project_id=s.project_id AND f.environment_id=s.environment_id " + "AND f.run_id=s.run_id AND (f.instance_key=s.node_id OR (f.instance_key='' AND f.node_id=s.node_id)))" +) +_STEP_ORDER = 'scheduled_at ASC NULLS LAST,node_id COLLATE "C",instance_key COLLATE "C"' +LEGACY_KEYS = 100_000 +LEGACY_SECONDS = 2 +_KIND: TypeAdapter[StepKind] = TypeAdapter(StepKind) + + +def unavailable(identifier: UUID) -> UnavailableResource: + return UnavailableResource(id=identifier, omissions=[Omission(path="", reason="classification_unavailable")]) + + +def available(row: Mapping[str, Any]) -> bool: + features = row.get("pinned_features") + return ( + row.get("classification_state") == "available" + and row.get("classification_policy") == admission.POLICY + and row.get("policy_blocked") is False + and isinstance(features, list) + and len(features) <= 256 + and all(type(value) is str for value in features) + and len(json.dumps(features, ensure_ascii=False).encode()) <= 16384 + and set(features) <= set(language_features.ADVERTISED_FEATURES) + and row.get("pinned_ir_version") in accepted_ir_versions(language_features.ADVERTISED_FEATURES) + ) + + +def milliseconds(start: datetime | None, end: datetime | None) -> int | None: + return int((end - start).total_seconds() * 1000) if start is not None and end is not None else None + + +def summary_of(row: Mapping[str, Any]) -> RunSummary | UnavailableResource: + identifier = row["run_id"] + if not available(row): + return unavailable(identifier) + try: + return RunSummary.model_validate( + dict( + id=identifier, + workflow=dict( + name=row["workflow_name"], + version=row["workflow_version"], + definition_version_id=row["definition_version_id"], + ), + activation_id=row["activation_id"], + activation=dict(name=row["activation_name"], revision=row["activation_revision"]), + status=row["status"], + paused=row["paused"], + test=row["test"], + origin=row["origin"], + caller=( + dict( + run_id=row["caller_run_id"], + node_id=row["caller_node_id"], + instance_key=row["caller_instance_key"], + ) + if row["caller_run_id"] is not None + else None + ), + retried_from_run_id=row["retried_from_run_id"], + started_at=row["started_at"], + updated_at=row["updated_at"], + ended_at=row["ended_at"], + duration_ms=milliseconds(row["started_at"], row["ended_at"]), + business_key=row["business_key"], + correlation_key=row["correlation_key"], + failed_step=( + dict( + node_id=row["failed_node_id"], + instance_key=row["failed_instance_key"], + error_code=row["failed_error_code"], + ) + if row["failed_node_id"] is not None + else None + ), + active_incidents=row["active_incidents"], + handled_errors=row["handled_errors"], + archived=row["archived"], + ) + ) + except (ValueError, TypeError, KeyError, OverflowError): + return unavailable(identifier) + + +def step_of(row: Mapping[str, Any], *, output: SafeProjection | None = None) -> StepFact | StepFactWithOutput: + try: + values: dict[str, Any] = { + key: row.get(key) + for key in ( + "node_id", + "instance_key", + "kind", + "status", + "scheduled_at", + "started_at", + "ended_at", + "attempts", + "worker_id", + "error_code", + "handled", + "child_run_id", + ) + } + values.update( + iteration=list(split_instance(row["instance_key"] or row["node_id"]).indexes), + duration_ms=milliseconds(row["started_at"], row["ended_at"]), + queue_wait_ms=milliseconds(row["scheduled_at"], row["started_at"]), + log_entries=0, + ) + if output is not None: + if output.available: + return StepFactWithOutput(**values, output=output.value) + values["omissions"] = [ + omission.model_copy(update={"path": "/output" + omission.path}) for omission in output.omissions + ] + return StepFact(**values) + except (ValueError, TypeError, KeyError, OverflowError): + raise legacy_unavailable() from None + + +def legacy_unavailable() -> CatalogError: + return CatalogError(409, "WV-LEGACY-UNAVAILABLE", "Step classification is unavailable") + + +def step_order(row: Mapping[str, Any]) -> tuple[bool, datetime, str, str]: + at = row["scheduled_at"] + return at is None, at or datetime.max.replace(tzinfo=UTC), row["node_id"], row["instance_key"] + + +@dataclass +class _Candidate: + order: tuple[bool, datetime, str, str] + row: dict[str, Any] = field(compare=False) + + def __lt__(self, other: "_Candidate") -> bool: + return self.order > other.order + + +class FactReadRepository(FactRepository): + async def summary_rows(self, query: RunSummaryQuery, position: RunPosition | None) -> list[dict[str, Any]]: + return await self._run_rows(query, position) + + async def legacy_rows(self, query: RunListQuery, position: RunPosition | UUID | None) -> list[dict[str, Any]]: + return await self._run_rows(query, position) + + async def _run_rows(self, query: RunSummaryQuery, position: RunPosition | UUID | None) -> list[dict[str, Any]]: + clauses = ["(:include_archived OR NOT coalesce(a.archived,false))"] + params: dict[str, Any] = dict(include_archived=query.include_archived, limit=query.limit + 1) + for name, column in ( + ("workflow", "workflow_name"), + ("version", "workflow_version"), + ("origin", "origin"), + ("caller_run_id", "caller_run_id"), + ("retried_from_run_id", "retried_from_run_id"), + ("business_key", "business_key"), + ("correlation_key", "correlation_key"), + ("activation_id", "activation_id"), + ): + value = getattr(query, name) + if value is not None: + clauses.append(f"f.{column}=:{name}") + params[name] = value + if query.status: + clauses.append("f.status=ANY(:statuses)") + params["statuses"] = query.status + if not query.include_test: + clauses.append("NOT f.test") + if query.top_level_only: + clauses.append("f.caller_run_id IS NULL") + for name, column in (("has_active_incident", "active_incidents"), ("has_handled_errors", "handled_errors")): + value = getattr(query, name) + if value is not None: + clauses.append(f"f.{column}>0" if value else f"f.{column}=0") + for name, operator in (("started_after", ">="), ("started_before", "<")): + value = getattr(query, name) + if value is not None: + clauses.append(f"f.started_at{operator}:{name}") + params[name] = value + column, direction, operator = { + "id": ("run_id", "ASC", ">"), + "started_desc": ("started_at", "DESC", "<"), + "started_asc": ("started_at", "ASC", ">"), + "updated_desc": ("updated_at", "DESC", "<"), + }[query.order] + if column == "run_id": + if position is not None: + if not isinstance(position, UUID): + raise ValueError("ID order requires an ID position") + clauses.append("f.run_id>:position_id") + params["position_id"] = position + order = "f.run_id ASC" + else: + if position is not None: + if not isinstance(position, RunPosition): + raise ValueError("Chronological order requires a time position") + params["position_id"] = position.id + if position.at is None: + clauses.append(f"f.{column} IS NULL AND f.run_id{operator}:position_id") + else: + params["position_at"] = position.at + clauses.append( + f"((f.{column},f.run_id){operator}(:position_at,:position_id) OR f.{column} IS NULL)" + ) + order = f"f.{column} {direction} NULLS LAST,f.run_id {direction}" + return await self.rows( + _RUN_SELECT + " AND " + " AND ".join(clauses) + " ORDER BY " + order + " LIMIT :limit", **params + ) + + async def metadata(self, run_id: UUID) -> dict[str, Any]: + rows = await self.rows(_RUN_SELECT + " AND f.run_id=:run", run=run_id) + if not rows: + raise CatalogError(404, "WV-NOT-FOUND", "Run not found") + row = rows[0] + if isinstance(summary_of(row), UnavailableResource): + raise legacy_unavailable() + kinds = row.get("node_kinds") + if not isinstance(kinds, dict) or len(kinds) > 10000 or len(json.dumps(kinds).encode()) > 1048576: + raise legacy_unavailable() + return row + + async def step_rows( + self, run_id: UUID, query: RunStepQuery, position: StepPosition | None + ) -> tuple[list[dict[str, Any]], bool]: + metadata = await self.metadata(run_id) + clauses = [_RUN_SCOPE, "f.run_id=:run", "left(f.node_id,1)<>'@'", "left(f.instance_key,1)<>'@'"] + params: dict[str, Any] = dict(run=run_id, limit=query.limit + 1) + if query.step is not None: + clauses.append("f.node_id=:step") + params["step"] = query.step + if position is not None: + params.update(node=position.node_id, instance=position.instance_key) + identity_predicate = ( + '(f.node_id COLLATE "C",f.instance_key COLLATE "C")>' + '(cast(:node AS text) COLLATE "C",cast(:instance AS text) COLLATE "C")' + ) + if position.scheduled_at is None: + clauses.append("(f.scheduled_at IS NULL AND " + identity_predicate + ")") + else: + params["scheduled"] = position.scheduled_at + clauses.append( + "(f.scheduled_at>:scheduled OR f.scheduled_at IS NULL OR (f.scheduled_at=:scheduled AND " + + identity_predicate + + "))" + ) + rows = await self.rows( + "SELECT f.* FROM step_facts f WHERE " + + " AND ".join(clauses) + + " ORDER BY " + + _STEP_ORDER + + " LIMIT :limit", + **params, + ) + for row in rows: + try: + identity = author_instance(row["instance_key"] or row["node_id"]) + except ValueError: + raise legacy_unavailable() from None + if ( + identity is None + or identity.node_id != row["node_id"] + or metadata["node_kinds"].get(row["node_id"]) != row["kind"] + ): + raise legacy_unavailable() + missing = await self.tx.session.scalar( + text("SELECT EXISTS(SELECT 1 FROM step_instances s WHERE " + _MISSING + ")"), {**self.params, "run": run_id} + ) + if not missing: + return rows, True + historical, complete = await self.historical_steps(run_id, query.step, position, query.limit) + return sorted([*rows, *historical], key=step_order)[: query.limit + 1], complete + + async def historical_steps( + self, run_id: UUID, step: str | None, position: StepPosition | None, limit: int + ) -> tuple[list[dict[str, Any]], bool]: + metadata = await self.metadata(run_id) + boundary = step_order(vars(position)) if position is not None else None + heap: list[_Candidate] = [] + stream = None + scanned = 0 + try: + async with asyncio.timeout(LEGACY_SECONDS): + stream = await self.tx.session.stream( + text( + "SELECT s.node_id,s.status FROM step_instances s WHERE " + + _MISSING + + ' ORDER BY s.node_id COLLATE "C"' + ), + {**self.params, "run": run_id}, + execution_options={"yield_per": 500}, + ) + async for batch in stream.mappings().partitions(500): + for source in batch: + scanned += 1 + if scanned > LEGACY_KEYS: + raise CatalogError(429, "WV-RUNTIME-LIMIT", "Legacy step scan exceeds its bounded budget") + identity = author_instance(source["node_id"]) + if identity is None: + continue + kind: StepKind = _KIND.validate_python( + metadata["node_kinds"].get(identity.node_id), strict=True + ) + if source["status"] not in {"completed", "waiting"}: + raise legacy_unavailable() + row = dict( + node_id=identity.node_id, + instance_key=identity.instance_key, + kind=kind, + status="succeeded" if source["status"] == "completed" else "waiting", + scheduled_at=None, + started_at=None, + ended_at=None, + attempts=0, + worker_id=None, + error_code=None, + handled=None, + child_run_id=None, + ) + order = step_order(row) + if (step is not None and identity.node_id != step) or ( + boundary is not None and order <= boundary + ): + continue + candidate = _Candidate(order, row) + if len(heap) < limit + 1: + heapq.heappush(heap, candidate) + elif order < heap[0].order: + heapq.heapreplace(heap, candidate) + await asyncio.sleep(0) + except TimeoutError: + raise CatalogError(429, "WV-RUNTIME-LIMIT", "Legacy step scan exceeds its bounded budget") from None + except (ValueError, TypeError, KeyError): + raise legacy_unavailable() from None + finally: + if stream is not None: + await stream.close() + return [candidate.row for candidate in sorted(heap, key=lambda value: value.order)], scanned == 0 + + async def outputs(self, run_id: UUID, rows: list[dict[str, Any]]) -> dict[str, SafeProjection]: + from firefly_weave.compiler.api import import_artifact + from firefly_weave.compiler.ir import ActionNode, HumanTaskNode, SignalNode, WaitNode + from firefly_weave.contracts.definitions import ActionDefinition, load_definition + from firefly_weave.contracts.values import JsonObject + from firefly_weave.operations.redaction import project + from firefly_weave.runtime.kernel import action_schemas, workflow + from firefly_weave.runtime.repository import RUN_FETCH_BYTES, RuntimeRepository + + await self.metadata(run_id) + retained = await RuntimeRepository(self.tx).run(run_id) + admission.require_available(retained) + ir = workflow(import_artifact(retained["artifact"])) + bundle = { + dependency.reference: dependency.document for dependency in ir.dependencies if dependency.kind == "Schema" + } + nodes = {node.id: node for node in ir.graph.nodes} + keys = [row["instance_key"] or row["node_id"] for row in rows] + withheld = SafeProjection(available=False, omissions=[Omission(path="", reason="classification_unavailable")]) + result = dict.fromkeys(keys, withheld) + if not keys: + return result + predicate = ( + "tenant_id=:tenant AND project_id=:project AND environment_id=:environment AND run_id=:run " + "AND left(node_id,1)<>'@' AND node_id=ANY(:keys)" + ) + outputs = await self.rows( + "WITH sized AS (SELECT node_id,output,sum(octet_length(output::text)) OVER () AS total_bytes " + "FROM step_instances WHERE " + predicate + ") " + "SELECT node_id,CASE WHEN total_bytes<=:budget THEN output ELSE NULL END AS output," + "total_bytes>:budget AS oversized FROM sized", + run=run_id, + keys=keys, + budget=RUN_FETCH_BYTES, + ) + if any(output["oversized"] for output in outputs): + raise CatalogError(429, "WV-RUNTIME-LIMIT", "Step outputs exceed the bounded fetch budget") + for output in outputs: + identity = author_instance(output["node_id"]) + if identity is None: + continue + node = nodes.get(identity.node_id) + schemas: list[JsonObject] = [] + if isinstance(node, ActionNode): + definition = load_definition( + next(dependency.document for dependency in ir.dependencies if dependency.digest == node.dependency) + ) + schemas = action_schemas(ir, cast(ActionDefinition, definition), "output", node=node) + elif isinstance(node, SignalNode): + schemas = [ir.schemas[node.schema_ref]] + elif isinstance(node, HumanTaskNode): + schemas = [{"type": "object", "properties": {"data": node.form_schema, "decision": {"type": "string"}}}] + elif isinstance(node, WaitNode): + schemas = [{"type": "null"}] + elif node is not None: + schemas = [ + ir.schemas[guard.schema_ref] + for guard in ir.guards + if guard.path in {node.path, node.path + "/value"} + and guard.purpose in {"transform_output", "decision_output", "branch_output"} + ] + if schemas: + projections = [project(output["output"], schema, bundle) for schema in schemas] + result[output["node_id"]] = next( + (projection for projection in projections if not projection.available), projections[0] + ) + return result diff --git a/src/firefly_weave/operations/facts.py b/src/firefly_weave/operations/facts.py index 79a254db..a1441b60 100644 --- a/src/firefly_weave/operations/facts.py +++ b/src/firefly_weave/operations/facts.py @@ -53,6 +53,8 @@ SCOPE = "tenant_id=:tenant AND project_id=:project AND environment_id=:environment" TERMINAL = {"succeeded", "failed", "cancelled", "timed_out"} +_AUTHOR_KEY: TypeAdapter[str] = TypeAdapter(InstanceKeyText) +_AUTHOR_NODE: TypeAdapter[str] = TypeAdapter(ResourceName) def author_instance(key: str) -> InstanceView | None: @@ -60,9 +62,9 @@ def author_instance(key: str) -> InstanceView | None: raise ValueError("Instance key must be text") if key.startswith("@"): return None - TypeAdapter(InstanceKeyText).validate_python(key, strict=True) + _AUTHOR_KEY.validate_python(key, strict=True) value = instance_view(key) - TypeAdapter(ResourceName).validate_python(value.node_id, strict=True) + _AUTHOR_NODE.validate_python(value.node_id, strict=True) return value diff --git a/src/firefly_weave/operations/run_views.py b/src/firefly_weave/operations/run_views.py index 45294aee..a27f9147 100644 --- a/src/firefly_weave/operations/run_views.py +++ b/src/firefly_weave/operations/run_views.py @@ -16,8 +16,7 @@ """Run summaries, run steps and run logs behind current `run.read` authority. -The wire contract is frozen in `contracts/run_views.py`. Until run facts exist, every authorized read answers -501 WV-UNAVAILABLE after its query and cursor were validated. +Summaries and timelines read bounded facts in the authorization transaction; logs remain unavailable. """ from uuid import UUID @@ -28,6 +27,7 @@ from firefly_weave.access.authorization import AuthorizationService from firefly_weave.access.models import Principal from firefly_weave.access.repository import load_principal +from firefly_weave.api.transport import encode_cursor_v2 from firefly_weave.contracts.access import Scope from firefly_weave.contracts.run_views import ( RunLogPage, @@ -39,7 +39,9 @@ ) from firefly_weave.contracts.values import JsonValue from firefly_weave.definitions.models import CatalogError -from firefly_weave.persistence.uow import UnitOfWork +from firefly_weave.operations.fact_positions import run_position, step_position +from firefly_weave.operations.fact_reads import FactReadRepository, step_of, summary_of +from firefly_weave.persistence.uow import Transaction, UnitOfWork type Position = tuple[JsonValue, str] | None @@ -53,21 +55,32 @@ class RunViewService: def __init__(self, uow: UnitOfWork, authorization: AuthorizationService) -> None: self.uow, self.authorization = uow, authorization - async def authorize(self, actor: Principal, scope: Scope, *, context: AuditContext) -> Principal: + async def authorize(self, actor: Principal, scope: Scope, tx: Transaction, *, context: AuditContext) -> Principal: """`run.read` on the token principal and again on the principal stored now.""" if scope.environment_id is None: raise CatalogError(422, "WV-SCOPE", "Runtime requires an environment") self.authorization.require(actor, scope, "run.read", context=context) - async with self.uow.open(scope, mutation=False) as tx: - current = await load_principal(tx.session, actor.id) + current = await load_principal(tx.session, actor.id) self.authorization.require(current, scope, "run.read", context=context) return current async def summaries( self, actor: Principal, scope: Scope, query: RunSummaryQuery, position: Position, *, context: AuditContext ) -> RunSummaryPage: - await self.authorize(actor, scope, context=context) - raise not_served("run_summaries.list") + typed = run_position(position) + self.precheck(actor, scope, context) + async with self.uow.open(scope, mutation=False) as tx: + await self.authorize(actor, scope, tx, context=context) + rows = await FactReadRepository(tx).summary_rows(query, typed) + visible = rows[: query.limit] + cursor = None + if len(rows) > query.limit: + last = visible[-1] + at = last["updated_at"] if query.order == "updated_desc" else last["started_at"] + cursor = encode_cursor_v2( + scope, query.cursor_collection(), at.isoformat() if at else None, str(last["run_id"]) + ) + return RunSummaryPage(items=[summary_of(row) for row in visible], next_cursor=cursor) async def steps( self, @@ -79,8 +92,29 @@ async def steps( *, context: AuditContext, ) -> StepFactPage: - await self.authorize(actor, scope, context=context) - raise not_served("runs.steps") + typed = step_position(position) + self.precheck(actor, scope, context) + async with self.uow.open(scope, mutation=False) as tx: + await self.authorize(actor, scope, tx, context=context) + repository = FactReadRepository(tx) + rows, complete = await repository.step_rows(run_id, query, typed) + visible = rows[: query.limit] + outputs = await repository.outputs(run_id, visible) if query.include == "output" else {} + cursor = None + if len(rows) > query.limit: + last = visible[-1] + at = last["scheduled_at"] + cursor = encode_cursor_v2( + scope, + query.cursor_collection(run_id), + [at.isoformat() if at else None, last["node_id"], last["instance_key"]], + last["instance_key"] or last["node_id"], + ) + return StepFactPage( + items=[step_of(row, output=outputs.get(row["instance_key"] or row["node_id"])) for row in visible], + next_cursor=cursor, + complete=complete, + ) async def logs( self, @@ -92,5 +126,12 @@ async def logs( *, context: AuditContext, ) -> RunLogPage: - await self.authorize(actor, scope, context=context) - raise not_served("runs.logs") + self.precheck(actor, scope, context) + async with self.uow.open(scope, mutation=False) as tx: + await self.authorize(actor, scope, tx, context=context) + raise not_served("runs.logs") + + def precheck(self, actor: Principal, scope: Scope, context: AuditContext) -> None: + if scope.environment_id is None: + raise CatalogError(422, "WV-SCOPE", "Runtime requires an environment") + self.authorization.require(actor, scope, "run.read", context=context) diff --git a/src/firefly_weave/operations/telemetry.py b/src/firefly_weave/operations/telemetry.py index f6bbc129..ef91b49f 100644 --- a/src/firefly_weave/operations/telemetry.py +++ b/src/firefly_weave/operations/telemetry.py @@ -59,7 +59,7 @@ "other", } ) -STATUSES = frozenset({"ok", "rejected", "failed", "blocked", "pending", "unknown", "other"}) +STATUSES = frozenset({"ok", "rejected", "failed", "blocked", "pending", "unknown", "unavailable", "other"}) ERRORS = frozenset( { "none", diff --git a/src/firefly_weave/operations/transport.py b/src/firefly_weave/operations/transport.py index db38b189..c6c2d7d3 100644 --- a/src/firefly_weave/operations/transport.py +++ b/src/firefly_weave/operations/transport.py @@ -208,12 +208,22 @@ async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: None, ) status = 500 + unavailable = False + problem = bytearray() started = time.monotonic() async def observed_send(message: Message) -> None: - nonlocal status + nonlocal status, unavailable if message["type"] == "http.response.start": status = message["status"] + elif message["type"] == "http.response.body" and status == 501: + problem.extend(message.get("body", b"")[: max(0, 8193 - len(problem))]) + if not message.get("more_body") and len(problem) <= 8192: + try: + decoded = json.loads(problem) + unavailable = isinstance(decoded, dict) and decoded.get("code") == "WV-UNAVAILABLE" + except (ValueError, RecursionError): + pass await send(message) try: @@ -223,7 +233,13 @@ async def observed_send(message: Message) -> None: telemetry.record( "request", operation=operation, - status="ok" if status < 400 else "rejected" if status < 500 else "failed", + status="unavailable" + if unavailable + else "ok" + if status < 400 + else "rejected" + if status < 500 + else "failed", duration=time.monotonic() - started, ) diff --git a/src/firefly_weave/runtime/service.py b/src/firefly_weave/runtime/service.py index bb64d056..de8d436f 100644 --- a/src/firefly_weave/runtime/service.py +++ b/src/firefly_weave/runtime/service.py @@ -28,6 +28,7 @@ from firefly_weave.access.models import Principal from firefly_weave.access.repository import load_principal from firefly_weave.access.service import audit +from firefly_weave.api.transport import encode_cursor, encode_cursor_v2 from firefly_weave.compiler.api import import_artifact from firefly_weave.compiler.canonical import canonical_digest from firefly_weave.compiler.expressions import ExpressionFailure @@ -35,6 +36,8 @@ from firefly_weave.contracts.catalog import Activation from firefly_weave.contracts.human_tasks import ManualControlRequest from firefly_weave.contracts.operations import CancelRunRequest +from firefly_weave.contracts.public import UnavailableResource +from firefly_weave.contracts.run_views import RunListQuery from firefly_weave.contracts.runtime import ( CapacityRunAcknowledgment, RunListFilters, @@ -44,6 +47,8 @@ ) from firefly_weave.definitions.models import CatalogError from firefly_weave.definitions.service import DefinitionService +from firefly_weave.operations.fact_positions import RunPosition +from firefly_weave.operations.fact_reads import FactReadRepository, summary_of from firefly_weave.operations.facts import RunStartFacts from firefly_weave.persistence.idempotency import Idempotency from firefly_weave.persistence.uow import Transaction @@ -607,18 +612,40 @@ async def list( limit: int = 50, cursor: UUID | None = None, filters: RunListFilters | None = None, + query: RunListQuery | None = None, + position: RunPosition | UUID | None = None, context: AuditContext, ) -> dict[str, Any]: from firefly_weave.access.repository import load_principal - from firefly_weave.persistence.paging import page_ids + + legacy_adapter = query is None and (filters is not None or cursor is not None) + if query is None: + values = (filters or RunListFilters()).model_dump(mode="json") + values["status"] = [values["status"]] if values["status"] is not None else [] + query = RunListQuery.model_validate_json( + json.dumps({**values, "limit": limit, "order": "id" if legacy_adapter else "started_desc"}) + ) + position = cursor + elif filters is not None or cursor is not None: + raise ValueError("Use either a query or legacy list arguments") + if query.order != "id" and isinstance(position, UUID): + raise ValueError("Chronological order requires a time position") + if query.order == "id" and isinstance(position, RunPosition): + raise ValueError("ID order requires an ID position") self.require(actor, scope, "run.read", context) async with self.definitions.transaction(scope, None, mutation=False) as tx: actor = await load_principal(tx.session, actor.id) self.require(actor, scope, "run.read", context) - ids = await page_ids(tx, "runs", limit, cursor, run_filters=(filters or RunListFilters()).model_dump()) + rows = await FactReadRepository(tx).legacy_rows(query, position) + limit = query.limit items = [] - for identifier in ids[:limit]: + for row in rows[:limit]: + identifier = row["run_id"] + summary = summary_of(row) + if isinstance(summary, UnavailableResource): + items.append(summary.model_dump(mode="json")) + continue try: items.append( (await self.read(actor, scope, identifier, context=context, tx=tx)).model_dump(mode="json") @@ -634,4 +661,18 @@ async def list( "omissions": [{"path": "", "reason": "classification_unavailable"}], } ) - return {"items": items, "next_cursor": str(ids[limit - 1]) if len(ids) > limit else None} + next_cursor = None + if len(rows) > limit: + last = rows[limit - 1] + if query.order == "id": + next_cursor = ( + str(last["run_id"]) + if legacy_adapter + else encode_cursor(scope, query.cursor_collection(), last["run_id"]) + ) + else: + at = last["updated_at"] if query.order == "updated_desc" else last["started_at"] + next_cursor = encode_cursor_v2( + scope, query.cursor_collection(), at.isoformat() if at else None, str(last["run_id"]) + ) + return {"items": items, "next_cursor": next_cursor} diff --git a/src/firefly_weave/sdk/client.py b/src/firefly_weave/sdk/client.py index a4849259..b56f6ca5 100644 --- a/src/firefly_weave/sdk/client.py +++ b/src/firefly_weave/sdk/client.py @@ -19,7 +19,7 @@ from __future__ import annotations import inspect -from collections.abc import Callable +from collections.abc import Callable, Mapping from typing import TYPE_CHECKING, Any, Literal, Protocol, cast from urllib.parse import urlsplit from uuid import UUID @@ -102,6 +102,7 @@ catalog_lock, ) from firefly_weave.contracts.run_lifecycle import RunLifecycle, RunLifecycleRequest, RunPurgeRequest +from firefly_weave.contracts.run_views import RunListOrder, RunListQuery from firefly_weave.contracts.runtime import ( CapacityRunAcknowledgment, RunView, @@ -221,7 +222,7 @@ async def invoke( body: BaseModel | None = None, revision: int | None = None, idempotency_key: str | None = None, - query: dict[str, str | int] | None = None, + query: Mapping[str, str | int | list[str]] | None = None, adapter: str | None = None, ) -> Any: """Typed registry boundary also used by the thin CLI; no arbitrary URL input.""" @@ -715,24 +716,36 @@ async def list_runs( correlation_key: str | None = None, status: str | None = None, include_archived: bool = False, + order: RunListOrder = "started_desc", + query: RunListQuery | None = None, ) -> Page[RunView | UnavailableResource]: + values: dict[str, str | int | list[str]] + if query is not None: + values = { + key: str(value).lower() if isinstance(value, bool) else value + for key, value in query.model_dump(mode="json", exclude_none=True).items() + if not isinstance(value, list) or value + } + else: + values = { + **self._page(limit, cursor), + "order": order, + "include_archived": str(include_archived).lower(), + **{ + key: value + for key, value in { + "business_key": business_key, + "correlation_key": correlation_key, + "status": status, + }.items() + if value is not None + }, + } return cast( Page[RunView | UnavailableResource], await self.invoke( "runs.list", - query={ - **self._page(limit, cursor), - **{ - key: value - for key, value in { - "business_key": business_key, - "correlation_key": correlation_key, - "status": status, - }.items() - if value is not None - }, - "include_archived": str(include_archived).lower(), - }, + query=values, ), ) diff --git a/tests/contracts/test_run_views_contract.py b/tests/contracts/test_run_views_contract.py index 431f9ece..4e20ffb1 100644 --- a/tests/contracts/test_run_views_contract.py +++ b/tests/contracts/test_run_views_contract.py @@ -579,7 +579,8 @@ def test_query_digest_coverage_includes_every_filter(): assert plain.cursor_collection() != plain.model_copy(update={"has_handled_errors": False}).cursor_collection() with pytest.raises(ValidationError): parse(RunSummaryQuery, {"order": "id"}) - assert RunListQuery().order == "id" + assert RunListQuery().order == "started_desc" + assert RunListQuery(order="id").order == "id" assert RunListQuery().cursor_collection().startswith("runs:") assert RunListQuery().cursor_collection() != plain.cursor_collection() assert ( diff --git a/tests/contracts/test_run_views_snapshot.py b/tests/contracts/test_run_views_snapshot.py index 9555989f..5cd7ecd5 100644 --- a/tests/contracts/test_run_views_snapshot.py +++ b/tests/contracts/test_run_views_snapshot.py @@ -72,7 +72,7 @@ def test_snapshot_documents_each_operation_exactly(): "method": "GET", "path": environment + "/run-summaries", "capability": "run.read", - "served": False, + "served": True, "query": [ "workflow", "version", @@ -94,16 +94,16 @@ def test_snapshot_documents_each_operation_exactly(): "limit", "cursor", ], - "responses": ["200", "401", "403", "404", "409", "412", "413", "422", "500", "501"], + "responses": ["200", "401", "403", "404", "409", "412", "413", "422", "500"], "response": "RunSummaryPage", }, "runs.steps": { "method": "GET", "path": environment + "/runs/{identifier}/steps", "capability": "run.read", - "served": False, + "served": True, "query": ["step", "include", "limit", "cursor"], - "responses": ["200", "401", "403", "404", "409", "412", "413", "422", "500", "501"], + "responses": ["200", "401", "403", "404", "409", "412", "413", "422", "500"], "response": "StepFactPage", }, "runs.logs": { diff --git a/tests/contracts/test_run_views_surface.py b/tests/contracts/test_run_views_surface.py index aeb4c875..31eeabb8 100644 --- a/tests/contracts/test_run_views_surface.py +++ b/tests/contracts/test_run_views_surface.py @@ -41,12 +41,13 @@ def native(identifier: str) -> dict: @pytest.mark.parametrize("identifier", sorted(EXPECTED)) -def test_operations_are_registered_as_not_served_run_reads(identifier): +def test_operation_availability_matches_served_reads(identifier): operation = OPERATIONS[identifier] path, response = EXPECTED[identifier] assert (operation.method, operation.canonical_path, operation.response) == ("GET", path, response) assert operation.capability == "run.read" - assert operation.served is False and operation.page is False and operation.request is None + assert operation.served is (identifier != "runs.logs") + assert operation.page is False and operation.request is None assert all(other.served for key, other in OPERATIONS.items() if key not in EXPECTED) @@ -56,11 +57,15 @@ def test_openapi_documents_query_501_and_bearer_security(identifier): query = [parameter for parameter in document["parameters"] if parameter["in"] == "query"] assert [parameter["name"] for parameter in query] == list(OPERATIONS[identifier].query.model_fields) assert all(parameter["required"] is False for parameter in query) - assert "501" in document["responses"] and "422" in document["responses"] - assert document["responses"]["501"]["content"]["application/problem+json"]["schema"]["$ref"].endswith("/Problem") + assert ("501" in document["responses"]) is (identifier == "runs.logs") + assert "422" in document["responses"] + if identifier == "runs.logs": + assert document["responses"]["501"]["content"]["application/problem+json"]["schema"]["$ref"].endswith( + "/Problem" + ) assert document["security"] == [{"bearer": []}] assert "Required capability: run.read" in document["description"] - assert "answers 501 WV-UNAVAILABLE" in document["description"] + assert ("answers 501 WV-UNAVAILABLE" in document["description"]) is (identifier == "runs.logs") def test_query_parameters_keep_their_constraints_and_defaults(): diff --git a/tests/fixtures/contracts/run-views.snapshot.json b/tests/fixtures/contracts/run-views.snapshot.json index db185c12..2afa100f 100644 --- a/tests/fixtures/contracts/run-views.snapshot.json +++ b/tests/fixtures/contracts/run-views.snapshot.json @@ -35,10 +35,9 @@ "412", "413", "422", - "500", - "501" + "500" ], - "served": false + "served": true }, "runs.logs": { "capability": "run.read", @@ -86,10 +85,9 @@ "412", "413", "422", - "500", - "501" + "500" ], - "served": false + "served": true } }, "schemas": { @@ -182,7 +180,7 @@ "$id": "run-list-query.schema.json", "$schema": "https://json-schema.org/draft/2020-12/schema", "additionalProperties": false, - "description": "`runs.list` filters, with ID order retained for existing callers.", + "description": "Chronological `runs.list` filters, with explicit ID order for legacy cursors.", "properties": { "activation_id": { "anyOf": [ @@ -283,7 +281,7 @@ }, "order": { "$ref": "#/$defs/RunListOrder", - "default": "id" + "default": "started_desc" }, "origin": { "anyOf": [ diff --git a/tests/integration/test_run_fact_reads.py b/tests/integration/test_run_fact_reads.py new file mode 100644 index 00000000..5811af1c --- /dev/null +++ b/tests/integration/test_run_fact_reads.py @@ -0,0 +1,853 @@ +# Copyright 2026 Firefly Software Foundation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# Author: Firefly Software Foundation +# SPDX-License-Identifier: Apache-2.0 + +"""Real PostgreSQL operational reads, classification, pagination and legacy coverage.""" + +import pytest +from sqlalchemy import text + +from firefly_weave.access.audit import AuditContext +from firefly_weave.access.authorization import AuthorizationService +from firefly_weave.contracts.run_views import RunStepQuery, RunSummaryQuery +from firefly_weave.operations.run_views import RunViewService +from firefly_weave.persistence.uow import UnitOfWork + + +@pytest.mark.integration +async def test_real_summary_and_steps_use_metadata_only(operations_case, monkeypatch): + from firefly_weave.runtime.repository import RuntimeRepository + + case = operations_case + run = await case.start() + service = RunViewService(UnitOfWork(case.sessions), AuthorizationService()) + + async def forbidden(*args, **kwargs): + pytest.fail("Metadata read fetched a full run") + + monkeypatch.setattr(RuntimeRepository, "run", forbidden) + page = await service.summaries(case.actor, case.scope, RunSummaryQuery(), None, context=AuditContext()) + assert [item.id for item in page.items] == [run.id] + assert page.items[0].workflow.name == "worker-flow" + steps = await service.steps(case.actor, case.scope, run.id, RunStepQuery(), None, context=AuditContext()) + assert steps.complete and len(steps.items) == 1 + assert "output" not in steps.items[0].model_dump(mode="json") + + +@pytest.mark.integration +async def test_summary_filters_classification_and_archive_restore(operations_case): + from firefly_weave.operations.fact_reads import FactReadRepository + + case = operations_case + run = await case.start() + async with case.owner.begin() as session: + await session.execute( + text( + "UPDATE run_facts SET business_key='invoice',correlation_key='thread'," + "handled_errors=2,active_incidents=1 WHERE run_id=:run" + ), + {"run": run.id}, + ) + positive = [ + {"workflow": "worker-flow"}, + {"workflow": "worker-flow", "version": "1.0.0"}, + {"status": ["waiting", "running"]}, + {"origin": "manual"}, + {"started_after": "2000-01-01T00:00:00Z"}, + {"started_before": "2100-01-01T00:00:00Z"}, + {"include_test": True}, + {"top_level_only": True}, + {"business_key": "invoice"}, + {"correlation_key": "thread"}, + {"has_active_incident": True}, + {"has_handled_errors": True}, + {"include_archived": True}, + {"activation_id": str(case.activation.id)}, + ] + negative = [ + {"workflow": "other"}, + {"workflow": "worker-flow", "version": "9.0.0"}, + {"status": ["failed"]}, + {"origin": "webhook"}, + {"started_after": "2100-01-01T00:00:00Z"}, + {"started_before": "2000-01-01T00:00:00Z"}, + {"business_key": "other"}, + {"correlation_key": "other"}, + {"has_active_incident": False}, + {"has_handled_errors": False}, + {"caller_run_id": str(run.id)}, + {"retried_from_run_id": str(run.id)}, + {"activation_id": str(run.id)}, + ] + async with case.tx() as tx: + repository = FactReadRepository(tx) + for values in positive: + query = RunSummaryQuery.model_validate_json(__import__("json").dumps(values)) + assert [row["run_id"] for row in await repository.summary_rows(query, None)] == [run.id], values + for values in negative: + query = RunSummaryQuery.model_validate_json(__import__("json").dumps(values)) + assert await repository.summary_rows(query, None) == [], values + + cases = [ + ("test=true", {"include_test": True}, {}), + ( + "test=false,origin='call',caller_run_id=run_id,caller_node_id='work',caller_instance_key=''", + {"caller_run_id": str(run.id)}, + {"top_level_only": True}, + ), + ( + "origin='retry',caller_run_id=NULL,caller_node_id=NULL,caller_instance_key=NULL,retried_from_run_id=run_id", + {"retried_from_run_id": str(run.id)}, + {"retried_from_run_id": str(case.activation.id)}, + ), + ( + "active_incidents=0,handled_errors=0", + {"has_active_incident": False, "has_handled_errors": False}, + {"has_active_incident": True, "has_handled_errors": True}, + ), + ] + for update, matches, excludes in cases: + async with case.owner.begin() as session: + await session.execute(text("UPDATE run_facts SET " + update + " WHERE run_id=:run"), {"run": run.id}) + async with case.tx() as tx: + repository = FactReadRepository(tx) + assert ( + len( + await repository.summary_rows( + RunSummaryQuery.model_validate_json(__import__("json").dumps(matches)), None + ) + ) + == 1 + ) + assert ( + await repository.summary_rows( + RunSummaryQuery.model_validate_json(__import__("json").dumps(excludes)), None + ) + == [] + ) + + +@pytest.mark.integration +async def test_legacy_steps_use_null_times_and_run_wide_coverage(operations_case): + from firefly_weave.operations.fact_positions import StepPosition + from firefly_weave.operations.fact_reads import FactReadRepository + + case = operations_case + run = await case.start() + async with case.owner.begin() as session: + await session.execute( + text( + "INSERT INTO step_instances SELECT tenant_id,project_id,environment_id," + "id,key,'completed','null'::jsonb FROM runs CROSS JOIN " + "unnest(ARRAY['echo[2]','echo[10]','@join:hidden']) AS key WHERE id=:run" + ), + {"run": run.id}, + ) + await session.execute( + text('UPDATE run_facts SET node_kinds=node_kinds || \'{"echo":"action"}\'::jsonb WHERE run_id=:run'), + {"run": run.id}, + ) + async with case.tx() as tx: + repository = FactReadRepository(tx) + rows, complete = await repository.step_rows(run.id, RunStepQuery(step="echo", limit=1), None) + assert not complete and len(rows) == 2 + assert [row["instance_key"] for row in rows] == ["echo[10]", "echo[2]"] + assert all(row["scheduled_at"] is None and row["started_at"] is None for row in rows) + tail, complete = await repository.step_rows(run.id, RunStepQuery(step="unknown"), None) + assert tail == [] and not complete + rows, complete = await repository.step_rows( + run.id, RunStepQuery(step="echo"), StepPosition(None, "echo", "echo[10]") + ) + assert [row["instance_key"] for row in rows] == ["echo[2]"] and not complete + + +@pytest.mark.integration +async def test_explicit_step_output_is_paged_and_classified_once(operations_case, monkeypatch): + from firefly_weave.runtime.repository import RuntimeRepository + + case = operations_case + run = await case.start() + async with case.owner.begin() as session: + await session.execute( + text( + "INSERT INTO step_instances SELECT tenant_id,project_id,environment_id," + "id,'work','completed','null'::jsonb FROM runs WHERE id=:run " + "ON CONFLICT(run_id,node_id) " + "DO UPDATE SET output='null'::jsonb,status='completed'" + ), + {"run": run.id}, + ) + reads = [] + original = RuntimeRepository.run + + async def counted(repository, identifier, **kwargs): + reads.append(identifier) + return await original(repository, identifier, **kwargs) + + monkeypatch.setattr(RuntimeRepository, "run", counted) + service = RunViewService(UnitOfWork(case.sessions), AuthorizationService()) + page = await service.steps( + case.actor, case.scope, run.id, RunStepQuery(include="output", limit=1), None, context=AuditContext() + ) + assert reads == [run.id] + assert "output" in page.items[0].model_dump() and page.items[0].output is None + assert not page.items[0].omissions + + +@pytest.mark.integration +async def test_archive_restore_visibility_for_summary_and_runs_orders(operations_case): + from firefly_weave.access.models import Grant + from firefly_weave.contracts.run_lifecycle import RunLifecycleRequest + from firefly_weave.contracts.run_views import RunListQuery + from firefly_weave.runtime.lifecycle import RunLifecycleService + + case = operations_case + run = await case.start() + await case.access.grant(case.admin, case.actor.id, Grant(role="execution_manager", scope=case.scope)) + actor = await case.access.load_principal(case.actor.id) + async with case.tx() as tx: + await case.runtime.cancel(tx, run.id, "archive fixture", actor=actor, scope=case.scope, context=AuditContext()) + service = RunViewService(UnitOfWork(case.sessions), AuthorizationService()) + lifecycle = RunLifecycleService(case.runtime) + for action, revision in (("archive", 0), ("restore", 1)): + await lifecycle.command( + actor, + case.scope, + run.id, + RunLifecycleRequest(expected_revision=revision, reason="fixture"), + action, + action=action, + context=AuditContext(), + ) + for order in ("started_desc", "started_asc", "updated_desc"): + for include in (False, True): + page = await service.summaries( + actor, + case.scope, + RunSummaryQuery(order=order, include_archived=include), + None, + context=AuditContext(), + ) + assert bool(page.items) is (include or action == "restore") + if page.items: + assert page.items[0].archived is (action == "archive") + for order in ("started_desc", "started_asc", "updated_desc", "id"): + for include in (False, True): + page = await case.runtime.list( + actor, case.scope, query=RunListQuery(order=order, include_archived=include), context=AuditContext() + ) + assert bool(page["items"]) is (include or action == "restore") + assert (await case.rows("SELECT archived FROM run_archives WHERE run_id=:run", run=run.id)) == [{"archived": False}] + + +@pytest.mark.integration +async def test_summaries_withhold_cached_or_current_unsupported_and_policy_blocks(operations_case, monkeypatch): + from firefly_weave.contracts import language_features + from firefly_weave.operations.fact_reads import summary_of + from firefly_weave.runtime import admission + + case = operations_case + run = await case.start() + service = RunViewService(UnitOfWork(case.sessions), AuthorizationService()) + + async def read(): + return (await service.summaries(case.actor, case.scope, RunSummaryQuery(), None, context=AuditContext())).items[ + 0 + ] + + original = await read() + assert not getattr(original, "unavailable", False) + monkeypatch.setattr(admission, "POLICY", "new-classification") + assert (await read()).unavailable + monkeypatch.undo() + features = language_features.ADVERTISED_FEATURES + async with case.owner.begin() as session: + await session.execute( + text("UPDATE run_facts SET pinned_features=cast(:features AS jsonb) WHERE run_id=:run"), + {"features": __import__("json").dumps([features[0]]), "run": run.id}, + ) + monkeypatch.setattr(language_features, "ADVERTISED_FEATURES", ()) + assert (await read()).unavailable + monkeypatch.setattr(language_features, "ADVERTISED_FEATURES", features) + async with case.owner.begin() as session: + await session.execute( + text("UPDATE run_facts SET classification_state='unsupported' WHERE run_id=:run"), {"run": run.id} + ) + assert (await read()).unavailable + assert (await case.rows("SELECT classification_state FROM run_facts WHERE run_id=:run", run=run.id))[0][ + "classification_state" + ] == "unsupported" + async with case.owner.begin() as session: + await session.execute( + text("UPDATE run_facts SET classification_state='available' WHERE run_id=:run"), {"run": run.id} + ) + await session.execute( + text( + "INSERT INTO run_policy_blocks(tenant_id,project_id,environment_id,run_id,reason_code) " + "SELECT tenant_id,project_id,environment_id,id,'WV-LEGACY-UNAVAILABLE' FROM runs WHERE id=:run" + ), + {"run": run.id}, + ) + listed = await case.runtime.list(case.actor, case.scope, context=AuditContext()) + assert listed["items"][0].get("unavailable") is True + assert (await read()).model_dump(mode="json") == { + "id": str(run.id), + "unavailable": True, + "omissions": [{"path": "", "reason": "classification_unavailable"}], + } + row = ( + await case.rows( + "SELECT *,false AS policy_blocked,false AS archived FROM run_facts WHERE run_id=:run", run=run.id + ) + )[0] + for changes in ( + {"started_at": None}, + {"updated_at": "bad"}, + {"status": None}, + {"workflow_name": "bad/name"}, + {"pinned_features": [None]}, + {"classification_policy": "old"}, + {"pinned_ir_version": "unknown"}, + ): + assert summary_of({**row, **changes}).unavailable + + +@pytest.fixture +def worker_runtime_fixture(worker_runtime_fixture, request): + import json + + mode = getattr(request, "param", None) + if mode not in {"parallel", "switch", "secret"}: + return worker_runtime_fixture + document = json.loads(worker_runtime_fixture["source"]) + branch = { + "steps": [{"id": "copy", "kind": "transform", "value": {"literal": 3}}], + "output": {"ref": "/steps/copy/output"}, + } + if mode == "parallel": + step = {"id": "fork", "kind": "parallel", "concurrency": 1, "branches": {"one": branch}} + elif mode == "switch": + step = { + "id": "fork", + "kind": "switch", + "cases": [{"when": {"literal": True}, **branch}], + "default": {"steps": [], "output": {"literal": None}}, + } + else: + step = { + "id": "secret", + "kind": "signal", + "name": "secret", + "timeoutSeconds": 300, + "payloadSchema": {"type": "string", "x-secret": True}, + } + document["spec"]["steps"] = [step, *document["spec"]["steps"]] + document["spec"]["output"] = {"literal": None} + document["spec"]["outputSchema"] = {} + return {**worker_runtime_fixture, "source": json.dumps(document)} + + +@pytest.mark.integration +@pytest.mark.parametrize("worker_runtime_fixture", ["parallel", "switch"], indirect=True) +async def test_current_parallel_and_switch_steps_exclude_synthetic_and_are_complete(operations_case, monkeypatch): + from firefly_weave.operations.fact_reads import FactReadRepository + + case = operations_case + run = await case.start() + assert await case.rows("SELECT node_id FROM step_instances WHERE run_id=:run AND left(node_id,1)='@'", run=run.id) + + async def forbidden(*args, **kwargs): + pytest.fail("Current author facts must not invoke legacy fallback") + + monkeypatch.setattr(FactReadRepository, "historical_steps", forbidden) + service = RunViewService(UnitOfWork(case.sessions), AuthorizationService()) + page = await service.steps(case.actor, case.scope, run.id, RunStepQuery(), None, context=AuditContext()) + assert page.complete and {item.node_id for item in page.items} == {"fork", "copy", "work"} + assert all(not item.instance_key.startswith("@") for item in page.items) + + +@pytest.mark.integration +@pytest.mark.parametrize("worker_runtime_fixture", ["parallel", "switch"], indirect=True) +async def test_legacy_parallel_and_switch_fallback_counts_only_author_instances(operations_case): + case = operations_case + run = await case.start() + async with case.owner.begin() as session: + await session.execute(text("DELETE FROM step_facts WHERE run_id=:run AND node_id='copy'"), {"run": run.id}) + service = RunViewService(UnitOfWork(case.sessions), AuthorizationService()) + page = await service.steps( + case.actor, case.scope, run.id, RunStepQuery(step="fork", include="output"), None, context=AuditContext() + ) + assert not page.complete and [item.node_id for item in page.items] == ["fork"] + assert all(not item.instance_key.startswith("@") for item in page.items) + + +@pytest.mark.integration +@pytest.mark.parametrize("worker_runtime_fixture", ["secret"], indirect=True) +async def test_step_output_redaction_hides_secret_canary(operations_case): + case = operations_case + run = await case.start() + async with case.owner.begin() as session: + await session.execute( + text( + "UPDATE step_instances SET output='\"secret-read-canary\"'::jsonb WHERE run_id=:run " + "AND node_id='secret'" + ), + {"run": run.id}, + ) + service = RunViewService(UnitOfWork(case.sessions), AuthorizationService()) + page = await service.steps( + case.actor, case.scope, run.id, RunStepQuery(include="output"), None, context=AuditContext() + ) + assert "secret-read-canary" not in page.model_dump_json() + assert "output" not in page.items[0].model_dump() + assert page.items[0].omissions[0].path == "/output" and page.items[0].omissions[0].reason == "classified_secret" + + +@pytest.mark.integration +async def test_ten_thousand_facts_walk_exact_sql_order(operations_case): + from firefly_weave.operations.fact_positions import RunPosition + from firefly_weave.operations.fact_reads import FactReadRepository + + case = operations_case + run = await case.start() + async with case.tx() as tx: + await case.runtime.cancel( + tx, run.id, "pagination fixture", actor=case.actor, scope=case.scope, context=AuditContext() + ) + async with case.owner.begin() as session: + await session.execute( + text( + "INSERT INTO runs SELECT (jsonb_populate_record(NULL::runs,to_jsonb(r) || " + "jsonb_build_object('id',md5(i::text)::uuid))).* FROM runs r CROSS JOIN " + "generate_series(1,9999) i WHERE r.id=:run" + ), + {"run": run.id}, + ) + await session.execute( + text( + "INSERT INTO run_facts SELECT (jsonb_populate_record(NULL::run_facts,to_jsonb(f) || " + "jsonb_build_object('run_id',md5(i::text)::uuid,'classification_state'," + "CASE WHEN i%17=0 THEN 'unavailable' WHEN i%19=0 THEN 'unsupported' ELSE 'available' END," + "'started_at',CASE WHEN i%991=0 THEN NULL ELSE timestamptz '2026-01-01Z' " + "+ (i/3)*interval '1 second' END," + "'updated_at',CASE WHEN i%991=0 THEN NULL ELSE timestamptz '2026-01-01Z' " + "+ (i/5)*interval '2 seconds' END," + "'test',i%23=0,'handled_errors',i%4,'active_incidents',i%3," + "'origin',CASE WHEN i%7=0 THEN 'call' WHEN i%11=0 THEN 'retry' ELSE 'manual' END," + "'caller_run_id',CASE WHEN i%7=0 THEN f.run_id ELSE NULL END," + "'caller_node_id',CASE WHEN i%7=0 THEN 'work' ELSE NULL END," + "'caller_instance_key',CASE WHEN i%7=0 THEN '' ELSE NULL END," + "'retried_from_run_id',CASE WHEN i%7<>0 AND i%11=0 THEN f.run_id ELSE NULL END," + "'ended_at',timestamptz '2026-01-02Z'))).* FROM run_facts f CROSS JOIN " + "generate_series(1,9999) i WHERE f.run_id=:run" + ), + {"run": run.id}, + ) + await session.execute( + text( + "INSERT INTO run_archives(tenant_id,project_id,environment_id,run_id,archived) " + "SELECT tenant_id,project_id,environment_id,run_id,true FROM run_facts WHERE run_id<>:run " + "AND get_byte(uuid_send(run_id),0)<8" + ), + {"run": run.id}, + ) + await session.execute( + text( + "INSERT INTO run_policy_blocks(tenant_id,project_id,environment_id,run_id,reason_code) " + "SELECT tenant_id,project_id,environment_id,run_id,'WV-LEGACY-UNAVAILABLE' FROM run_facts " + "WHERE run_id<>:run AND get_byte(uuid_send(run_id),0)=8" + ), + {"run": run.id}, + ) + async with case.tx() as tx: + repository = FactReadRepository(tx) + for order, column, direction in ( + ("started_desc", "started_at", "DESC"), + ("started_asc", "started_at", "ASC"), + ("updated_desc", "updated_at", "DESC"), + ): + expected = [ + row["run_id"] + for row in await case.rows( + f"SELECT run_id FROM run_facts ORDER BY {column} {direction} NULLS LAST,run_id {direction}" + ) + ] + assert len(expected) == 10000 + for limit in (1, 50, 100): + query = RunSummaryQuery(order=order, limit=limit, include_test=True, include_archived=True) + found, position = [], None + while True: + rows = await repository.summary_rows(query, position) + assert len(rows) <= limit + 1 + visible = rows[:limit] + found.extend(row["run_id"] for row in visible) + if len(rows) <= limit: + break + last = visible[-1] + position = RunPosition(last[column], last["run_id"]) + assert found == expected + tail = await repository.summary_rows( + RunSummaryQuery(order="started_asc"), RunPosition(None, __import__("uuid").UUID(int=2**128 - 1)) + ) + assert tail == [] + + +@pytest.mark.integration +async def test_native_routes_sdk_and_scope_bound_cursors(operations_case, authenticated_client): + from firefly_weave.api.transport import encode_cursor, encode_cursor_v2 + from firefly_weave.contracts.run_views import RunListQuery + from firefly_weave.contracts.runtime import RunListFilters + from firefly_weave.sdk.client import WeaveClient + + case = operations_case + first, second = await case.start("first"), await case.start("second") + client, tokens, scopes = authenticated_client + prefix = ( + f"/api/v1/tenants/{case.scope.tenant_id}/projects/{case.scope.project_id}" + f"/environments/{case.scope.environment_id}" + ) + headers = {"Authorization": "Bearer " + tokens[0]} + response = await client.get(prefix + "/run-summaries?limit=1", headers=headers) + assert response.status_code == 200 and response.json()["items"][0]["id"] == str(second.id) + cursor = response.json()["next_cursor"] + assert (await client.get(prefix + "/run-summaries", params={"limit": 1, "cursor": cursor}, headers=headers)).json()[ + "items" + ][0]["id"] == str(first.id) + for path, status in ((f"/runs/{first.id}/steps", 200), (f"/runs/{first.id}/logs", 501)): + assert (await client.get(prefix + path, headers=headers)).status_code == status + bad = encode_cursor_v2(scopes[1], RunSummaryQuery().cursor_collection(), None, str(first.id)) + assert (await client.get(prefix + "/run-summaries", params={"cursor": bad}, headers=headers)).status_code == 422 + assert ( + await client.get(prefix + "/run-summaries", params={"cursor": cursor, "include_test": "true"}, headers=headers) + ).status_code == 422 + async with WeaveClient( + "https://platform.example", lambda: tokens[0], case.scope, transport=client._transport + ) as sdk: + page = await sdk.list_runs(query=RunListQuery(limit=1, status=["running", "waiting"])) + assert page.items[0].id == second.id + page2 = await sdk.list_runs(query=RunListQuery(limit=1, status=["waiting", "running"], cursor=page.next_cursor)) + assert page2.items[0].id == first.id and page2.next_cursor is None + ordered = sorted([first.id, second.id]) + legacy = await case.runtime.list( + case.actor, case.scope, limit=1, filters=RunListFilters(), context=AuditContext() + ) + assert legacy["items"][0]["id"] == str(ordered[0]) and legacy["next_cursor"] == str(ordered[0]) + legacy_next = await case.runtime.list( + case.actor, case.scope, limit=1, filters=RunListFilters(), cursor=ordered[0], context=AuditContext() + ) + assert legacy_next["items"][0]["id"] == str(ordered[1]) + old = encode_cursor(case.scope, RunListFilters().cursor_collection(), ordered[0]) + page = await sdk.list_runs(order="id", cursor=old) + assert [item.id for item in page.items] == [ordered[1]] + + +@pytest.mark.integration +@pytest.mark.parametrize("count", [499, 500, 501]) +async def test_real_legacy_batches_and_c_collation(operations_case, count): + from firefly_weave.operations.fact_positions import StepPosition + from firefly_weave.operations.fact_reads import FactReadRepository + + case = operations_case + run = await case.start() + async with case.owner.begin() as session: + await session.execute( + text( + "INSERT INTO step_instances SELECT tenant_id,project_id,environment_id,id," + "'work['||i||']','completed','null'::jsonb FROM runs CROSS JOIN generate_series(0,:count-1) i " + "WHERE id=:run" + ), + {"run": run.id, "count": count}, + ) + async with case.tx() as tx: + repository = FactReadRepository(tx) + rows, complete = await repository.step_rows(run.id, RunStepQuery(limit=1), None) + assert not complete and rows[0]["instance_key"] == "" and rows[1]["instance_key"] == "work[0]" + rows, complete = await repository.step_rows( + run.id, RunStepQuery(limit=2), StepPosition(None, "work", "work[1]") + ) + assert [row["instance_key"] for row in rows] == ["work[200]", "work[201]", "work[202]"] + + +@pytest.mark.integration +@pytest.mark.parametrize("cancel", [False, True]) +async def test_real_legacy_cursor_closes_on_timeout_and_cancellation(operations_case, monkeypatch, cancel): + import asyncio + + from sqlalchemy.ext.asyncio import AsyncSession + + from firefly_weave.operations.fact_reads import FactReadRepository + + case = operations_case + run = await case.start() + async with case.owner.begin() as session: + await session.execute( + text( + "INSERT INTO step_instances SELECT tenant_id,project_id,environment_id,id," + "'work[1]','completed','null'::jsonb FROM runs WHERE id=:run" + ), + {"run": run.id}, + ) + opened, closed = asyncio.Event(), [] + original = AsyncSession.stream + + async def delayed(session, statement, *args, **kwargs): + stream = await original(session, statement, *args, **kwargs) + + class Wrapped: + def mappings(self): + return self + + async def partitions(self, size): + async for batch in stream.mappings().partitions(size): + opened.set() + await asyncio.Event().wait() + yield batch + + async def close(self): + await stream.close() + closed.append(stream.closed) + + return Wrapped() + + monkeypatch.setattr(AsyncSession, "stream", delayed) + async with case.tx() as tx: + task = asyncio.create_task(FactReadRepository(tx).historical_steps(run.id, None, None, 1)) + await opened.wait() + if cancel: + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + else: + from firefly_weave.definitions.models import CatalogError + + with pytest.raises(CatalogError) as failure: + await task + assert (failure.value.status, failure.value.code) == (429, "WV-RUNTIME-LIMIT") + assert closed == [True] + assert await tx.session.scalar(text("SELECT 1")) == 1 + + +@pytest.mark.integration +@pytest.mark.parametrize("count", [100000, 100001]) +async def test_real_legacy_key_cap_independently_of_deadline(operations_case, monkeypatch, count): + import json + import os + import time + from pathlib import Path + + from firefly_weave.definitions.models import CatalogError + from firefly_weave.operations import fact_reads + + case = operations_case + run = await case.start() + async with case.tx() as tx: + await case.runtime.cancel( + tx, run.id, "legacy cap fixture", actor=case.actor, scope=case.scope, context=AuditContext() + ) + async with case.owner.begin() as session: + await session.execute( + text( + "INSERT INTO step_instances SELECT tenant_id,project_id,environment_id,id," + "'work['||i||']','completed','null'::jsonb FROM runs CROSS JOIN generate_series(0,:count-1) i " + "WHERE id=:run" + ), + {"run": run.id, "count": count}, + ) + await session.execute( + text( + "INSERT INTO step_instances SELECT tenant_id,project_id,environment_id,id," + "'@control:'||i,'completed','null'::jsonb FROM runs CROSS JOIN generate_series(1,501) i WHERE id=:run" + ), + {"run": run.id}, + ) + # Isolate the key cap from the separately tested production two-second deadline. + monkeypatch.setattr(fact_reads, "LEGACY_SECONDS", 60) + started = time.monotonic() + async with case.tx() as tx: + repository = fact_reads.FactReadRepository(tx) + if count == 100000: + rows, complete = await repository.historical_steps(run.id, None, None, 1) + assert not complete and [row["instance_key"] for row in rows] == ["work[0]", "work[10000]"] + monkeypatch.setattr(fact_reads, "LEGACY_SECONDS", 2) + deadline_rows, deadline_complete = await repository.historical_steps(run.id, None, None, 1) + assert deadline_rows == rows and deadline_complete is False + else: + with pytest.raises(CatalogError) as failure: + await repository.historical_steps(run.id, None, None, 1) + assert (failure.value.status, failure.value.code) == (429, "WV-RUNTIME-LIMIT") + assert await tx.session.scalar(text("SELECT 1")) == 1 + if target := os.environ.get("WEAVE_FACT_READ_EVIDENCE"): + with Path(target).open("a") as evidence: + evidence.write( + json.dumps( + { + "keys": count, + "scan_seconds": time.monotonic() - started, + "test_deadline_seconds": 60, + "production_deadline_verified_separately": True, + } + ) + + "\n" + ) + + +@pytest.mark.integration +async def test_current_and_legacy_step_order_matches_c_collation_and_null_tail(operations_case): + from datetime import UTC, datetime + + from firefly_weave.operations.fact_positions import StepPosition + from firefly_weave.operations.fact_reads import FactReadRepository + + case = operations_case + run = await case.start() + keys = ["send[2]", "send[10]", "send[2]#agent.tool~1", "A", "a"] + at = datetime(2026, 1, 1, tzinfo=UTC) + async with case.owner.begin() as session: + await session.execute( + text( + "UPDATE run_facts SET node_kinds=node_kinds || " + '\'{"send":"action","A":"transform","a":"transform"}\'::jsonb WHERE run_id=:run' + ), + {"run": run.id}, + ) + for key in keys: + await session.execute( + text( + "INSERT INTO step_instances SELECT tenant_id,project_id,environment_id,id," + ":key,'completed','null'::jsonb FROM runs WHERE id=:run" + ), + {"run": run.id, "key": key}, + ) + for key in ["A", "a"]: + await session.execute( + text( + "INSERT INTO step_facts(tenant_id,project_id,environment_id,run_id,node_id,kind,status," + "scheduled_at) " + "SELECT tenant_id,project_id,environment_id,id,:key,'transform','succeeded',:at " + "FROM runs WHERE id=:run" + ), + {"run": run.id, "key": key, "at": at}, + ) + async with case.tx() as tx: + repo = FactReadRepository(tx) + found, position = [], None + while True: + rows, complete = await repo.step_rows(run.id, RunStepQuery(limit=1), position) + assert not complete + if not rows: + break + row = rows[0] + found.append(row["instance_key"] or row["node_id"]) + position = StepPosition(row["scheduled_at"], row["node_id"], row["instance_key"]) + assert found == ["A", "a", "work", "send[10]", "send[2]", "send[2]#agent.tool~1"] + async with case.owner.begin() as session: + await session.execute( + text( + "INSERT INTO step_instances SELECT tenant_id,project_id,environment_id,id," + "'broken[','completed','null'::jsonb FROM runs WHERE id=:run" + ), + {"run": run.id}, + ) + from firefly_weave.definitions.models import CatalogError + + async with case.tx() as tx: + with pytest.raises(CatalogError) as failure: + await FactReadRepository(tx).step_rows(run.id, RunStepQuery(step="absent"), None) + assert failure.value.code == "WV-LEGACY-UNAVAILABLE" + + +@pytest.mark.integration +async def test_newer_runs_do_not_displace_started_desc_continuation(operations_case): + from firefly_weave.api.transport import decode_cursor_v2 + + case = operations_case + first, second = await case.start("first"), await case.start("second") + service = RunViewService(UnitOfWork(case.sessions), AuthorizationService()) + query = RunSummaryQuery(limit=1) + page = await service.summaries(case.actor, case.scope, query, None, context=AuditContext()) + assert page.items[0].id == second.id + third = await case.start("third") + next_page = await service.summaries( + case.actor, + case.scope, + query, + decode_cursor_v2(page.next_cursor, case.scope, query.cursor_collection()), + context=AuditContext(), + ) + assert next_page.items[0].id == first.id and next_page.next_cursor is None + refreshed = await service.summaries(case.actor, case.scope, query, None, context=AuditContext()) + assert refreshed.items[0].id == third.id + + +@pytest.mark.integration +async def test_metadata_statements_never_select_run_payloads(operations_case): + from sqlalchemy import event + + case = operations_case + run = await case.start() + statements = [] + engine = case.sessions.kw["bind"].sync_engine + + def capture(connection, cursor, statement, parameters, context, executemany): + statements.append(" ".join(statement.lower().split())) + + event.listen(engine, "before_cursor_execute", capture) + try: + service = RunViewService(UnitOfWork(case.sessions), AuthorizationService()) + await service.summaries(case.actor, case.scope, RunSummaryQuery(), None, context=AuditContext()) + await service.steps(case.actor, case.scope, run.id, RunStepQuery(), None, context=AuditContext()) + finally: + event.remove(engine, "before_cursor_execute", capture) + assert any("from run_facts" in sql for sql in statements) + assert not any("from runs " in sql or "select runs.*" in sql or "s.output" in sql for sql in statements) + assert not any("insert " in sql or "update run_facts" in sql for sql in statements) + + +@pytest.mark.integration +async def test_malformed_current_author_key_is_unavailable(operations_case): + from firefly_weave.definitions.models import CatalogError + + case = operations_case + run = await case.start() + async with case.owner.begin() as session: + await session.execute(text("UPDATE step_facts SET instance_key='work[' WHERE run_id=:run"), {"run": run.id}) + service = RunViewService(UnitOfWork(case.sessions), AuthorizationService()) + with pytest.raises(CatalogError) as failure: + await service.steps(case.actor, case.scope, run.id, RunStepQuery(), None, context=AuditContext()) + assert (failure.value.status, failure.value.code) == (409, "WV-LEGACY-UNAVAILABLE") + + +@pytest.mark.integration +async def test_unavailable_timeline_does_not_fetch_requested_output(operations_case, monkeypatch): + from firefly_weave.definitions.models import CatalogError + from firefly_weave.runtime.repository import RuntimeRepository + + case = operations_case + run = await case.start() + async with case.owner.begin() as session: + await session.execute( + text("UPDATE run_facts SET classification_state='unsupported' WHERE run_id=:run"), {"run": run.id} + ) + + async def forbidden(*args, **kwargs): + pytest.fail("Unavailable metadata fetched source payloads") + + monkeypatch.setattr(RuntimeRepository, "run", forbidden) + service = RunViewService(UnitOfWork(case.sessions), AuthorizationService()) + with pytest.raises(CatalogError) as failure: + await service.steps( + case.actor, case.scope, run.id, RunStepQuery(include="output"), None, context=AuditContext() + ) + assert failure.value.code == "WV-LEGACY-UNAVAILABLE" diff --git a/tests/unit/operations/test_run_views.py b/tests/unit/operations/test_run_views.py index f0008511..c7f319d1 100644 --- a/tests/unit/operations/test_run_views.py +++ b/tests/unit/operations/test_run_views.py @@ -14,7 +14,7 @@ # Author: Firefly Software Foundation # SPDX-License-Identifier: Apache-2.0 -"""Run summaries, steps and logs authorize run.read twice, validate, then answer 501 until served.""" +"""Scoped operational reads preserve current authority and classified metadata.""" import json from contextlib import asynccontextmanager @@ -42,12 +42,19 @@ class ReadUnit: def __init__(self) -> None: self.opened = 0 + self.inside = False + self.tx = SimpleNamespace(session=object(), scope=SCOPE) + self.reads = [] @asynccontextmanager async def open(self, scope, *, mutation=True): assert scope == SCOPE and mutation is False self.opened += 1 - yield SimpleNamespace(session=object()) + self.inside = True + try: + yield self.tx + finally: + self.inside = False def principal(role: str | None = "viewer", *, active: bool = True, scope: Scope = SCOPE) -> Principal: @@ -59,6 +66,7 @@ def service_with(monkeypatch, current: Principal | None) -> tuple[RunViewService unit = ReadUnit() async def load(session, identifier): + assert unit.inside and session is unit.tx.session assert current is not None return current.model_copy(update={"id": identifier}) @@ -100,7 +108,7 @@ async def call(service: RunViewService, actor: Principal, operation: str): OPERATIONS = ("run_summaries.list", "runs.steps", "runs.logs") -@pytest.mark.parametrize("operation", OPERATIONS) +@pytest.mark.parametrize("operation", ["runs.logs"]) @pytest.mark.parametrize("role", ["viewer", "execution_manager"]) async def test_run_readers_reach_the_not_served_answer(monkeypatch, operation, role): actor = principal(role) @@ -141,7 +149,7 @@ async def test_environment_scope_is_required(monkeypatch): "method,path,query", [ ("summaries", "/run-summaries", "workflow=invoice-approval&include_test=true&order=started_desc&limit=50"), - ("steps", f"/runs/{RUN}/steps", "include=output&limit=500"), + ("steps", f"/runs/{RUN}/steps", "limit=500"), ("logs", f"/runs/{RUN}/logs", "level=warning&node_id=send"), ], ) @@ -149,9 +157,13 @@ async def test_controller_validates_then_delegates(monkeypatch, method, path, qu actor = principal("viewer") service, _ = service_with(monkeypatch, actor) controller = RunViewController(service) - with pytest.raises(CatalogError) as failure: - await getattr(controller, method)(request(path, query, actor, identifier=method != "summaries")) - assert failure.value.status == 501 + if method == "logs": + with pytest.raises(CatalogError) as failure: + await controller.logs(request(path, query, actor)) + assert failure.value.status == 501 + else: + response = await getattr(controller, method)(request(path, query, actor, identifier=method != "summaries")) + assert response.status_code == 200 @pytest.mark.parametrize( @@ -194,9 +206,8 @@ async def test_controller_rejects_cursors_from_another_filter_or_run(monkeypatch steps = encode_cursor_v2(SCOPE, RunStepQuery().cursor_collection(uuid4()), [None, "send"], "send[3]") with pytest.raises(ValueError, match="cursor"): await controller.steps(request("/", f"cursor={steps}", actor)) - accepted = encode_cursor_v2(SCOPE, RunStepQuery().cursor_collection(RUN), [None, "send"], "send[3]") - with pytest.raises(CatalogError): - await controller.steps(request("/", f"cursor={accepted}", actor)) + accepted = encode_cursor_v2(SCOPE, RunStepQuery().cursor_collection(RUN), [None, "send", "send[3]"], "send[3]") + assert (await controller.steps(request("/", f"cursor={accepted}", actor))).status_code == 200 assert unit.opened == 1 @@ -210,3 +221,227 @@ async def test_not_served_problem_reaches_clients_as_501(): "code": "WV-UNAVAILABLE", "message": "This server does not serve runs.logs yet", } + + +@pytest.fixture(autouse=True) +def empty_repository(monkeypatch): + from firefly_weave.operations.fact_reads import FactReadRepository + + async def summaries(repository, query, position): + return [] + + async def steps(repository, run_id, query, position): + return [], True + + monkeypatch.setattr(FactReadRepository, "summary_rows", summaries) + monkeypatch.setattr(FactReadRepository, "step_rows", steps) + + +async def test_fresh_authority_and_fact_read_share_transaction(monkeypatch): + from firefly_weave.operations.fact_reads import FactReadRepository + + actor = principal() + service, unit = service_with(monkeypatch, actor) + events = [] + + async def load(session, identifier): + assert unit.inside + events.append(("principal", session)) + return actor + + async def rows(repository, query, position): + assert unit.inside + events.append(("facts", repository.tx.session)) + return [] + + monkeypatch.setattr("firefly_weave.operations.run_views.load_principal", load) + monkeypatch.setattr(FactReadRepository, "summary_rows", rows) + result = await service.summaries(actor, SCOPE, RunSummaryQuery(), None, context=CONTEXT) + assert result.items == [] and unit.opened == 1 + assert [event[0] for event in events] == ["principal", "facts"] + assert events[0][1] is events[1][1] + + +async def test_invalid_typed_cursor_precedes_run_lookup(monkeypatch): + service, unit = service_with(monkeypatch, principal()) + with pytest.raises(ValueError): + await service.steps(principal(), SCOPE, RUN, RunStepQuery(), ([None, "send"], "send"), context=CONTEXT) + assert unit.opened == 0 + + +async def test_sdk_list_retains_old_keywords_and_encodes_repeated_status(): + import httpx + + from firefly_weave.contracts.run_views import RunListQuery + from firefly_weave.sdk.client import WeaveClient + + calls = [] + + def receive(request): + calls.append(request) + return httpx.Response(200, json={"items": [], "next_cursor": None}) + + async with WeaveClient( + "https://platform.example", lambda: "token", SCOPE, transport=httpx.MockTransport(receive) + ) as client: + await client.list_runs( + limit=12, + business_key="invoice", + correlation_key="thread", + status="waiting", + include_archived=True, + order="id", + ) + await client.list_runs(query=RunListQuery(status=["waiting", "failed"], include_test=True, limit=5)) + assert dict(calls[0].url.params) == { + "limit": "12", + "business_key": "invoice", + "correlation_key": "thread", + "status": "waiting", + "include_archived": "true", + "order": "id", + } + assert calls[1].url.params.get_list("status") == ["failed", "waiting"] + assert calls[1].url.params["include_test"] == "true" + assert calls[1].url.params["order"] == "started_desc" + + +async def test_runs_route_uses_strict_filters_and_preserves_explicit_id_cursor(): + from firefly_weave.api.runs import RunController + from firefly_weave.api.transport import encode_cursor, encode_cursor_v2 + from firefly_weave.contracts.run_views import RunListQuery + from firefly_weave.contracts.runtime import RunListFilters + from firefly_weave.operations.fact_positions import RunPosition + + calls = [] + + class Service: + async def list(self, actor, scope, **kwargs): + calls.append(kwargs) + return {"items": [], "next_cursor": None} + + controller = RunController(Service(), None) + cursor = encode_cursor(SCOPE, RunListFilters(status="waiting").cursor_collection(), RUN) + response = await controller.discover( + request( + "/api/v1/tenants/t/projects/p/environments/e/runs", f"order=id&status=waiting&cursor={cursor}", principal() + ) + ) + assert response.status_code == 200 and calls[-1]["position"] == RUN + query = RunListQuery(status=["failed", "waiting"]) + cursor = encode_cursor_v2(SCOPE, query.cursor_collection(), None, str(RUN)) + await controller.discover( + request( + "/api/v1/tenants/t/projects/p/environments/e/runs", + f"status=waiting&status=failed&cursor={cursor}", + principal(), + ) + ) + assert calls[-1]["position"] == RunPosition(None, RUN) + for value in ( + "top_level_only=true&origin=call", + "include_archived=yes", + "colour=red", + "limit=101", + "workflow=one&workflow=two", + "order=started_desc&cursor=" + encode_cursor(SCOPE, "runs", RUN), + ): + with pytest.raises((ValueError, CatalogError)): + await controller.discover(request("/api/v1/tenants/t/projects/p/environments/e/runs", value, principal())) + assert len(calls) == 2 + + +@pytest.mark.parametrize("count", [499, 500, 501, 100000, 100001]) +async def test_historical_stream_is_bounded_and_always_closed(monkeypatch, count): + from firefly_weave.operations import fact_reads + + monkeypatch.setattr(fact_reads, "LEGACY_SECONDS", 60) + closed = [] + maximum_heap = [] + + class Stream: + def mappings(self): + return self + + async def partitions(self, size): + assert size == 500 + for start in range(0, count, size): + yield [ + {"node_id": f"send[{index}]", "status": "completed"} + for index in range(start, min(count, start + size)) + ] + + async def close(self): + closed.append(True) + + class Session: + async def stream(self, statement, params, **kwargs): + assert kwargs["execution_options"] == {"yield_per": 500} + return Stream() + + async def metadata(repository, run_id): + return {"node_kinds": {"send": "action"}} + + original = fact_reads.heapq.heappush + + def pushed(heap, item): + original(heap, item) + maximum_heap.append(len(heap)) + + monkeypatch.setattr(fact_reads.FactReadRepository, "metadata", metadata) + monkeypatch.setattr(fact_reads.heapq, "heappush", pushed) + repository = fact_reads.FactReadRepository(SimpleNamespace(session=Session(), scope=SCOPE)) + if count > 100000: + with pytest.raises(CatalogError) as failure: + await repository.historical_steps(RUN, None, None, 1) + assert (failure.value.status, failure.value.code) == (429, "WV-RUNTIME-LIMIT") + else: + rows, complete = await repository.historical_steps(RUN, None, None, 1) + assert not complete and [row["instance_key"] for row in rows] == [ + "send[0]", + "send[10000]" if count == 100000 else "send[100]", + ] + assert closed == [True] and max(maximum_heap) == 2 + + +@pytest.mark.parametrize("cancel", [False, True]) +async def test_historical_stream_timeout_and_cancellation_close_cursor(monkeypatch, cancel): + import asyncio + + from firefly_weave.operations import fact_reads + + started = asyncio.Event() + closed = [] + + class Stream: + def mappings(self): + return self + + async def partitions(self, size): + started.set() + await asyncio.Event().wait() + yield [] + + async def close(self): + closed.append(True) + + class Session: + async def stream(self, *args, **kwargs): + return Stream() + + async def metadata(*args): + return {"node_kinds": {}} + + monkeypatch.setattr(fact_reads.FactReadRepository, "metadata", metadata) + repository = fact_reads.FactReadRepository(SimpleNamespace(session=Session(), scope=SCOPE)) + task = asyncio.create_task(repository.historical_steps(RUN, None, None, 1)) + await started.wait() + if cancel: + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + else: + with pytest.raises(CatalogError) as failure: + await task + assert (failure.value.status, failure.value.code) == (429, "WV-RUNTIME-LIMIT") + assert closed == [True] diff --git a/tests/unit/operations/test_transport.py b/tests/unit/operations/test_transport.py index c3982b32..26001223 100644 --- a/tests/unit/operations/test_transport.py +++ b/tests/unit/operations/test_transport.py @@ -280,3 +280,44 @@ async def send(message): compatibility.ready = True status, headers, code = await call("POST") assert (status, code, headers[b"retry-after"]) == (503, "WV-COMPATIBILITY", b"9") + + +@pytest.mark.parametrize( + "status,code,expected", + [(501, "WV-UNAVAILABLE", "unavailable"), (501, "WV-OTHER", "failed"), (503, "WV-UNAVAILABLE", "failed")], +) +async def test_unserved_operations_are_telemetry_unavailable(status, code, expected): + from contextlib import nullcontext + from types import SimpleNamespace + + from firefly_weave.operations.telemetry import attributes + from firefly_weave.operations.transport import BodyBoundary + + records = [] + telemetry = SimpleNamespace( + span=lambda *args, **kwargs: nullcontext(), record=lambda *args, **kwargs: records.append(kwargs) + ) + + async def downstream(scope, receive, send): + await send({"type": "http.response.start", "status": status, "headers": []}) + await send({"type": "http.response.body", "body": json.dumps({"code": code}).encode()}) + + async def receive(): + return {"type": "http.request", "body": b""} + + async def send(message): + pass + + await BodyBoundary(downstream)( + { + "type": "http", + "method": "GET", + "path": "/runs/id/logs", + "headers": [], + "app": SimpleNamespace(state=SimpleNamespace(telemetry_service=telemetry)), + }, + receive, + send, + ) + assert records[0]["status"] == expected + assert attributes(status=expected)["status"] == expected diff --git a/tests/unit/runtime/test_run_filters.py b/tests/unit/runtime/test_run_filters.py index 650828aa..56fc521d 100644 --- a/tests/unit/runtime/test_run_filters.py +++ b/tests/unit/runtime/test_run_filters.py @@ -47,3 +47,16 @@ def test_filter_cursor_rejects_changed_business_context_and_archive_selection(): def test_filters_reject_invalid_status_and_unbounded_keys(value): with pytest.raises(ValidationError): RunListFilters.model_validate(value) + + +def test_explicit_id_queries_retain_exact_legacy_cursor_collections(): + from firefly_weave.contracts.run_views import RunListQuery + + old = RunListFilters(business_key="invoice", correlation_key="thread", status="waiting", include_archived=True) + query = RunListQuery( + order="id", business_key="invoice", correlation_key="thread", status=["waiting"], include_archived=True + ) + assert query.cursor_collection() == old.cursor_collection() + assert query.model_copy(update={"include_test": True}).cursor_collection() != old.cursor_collection() + assert query.model_copy(update={"status": ["waiting", "failed"]}).cursor_collection() != old.cursor_collection() + assert query.model_copy(update={"order": "started_desc"}).cursor_collection() != old.cursor_collection() From e0d0bc29ea4503c335188bc1a9692837696a085c Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Sun, 11 Oct 2026 12:45:10 -0700 Subject: [PATCH 07/63] Bound run list pages and keep chronological pages on their indexes - runs.list measures each visible page against the 8 MiB logical byte ceiling before it reads full runs, and withholds only policy-blocked items ahead of the authoritative read. - Chronological run pages read known times and the unknown-time tail as separate keyset scans, so the scoped time indexes serve every order without sorting the environment. - Step cursors carry the scheduled time and the full instance key only, so the longest valid key still fits. - Step outputs share the 8 MiB page budget, and outputs of agent, tool or yield instance keys stay withheld. - Steps of a run waiting for a platform that runs it answer WV-IR-UNSUPPORTED, as runs.read does. - The legacy step stream is unsorted, so its key cap also bounds the database work. - Token and current authority are each checked and audited once. - The keyword list adapter always pages by ID, and the SDK rejects a query combined with keyword filters. - Run pages select summary columns only, and a timeline reads its run metadata once. --- CHANGELOG.md | 11 + docs/operations/upgrades.md | 22 ++ docs/reference/api.md | 23 +- .../operations/fact_positions.py | 23 +- src/firefly_weave/operations/fact_reads.py | 163 +++++++--- src/firefly_weave/operations/run_views.py | 9 +- src/firefly_weave/runtime/service.py | 23 +- src/firefly_weave/sdk/client.py | 4 + tests/integration/test_run_fact_reads.py | 288 +++++++++++++++++- tests/unit/operations/test_fact_positions.py | 43 ++- tests/unit/operations/test_run_views.py | 129 +++++++- tests/unit/runtime/test_run_filters.py | 52 +++- 12 files changed, 675 insertions(+), 115 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 799ff9ce..9aa637c8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,17 @@ SPDX-License-Identifier: Apache-2.0 supported with explicit `order=id`; chronological orders use v2 cursors. Update-time pagination is a live view and may require a refresh after changes. Restored executions reappear in every order. +- `runs.list` accepts the run summary filters and leaves out test runs unless + `include_test=true`. It rejects unknown query parameters, and an out-of-range + `limit` answers `WV-VALIDATION` instead of `WV-PAGE`. Pages stay limited to + 8 MiB of retained run data. +- Step timeline outputs are limited to 8 MiB per page (`429 WV-PAGE-LIMIT`). + Outputs of agent, tool or yield instance keys stay withheld until they have + their own schemas. A run waiting for a platform that runs it answers + `422 WV-IR-UNSUPPORTED` as `runs.read` does. +- The Python SDK's `list_runs` rejects a `query` combined with keyword filters. +- Request telemetry records authorized `501 WV-UNAVAILABLE` answers with the + status `unavailable` instead of `failed`. - Keep typing in the selected form field when deferred row autofocus completes. Rapid row additions focus the newest item without overriding a later choice. diff --git a/docs/operations/upgrades.md b/docs/operations/upgrades.md index 3ca5b030..7110fee1 100644 --- a/docs/operations/upgrades.md +++ b/docs/operations/upgrades.md @@ -155,6 +155,28 @@ default to `started_desc`, with `started_asc` and `updated_desc` also available. Clients that continue a legacy v1 cursor must send `order=id` and retain its original filters. New filters bind ID cursors to the normalized query; time orders use v2 cursors. Restart pagination after changing filters or order. +The `weave runs list` command does not send `order=id` yet, so it cannot +continue a cursor saved before the upgrade; start such a listing again. + +`runs.list` also changes how it reads requests: + +- It accepts the run summary filters and leaves out test runs unless the + request sends `include_test=true`. +- It rejects unknown query parameters with `422 WV-VALIDATION`; the earlier + route ignored them. +- A `limit` outside 1 to 100 answers `422 WV-VALIDATION` instead of + `422 WV-PAGE`. +- An older server reads only `limit`, `cursor`, `business_key`, + `correlation_key`, a single `status` and `include_archived`. It ignores the + other filters, and it keeps only the last of several `status` values. Upgrade + servers before relying on the new filters. + +The Python SDK's `list_runs` takes either a `query` or the keyword filters; +passing both raises `ValueError` instead of silently dropping the keywords. + +Request telemetry records an authorized `501 WV-UNAVAILABLE` answer, such as +`runs.logs`, with the status `unavailable` instead of `failed`. Alerts that +count `weave.requests` with `status=failed` no longer include these answers. The update-time order is a live view: a later update can move an item between pages. Refresh from the first page to see the latest ordering. Unknown historical diff --git a/docs/reference/api.md b/docs/reference/api.md index ada3db09..1e72996c 100644 --- a/docs/reference/api.md +++ b/docs/reference/api.md @@ -385,7 +385,9 @@ capabilities, bindings, Actions, and connection settings, plus **Lists.** Canonical lists return `{ "items": [...], "next_cursor": null | "opaque" }`. -- `limit` is 1 to 100 (50 by default), and items come in stable ID order. +- `limit` is 1 to 100 (50 by default). Items come in stable ID order unless a + list documents another order: `runs.list` and run summaries are newest + started first by default (see below). - Pass `next_cursor` back as `cursor` for the next page. A cursor is bound to its tenant, project, environment, collection, and any run or schedule filter; a cursor from another scope or filter is rejected. @@ -430,7 +432,11 @@ with `WV-UNAVAILABLE`. - Each cursor is bound to every filter and to the order. After you change a filter, start again from the first page. - Test runs are left out unless you send `include_test=true`, and archived runs - unless you send `include_archived=true`. + unless you send `include_archived=true`. This applies to `runs.list` too. +- `runs.list` and run summaries reject unknown query parameters, and a `limit` + outside 1 to 100 answers `422` with `WV-VALIDATION`. +- A `runs.list` page is limited to 8 MiB of retained run data. A larger page + answers `429` with `WV-PAGE-LIMIT`; request a smaller `limit`. - A contradictory filter, such as `version` without `workflow`, answers `422` with `WV-FILTER`; the message names the rule that failed. - Send times as RFC 3339 with an offset. Use `Z`, or encode `+` as `%2B` in the @@ -438,13 +444,20 @@ with `WV-UNAVAILABLE`. Step timelines do not read payloads unless `include=output` is requested. Only the displayed outputs are fetched, after checking the bounded run artifact and -classifying each output. An omitted output carries an omission reason; a -returned JSON `null` is distinct from an omitted output. Synthetic control +classifying each output. The outputs on one page are limited to 8 MiB in total; +a larger page answers `429 WV-PAGE-LIMIT`, so request a smaller `limit`. An +omitted output carries an omission reason; a returned JSON `null` is distinct +from an omitted output. The output of an instance key with an activation +segment (`#`) or a yield count (`~`) is omitted as `classification_unavailable`, +because its step's schema does not describe it. Synthetic control instances do not appear. `complete=false` means the run has author step instances without facts, even when they fall outside the current page or filter. Their unknown timestamps remain null. Legacy metadata is streamed in batches of 500, up to 100,000 keys and two seconds per request; exceeding either budget -answers `429 WV-RUNTIME-LIMIT`. +answers `429 WV-RUNTIME-LIMIT`. A run this platform cannot run yet answers +`422 WV-IR-UNSUPPORTED`, as `runs.read` does; a run with unavailable legacy +evidence answers `409 WV-LEGACY-UNAVAILABLE`. In `runs.list`, only a recorded +policy block withholds a run before its own read classifies it. **Answers that say "unavailable".** Cancellation and task completion can return an unavailable acknowledgment when the platform cannot classify the historical diff --git a/src/firefly_weave/operations/fact_positions.py b/src/firefly_weave/operations/fact_positions.py index babd211c..c60abb73 100644 --- a/src/firefly_weave/operations/fact_positions.py +++ b/src/firefly_weave/operations/fact_positions.py @@ -23,14 +23,14 @@ from pydantic import TypeAdapter from firefly_weave.contracts.definitions import ResourceName -from firefly_weave.contracts.instance_keys import InstanceKeyTextOrEmpty, instance_view +from firefly_weave.contracts.instance_keys import InstanceKeyText, instance_view from firefly_weave.contracts.run_views import utc_time from firefly_weave.contracts.values import JsonValue type Position = tuple[JsonValue, str] | None _NODE: TypeAdapter[str] = TypeAdapter(ResourceName) -_INSTANCE: TypeAdapter[str] = TypeAdapter(InstanceKeyTextOrEmpty) +_INSTANCE: TypeAdapter[str] = TypeAdapter(InstanceKeyText) @dataclass(frozen=True) @@ -61,16 +61,13 @@ def run_position(position: Position) -> RunPosition | None: def step_position(position: Position) -> StepPosition | None: - """Validate a decoded step cursor and its repeated instance identity.""" + """Validate a decoded step cursor: the scheduled time and the full author instance key it ends at. + + The key alone determines the step ID and the stored instance key ('' for the step itself). + """ if position is None: return None - value, identifier = position - if type(value) is not list or len(value) != 3: - raise ValueError("Invalid step position") - at, node, key = value - node = _NODE.validate_python(node, strict=True) - key = _INSTANCE.validate_python(key, strict=True) - identity = instance_view(key or node) - if identity.node_id != node or identity.instance_key != key or identifier != (key or node): - raise ValueError("Invalid step position identity") - return StepPosition(utc_time(at), node, key) + at, identifier = position + identity = instance_view(_INSTANCE.validate_python(identifier, strict=True)) + _NODE.validate_python(identity.node_id, strict=True) + return StepPosition(utc_time(at), identity.node_id, identity.instance_key) diff --git a/src/firefly_weave/operations/fact_reads.py b/src/firefly_weave/operations/fact_reads.py index b3dcfe6c..2dd2462e 100644 --- a/src/firefly_weave/operations/fact_reads.py +++ b/src/firefly_weave/operations/fact_reads.py @@ -41,20 +41,32 @@ StepFactWithOutput, StepKind, ) -from firefly_weave.definitions.models import CatalogError +from firefly_weave.definitions.models import CatalogError, ir_unsupported from firefly_weave.operations.fact_positions import RunPosition, StepPosition from firefly_weave.operations.facts import FactRepository, author_instance from firefly_weave.operations.redaction import Omission, SafeProjection +from firefly_weave.persistence.paging import PAGE_BYTES +from firefly_weave.persistence.uow import Transaction from firefly_weave.runtime import admission _RUN_SCOPE = "f.tenant_id=:tenant AND f.project_id=:project AND f.environment_id=:environment" -_RUN_SELECT = ( - "SELECT f.*,coalesce(a.archived,false) AS archived,EXISTS(SELECT 1 FROM run_policy_blocks b " +# Summary columns only: node kinds (up to 1 MiB) are read by the one timeline metadata query that needs them. +_RUN_COLUMNS = ( + "f.run_id,f.definition_version_id,f.workflow_name,f.workflow_version,f.activation_id,f.activation_name," + "f.activation_revision,f.status,f.paused,f.test,f.origin,f.caller_run_id,f.caller_node_id," + "f.caller_instance_key,f.retried_from_run_id,f.business_key,f.correlation_key,f.started_at,f.updated_at," + "f.ended_at,f.failed_node_id,f.failed_instance_key,f.failed_error_code,f.active_incidents,f.handled_errors," + "f.classification_state,f.classification_policy,f.pinned_ir_version,f.pinned_features" +) +_RUN_FROM = ( + ",coalesce(a.archived,false) AS archived,EXISTS(SELECT 1 FROM run_policy_blocks b " "WHERE b.tenant_id=f.tenant_id AND b.project_id=f.project_id AND b.environment_id=f.environment_id " "AND b.run_id=f.run_id) AS policy_blocked FROM run_facts f LEFT JOIN run_archives a ON " "a.tenant_id=f.tenant_id AND a.project_id=f.project_id AND a.environment_id=f.environment_id " "AND a.run_id=f.run_id WHERE " + _RUN_SCOPE ) +_RUN_SELECT = "SELECT " + _RUN_COLUMNS + _RUN_FROM +_METADATA_SELECT = "SELECT " + _RUN_COLUMNS + ",f.node_kinds" + _RUN_FROM _MISSING = ( "s.tenant_id=:tenant AND s.project_id=:project AND s.environment_id=:environment AND s.run_id=:run " "AND left(s.node_id,1)<>'@' AND NOT EXISTS(SELECT 1 FROM step_facts f " @@ -71,21 +83,54 @@ def unavailable(identifier: UUID) -> UnavailableResource: return UnavailableResource(id=identifier, omissions=[Omission(path="", reason="classification_unavailable")]) -def available(row: Mapping[str, Any]) -> bool: +def _pinned_features(row: Mapping[str, Any]) -> list[str] | None: features = row.get("pinned_features") + if ( + isinstance(features, list) + and len(features) <= 256 + and all(type(value) is str for value in features) + and len(json.dumps(features, ensure_ascii=False).encode()) <= 16384 + ): + return features + return None + + +def _supported(row: Mapping[str, Any], features: list[str]) -> bool: + advertised = language_features.ADVERTISED_FEATURES + return set(features) <= set(advertised) and row.get("pinned_ir_version") in accepted_ir_versions(advertised) + + +def available(row: Mapping[str, Any]) -> bool: + features = _pinned_features(row) return ( row.get("classification_state") == "available" and row.get("classification_policy") == admission.POLICY and row.get("policy_blocked") is False - and isinstance(features, list) - and len(features) <= 256 - and all(type(value) is str for value in features) - and len(json.dumps(features, ensure_ascii=False).encode()) <= 16384 - and set(features) <= set(language_features.ADVERTISED_FEATURES) - and row.get("pinned_ir_version") in accepted_ir_versions(language_features.ADVERTISED_FEATURES) + and features is not None + and _supported(row, features) ) +def withheld(row: Mapping[str, Any]) -> CatalogError: + """The answer for a run whose metadata is withheld, matching what ``runs.read`` answers for it. + + A run this platform does not run yet (cached as unsupported, or pinning an IR version or feature no longer + advertised) answers ``ir_unsupported``; it is transient. Legacy or blocked evidence stays unavailable. + """ + features = _pinned_features(row) + if ( + features is not None + and row.get("policy_blocked") is False + and row.get("classification_policy") == admission.POLICY + and ( + row.get("classification_state") == "unsupported" + or (row.get("classification_state") == "available" and not _supported(row, features)) + ) + ): + return ir_unsupported(tuple(sorted(set(features) - set(language_features.ADVERTISED_FEATURES)))) + return legacy_unavailable() + + def milliseconds(start: datetime | None, end: datetime | None) -> int | None: return int((end - start).total_seconds() * 1000) if start is not None and end is not None else None @@ -183,6 +228,11 @@ def legacy_unavailable() -> CatalogError: return CatalogError(409, "WV-LEGACY-UNAVAILABLE", "Step classification is unavailable") +def _segmented(key: str) -> bool: + parsed = split_instance(key) + return bool(parsed.segments) or parsed.count is not None + + def step_order(row: Mapping[str, Any]) -> tuple[bool, datetime, str, str]: at = row["scheduled_at"] return at is None, at or datetime.max.replace(tzinfo=UTC), row["node_id"], row["instance_key"] @@ -198,6 +248,11 @@ def __lt__(self, other: "_Candidate") -> bool: class FactReadRepository(FactRepository): + def __init__(self, tx: Transaction) -> None: + super().__init__(tx) + # One read unit is one snapshot, so a timeline reads its run metadata once. + self._metadata: dict[UUID, dict[str, Any]] = {} + async def summary_rows(self, query: RunSummaryQuery, position: RunPosition | None) -> list[dict[str, Any]]: return await self._run_rows(query, position) @@ -249,34 +304,66 @@ async def _run_rows(self, query: RunSummaryQuery, position: RunPosition | UUID | raise ValueError("ID order requires an ID position") clauses.append("f.run_id>:position_id") params["position_id"] = position - order = "f.run_id ASC" - else: + return await self.rows( + _RUN_SELECT + " AND " + " AND ".join(clauses) + " ORDER BY f.run_id ASC LIMIT :limit", **params + ) + if position is not None and not isinstance(position, RunPosition): + raise ValueError("Chronological order requires a time position") + # Known times, then the unknown-time tail, as two keyset scans in one snapshot. The known-time scan excludes + # nulls, so the scoped DESC (nulls first) time indexes serve it in both directions with the row comparison + # as an index bound; the tail is read only when known times do not fill the page. + found: list[dict[str, Any]] = [] + if position is None or position.at is not None: + known = [*clauses, f"f.{column} IS NOT NULL"] + bounds: dict[str, Any] = {} if position is not None: - if not isinstance(position, RunPosition): - raise ValueError("Chronological order requires a time position") - params["position_id"] = position.id - if position.at is None: - clauses.append(f"f.{column} IS NULL AND f.run_id{operator}:position_id") - else: - params["position_at"] = position.at - clauses.append( - f"((f.{column},f.run_id){operator}(:position_at,:position_id) OR f.{column} IS NULL)" - ) - order = f"f.{column} {direction} NULLS LAST,f.run_id {direction}" - return await self.rows( - _RUN_SELECT + " AND " + " AND ".join(clauses) + " ORDER BY " + order + " LIMIT :limit", **params + known.append(f"(f.{column},f.run_id){operator}(:position_at,:position_id)") + bounds = dict(position_at=position.at, position_id=position.id) + found = await self.rows( + _RUN_SELECT + + " AND " + + " AND ".join(known) + + f" ORDER BY f.{column} {direction},f.run_id {direction} LIMIT :limit", + **params, + **bounds, + ) + if len(found) > query.limit: + return found + unknown = [*clauses, f"f.{column} IS NULL"] + bounds = {"limit": query.limit + 1 - len(found)} + if position is not None and position.at is None: + unknown.append(f"f.run_id{operator}:position_id") + bounds["position_id"] = position.id + return found + await self.rows( + _RUN_SELECT + " AND " + " AND ".join(unknown) + f" ORDER BY f.run_id {direction} LIMIT :limit", + **{**params, **bounds}, ) + async def page_bytes(self, run_ids: list[UUID]) -> int: + """Logical bytes of the retained runs on one page, measured before any of them is fetched.""" + if not run_ids: + return 0 + rows = await self.rows( + "SELECT coalesce(sum(public.weave_runs_bytes(r)),0) AS bytes FROM runs r " + "WHERE r.tenant_id=:tenant AND r.project_id=:project AND r.environment_id=:environment " + "AND r.id=ANY(:ids)", + ids=run_ids, + ) + return int(rows[0]["bytes"]) + async def metadata(self, run_id: UUID) -> dict[str, Any]: - rows = await self.rows(_RUN_SELECT + " AND f.run_id=:run", run=run_id) + if run_id in self._metadata: + return self._metadata[run_id] + rows = await self.rows(_METADATA_SELECT + " AND f.run_id=:run", run=run_id) if not rows: raise CatalogError(404, "WV-NOT-FOUND", "Run not found") row = rows[0] if isinstance(summary_of(row), UnavailableResource): - raise legacy_unavailable() + raise withheld(row) kinds = row.get("node_kinds") if not isinstance(kinds, dict) or len(kinds) > 10000 or len(json.dumps(kinds).encode()) > 1048576: raise legacy_unavailable() + self._metadata[run_id] = row return row async def step_rows( @@ -340,12 +427,9 @@ async def historical_steps( scanned = 0 try: async with asyncio.timeout(LEGACY_SECONDS): + # Unsorted on purpose: the bounded heap orders candidates, so the key cap also bounds the database. stream = await self.tx.session.stream( - text( - "SELECT s.node_id,s.status FROM step_instances s WHERE " - + _MISSING - + ' ORDER BY s.node_id COLLATE "C"' - ), + text("SELECT s.node_id,s.status FROM step_instances s WHERE " + _MISSING), {**self.params, "run": run_id}, execution_options={"yield_per": 500}, ) @@ -403,7 +487,7 @@ async def outputs(self, run_id: UUID, rows: list[dict[str, Any]]) -> dict[str, S from firefly_weave.contracts.values import JsonObject from firefly_weave.operations.redaction import project from firefly_weave.runtime.kernel import action_schemas, workflow - from firefly_weave.runtime.repository import RUN_FETCH_BYTES, RuntimeRepository + from firefly_weave.runtime.repository import RuntimeRepository await self.metadata(run_id) retained = await RuntimeRepository(self.tx).run(run_id) @@ -414,8 +498,13 @@ async def outputs(self, run_id: UUID, rows: list[dict[str, Any]]) -> dict[str, S } nodes = {node.id: node for node in ir.graph.nodes} keys = [row["instance_key"] or row["node_id"] for row in rows] - withheld = SafeProjection(available=False, omissions=[Omission(path="", reason="classification_unavailable")]) - result = dict.fromkeys(keys, withheld) + unclassified = SafeProjection( + available=False, omissions=[Omission(path="", reason="classification_unavailable")] + ) + result = dict.fromkeys(keys, unclassified) + # An agent turn, tool call or loop yield has no schema of its own yet; the step's schema would classify + # another value, so those outputs stay withheld and are never fetched. + keys = [key for key in keys if not _segmented(key)] if not keys: return result predicate = ( @@ -429,10 +518,10 @@ async def outputs(self, run_id: UUID, rows: list[dict[str, Any]]) -> dict[str, S "total_bytes>:budget AS oversized FROM sized", run=run_id, keys=keys, - budget=RUN_FETCH_BYTES, + budget=PAGE_BYTES, ) if any(output["oversized"] for output in outputs): - raise CatalogError(429, "WV-RUNTIME-LIMIT", "Step outputs exceed the bounded fetch budget") + raise CatalogError(429, "WV-PAGE-LIMIT", "Selected page exceeds the logical byte limit") for output in outputs: identity = author_instance(output["node_id"]) if identity is None: diff --git a/src/firefly_weave/operations/run_views.py b/src/firefly_weave/operations/run_views.py index a27f9147..9cd0d6f8 100644 --- a/src/firefly_weave/operations/run_views.py +++ b/src/firefly_weave/operations/run_views.py @@ -56,10 +56,7 @@ def __init__(self, uow: UnitOfWork, authorization: AuthorizationService) -> None self.uow, self.authorization = uow, authorization async def authorize(self, actor: Principal, scope: Scope, tx: Transaction, *, context: AuditContext) -> Principal: - """`run.read` on the token principal and again on the principal stored now.""" - if scope.environment_id is None: - raise CatalogError(422, "WV-SCOPE", "Runtime requires an environment") - self.authorization.require(actor, scope, "run.read", context=context) + """`run.read` on the principal stored now, inside the read unit; `precheck` already checked the token.""" current = await load_principal(tx.session, actor.id) self.authorization.require(current, scope, "run.read", context=context) return current @@ -104,10 +101,11 @@ async def steps( if len(rows) > query.limit: last = visible[-1] at = last["scheduled_at"] + # The full instance key determines the step ID, so the longest valid key still fits the cursor. cursor = encode_cursor_v2( scope, query.cursor_collection(run_id), - [at.isoformat() if at else None, last["node_id"], last["instance_key"]], + at.isoformat() if at else None, last["instance_key"] or last["node_id"], ) return StepFactPage( @@ -132,6 +130,7 @@ async def logs( raise not_served("runs.logs") def precheck(self, actor: Principal, scope: Scope, context: AuditContext) -> None: + """`run.read` on the token principal, before any read unit opens.""" if scope.environment_id is None: raise CatalogError(422, "WV-SCOPE", "Runtime requires an environment") self.authorization.require(actor, scope, "run.read", context=context) diff --git a/src/firefly_weave/runtime/service.py b/src/firefly_weave/runtime/service.py index de8d436f..d46e3284 100644 --- a/src/firefly_weave/runtime/service.py +++ b/src/firefly_weave/runtime/service.py @@ -36,7 +36,6 @@ from firefly_weave.contracts.catalog import Activation from firefly_weave.contracts.human_tasks import ManualControlRequest from firefly_weave.contracts.operations import CancelRunRequest -from firefly_weave.contracts.public import UnavailableResource from firefly_weave.contracts.run_views import RunListQuery from firefly_weave.contracts.runtime import ( CapacityRunAcknowledgment, @@ -48,7 +47,7 @@ from firefly_weave.definitions.models import CatalogError from firefly_weave.definitions.service import DefinitionService from firefly_weave.operations.fact_positions import RunPosition -from firefly_weave.operations.fact_reads import FactReadRepository, summary_of +from firefly_weave.operations.fact_reads import FactReadRepository, unavailable from firefly_weave.operations.facts import RunStartFacts from firefly_weave.persistence.idempotency import Idempotency from firefly_weave.persistence.uow import Transaction @@ -617,14 +616,14 @@ async def list( context: AuditContext, ) -> dict[str, Any]: from firefly_weave.access.repository import load_principal + from firefly_weave.persistence.paging import PAGE_BYTES - legacy_adapter = query is None and (filters is not None or cursor is not None) + # The keyword arguments are the original contract: ID order and raw UUID positions in both directions. + legacy_adapter = query is None if query is None: values = (filters or RunListFilters()).model_dump(mode="json") values["status"] = [values["status"]] if values["status"] is not None else [] - query = RunListQuery.model_validate_json( - json.dumps({**values, "limit": limit, "order": "id" if legacy_adapter else "started_desc"}) - ) + query = RunListQuery.model_validate_json(json.dumps({**values, "limit": limit, "order": "id"})) position = cursor elif filters is not None or cursor is not None: raise ValueError("Use either a query or legacy list arguments") @@ -637,14 +636,18 @@ async def list( async with self.definitions.transaction(scope, None, mutation=False) as tx: actor = await load_principal(tx.session, actor.id) self.require(actor, scope, "run.read", context) - rows = await FactReadRepository(tx).legacy_rows(query, position) + repository = FactReadRepository(tx) + rows = await repository.legacy_rows(query, position) limit = query.limit + # Measure the whole visible page before any full run is fetched; the lookahead row is not charged. + if await repository.page_bytes([row["run_id"] for row in rows[:limit]]) > PAGE_BYTES: + raise CatalogError(429, "WV-PAGE-LIMIT", "Selected page exceeds the logical byte limit") items = [] for row in rows[:limit]: identifier = row["run_id"] - summary = summary_of(row) - if isinstance(summary, UnavailableResource): - items.append(summary.model_dump(mode="json")) + # Only a policy block withholds a full-run item here; the authoritative read classifies the rest. + if row["policy_blocked"]: + items.append(unavailable(identifier).model_dump(mode="json")) continue try: items.append( diff --git a/src/firefly_weave/sdk/client.py b/src/firefly_weave/sdk/client.py index b56f6ca5..019db3e3 100644 --- a/src/firefly_weave/sdk/client.py +++ b/src/firefly_weave/sdk/client.py @@ -719,8 +719,12 @@ async def list_runs( order: RunListOrder = "started_desc", query: RunListQuery | None = None, ) -> Page[RunView | UnavailableResource]: + """List runs with either a typed ``query`` or the keyword filters; mixing them raises ``ValueError``.""" values: dict[str, str | int | list[str]] if query is not None: + keywords = (limit, cursor, business_key, correlation_key, status, include_archived, order) + if keywords != (50, None, None, None, None, False, "started_desc"): + raise ValueError("Use either a query or keyword list arguments") values = { key: str(value).lower() if isinstance(value, bool) else value for key, value in query.model_dump(mode="json", exclude_none=True).items() diff --git a/tests/integration/test_run_fact_reads.py b/tests/integration/test_run_fact_reads.py index 5811af1c..f7594295 100644 --- a/tests/integration/test_run_fact_reads.py +++ b/tests/integration/test_run_fact_reads.py @@ -816,13 +816,17 @@ def capture(connection, cursor, statement, parameters, context, executemany): @pytest.mark.integration -async def test_malformed_current_author_key_is_unavailable(operations_case): +@pytest.mark.parametrize("stored", ["work[", "work"]) +async def test_malformed_current_author_key_is_unavailable(operations_case, stored): from firefly_weave.definitions.models import CatalogError case = operations_case run = await case.start() + # A step's own ID is stored as an empty instance key; any other spelling cannot be paged exactly. async with case.owner.begin() as session: - await session.execute(text("UPDATE step_facts SET instance_key='work[' WHERE run_id=:run"), {"run": run.id}) + await session.execute( + text("UPDATE step_facts SET instance_key=:stored WHERE run_id=:run"), {"run": run.id, "stored": stored} + ) service = RunViewService(UnitOfWork(case.sessions), AuthorizationService()) with pytest.raises(CatalogError) as failure: await service.steps(case.actor, case.scope, run.id, RunStepQuery(), None, context=AuditContext()) @@ -830,16 +834,37 @@ async def test_malformed_current_author_key_is_unavailable(operations_case): @pytest.mark.integration -async def test_unavailable_timeline_does_not_fetch_requested_output(operations_case, monkeypatch): +@pytest.mark.parametrize( + "mode,status,code", + [ + ("cached_unsupported", 422, "WV-IR-UNSUPPORTED"), + ("feature_withdrawn", 422, "WV-IR-UNSUPPORTED"), + ("unavailable", 409, "WV-LEGACY-UNAVAILABLE"), + ], +) +async def test_unavailable_timeline_does_not_fetch_requested_output(operations_case, monkeypatch, mode, status, code): + import json + + from firefly_weave.contracts import language_features from firefly_weave.definitions.models import CatalogError from firefly_weave.runtime.repository import RuntimeRepository case = operations_case run = await case.start() + features = language_features.ADVERTISED_FEATURES async with case.owner.begin() as session: - await session.execute( - text("UPDATE run_facts SET classification_state='unsupported' WHERE run_id=:run"), {"run": run.id} - ) + if mode == "feature_withdrawn": + await session.execute( + text("UPDATE run_facts SET pinned_features=cast(:features AS jsonb) WHERE run_id=:run"), + {"features": json.dumps([features[0]]), "run": run.id}, + ) + else: + await session.execute( + text("UPDATE run_facts SET classification_state=:state WHERE run_id=:run"), + {"run": run.id, "state": "unsupported" if mode == "cached_unsupported" else "unavailable"}, + ) + if mode == "feature_withdrawn": + monkeypatch.setattr(language_features, "ADVERTISED_FEATURES", features[1:]) async def forbidden(*args, **kwargs): pytest.fail("Unavailable metadata fetched source payloads") @@ -850,4 +875,253 @@ async def forbidden(*args, **kwargs): await service.steps( case.actor, case.scope, run.id, RunStepQuery(include="output"), None, context=AuditContext() ) - assert failure.value.code == "WV-LEGACY-UNAVAILABLE" + # A run waiting for a platform that runs it answers as runs.read does; legacy evidence stays unavailable. + assert (failure.value.status, failure.value.code) == (status, code) + if status == 422: + missing = [features[0]] if mode == "feature_withdrawn" else [] + assert failure.value.result == {"reason": "ir_unsupported", "missing_features": missing} + + +@pytest.mark.integration +@pytest.mark.parametrize("order", ["id", "started_desc"]) +async def test_run_list_pages_keep_the_logical_byte_ceiling(operations_case, order): + from firefly_weave.contracts.run_views import RunListQuery + from firefly_weave.definitions.models import CatalogError + from firefly_weave.persistence.paging import PAGE_BYTES + + case = operations_case + first, second = await case.start("first"), await case.start("second") + page = sorted([first.id, second.id]) if order == "id" else [second.id, first.id] + + async def pad(characters): + # Only the last item of the page grows, so a one-item page never charges it. + async with case.tx() as tx: + await tx.session.execute( + text("UPDATE runs SET state=jsonb_set(state,'{input}',to_jsonb(repeat('x',:count))) WHERE id=:run"), + {"count": characters, "run": page[-1]}, + ) + measured = await case.rows( + "SELECT public.weave_runs_bytes(r) AS bytes FROM runs r WHERE id=ANY(:ids)", ids=page + ) + return sum(row["bytes"] for row in measured) + + async def listed(limit): + query = RunListQuery(order=order, limit=limit) + return await case.runtime.list(case.actor, case.scope, query=query, context=AuditContext()) + + baseline = await pad(0) + assert await pad(PAGE_BYTES - baseline) == PAGE_BYTES + full = await listed(2) + assert [item["id"] for item in full["items"]] == [str(identifier) for identifier in page] + assert all("state" in item for item in full["items"]) + assert await pad(PAGE_BYTES - baseline + 1) == PAGE_BYTES + 1 + with pytest.raises(CatalogError) as rejected: + await listed(2) + assert (rejected.value.status, rejected.value.code) == (429, "WV-PAGE-LIMIT") + assert [item["id"] for item in (await listed(1))["items"]] == [str(page[0])] + + +@pytest.mark.integration +async def test_runs_list_withholds_only_policy_blocked_items(operations_case): + from firefly_weave.runtime import admission + + case = operations_case + run = await case.start() + service = RunViewService(UnitOfWork(case.sessions), AuthorizationService()) + for update in ("classification_policy='retired-policy'", "classification_state='unsupported'"): + async with case.owner.begin() as session: + await session.execute( + text( + "UPDATE run_facts SET classification_policy=:policy,classification_state='available' " + "WHERE run_id=:run" + ), + {"policy": admission.POLICY, "run": run.id}, + ) + await session.execute(text("UPDATE run_facts SET " + update + " WHERE run_id=:run"), {"run": run.id}) + summaries = await service.summaries(case.actor, case.scope, RunSummaryQuery(), None, context=AuditContext()) + assert summaries.items[0].unavailable, update + # The cached projection never decides a full-run item: the authoritative read classifies it. + listed = await case.runtime.list(case.actor, case.scope, context=AuditContext()) + assert listed["items"][0]["id"] == str(run.id) and "state" in listed["items"][0], update + async with case.owner.begin() as session: + await session.execute( + text( + "INSERT INTO run_policy_blocks(tenant_id,project_id,environment_id,run_id,reason_code) " + "SELECT tenant_id,project_id,environment_id,id,'WV-LEGACY-UNAVAILABLE' FROM runs WHERE id=:run" + ), + {"run": run.id}, + ) + listed = await case.runtime.list(case.actor, case.scope, context=AuditContext()) + assert listed["items"] == [ + {"id": str(run.id), "unavailable": True, "omissions": [{"path": "", "reason": "classification_unavailable"}]} + ] + + +@pytest.mark.integration +async def test_step_output_pages_keep_the_logical_byte_ceiling(operations_case): + from firefly_weave.definitions.models import CatalogError + from firefly_weave.persistence.paging import PAGE_BYTES + + case = operations_case + run = await case.start() + service = RunViewService(UnitOfWork(case.sessions), AuthorizationService()) + + async def output_bytes(characters): + async with case.owner.begin() as session: + await session.execute( + text( + "INSERT INTO step_instances SELECT tenant_id,project_id,environment_id,id,'work','completed'," + "to_jsonb(repeat('x',:count)) FROM runs WHERE id=:run ON CONFLICT(run_id,node_id) " + "DO UPDATE SET output=excluded.output,status='completed'" + ), + {"run": run.id, "count": characters}, + ) + rows = await case.rows( + "SELECT octet_length(output::text) AS bytes FROM step_instances WHERE run_id=:run AND node_id='work'", + run=run.id, + ) + return rows[0]["bytes"] + + async def page(): + query = RunStepQuery(include="output") + return await service.steps(case.actor, case.scope, run.id, query, None, context=AuditContext()) + + assert await output_bytes(PAGE_BYTES - 2) == PAGE_BYTES + served = await page() + assert [item.node_id for item in served.items] == ["work"] + assert await output_bytes(PAGE_BYTES - 1) == PAGE_BYTES + 1 + with pytest.raises(CatalogError) as rejected: + await page() + assert (rejected.value.status, rejected.value.code) == (429, "WV-PAGE-LIMIT") + + +@pytest.mark.integration +async def test_segmented_instance_outputs_are_withheld(operations_case): + case = operations_case + run = await case.start() + canary = 271828182845 + segmented = ["work#agent.tool~1", "work[0]#review", "work~2"] + async with case.owner.begin() as session: + await session.execute( + text( + "INSERT INTO step_instances SELECT tenant_id,project_id,environment_id,id,'work','completed'," + "'7'::jsonb FROM runs WHERE id=:run ON CONFLICT(run_id,node_id) " + "DO UPDATE SET output=excluded.output,status='completed'" + ), + {"run": run.id}, + ) + for key in segmented: + await session.execute( + text( + "INSERT INTO step_instances SELECT tenant_id,project_id,environment_id,id,:key,'completed'," + "to_jsonb(cast(:canary AS bigint)) FROM runs WHERE id=:run" + ), + {"run": run.id, "key": key, "canary": canary}, + ) + service = RunViewService(UnitOfWork(case.sessions), AuthorizationService()) + page = await service.steps( + case.actor, case.scope, run.id, RunStepQuery(include="output"), None, context=AuditContext() + ) + items = {item["instance_key"] or item["node_id"]: item for item in page.model_dump(mode="json")["items"]} + assert set(items) == {"work", *segmented} + assert items["work"]["output"] == 7 + # An agent turn, tool call or yield has no schema of its own yet, so the step's schema cannot classify it. + for key in segmented: + assert "output" not in items[key], key + assert items[key]["omissions"] == [{"path": "/output", "reason": "classification_unavailable"}], key + assert str(canary) not in page.model_dump_json() + + +@pytest.mark.integration +async def test_real_legacy_deadline_while_postgresql_holds_the_first_fetch(operations_case, monkeypatch): + import asyncio + + from sqlalchemy.ext.asyncio import AsyncSession + + from firefly_weave.definitions.models import CatalogError + from firefly_weave.operations import fact_reads + + case = operations_case + run = await case.start() + async with case.owner.begin() as session: + await session.execute( + text( + "INSERT INTO step_instances SELECT tenant_id,project_id,environment_id,id," + "'work[1]','completed','null'::jsonb FROM runs WHERE id=:run" + ), + {"run": run.id}, + ) + opened, closed = [], [] + original = AsyncSession.stream + + async def tracked(session, statement, *args, **kwargs): + stream = await original(session, statement, *args, **kwargs) + opened.append(stream) + + class Tracked: + def mappings(self): + return stream.mappings() + + async def close(self): + await stream.close() + closed.append(stream.closed) + + return Tracked() + + monkeypatch.setattr(AsyncSession, "stream", tracked) + monkeypatch.setattr(fact_reads, "LEGACY_SECONDS", 0.5) + async with case.owner.begin() as blocker: + # The stream's first round trip waits on PostgreSQL itself, not on Python between batches. + await blocker.execute(text("LOCK TABLE step_instances IN ACCESS EXCLUSIVE MODE")) + with pytest.raises(CatalogError) as failure: + async with asyncio.timeout(10), UnitOfWork(case.sessions).open(case.scope, mutation=False) as tx: + backend = await tx.session.scalar(text("SELECT pg_backend_pid()")) + await fact_reads.FactReadRepository(tx).historical_steps(run.id, None, None, 1) + # The read unit rolled back cleanly: the refusal is the answer, not a rollback error. + assert (failure.value.status, failure.value.code) == (429, "WV-RUNTIME-LIMIT") + async with case.owner() as observer: + waiting = await observer.scalar( + text("SELECT count(*) FROM pg_locks WHERE pid=:pid AND NOT granted"), {"pid": backend} + ) + state = await observer.scalar(text("SELECT state FROM pg_stat_activity WHERE pid=:pid"), {"pid": backend}) + # While the lock is still held, PostgreSQL no longer runs or waits for the abandoned read. + assert waiting == 0 and state in {None, "idle"} + assert len(closed) == len(opened) and all(closed) + + +@pytest.mark.integration +async def test_list_pages_select_summary_columns_and_timelines_read_metadata_once(operations_case): + from sqlalchemy import event + + case = operations_case + run = await case.start() + async with case.owner.begin() as session: + await session.execute( + text( + "INSERT INTO step_instances SELECT tenant_id,project_id,environment_id,id," + "'work[1]','completed','7'::jsonb FROM runs WHERE id=:run" + ), + {"run": run.id}, + ) + statements = [] + engine = case.sessions.kw["bind"].sync_engine + + def capture(connection, cursor, statement, parameters, context, executemany): + statements.append(" ".join(statement.lower().split())) + + event.listen(engine, "before_cursor_execute", capture) + try: + service = RunViewService(UnitOfWork(case.sessions), AuthorizationService()) + await service.summaries(case.actor, case.scope, RunSummaryQuery(), None, context=AuditContext()) + await case.runtime.list(case.actor, case.scope, context=AuditContext()) + page = await service.steps( + case.actor, case.scope, run.id, RunStepQuery(include="output"), None, context=AuditContext() + ) + finally: + event.remove(engine, "before_cursor_execute", capture) + assert not page.complete and {item.instance_key for item in page.items} == {"", "work[1]"} + facts = [sql for sql in statements if "from run_facts f" in sql] + metadata = [sql for sql in facts if "node_kinds" in sql] + # Pages select only summary columns; one timeline request reads its run metadata once. + assert len(metadata) == 1 and len(facts) - len(metadata) >= 2 + assert not any("f.*" in sql for sql in facts) diff --git a/tests/unit/operations/test_fact_positions.py b/tests/unit/operations/test_fact_positions.py index c4ed64b6..ce05f79c 100644 --- a/tests/unit/operations/test_fact_positions.py +++ b/tests/unit/operations/test_fact_positions.py @@ -70,40 +70,37 @@ def test_positions_normalize_times_preserve_null_and_are_immutable(): with pytest.raises(FrozenInstanceError): run.at = None assert run_position((None, RUN)).at is None - step = step_position(([None, "send", "send[3]#2~1"], "send[3]#2~1")) + step = step_position((None, "send[3]#2~1")) assert (step.scheduled_at, step.node_id, step.instance_key) == (None, "send", "send[3]#2~1") with pytest.raises(FrozenInstanceError): step.node_id = "other" - plain = step_position((["2026-10-09T02:00:00+02:00", "send", ""], "send")) - assert (plain.scheduled_at, plain.instance_key) == (datetime(2026, 10, 9, tzinfo=UTC), "") + plain = step_position(("2026-10-09T02:00:00+02:00", "send")) + assert (plain.scheduled_at, plain.node_id, plain.instance_key) == (datetime(2026, 10, 9, tzinfo=UTC), "send", "") @pytest.mark.parametrize( "value,identifier", [ - ([None, "send", "other[3]"], "send[3]"), - ([None, "send", "send[3]"], "send"), - ([None, "send", "send"], "send"), - ([None, "@join:send", ""], "@join:send"), - ([None, "send", "send[03]"], "send[03]"), - ([None, 3, ""], "3"), - ([None, True, ""], "true"), - ([None, {}, ""], "x"), - ([None, [], ""], "x"), - ([None, "send", 3], "x"), - ([None, "send", True], "x"), - ([None, "send", {}], "x"), - ([None, "send", []], "x"), - ([True, "send", ""], "send"), - ([3, "send", ""], "send"), - ([{}, "send", ""], "send"), - ([[], "send", ""], "send"), - (None, "send"), + (None, "send[03]"), + (None, "send[3]#"), + (None, "send[3]\n"), + (None, "@join:send"), + (None, "@branch:send:1[2]"), + (None, ""), + (None, "s" * 513), + (None, None), + (None, 3), + (None, True), + (None, {}), + (None, []), + (True, "send"), + (3, "send"), ({}, "send"), ([], "send"), + ("2026-10-09", "send"), + ([None, "send", ""], "send"), + ([None, "send", "send[3]"], "send[3]"), ([None, "send"], "send"), - ([None, "send", "", "extra"], "send"), - ([None, "send", ""], None), ], ) def test_step_cursor_shape_and_identity_are_validated(value, identifier): diff --git a/tests/unit/operations/test_run_views.py b/tests/unit/operations/test_run_views.py index c7f319d1..693ed94b 100644 --- a/tests/unit/operations/test_run_views.py +++ b/tests/unit/operations/test_run_views.py @@ -122,19 +122,38 @@ async def test_run_readers_reach_the_not_served_answer(monkeypatch, operation, r @pytest.mark.parametrize("operation", OPERATIONS) @pytest.mark.parametrize( - "token,current", + "token,current,opened", [ - (principal(None), principal("viewer")), - (principal("operator"), principal("operator")), - (principal("viewer", scope=Scope(tenant_id=SCOPE.tenant_id, project_id=uuid4())), principal("viewer")), - (principal("viewer"), principal("viewer", active=False)), - (principal("viewer"), principal(None)), + (principal(None), principal("viewer"), 0), + (principal("operator"), principal("operator"), 0), + (principal("viewer", scope=Scope(tenant_id=SCOPE.tenant_id, project_id=uuid4())), principal("viewer"), 0), + (principal("viewer"), principal("viewer", active=False), 1), + (principal("viewer"), principal(None), 1), ], ) -async def test_token_and_current_principal_both_need_run_read(monkeypatch, operation, token, current): - service, _ = service_with(monkeypatch, current) +async def test_token_and_current_principal_both_need_run_read(monkeypatch, operation, token, current, opened): + service, unit = service_with(monkeypatch, current) with pytest.raises(AccessDenied): await call(service, token, operation) + # A token without run.read is refused before any read unit opens; the stored principal is checked inside one. + assert unit.opened == opened + + +@pytest.mark.parametrize("operation", OPERATIONS) +async def test_each_authority_is_checked_and_audited_once(monkeypatch, caplog, operation): + import logging + + actor = principal("viewer") + service, unit = service_with(monkeypatch, actor) + with caplog.at_level(logging.INFO, logger="weave.authorization"): + try: + await call(service, actor, operation) + except CatalogError as error: + assert (operation, error.status) == ("runs.logs", 501) + records = [json.loads(record.getMessage()) for record in caplog.records if record.name == "weave.authorization"] + # One decision for the token principal and one for the principal stored now. + assert [(record["capability"], record["outcome"]) for record in records] == [("run.read", "allow")] * 2 + assert unit.opened == 1 async def test_environment_scope_is_required(monkeypatch): @@ -203,10 +222,10 @@ async def test_controller_rejects_cursors_from_another_filter_or_run(monkeypatch summaries = encode_cursor_v2(SCOPE, included, "2026-10-07T12:00:00Z", str(uuid4())) with pytest.raises(ValueError, match="cursor"): await controller.summaries(request("/", f"cursor={summaries}", actor, identifier=False)) - steps = encode_cursor_v2(SCOPE, RunStepQuery().cursor_collection(uuid4()), [None, "send"], "send[3]") + steps = encode_cursor_v2(SCOPE, RunStepQuery().cursor_collection(uuid4()), None, "send[3]") with pytest.raises(ValueError, match="cursor"): await controller.steps(request("/", f"cursor={steps}", actor)) - accepted = encode_cursor_v2(SCOPE, RunStepQuery().cursor_collection(RUN), [None, "send", "send[3]"], "send[3]") + accepted = encode_cursor_v2(SCOPE, RunStepQuery().cursor_collection(RUN), None, "send[3]") assert (await controller.steps(request("/", f"cursor={accepted}", actor))).status_code == 200 assert unit.opened == 1 @@ -262,6 +281,52 @@ async def rows(repository, query, position): assert events[0][1] is events[1][1] +@pytest.mark.parametrize( + "node,key", + [ + # The longest valid instance key: a long step ID, fifteen maximal loop indexes and a yield count. + ("s" * 240, "s" * 240 + "[9007199254740991]" * 15 + "~1"), + ("s" * 512, ""), + ], +) +async def test_step_cursor_fits_the_longest_valid_instance_key(monkeypatch, node, key): + from datetime import UTC, datetime + + from firefly_weave.api.transport import decode_cursor_v2 + from firefly_weave.contracts.instance_keys import split_instance + from firefly_weave.operations.fact_positions import StepPosition, step_position + from firefly_weave.operations.fact_reads import FactReadRepository + + assert len(key or node) == 512 + split_instance(key or node) + at = datetime(2026, 10, 9, 23, 59, 59, 999999, tzinfo=UTC) + row = dict( + node_id=node, + instance_key=key, + kind="action", + status="succeeded", + scheduled_at=at, + started_at=None, + ended_at=None, + attempts=1, + worker_id=None, + error_code=None, + handled=None, + child_run_id=None, + ) + + async def steps(repository, run_id, query, position): + return [row, {**row, "node_id": "t", "instance_key": ""}], True + + monkeypatch.setattr(FactReadRepository, "step_rows", steps) + service, _ = service_with(monkeypatch, principal()) + query = RunStepQuery(limit=1) + page = await service.steps(principal(), SCOPE, RUN, query, None, context=CONTEXT) + assert page.next_cursor is not None and len(page.next_cursor) <= 2048 + position = step_position(decode_cursor_v2(page.next_cursor, SCOPE, query.cursor_collection(RUN))) + assert position == StepPosition(at, node, key) + + async def test_invalid_typed_cursor_precedes_run_lookup(monkeypatch): service, unit = service_with(monkeypatch, principal()) with pytest.raises(ValueError): @@ -306,6 +371,39 @@ def receive(request): assert calls[1].url.params["order"] == "started_desc" +@pytest.mark.parametrize( + "keywords", + [ + {"limit": 7}, + {"cursor": "abc"}, + {"business_key": "kept"}, + {"correlation_key": "kept"}, + {"status": "waiting"}, + {"include_archived": True}, + {"order": "id"}, + {"limit": 7, "cursor": "abc", "business_key": "kept"}, + ], +) +async def test_sdk_list_rejects_a_query_mixed_with_keyword_filters(keywords): + import httpx + + from firefly_weave.contracts.run_views import RunListQuery + from firefly_weave.sdk.client import WeaveClient + + calls = [] + + def receive(request): + calls.append(request) + return httpx.Response(200, json={"items": [], "next_cursor": None}) + + async with WeaveClient( + "https://platform.example", lambda: "token", SCOPE, transport=httpx.MockTransport(receive) + ) as client: + with pytest.raises(ValueError, match="either a query"): + await client.list_runs(query=RunListQuery(limit=3), **keywords) + assert calls == [] + + async def test_runs_route_uses_strict_filters_and_preserves_explicit_id_cursor(): from firefly_weave.api.runs import RunController from firefly_weave.api.transport import encode_cursor, encode_cursor_v2 @@ -358,6 +456,7 @@ async def test_historical_stream_is_bounded_and_always_closed(monkeypatch, count monkeypatch.setattr(fact_reads, "LEGACY_SECONDS", 60) closed = [] maximum_heap = [] + statements = [] class Stream: def mappings(self): @@ -365,11 +464,10 @@ def mappings(self): async def partitions(self, size): assert size == 500 + # PostgreSQL returns unsorted keys; the bounded heap alone orders the page. + keys = list(reversed(range(count))) for start in range(0, count, size): - yield [ - {"node_id": f"send[{index}]", "status": "completed"} - for index in range(start, min(count, start + size)) - ] + yield [{"node_id": f"send[{index}]", "status": "completed"} for index in keys[start : start + size]] async def close(self): closed.append(True) @@ -377,6 +475,7 @@ async def close(self): class Session: async def stream(self, statement, params, **kwargs): assert kwargs["execution_options"] == {"yield_per": 500} + statements.append(str(statement)) return Stream() async def metadata(repository, run_id): @@ -402,6 +501,8 @@ def pushed(heap, item): "send[10000]" if count == 100000 else "send[100]", ] assert closed == [True] and max(maximum_heap) == 2 + # Sorting in SQL would make PostgreSQL read and sort every missing key before the cap could stop it. + assert len(statements) == 1 and "ORDER BY" not in statements[0].upper() @pytest.mark.parametrize("cancel", [False, True]) diff --git a/tests/unit/runtime/test_run_filters.py b/tests/unit/runtime/test_run_filters.py index 56fc521d..d766fb02 100644 --- a/tests/unit/runtime/test_run_filters.py +++ b/tests/unit/runtime/test_run_filters.py @@ -16,7 +16,7 @@ """Bounded exact run discovery and filter-bound cursors.""" -from uuid import uuid4 +from uuid import UUID, uuid4 import pytest from pydantic import ValidationError @@ -60,3 +60,53 @@ def test_explicit_id_queries_retain_exact_legacy_cursor_collections(): assert query.model_copy(update={"include_test": True}).cursor_collection() != old.cursor_collection() assert query.model_copy(update={"status": ["waiting", "failed"]}).cursor_collection() != old.cursor_collection() assert query.model_copy(update={"order": "started_desc"}).cursor_collection() != old.cursor_collection() + + +async def test_keyword_list_adapter_always_pages_by_id(monkeypatch): + from contextlib import asynccontextmanager + from types import SimpleNamespace + + from firefly_weave.access.audit import AuditContext + from firefly_weave.access.models import Grant, Principal + from firefly_weave.operations.fact_reads import FactReadRepository + from firefly_weave.runtime.service import RuntimeService + + scope = Scope(tenant_id=uuid4(), project_id=uuid4(), environment_id=uuid4()) + actor = Principal(id=uuid4(), kind="human", grants=(Grant(role="viewer", scope=scope),)) + identifiers = sorted(uuid4() for _ in range(3)) + queries = [] + + class Definitions: + def require(self, *args): + pass + + @asynccontextmanager + async def transaction(self, scope, tx, *, mutation=True): + yield SimpleNamespace(session=object(), scope=scope) + + async def load(session, identifier): + return actor + + async def rows(repository, query, position): + queries.append((query.order, position)) + # Policy-blocked rows answer placeholders, so the adapter is observed without full-run reads. + return [ + {"run_id": identifier, "policy_blocked": True} + for identifier in identifiers + if not position or identifier > position + ] + + async def measured(repository, run_ids): + return 0 + + monkeypatch.setattr("firefly_weave.access.repository.load_principal", load) + monkeypatch.setattr(FactReadRepository, "legacy_rows", rows) + monkeypatch.setattr(FactReadRepository, "page_bytes", measured, raising=False) + service = RuntimeService(Definitions()) + first = await service.list(actor, scope, limit=2, context=AuditContext()) + # The keyword contract returns a raw ID position that the same keywords can continue. + assert first["next_cursor"] == str(identifiers[1]) + rest = await service.list(actor, scope, limit=2, cursor=UUID(first["next_cursor"]), context=AuditContext()) + assert [item["id"] for item in rest["items"]] == [str(identifiers[2])] and rest["next_cursor"] is None + await service.list(actor, scope, limit=2, filters=RunListFilters(), context=AuditContext()) + assert queries == [("id", None), ("id", identifiers[1]), ("id", None)] From c2e2ef490d183cec1be241be0429910b552f6703 Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Sun, 11 Oct 2026 13:01:49 -0700 Subject: [PATCH 08/63] Keep the current run read check behind the token check --- src/firefly_weave/operations/run_views.py | 12 +++++++----- tests/integration/test_run_fact_reads.py | 3 ++- tests/unit/operations/test_run_views.py | 10 ++++++++++ 3 files changed, 19 insertions(+), 6 deletions(-) diff --git a/src/firefly_weave/operations/run_views.py b/src/firefly_weave/operations/run_views.py index 9cd0d6f8..b5cb68e6 100644 --- a/src/firefly_weave/operations/run_views.py +++ b/src/firefly_weave/operations/run_views.py @@ -55,8 +55,10 @@ class RunViewService: def __init__(self, uow: UnitOfWork, authorization: AuthorizationService) -> None: self.uow, self.authorization = uow, authorization - async def authorize(self, actor: Principal, scope: Scope, tx: Transaction, *, context: AuditContext) -> Principal: - """`run.read` on the principal stored now, inside the read unit; `precheck` already checked the token.""" + async def _authorize_current( + self, actor: Principal, scope: Scope, tx: Transaction, *, context: AuditContext + ) -> Principal: + """`run.read` on the principal stored now, inside the read unit; callers run `precheck` first.""" current = await load_principal(tx.session, actor.id) self.authorization.require(current, scope, "run.read", context=context) return current @@ -67,7 +69,7 @@ async def summaries( typed = run_position(position) self.precheck(actor, scope, context) async with self.uow.open(scope, mutation=False) as tx: - await self.authorize(actor, scope, tx, context=context) + await self._authorize_current(actor, scope, tx, context=context) rows = await FactReadRepository(tx).summary_rows(query, typed) visible = rows[: query.limit] cursor = None @@ -92,7 +94,7 @@ async def steps( typed = step_position(position) self.precheck(actor, scope, context) async with self.uow.open(scope, mutation=False) as tx: - await self.authorize(actor, scope, tx, context=context) + await self._authorize_current(actor, scope, tx, context=context) repository = FactReadRepository(tx) rows, complete = await repository.step_rows(run_id, query, typed) visible = rows[: query.limit] @@ -126,7 +128,7 @@ async def logs( ) -> RunLogPage: self.precheck(actor, scope, context) async with self.uow.open(scope, mutation=False) as tx: - await self.authorize(actor, scope, tx, context=context) + await self._authorize_current(actor, scope, tx, context=context) raise not_served("runs.logs") def precheck(self, actor: Principal, scope: Scope, context: AuditContext) -> None: diff --git a/tests/integration/test_run_fact_reads.py b/tests/integration/test_run_fact_reads.py index f7594295..e03684d3 100644 --- a/tests/integration/test_run_fact_reads.py +++ b/tests/integration/test_run_fact_reads.py @@ -1086,7 +1086,8 @@ async def close(self): state = await observer.scalar(text("SELECT state FROM pg_stat_activity WHERE pid=:pid"), {"pid": backend}) # While the lock is still held, PostgreSQL no longer runs or waits for the abandoned read. assert waiting == 0 and state in {None, "idle"} - assert len(closed) == len(opened) and all(closed) + # The lock holds the statement inside stream() itself, so the deadline fires before any cursor exists. + assert opened == [] @pytest.mark.integration diff --git a/tests/unit/operations/test_run_views.py b/tests/unit/operations/test_run_views.py index 693ed94b..9cd06724 100644 --- a/tests/unit/operations/test_run_views.py +++ b/tests/unit/operations/test_run_views.py @@ -156,6 +156,16 @@ async def test_each_authority_is_checked_and_audited_once(monkeypatch, caplog, o assert unit.opened == 1 +def test_the_current_principal_check_cannot_be_called_without_the_token_check(): + # Only the read operations and the token check are public; each read runs both checks in order. + assert {name for name in vars(RunViewService) if not name.startswith("_")} == { + "summaries", + "steps", + "logs", + "precheck", + } + + async def test_environment_scope_is_required(monkeypatch): service, unit = service_with(monkeypatch, principal("viewer")) project = Scope(tenant_id=SCOPE.tenant_id, project_id=SCOPE.project_id) From 198d03d47c05e84909fb5a477325ef8838609413 Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Sun, 11 Oct 2026 13:56:39 -0700 Subject: [PATCH 09/63] Serve scoped incident facts and direct incident reads --- CHANGELOG.md | 12 + docs/operations/upgrades.md | 38 + docs/reference/api.md | 36 +- docs/reference/incident-operations.md | 1 + docs/reference/sdk.md | 2 +- src/firefly_weave/api/operations.py | 69 +- .../contracts/incident_queries.py | 89 +++ src/firefly_weave/contracts/operations.py | 37 +- src/firefly_weave/contracts/schema_export.py | 11 +- src/firefly_weave/contracts/surface.py | 11 + src/firefly_weave/operations/fact_reads.py | 139 ++++ src/firefly_weave/operations/incidents.py | 82 +- src/firefly_weave/sdk/client.py | 34 +- tests/contracts/test_incident_queries.py | 281 +++++++ tests/integration/test_incident_fact_reads.py | 737 ++++++++++++++++++ 15 files changed, 1529 insertions(+), 50 deletions(-) create mode 100644 src/firefly_weave/contracts/incident_queries.py create mode 100644 tests/contracts/test_incident_queries.py create mode 100644 tests/integration/test_incident_fact_reads.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 9aa637c8..2b43042b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -38,6 +38,18 @@ SPDX-License-Identifier: Apache-2.0 - The Python SDK's `list_runs` rejects a `query` combined with keyword filters. - Request telemetry records authorized `501 WV-UNAVAILABLE` answers with the status `unavailable` instead of `failed`. +- Serve incident lists from incident facts without loading runs, newest opened + first. They filter by status, code, workflow, step, opening time, and test + runs (left out unless `include_test=true`). Explicit `order=id` continues + earlier cursors. Unknown query parameters are rejected, and an out-of-range + `limit` answers `WV-VALIDATION` instead of `WV-PAGE`. +- New operation `incidents.read` (`weave runs incidents read`, SDK + `read_incident`) serves direct incident links and sends the revision as `ETag`. +- Incident views add optional `opened_at`, `opened_at_estimated`, `updated_at`, + `workflow`, `run_status`, `test`, and `instance_key`; for a repeated step + instance, `node_id` is the static step ID. Earlier payloads keep their bytes. +- The Python SDK's `list_incidents` and `run_incidents` accept a typed `query` + and reject one combined with keyword arguments. - Keep typing in the selected form field when deferred row autofocus completes. Rapid row additions focus the newest item without overriding a later choice. diff --git a/docs/operations/upgrades.md b/docs/operations/upgrades.md index 7110fee1..22432e2e 100644 --- a/docs/operations/upgrades.md +++ b/docs/operations/upgrades.md @@ -190,6 +190,44 @@ the time budget can be reached first. Synthetic control instances do not count toward completeness or the key cap. Step outputs remain optional and classified. `runs.logs` still answers authorized requests with `501 WV-UNAVAILABLE`. +## Incident list compatibility + +`incidents.list` and the paged `incidents.run_list` now read incident facts +instead of loading each incident's run. Their default order changes to +`opened_desc`, newest opened first, with `opened_asc` also available; incidents +whose opening time is unknown come last. Clients that continue a cursor saved +before the upgrade must send `order=id` without filters. The +`weave runs incidents list` and `run-list` commands do not send `order=id` yet, +so they cannot continue such a cursor; start the listing again. + +The incident lists also change how they read requests: + +- They accept `status` (repeatable), `code`, `workflow`, `node_id`, an opening + time range, and `include_test`. Without `status` they still list every + status. +- Incidents of test runs are left out unless the request sends + `include_test=true`. +- They reject unknown query parameters with `422 WV-VALIDATION`; the earlier + routes ignored them. A `limit` outside 1 to 100 answers `422 WV-VALIDATION` + instead of `422 WV-PAGE`. +- An older server ignores the filters and the order and lists every incident in + ID order. Upgrade servers before relying on them. + +Incident views gain optional fields that are left out while unknown, false, or +empty, so earlier responses and stored resolution receipts keep their bytes. +For an incident of a repeated step instance, `node_id` becomes the static step +ID and `instance_key` carries the full key; other incidents keep the same +`node_id`. Whether an incident is withheld now comes from its run's metadata +and policy blocks, so a list never loads runs. The new `incidents.read` +operation serves direct links and sends the revision as `ETag`. The Python +SDK adds `read_incident`, and `list_incidents` and `run_incidents` take either +a `query` or the keyword arguments; passing both raises `ValueError`. + +Historical opening times come from each incident's opening event. When that +event was not retained, the upgrade used the run's last recorded event and +marks the time `opened_at_estimated`; without either, the opening time stays +unknown. + ## Operational fact storage The `0031_operations_facts` migration adds five scoped operational projections diff --git a/docs/reference/api.md b/docs/reference/api.md index 1e72996c..bc74575b 100644 --- a/docs/reference/api.md +++ b/docs/reference/api.md @@ -387,7 +387,7 @@ capabilities, bindings, Actions, and connection settings, plus - `limit` is 1 to 100 (50 by default). Items come in stable ID order unless a list documents another order: `runs.list` and run summaries are newest - started first by default (see below). + started first, and incident lists newest opened first, by default (see below). - Pass `next_cursor` back as `cursor` for the next page. A cursor is bound to its tenant, project, environment, collection, and any run or schedule filter; a cursor from another scope or filter is rejected. @@ -459,6 +459,39 @@ answers `429 WV-RUNTIME-LIMIT`. A run this platform cannot run yet answers evidence answers `409 WV-LEGACY-UNAVAILABLE`. In `runs.list`, only a recorded policy block withholds a run before its own read classifies it. +**Incidents.** `incidents.list` and the paged `incidents.run_list` read +incident facts after checking both token and current permissions; they never +load run payloads. `incidents.read` (`GET …/incidents/{identifier}`) reads one +incident for a direct link and sends its revision as the `ETag`, ready for +`incidents.resolve`. + +- Filters: `status` (repeatable: `active`, `closed`, `resolved`; every status + when omitted), `code`, `workflow`, `node_id`, `opened_after`, + `opened_before`, and `include_test`. `node_id` is a static step ID and + matches every instance of that step. Opening times follow the run summary + rules: RFC 3339 with an offset, `opened_after` earlier than `opened_before`, + and a range of at most 400 days. +- Incident lists default to `order=opened_desc` and also accept `opened_asc` + and `order=id`. Incidents whose opening time is unknown come last. ID order + without filters keeps the earlier cursors, so send explicit `order=id` to + continue one; every other cursor is bound to the run, each filter, and the + order. +- Incidents of test runs are left out unless you send `include_test=true`. + Incidents of archived runs stay listed. +- Incident views add `opened_at`, `updated_at`, `workflow` (`name` and + `version`), `run_status`, `test`, and, for an incident of a repeated step + instance, `instance_key`; `node_id` is then the static step ID. Each is left + out while unknown, false, or empty. `opened_at_estimated: true` marks an + opening time taken from the run's last recorded event because the incident's + own opening event was not retained; an unknown opening time is never + estimated. +- An incident whose run metadata cannot be classified, or whose run has a + recorded policy block, is an unavailable placeholder. `incidents.read` then + sends no `ETag`. +- A page is limited to 8 MiB of retained incident data; a larger page answers + `429` with `WV-PAGE-LIMIT`. Unknown query parameters are rejected, and a + `limit` outside 1 to 100 answers `422` with `WV-VALIDATION`. + **Answers that say "unavailable".** Cancellation and task completion can return an unavailable acknowledgment when the platform cannot classify the historical data or compare payloads; task completion then includes @@ -611,6 +644,7 @@ operation in the [full API reference](api-explorer.md); to generate a client, | `runs.logs` | `GET /api/v1/tenants/{tenant}/projects/{project}/environments/{environment}/runs/{identifier}/logs` | run.read | | `incidents.run_list` | `GET /api/v1/tenants/{tenant}/projects/{project}/environments/{environment}/runs/{identifier}/incidents` | incident.read | | `incidents.list` | `GET /api/v1/tenants/{tenant}/projects/{project}/environments/{environment}/incidents` | incident.read | +| `incidents.read` | `GET /api/v1/tenants/{tenant}/projects/{project}/environments/{environment}/incidents/{identifier}` | incident.read | | `incidents.resolve` | `POST /api/v1/tenants/{tenant}/projects/{project}/environments/{environment}/incidents/{identifier}/resolve` | incident.resolve | | `releases.create` | `POST /api/v1/tenants/{tenant}/projects/{project}/environments/{environment}/worker-releases` | release.activate | | `releases.list` | `GET /api/v1/tenants/{tenant}/projects/{project}/environments/{environment}/worker-releases` | catalog.read | diff --git a/docs/reference/incident-operations.md b/docs/reference/incident-operations.md index 96e08682..41de245f 100644 --- a/docs/reference/incident-operations.md +++ b/docs/reference/incident-operations.md @@ -211,6 +211,7 @@ repeated. Platform administration alone grants no execution rights. | --- | --- | --- | --- | | `GET /runs/{run}/incidents` | `weave runs incidents run-list` | `incident.read` | Incident views, including closed history | | `GET /incidents` | `weave runs incidents list` | `incident.read` | Incidents across the environment | +| `GET /incidents/{incident}` | `weave runs incidents read` | `incident.read` | One incident, with its revision as `ETag` | | `POST /incidents/{incident}/resolve` | `weave runs incidents resolve` | `incident.resolve` | An immutable, revision-fenced resolution | | `POST /runs/{run}/cancel` | `weave runs cancel` | `run.cancel` | Terminal cancellation with the external-effects flag | | `POST /runs/{run}/retry` | `weave runs retry` | `run.retry` and normal start admission | A new run linked by `parent_run_id` | diff --git a/docs/reference/sdk.md b/docs/reference/sdk.md index ce707c09..6f51207a 100644 --- a/docs/reference/sdk.md +++ b/docs/reference/sdk.md @@ -351,7 +351,7 @@ capability. | `create_connection`, `read_connection`, `test_connection`, `list_connections` | Immutable integration connections that hold only secret handles, and explicit checks | | `start_run`, `read_run`, `signal`, `cancel`, `retry`, `list_runs` | Durable runs | | `pause_run`, `resume_run`, `run_lifecycle`, `archive_run`, `restore_run`, `purge_run` | Run lifecycle management | -| `history`, `export_run`, `replay`, `run_incidents`, `list_incidents`, `resolve_incident` | Bounded evidence and audited recovery | +| `history`, `export_run`, `replay`, `run_incidents`, `list_incidents`, `read_incident`, `resolve_incident` | Bounded evidence and audited recovery | | `human_tasks`, `read_human_task`, `claim_human_task`, `release_human_task`, `complete_human_task`, `reassign_human_task` | Human task inbox and decisions | | `assignment_bindings`, `put_assignment_binding`, `put_human_group` | Who may work on which human task | | `register_release`, `read_release`, `list_releases`, `register_worker`, `read_worker`, `list_workers`, `revoke_worker`, `grant_connection` | Trusted worker releases and instances | diff --git a/src/firefly_weave/api/operations.py b/src/firefly_weave/api/operations.py index 250e8b18..66d5d981 100644 --- a/src/firefly_weave/api/operations.py +++ b/src/firefly_weave/api/operations.py @@ -25,10 +25,21 @@ from firefly_weave.api.access import request_scope from firefly_weave.api.surface import operation -from firefly_weave.api.transport import canonical_path, etag, page_request, page_response, parse_revision +from firefly_weave.api.transport import ( + canonical_path, + decode_cursor, + decode_cursor_v2, + etag, + parse_query, + parse_revision, +) +from firefly_weave.contracts.access import Scope +from firefly_weave.contracts.incident_queries import IncidentQuery from firefly_weave.contracts.operations import CancelRunRequest, HistoryExport, IncidentResolution +from firefly_weave.contracts.public import UnavailableResource from firefly_weave.contracts.runtime import StartRunRequest from firefly_weave.definitions.models import CatalogError +from firefly_weave.operations.fact_positions import RunPosition, run_position from firefly_weave.operations.history import HistoryService from firefly_weave.operations.incidents import IncidentService from firefly_weave.runtime.service import RuntimeService @@ -48,18 +59,8 @@ def __init__(self, incidents: IncidentService, runtime: RuntimeService, history: async def list(self, request: Request) -> JSONResponse: scope = request_scope(request, environment=True) if canonical_path(request.url.path): - identifier = UUID(request.path_params["identifier"]) - collection = "incidents.run/" + str(identifier) - limit, cursor = page_request(request, scope, collection) - page = await self.incidents.list_all( - request.state.principal, - scope, - run_id=identifier, - limit=limit, - cursor=cursor, - context=request.state.audit_context, - ) - return JSONResponse(page_response(page, scope, collection, request.url.path)) + return await self._page(request, scope, UUID(request.path_params["identifier"])) + # The earlier route keeps its plain array of the run's incidents. async with self.runtime.definitions.transaction(scope, None, mutation=False) as tx: result = await self.incidents.list( tx, @@ -168,9 +169,45 @@ def _limit(request: Request, default: int) -> int: @operation("incidents.list") async def discover(self, request: Request) -> JSONResponse: + return await self._page(request, request_scope(request, environment=True), None) + + @operation("incidents.read") + async def read_incident(self, request: Request) -> JSONResponse: scope = request_scope(request, environment=True) - limit, cursor = page_request(request, scope, "incidents") + result = await self.incidents.read( + request.state.principal, + scope, + UUID(request.path_params["identifier"]), + context=request.state.audit_context, + ) + # The revision tag lets a reader resolve what it read; a withheld incident has no revision to offer. + headers = {} if isinstance(result, UnavailableResource) else {"ETag": etag(result.revision, request.url.path)} + return JSONResponse(result.model_dump(mode="json"), headers=headers) + + async def _page(self, request: Request, scope: Scope, run_id: UUID | None) -> JSONResponse: + query = parse_query(request.query_params, IncidentQuery) + canonical = canonical_path(request.url.path) + collection = query.cursor_collection(run_id) + position: RunPosition | UUID | None + if query.order == "id": + # Explicit ID order continues the earlier cursors: scope-bound here, raw IDs on the earlier route. + position = ( + decode_cursor(query.cursor, scope, collection) + if canonical + else UUID(query.cursor) + if query.cursor + else None + ) + else: + position = run_position(decode_cursor_v2(query.cursor, scope, collection)) result = await self.incidents.list_all( - request.state.principal, scope, limit=limit, cursor=cursor, context=request.state.audit_context + request.state.principal, + scope, + run_id=run_id, + query=query, + position=position, + context=request.state.audit_context, ) - return JSONResponse(page_response(result, scope, "incidents", request.url.path)) + if query.order == "id" and not canonical and result["next_cursor"] is not None: + result["next_cursor"] = str(decode_cursor(result["next_cursor"], scope, collection)) + return JSONResponse(result) diff --git a/src/firefly_weave/contracts/incident_queries.py b/src/firefly_weave/contracts/incident_queries.py new file mode 100644 index 00000000..326fc5c4 --- /dev/null +++ b/src/firefly_weave/contracts/incident_queries.py @@ -0,0 +1,89 @@ +# Copyright 2026 Firefly Software Foundation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# Author: Firefly Software Foundation +# SPDX-License-Identifier: Apache-2.0 + +"""Incident list queries for `incidents.list` and `incidents.run_list`. + +Lists are newest opened first by default; incidents with an unknown opening time come last. Explicit ID order without +filters keeps the cursors issued before incidents had filters. `node_id` is a static step ID and matches every +instance of that step. +""" + +from __future__ import annotations + +from datetime import datetime +from typing import Annotated, Literal +from uuid import UUID + +from pydantic import AwareDatetime, Field, field_validator, model_validator + +from firefly_weave.contracts.definitions import ContractModel, ResourceName +from firefly_weave.contracts.run_views import ( + MAX_TIME_RANGE, + Cursor, + ErrorCode, + SummaryLimit, + _digest, + _filter_error, + utc_time, +) + +type IncidentStatus = Literal["active", "closed", "resolved"] +type IncidentOrder = Literal["opened_desc", "opened_asc", "id"] +type IncidentStatusFilter = Annotated[list[IncidentStatus], Field(max_length=3)] + + +class IncidentQuery(ContractModel): + """Incident list query parameters; the cursor is bound to every filter, the order and the run.""" + + status: IncidentStatusFilter = Field(default_factory=list) + code: ErrorCode | None = None + workflow: ResourceName | None = None + node_id: ResourceName | None = None + opened_after: AwareDatetime | None = None + opened_before: AwareDatetime | None = None + include_test: bool = False + order: IncidentOrder = "opened_desc" + limit: SummaryLimit = 50 + cursor: Cursor | None = None + + @field_validator("status") + @classmethod + def canonical_status(cls, value: list[IncidentStatus]) -> list[IncidentStatus]: + return sorted(set(value)) + + @field_validator("opened_after", "opened_before", mode="before") + @classmethod + def utc(cls, value: object) -> datetime | None: + return utc_time(value) + + @model_validator(mode="after") + def coherent(self) -> IncidentQuery: + if self.opened_after is not None and self.opened_before is not None: + if self.opened_after >= self.opened_before: + raise _filter_error("opened_after must be earlier than opened_before") + if self.opened_before - self.opened_after > MAX_TIME_RANGE: + raise _filter_error("The time range is longer than 400 days") + return self + + def unfiltered(self) -> bool: + return not self.model_dump(exclude={"order", "limit", "cursor"}, exclude_defaults=True) + + def cursor_collection(self, run_id: UUID | None = None) -> str: + collection = "incidents" if run_id is None else f"incidents.run/{run_id}" + if self.order == "id" and self.unfiltered(): + return collection + filters = self.model_dump(mode="json", exclude={"order", "limit", "cursor"}) + return f"{collection}:{_digest(filters)}:{self.order}" diff --git a/src/firefly_weave/contracts/operations.py b/src/firefly_weave/contracts/operations.py index d5731076..1025894f 100644 --- a/src/firefly_weave/contracts/operations.py +++ b/src/firefly_weave/contracts/operations.py @@ -16,15 +16,17 @@ """Operator requests and immutable incident resolution receipts.""" -from typing import Literal +from typing import Literal, Self from uuid import UUID -from pydantic import AwareDatetime, Field, field_validator +from pydantic import AwareDatetime, Field, field_validator, model_validator from firefly_weave.contracts.access import Scope from firefly_weave.contracts.catalog import Digest -from firefly_weave.contracts.definitions import ContractModel +from firefly_weave.contracts.definitions import ContractModel, ResourceName, SemVer from firefly_weave.contracts.diagnostics import SourceRange +from firefly_weave.contracts.instance_keys import InstanceKeyTextOrEmpty +from firefly_weave.contracts.run_views import RunStatus, _same_step from firefly_weave.contracts.values import JsonData from firefly_weave.operations.redaction import Omission, SafeProjection from firefly_weave.runtime.models import RunState, RuntimeEvent @@ -45,7 +47,19 @@ def nonblank(cls, value: str | None) -> str | None: return value +class IncidentWorkflow(ContractModel): + name: ResourceName + version: SemVer + + class IncidentView(ContractModel): + """An incident. Read from operational facts, `node_id` is the static step ID and `instance_key` the full key. + + The fields after `external_effects_may_continue` are additive: each is left out while it is unknown, false or + empty, so earlier payloads keep their bytes. `opened_at_estimated` marks an opening time inferred from the run's + last recorded event because the incident's own opening event was not retained. + """ + id: UUID run_id: UUID incident_key: str @@ -59,6 +73,23 @@ class IncidentView(ContractModel): resolved_at: AwareDatetime | None = None resolution: IncidentResolution | None = None external_effects_may_continue: bool = False + instance_key: InstanceKeyTextOrEmpty = Field(default="", exclude_if=lambda value: not value) + opened_at: AwareDatetime | None = Field(default=None, exclude_if=lambda value: value is None) + opened_at_estimated: bool = Field(default=False, exclude_if=lambda value: not value) + updated_at: AwareDatetime | None = Field(default=None, exclude_if=lambda value: value is None) + workflow: IncidentWorkflow | None = Field(default=None, exclude_if=lambda value: value is None) + run_status: RunStatus | None = Field(default=None, exclude_if=lambda value: value is None) + test: bool = Field(default=False, exclude_if=lambda value: not value) + + @model_validator(mode="after") + def coherent(self) -> Self: + if self.instance_key: + if self.node_id is None: + raise ValueError("A run-level incident has no instance_key") + _same_step(self.node_id, self.instance_key) + if self.opened_at_estimated and self.opened_at is None: + raise ValueError("Only a known opening time can be estimated") + return self class CancelRunRequest(ContractModel): diff --git a/src/firefly_weave/contracts/schema_export.py b/src/firefly_weave/contracts/schema_export.py index 505836fe..dbe9c89f 100644 --- a/src/firefly_weave/contracts/schema_export.py +++ b/src/firefly_weave/contracts/schema_export.py @@ -152,6 +152,7 @@ def contract_models() -> dict[str, SchemaModel]: TaskGroupRequest, ) from firefly_weave.contracts.identity import IdentityView + from firefly_weave.contracts.incident_queries import IncidentQuery from firefly_weave.contracts.language import LanguageManifest from firefly_weave.contracts.lumi import ( LumiAskRequest, @@ -169,7 +170,13 @@ def contract_models() -> dict[str, SchemaModel]: PrincipalRecord, PrincipalStatusRequest, ) - from firefly_weave.contracts.operations import EventPage, HistoryExport, RecordedEvidence, ReplayReport + from firefly_weave.contracts.operations import ( + EventPage, + HistoryExport, + IncidentView, + RecordedEvidence, + ReplayReport, + ) from firefly_weave.contracts.providers import ( ProviderEvent, ProviderIngressResponse, @@ -302,6 +309,8 @@ def contract_models() -> dict[str, SchemaModel]: "run-step-query": RunStepQuery, "run-log-page": RunLogPage, "run-log-query": RunLogQuery, + "incident-view": IncidentView, + "incident-query": IncidentQuery, "human-task": HumanTask, "human-task-command": HumanTaskCommand, "complete-human-task": CompleteHumanTask, diff --git a/src/firefly_weave/contracts/surface.py b/src/firefly_weave/contracts/surface.py index 595af133..bb22f6ea 100644 --- a/src/firefly_weave/contracts/surface.py +++ b/src/firefly_weave/contracts/surface.py @@ -95,6 +95,7 @@ TaskGroupRequest, ) from firefly_weave.contracts.identity import IdentityView +from firefly_weave.contracts.incident_queries import IncidentQuery from firefly_weave.contracts.integration_events import DeliveryAttempt, DeliveryView, Subscription, SubscriptionRequest from firefly_weave.contracts.language import LanguageManifest from firefly_weave.contracts.lumi import ( @@ -1438,6 +1439,7 @@ def native(self) -> OpenAPIOperation: Page[IncidentView | UnavailableResource], "incident.read", page=True, + query=IncidentQuery, ), Operation( "incidents.list", @@ -1446,6 +1448,15 @@ def native(self) -> OpenAPIOperation: Page[IncidentView | UnavailableResource], "incident.read", page=True, + query=IncidentQuery, + ), + Operation( + "incidents.read", + ENVIRONMENT + "/incidents/{identifier}", + "GET", + IncidentView | UnavailableResource, + "incident.read", + etag=True, ), Operation( "incidents.resolve", diff --git a/src/firefly_weave/operations/fact_reads.py b/src/firefly_weave/operations/fact_reads.py index 2dd2462e..7f86714e 100644 --- a/src/firefly_weave/operations/fact_reads.py +++ b/src/firefly_weave/operations/fact_reads.py @@ -30,7 +30,9 @@ from firefly_weave.compiler.ir import accepted_ir_versions from firefly_weave.contracts import language_features +from firefly_weave.contracts.incident_queries import IncidentQuery from firefly_weave.contracts.instance_keys import split_instance +from firefly_weave.contracts.operations import IncidentView from firefly_weave.contracts.public import UnavailableResource from firefly_weave.contracts.run_views import ( RunListQuery, @@ -74,6 +76,29 @@ "AND f.run_id=s.run_id AND (f.instance_key=s.node_id OR (f.instance_key='' AND f.node_id=s.node_id)))" ) _STEP_ORDER = 'scheduled_at ASC NULLS LAST,node_id COLLATE "C",instance_key COLLATE "C"' +_INCIDENT_SCOPE = "i.tenant_id=:tenant AND i.project_id=:project AND i.environment_id=:environment" +_INCIDENT_FACTS = ( + " LEFT JOIN incident_facts f ON f.tenant_id=i.tenant_id AND f.project_id=i.project_id " + "AND f.environment_id=i.environment_id AND f.incident_id=i.id" +) +# Ordering keys only: filters and sorting never read the incident's resolution or its run. +_INCIDENT_KEYS = "SELECT i.id,f.opened_at FROM incidents i" + _INCIDENT_FACTS + " WHERE " + _INCIDENT_SCOPE +# One statement for the selected incidents: their columns, measured together against the page budget before the +# resolutions are returned, their facts and their run's classification metadata. +_INCIDENT_DETAILS = ( + "WITH page AS (SELECT i.*,sum(octet_length(to_jsonb(i)::text)) OVER () AS page_bytes FROM incidents i " + "WHERE " + _INCIDENT_SCOPE + " AND i.id=ANY(:ids)) " + "SELECT i.id,i.run_id,i.incident_key,i.node_id,i.generation,i.origin_code,i.code,i.status,i.revision," + "i.actor_id,i.resolved_at,CASE WHEN i.page_bytes<=:budget THEN i.resolution END AS resolution," + "i.page_bytes>:budget AS oversized,f.incident_id IS NOT NULL AS projected,f.node_id AS static_node_id," + "f.instance_key,f.opened_at,f.updated_at,f.opened_at_estimated,f.workflow_name,f.workflow_version,f.test," + "r.status AS run_status,r.classification_state,r.classification_policy,r.pinned_ir_version,r.pinned_features," + "EXISTS(SELECT 1 FROM run_policy_blocks b WHERE b.tenant_id=i.tenant_id AND b.project_id=i.project_id " + "AND b.environment_id=i.environment_id AND b.run_id=i.run_id) AS policy_blocked FROM page i" + + _INCIDENT_FACTS + + " LEFT JOIN run_facts r ON r.tenant_id=i.tenant_id AND r.project_id=i.project_id " + "AND r.environment_id=i.environment_id AND r.run_id=i.run_id" +) LEGACY_KEYS = 100_000 LEGACY_SECONDS = 2 _KIND: TypeAdapter[StepKind] = TypeAdapter(StepKind) @@ -224,6 +249,50 @@ def step_of(row: Mapping[str, Any], *, output: SafeProjection | None = None) -> raise legacy_unavailable() from None +def _json(value: object) -> str: + return value.isoformat() if isinstance(value, datetime) else str(value) + + +def incident_of(row: Mapping[str, Any]) -> IncidentView | UnavailableResource: + """An incident from its own columns and its facts; withheld unless its run's metadata is available. + + The facts carry the static step ID and full instance key normalized when they were written. An incident whose + facts are missing, or whose step identity could not be normalized, is unavailable rather than guessed. + """ + identifier = row["id"] + source = row["node_id"] + if not row["projected"] or not available(row) or (source is None) != (row["static_node_id"] is None): + return unavailable(identifier) + resolution = row["resolution"] + values = dict( + id=identifier, + run_id=row["run_id"], + incident_key=row["incident_key"], + node_id=row["static_node_id"], + generation=row["generation"], + origin_code=row["origin_code"], + code=row["code"], + status=row["status"], + revision=row["revision"], + actor_id=row["actor_id"], + resolved_at=row["resolved_at"], + resolution=resolution, + external_effects_may_continue=isinstance(resolution, dict) and resolution.get("kind") == "terminate", + instance_key=row["instance_key"], + opened_at=row["opened_at"], + # A time is only estimated when it is known; an unknown opening time stays unknown. + opened_at_estimated=row["opened_at_estimated"] is True and row["opened_at"] is not None, + updated_at=row["updated_at"], + workflow=dict(name=row["workflow_name"], version=row["workflow_version"]), + run_status=row["run_status"], + test=row["test"], + ) + try: + return IncidentView.model_validate_json(json.dumps(values, default=_json)) + except (ValueError, TypeError, KeyError, OverflowError): + return unavailable(identifier) + + def legacy_unavailable() -> CatalogError: return CatalogError(409, "WV-LEGACY-UNAVAILABLE", "Step classification is unavailable") @@ -339,6 +408,76 @@ async def _run_rows(self, query: RunSummaryQuery, position: RunPosition | UUID | **{**params, **bounds}, ) + async def incident_rows( + self, query: IncidentQuery, position: RunPosition | UUID | None, run_id: UUID | None + ) -> list[dict[str, Any]]: + """The ordering keys of one incident page and its lookahead: incident IDs and opening times.""" + clauses: list[str] = [] + params: dict[str, Any] = dict(limit=query.limit + 1) + for name, column, value in ( + ("run", "i.run_id", run_id), + ("code", "i.code", query.code), + ("workflow", "f.workflow_name", query.workflow), + ("node", "f.node_id", query.node_id), + ): + if value is not None: + clauses.append(f"{column}=:{name}") + params[name] = value + if query.status: + clauses.append("i.status=ANY(:statuses)") + params["statuses"] = query.status + if not query.include_test: + clauses.append("NOT coalesce(f.test,false)") + for name, operator in (("opened_after", ">="), ("opened_before", "<")): + value = getattr(query, name) + if value is not None: + clauses.append(f"f.opened_at{operator}:{name}") + params[name] = value + if query.order == "id": + if isinstance(position, RunPosition): + raise ValueError("ID order requires an ID position") + if position is not None: + clauses.append("i.id>:position_id") + params["position_id"] = position + order = "i.id ASC" + else: + if isinstance(position, UUID): + raise ValueError("Chronological order requires a time position") + direction, operator = ("DESC", "<") if query.order == "opened_desc" else ("ASC", ">") + if position is not None: + params["position_id"] = position.id + if position.at is None: + clauses.append(f"(f.opened_at IS NULL AND i.id{operator}:position_id)") + else: + # Unknown opening times come after every known one, in both directions. + clauses.append(f"((f.opened_at,i.id){operator}(:position_at,:position_id) OR f.opened_at IS NULL)") + params["position_at"] = position.at + order = f"f.opened_at {direction} NULLS LAST,i.id {direction}" + return await self.rows( + _INCIDENT_KEYS + "".join(" AND " + clause for clause in clauses) + f" ORDER BY {order} LIMIT :limit", + **params, + ) + + async def incident_details(self, ids: list[UUID]) -> list[dict[str, Any]]: + """The selected incidents with their facts; a page above the logical byte limit is refused unread.""" + if not ids: + return [] + rows = await self.rows(_INCIDENT_DETAILS, ids=ids, budget=PAGE_BYTES) + if any(row["oversized"] for row in rows): + raise CatalogError(429, "WV-PAGE-LIMIT", "Selected page exceeds the logical byte limit") + return rows + + async def run_exists(self, run_id: UUID) -> bool: + return bool( + await self.tx.session.scalar( + text( + "SELECT EXISTS(SELECT 1 FROM runs r WHERE r.tenant_id=:tenant AND r.project_id=:project " + "AND r.environment_id=:environment AND r.id=:run)" + ), + {**self.params, "run": run_id}, + ) + ) + async def page_bytes(self, run_ids: list[UUID]) -> int: """Logical bytes of the retained runs on one page, measured before any of them is fetched.""" if not run_ids: diff --git a/src/firefly_weave/operations/incidents.py b/src/firefly_weave/operations/incidents.py index 33d75c79..73ce4039 100644 --- a/src/firefly_weave/operations/incidents.py +++ b/src/firefly_weave/operations/incidents.py @@ -26,14 +26,19 @@ from firefly_weave.access.audit import AuditContext from firefly_weave.access.models import Principal from firefly_weave.access.service import AccessService, audit +from firefly_weave.api.transport import encode_cursor, encode_cursor_v2 from firefly_weave.compiler.api import import_artifact from firefly_weave.compiler.canonical import canonical_digest from firefly_weave.compiler.expressions import measure_value from firefly_weave.compiler.ir import ActionNode from firefly_weave.contracts.access import Scope from firefly_weave.contracts.definitions import ActionDefinition, load_definition +from firefly_weave.contracts.incident_queries import IncidentQuery from firefly_weave.contracts.operations import IncidentResolution, IncidentView +from firefly_weave.contracts.public import UnavailableResource from firefly_weave.definitions.models import CatalogError +from firefly_weave.operations.fact_positions import RunPosition +from firefly_weave.operations.fact_reads import FactReadRepository, incident_of from firefly_weave.operations.facts import task_fact_update from firefly_weave.persistence.uow import Transaction from firefly_weave.runtime.kernel import transition_async as transition @@ -259,6 +264,18 @@ async def resolve( await settle(repository, await repository.run(previous.id)) return resolved + async def read( + self, actor: Principal, scope: Scope, identifier: UUID, *, context: AuditContext + ) -> IncidentView | UnavailableResource: + """One incident from its facts; the run's classification metadata decides whether it is withheld.""" + self.runtime.require(actor, scope, "incident.read", context) + async with self.runtime.definitions.transaction(scope, None, mutation=False) as tx: + await self._require_current(actor, scope, tx, context) + rows = await FactReadRepository(tx).incident_details([identifier]) + if not rows: + raise CatalogError(404, "WV-NOT-FOUND", "Incident not found") + return incident_of(rows[0]) + async def list_all( self, actor: Principal, @@ -267,32 +284,47 @@ async def list_all( limit: int = 50, cursor: UUID | None = None, run_id: UUID | None = None, + query: IncidentQuery | None = None, + position: RunPosition | UUID | None = None, context: AuditContext, ) -> dict[str, Any]: - from firefly_weave.persistence.paging import page_ids + """A page of incidents read from facts, never from run payloads. + The keyword arguments are the original contract: every incident in ID order, with raw ID cursors in both + directions. A typed query selects filters and order, and its decoded position continues a page. + """ + keywords = query is None + if query is None: + if type(limit) is not int or not 1 <= limit <= 100: + raise CatalogError(422, "WV-PAGE", "A bounded page is required") + query, position = IncidentQuery(order="id", limit=limit), cursor + elif cursor is not None or limit != 50: + raise ValueError("Use either a query or legacy list arguments") + if query.order == "id" and isinstance(position, RunPosition): + raise ValueError("ID order requires an ID position") + if query.order != "id" and isinstance(position, UUID): + raise ValueError("Chronological order requires a time position") + self.runtime.require(actor, scope, "incident.read", context) async with self.runtime.definitions.transaction(scope, None, mutation=False) as tx: - actor = await self.access.load_principal(actor.id, tx=tx) - self.runtime.require(actor, scope, "incident.read", context) - if run_id is not None: - await RuntimeRepository(tx, self.runtime.definitions.outbox).run(run_id) - ids = await page_ids(tx, "incidents", limit, cursor, run_id=run_id) - items = [] - repository = RuntimeRepository(tx, self.runtime.definitions.outbox) - for identifier in ids[:limit]: - row = (await repository.rows(f"SELECT * FROM incidents WHERE {SCOPE} AND id=:id", id=identifier))[0] - try: - view_of(await repository.run(row["run_id"])) - items.append(incident_view(row).model_dump(mode="json")) - except CatalogError as error: - # A run waiting for an upgrade (ir_unsupported) is listed like legacy evidence. - if error.code not in {"WV-LEGACY-UNAVAILABLE", "WV-IR-UNSUPPORTED"}: - raise - items.append( - { - "id": str(identifier), - "unavailable": True, - "omissions": [{"path": "", "reason": "classification_unavailable"}], - } - ) - return {"items": items, "next_cursor": str(ids[limit - 1]) if len(ids) > limit else None} + await self._require_current(actor, scope, tx, context) + repository = FactReadRepository(tx) + if run_id is not None and not await repository.run_exists(run_id): + raise CatalogError(404, "WV-NOT-FOUND", "Run not found") + rows = await repository.incident_rows(query, position, run_id) + visible = rows[: query.limit] + details = {row["id"]: row for row in await repository.incident_details([row["id"] for row in visible])} + items = [incident_of(details[row["id"]]).model_dump(mode="json") for row in visible] + next_cursor = None + if len(rows) > query.limit: + last = visible[-1] + collection = query.cursor_collection(run_id) + if query.order == "id": + next_cursor = str(last["id"]) if keywords else encode_cursor(scope, collection, last["id"]) + else: + at = last["opened_at"] + next_cursor = encode_cursor_v2(scope, collection, at.isoformat() if at else None, str(last["id"])) + return {"items": items, "next_cursor": next_cursor} + + async def _require_current(self, actor: Principal, scope: Scope, tx: Transaction, context: AuditContext) -> None: + """`incident.read` on the principal stored now, in the read transaction; the token was checked first.""" + self.runtime.require(await self.access.load_principal(actor.id, tx=tx), scope, "incident.read", context) diff --git a/src/firefly_weave/sdk/client.py b/src/firefly_weave/sdk/client.py index 019db3e3..183b3397 100644 --- a/src/firefly_weave/sdk/client.py +++ b/src/firefly_weave/sdk/client.py @@ -56,6 +56,7 @@ TaskGroup, TaskGroupRequest, ) +from firefly_weave.contracts.incident_queries import IncidentOrder, IncidentQuery from firefly_weave.contracts.integration_events import DeliveryAttempt, DeliveryView, Subscription, SubscriptionRequest from firefly_weave.contracts.language import LanguageManifest from firefly_weave.contracts.lumi import ( @@ -541,16 +542,22 @@ async def run_incidents( *, limit: int = 50, cursor: str | None = None, + order: IncidentOrder = "opened_desc", + query: IncidentQuery | None = None, ) -> Page[IncidentView | UnavailableResource]: + """One run's incidents with either a typed ``query`` or the keyword arguments; mixing them raises.""" return cast( Page[IncidentView | UnavailableResource], await self.invoke( "incidents.run_list", identifier=identifier, - query=self._page(limit, cursor), + query=self._incident_query(limit, cursor, order, query), ), ) + async def read_incident(self, identifier: UUID) -> IncidentView | UnavailableResource: + return cast(IncidentView | UnavailableResource, await self.invoke("incidents.read", identifier=identifier)) + async def resolve_incident(self, identifier: UUID, request: IncidentResolution, *, revision: int) -> IncidentView: return cast( IncidentView, await self.invoke("incidents.resolve", identifier=identifier, body=request, revision=revision) @@ -763,13 +770,34 @@ async def list_triggers(self, *, limit: int = 50, cursor: str | None = None) -> return cast(Page[Trigger], await self.invoke("triggers.list", query=self._page(limit, cursor))) async def list_incidents( - self, *, limit: int = 50, cursor: str | None = None + self, + *, + limit: int = 50, + cursor: str | None = None, + order: IncidentOrder = "opened_desc", + query: IncidentQuery | None = None, ) -> Page[IncidentView | UnavailableResource]: + """Incidents with either a typed ``query`` or the keyword arguments; mixing them raises ``ValueError``.""" return cast( Page[IncidentView | UnavailableResource], - await self.invoke("incidents.list", query=self._page(limit, cursor)), + await self.invoke("incidents.list", query=self._incident_query(limit, cursor, order, query)), ) + @classmethod + def _incident_query( + cls, limit: int, cursor: str | None, order: IncidentOrder, query: IncidentQuery | None + ) -> dict[str, str | int | list[str]]: + if query is None: + return {**cls._page(limit, cursor), "order": order} + if (limit, cursor, order) != (50, None, "opened_desc"): + raise ValueError("Use either a query or keyword list arguments") + # Statuses travel as repeated parameters; empty filters are left out. + return { + key: str(value).lower() if isinstance(value, bool) else value + for key, value in query.model_dump(mode="json", exclude_none=True).items() + if not isinstance(value, list) or value + } + async def list_schedules(self, *, limit: int = 50, cursor: str | None = None) -> Page[ScheduleView]: return cast(Page[ScheduleView], await self.invoke("schedules.list", query=self._page(limit, cursor))) diff --git a/tests/contracts/test_incident_queries.py b/tests/contracts/test_incident_queries.py new file mode 100644 index 00000000..272bff42 --- /dev/null +++ b/tests/contracts/test_incident_queries.py @@ -0,0 +1,281 @@ +# Copyright 2026 Firefly Software Foundation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# Author: Firefly Software Foundation +# SPDX-License-Identifier: Apache-2.0 + +"""Incident queries and enriched incident views; earlier incident payloads keep their bytes.""" + +import json +from uuid import uuid4 + +import pytest +from pydantic import ValidationError +from starlette.datastructures import QueryParams + +from firefly_weave.contracts.operations import IncidentView + +# The exact bytes an incident view serialized to before incidents carried facts. +OLD_RESOLVED = ( + '{"id":"0f8b5c1e-7d1a-4a52-9c39-3b1f4f7e2a10","run_id":"5a7e2c44-91b3-4c0e-8f6d-2d3c4b5a6e7f",' + '"incident_key":"work:1","node_id":"work","generation":1,"origin_code":"WV-TASK-AMBIGUOUS",' + '"code":"WV-TASK-AMBIGUOUS","status":"resolved","revision":2,"actor_id":"9d2f6a3b-1c4e-4f7a-8b9c-0d1e2f3a4b5c",' + '"resolved_at":"2026-10-07T12:00:00Z","resolution":{"receipt_id":"3c2b1a09-8f7e-4d6c-9b5a-4e3d2c1b0a99",' + '"kind":"terminate","reason":"Stop the run","output":null,"evidence_reference":null},' + '"external_effects_may_continue":true}' +) +OLD_ACTIVE = ( + '{"id":"0f8b5c1e-7d1a-4a52-9c39-3b1f4f7e2a10","run_id":"5a7e2c44-91b3-4c0e-8f6d-2d3c4b5a6e7f",' + '"incident_key":"work:1","node_id":"work","generation":1,"origin_code":"WV-TASK-AMBIGUOUS",' + '"code":"WV-TASK-AMBIGUOUS","status":"active","revision":1,"actor_id":null,"resolved_at":null,' + '"resolution":null,"external_effects_may_continue":false}' +) +ENRICHED = { + **json.loads(OLD_ACTIVE), + "node_id": "send", + "instance_key": "send[2]", + "opened_at": "2026-10-07T11:00:00Z", + "updated_at": "2026-10-07T11:30:00Z", + "workflow": {"name": "invoice-approval", "version": "1.3.0"}, + "run_status": "suspended", + "test": True, +} +ADDITIVE = {"instance_key", "opened_at", "opened_at_estimated", "updated_at", "workflow", "run_status", "test"} + + +def query(model, values): + return model.model_validate_json(json.dumps(values)) + + +@pytest.mark.parametrize("raw", [OLD_RESOLVED, OLD_ACTIVE]) +def test_earlier_incident_payloads_keep_their_bytes(raw): + old = IncidentView.model_validate_json(raw) + assert old.model_dump_json() == raw + assert "opened_at_estimated" not in old.model_dump(mode="json") + estimated = old.model_copy(update={"opened_at_estimated": True}) + assert estimated.model_dump(mode="json")["opened_at_estimated"] is True + + +@pytest.mark.parametrize("estimated", [False, True]) +def test_exact_and_estimated_opening_times_round_trip_through_generated_schemas(estimated): + from jsonschema import Draft202012Validator + + from firefly_weave.contracts.schema_export import export_schemas + + payload = {**ENRICHED, **({"opened_at_estimated": True} if estimated else {})} + view = IncidentView.model_validate_json(json.dumps(payload)) + assert view.model_dump(mode="json") == payload + schema = export_schemas()["incident-view"] + assert not list(Draft202012Validator(schema).iter_errors(payload)) + assert list(Draft202012Validator(schema).iter_errors({**payload, "instance_key": "send[02]"})) + assert set(schema["properties"]) >= ADDITIVE + assert not ADDITIVE & set(schema["required"]) + + +def test_unknown_opening_time_is_never_estimated(): + with pytest.raises(ValidationError): + IncidentView.model_validate_json(json.dumps({**json.loads(OLD_ACTIVE), "opened_at_estimated": True})) + + +@pytest.mark.parametrize( + "node_id,instance_key,valid", + [ + ("send", "send[2]", True), + ("support", "support[3]#2.review", True), + ("send", "", True), + (None, "", True), + ("send", "send", False), + ("send[2]", "send[2]", False), + ("other", "send[2]", False), + (None, "send[2]", False), + ("send", "send[", False), + ], +) +def test_instance_key_is_the_full_key_of_the_static_step(node_id, instance_key, valid): + payload = {**json.loads(OLD_ACTIVE), "node_id": node_id, "instance_key": instance_key} + if valid: + view = IncidentView.model_validate_json(json.dumps(payload)) + assert ("instance_key" in view.model_dump(mode="json")) is bool(instance_key) + else: + with pytest.raises(ValidationError): + IncidentView.model_validate_json(json.dumps(payload)) + + +def test_query_defaults_list_every_status_newest_opened_first(): + from firefly_weave.contracts.incident_queries import IncidentQuery + + default = IncidentQuery() + assert (default.status, default.order, default.limit, default.include_test, default.cursor) == ( + [], + "opened_desc", + 50, + False, + None, + ) + assert list(IncidentQuery.model_fields) == [ + "status", + "code", + "workflow", + "node_id", + "opened_after", + "opened_before", + "include_test", + "order", + "limit", + "cursor", + ] + + +@pytest.mark.parametrize( + "values", + [ + {"opened_after": "2026-10-07T12:00:00Z", "opened_before": "2026-10-07T12:00:00Z"}, + {"opened_after": "2026-10-07T12:00:00Z", "opened_before": "2026-10-07T11:59:59Z"}, + {"opened_after": "2025-08-01T00:00:00Z", "opened_before": "2026-10-07T00:00:00Z"}, + ], +) +def test_contradictory_opening_ranges_are_filter_errors(values): + from firefly_weave.contracts.incident_queries import IncidentQuery + + with pytest.raises(ValidationError) as failure: + query(IncidentQuery, values) + assert [item["type"] for item in failure.value.errors()] == ["weave_filter"] + + +@pytest.mark.parametrize( + "values", + [ + {"status": ["active", "closed", "resolved", "active"]}, + {"status": ["open"]}, + {"opened_after": "2026-10-07T12:00:00"}, + {"opened_before": "2026-10-07 12:00:00Z"}, + {"opened_after": "9999-12-31T23:59:59-23:59"}, + {"code": ""}, + {"code": "C" * 101}, + {"workflow": "bad name"}, + {"node_id": "send[2]"}, + {"order": "started_desc"}, + {"limit": 0}, + {"limit": 101}, + {"cursor": "not/base64"}, + ], +) +def test_invalid_query_values_are_validation_errors(values): + from firefly_weave.contracts.incident_queries import IncidentQuery + + with pytest.raises(ValidationError) as failure: + query(IncidentQuery, values) + assert "weave_filter" not in {item["type"] for item in failure.value.errors()} + + +def test_opening_range_accepts_exactly_four_hundred_days_in_utc(): + from firefly_weave.contracts.incident_queries import IncidentQuery + + values = {"opened_after": "2026-01-01T01:00:00+01:00", "opened_before": "2027-02-05T00:00:00Z"} + parsed = query(IncidentQuery, values) + assert parsed.opened_after is not None and parsed.opened_after.isoformat() == "2026-01-01T00:00:00+00:00" + with pytest.raises(ValidationError): + query(IncidentQuery, {**values, "opened_before": "2027-02-05T00:00:00.000001Z"}) + + +def test_route_decoding_reports_filter_errors_and_repeated_statuses(): + from firefly_weave.api.transport import parse_query + from firefly_weave.contracts.incident_queries import IncidentQuery + from firefly_weave.definitions.models import CatalogError + + parsed = parse_query(QueryParams("status=resolved&status=active&status=resolved&order=opened_asc"), IncidentQuery) + assert (parsed.status, parsed.order) == (["active", "resolved"], "opened_asc") + with pytest.raises(CatalogError) as failure: + parse_query(QueryParams("opened_after=2026-10-07T12:00:00Z&opened_before=2026-10-07T12:00:00Z"), IncidentQuery) + assert (failure.value.status, failure.value.code) == (422, "WV-FILTER") + for raw in ("colour=red", "limit=1&limit=2", "include_test=yes"): + with pytest.raises(ValueError): + parse_query(QueryParams(raw), IncidentQuery) + + +def test_cursor_collection_keeps_id_cursors_and_binds_filters_order_and_run(): + from firefly_weave.contracts.incident_queries import IncidentQuery + + run = uuid4() + legacy = IncidentQuery(order="id") + # Explicit ID order without filters continues cursors issued before incidents had filters. + assert legacy.cursor_collection() == "incidents" + assert legacy.cursor_collection(run) == "incidents.run/" + str(run) + assert IncidentQuery(order="id", limit=7).cursor_collection() == "incidents" + collections = { + IncidentQuery().cursor_collection(), + IncidentQuery(order="opened_asc").cursor_collection(), + IncidentQuery().cursor_collection(run), + IncidentQuery().cursor_collection(uuid4()), + IncidentQuery(order="id", status=["active"]).cursor_collection(), + IncidentQuery(order="id", include_test=True).cursor_collection(), + IncidentQuery(status=["active"]).cursor_collection(), + IncidentQuery(code="WV-TASK-AMBIGUOUS").cursor_collection(), + IncidentQuery(workflow="invoice-approval").cursor_collection(), + IncidentQuery(node_id="send").cursor_collection(), + IncidentQuery(include_test=True).cursor_collection(), + query(IncidentQuery, {"opened_after": "2026-10-07T12:00:00Z"}).cursor_collection(), + query(IncidentQuery, {"opened_before": "2026-10-07T12:00:00Z"}).cursor_collection(), + } + assert len(collections) == 13 and "incidents" not in collections + assert ( + query(IncidentQuery, {"status": ["resolved", "active"], "limit": 3}).cursor_collection() + == query(IncidentQuery, {"status": ["active", "resolved", "active"]}).cursor_collection() + ) + + +def test_incident_operations_declare_the_query_and_the_direct_read(): + from firefly_weave.contracts.incident_queries import IncidentQuery + from firefly_weave.contracts.openapi import export_openapi + from firefly_weave.contracts.public import UnavailableResource + from firefly_weave.contracts.surface import OPERATIONS + + environment = "/api/v1/tenants/{tenant}/projects/{project}/environments/{environment}" + spec = export_openapi() + for identifier in ("incidents.list", "incidents.run_list"): + operation = OPERATIONS[identifier] + assert operation.query is IncidentQuery and operation.page + parameters = spec["paths"][operation.canonical_path]["get"]["parameters"] + assert [item["name"] for item in parameters if item["in"] == "query"] == list(IncidentQuery.model_fields) + read = OPERATIONS["incidents.read"] + assert (read.method, read.canonical_path, read.capability, read.response, read.etag) == ( + "GET", + environment + "/incidents/{identifier}", + "incident.read", + IncidentView | UnavailableResource, + True, + ) + assert read.request is None and read.query is None and not read.page + document = spec["paths"][read.canonical_path]["get"] + assert "ETag" in document["responses"]["200"]["headers"] + + +def test_incident_schemas_are_exported(): + from firefly_weave.contracts.schema_export import export_schemas + + schemas = export_schemas() + assert schemas["incident-query"]["properties"]["order"]["default"] == "opened_desc" + definitions = schemas["incident-query"]["$defs"] + assert definitions["IncidentStatusFilter"]["maxItems"] == 3 + assert definitions["IncidentStatus"]["enum"] == ["active", "closed", "resolved"] + assert set(schemas["incident-view"]["required"]) == { + "id", + "run_id", + "incident_key", + "node_id", + "generation", + "origin_code", + "code", + "status", + "revision", + } diff --git a/tests/integration/test_incident_fact_reads.py b/tests/integration/test_incident_fact_reads.py new file mode 100644 index 00000000..92c280ad --- /dev/null +++ b/tests/integration/test_incident_fact_reads.py @@ -0,0 +1,737 @@ +# Copyright 2026 Firefly Software Foundation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# Author: Firefly Software Foundation +# SPDX-License-Identifier: Apache-2.0 + +"""Real PostgreSQL incident reads: facts, filters, classification, authority and pagination without run payloads.""" + +from datetime import timedelta +from uuid import UUID, uuid4 + +import pytest +from sqlalchemy import event, text + +from firefly_weave.access.audit import AuditContext +from firefly_weave.access.authorization import AccessDenied +from firefly_weave.contracts.incident_queries import IncidentQuery +from firefly_weave.contracts.operations import IncidentResolution, IncidentView +from firefly_weave.contracts.public import UnavailableResource +from firefly_weave.definitions.models import CatalogError +from firefly_weave.operations.incidents import IncidentService +from firefly_weave.runtime.repository import RuntimeRepository +from firefly_weave.runtime.service import RuntimeService + +pytestmark = pytest.mark.integration + +UNAVAILABLE = [{"path": "", "reason": "classification_unavailable"}] + + +@pytest.fixture +def worker_runtime_fixture(worker_runtime_fixture): + from firefly_weave.compiler.catalog import CatalogSnapshot + from firefly_weave.contracts.definitions import load_definition + + # With a retry policy, a lost connection opens an incident as soon as the worker reports it. + action = worker_runtime_fixture["action"].model_dump(by_alias=True) + action["spec"]["retry"] = {"maxAttempts": 3, "initialDelaySeconds": 1, "maxDelaySeconds": 2} + action = load_definition(action) + contract = worker_runtime_fixture["catalog"].resolve("TaskCapability", "echo@1.2.0").definition.value + return { + **worker_runtime_fixture, + "action": action, + "catalog": CatalogSnapshot.from_definitions([action], tasks=[contract]), + } + + +def incidents(case) -> IncidentService: + return IncidentService(case.runtime, case.access, case.tasks) + + +async def open_incident(case, task_service, worker_ids, key="incident"): + from firefly_weave.contracts.workers import TaskError + + run = await case.start(key) + async with case.tx() as tx: + lease = (await task_service.claim(tx, worker_ids[0], 1))[0] + async with case.tx() as tx: + await task_service.fail(tx, lease.proof, TaskError(completion_id=uuid4(), code="CONNECTION_LOST")) + rows = await case.rows("SELECT * FROM incidents WHERE run_id=:run", run=run.id) + assert len(rows) == 1 and rows[0]["status"] == "active" + return run, rows[0] + + +async def seed(case, run_id, count, *, prefix="seed"): + """Incidents with tied and unknown opening times: every fifth minute repeats and every seventh is unknown.""" + async with case.owner.begin() as session: + await session.execute( + text( + "INSERT INTO incidents(id,tenant_id,project_id,environment_id,run_id,incident_key,node_id,generation," + "origin_code,code,status,revision) SELECT md5(:prefix||r.id::text||g)::uuid,r.tenant_id," + "r.project_id,r.environment_id,r.id,:prefix||':'||g,'work',g,'WV-TASK-FAILED'," + "CASE WHEN g%4=0 THEN 'WV-TASK-AMBIGUOUS' ELSE 'WV-TASK-FAILED' END," + "(ARRAY['active','closed','resolved'])[1+g%3],1 FROM runs r,generate_series(1,:count) g " + "WHERE r.id=:run" + ), + {"prefix": prefix, "run": run_id, "count": count}, + ) + await session.execute( + text( + "INSERT INTO incident_facts(tenant_id,project_id,environment_id,incident_id,node_id,instance_key," + "opened_at,updated_at,workflow_name,workflow_version,test,opened_at_estimated) SELECT i.tenant_id," + "i.project_id,i.environment_id,i.id,'work','',CASE WHEN i.generation%7=0 THEN NULL ELSE " + "timestamptz '2026-10-01T00:00:00Z'+(i.generation%5)*interval '1 minute' END," + "timestamptz '2026-10-02T00:00:00Z',f.workflow_name,f.workflow_version,false,i.generation%11=0 " + "FROM incidents i JOIN run_facts f ON f.run_id=i.run_id WHERE i.run_id=:run " + "AND i.incident_key LIKE :prefix||':%'" + ), + {"prefix": prefix, "run": run_id}, + ) + + +def capture(case): + statements: list[str] = [] + engine = case.sessions.kw["bind"].sync_engine + + def record(connection, cursor, statement, parameters, context, executemany): + statements.append(" ".join(statement.lower().split())) + + event.listen(engine, "before_cursor_execute", record) + return statements, lambda: event.remove(engine, "before_cursor_execute", record) + + +def forbid_run_payloads(monkeypatch): + async def forbidden(*args, **kwargs): + raise AssertionError("incident read loaded a run payload") + + monkeypatch.setattr(RuntimeRepository, "run", forbidden) + monkeypatch.setattr(RuntimeService, "read", forbidden) + + +def position_of(scope, query, run_id, cursor): + from firefly_weave.api.transport import decode_cursor, decode_cursor_v2 + from firefly_weave.operations.fact_positions import run_position + + collection = query.cursor_collection(run_id) + if query.order == "id": + return decode_cursor(cursor, scope, collection) + return run_position(decode_cursor_v2(cursor, scope, collection)) + + +async def walk(case, values, *, run_id=None): + service, position, found = incidents(case), None, [] + query = IncidentQuery(**values) + while True: + page = await service.list_all( + case.actor, case.scope, run_id=run_id, query=query, position=position, context=AuditContext() + ) + assert len(page["items"]) <= query.limit + found += [item["id"] for item in page["items"]] + if page["next_cursor"] is None: + return found + position = position_of(case.scope, query, run_id, page["next_cursor"]) + + +async def expected_order(case, order, *, statuses=None, run_id=None): + direction = {"opened_desc": "DESC", "opened_asc": "ASC"}.get(order) + sort = f"f.opened_at {direction} NULLS LAST,i.id {direction}" if direction else "i.id ASC" + rows = await case.rows( + "SELECT i.id FROM incidents i LEFT JOIN incident_facts f ON f.incident_id=i.id " + "WHERE (cast(:statuses AS text[]) IS NULL OR i.status=ANY(cast(:statuses AS text[]))) " + "AND (cast(:run AS uuid) IS NULL OR i.run_id=:run) ORDER BY " + sort, + statuses=statuses, + run=run_id, + ) + return [str(row["id"]) for row in rows] + + +async def test_incident_list_never_loads_run_payload(monkeypatch, operations_case, task_service, worker_ids): + case = operations_case + service = incidents(case) + forbid_run_payloads(monkeypatch) + page = await service.list_all( + case.actor, case.scope, query=IncidentQuery(status=["active"]), position=None, context=AuditContext() + ) + assert page["items"] == [] + monkeypatch.undo() + run, incident = await open_incident(case, task_service, worker_ids) + forbid_run_payloads(monkeypatch) + for run_id in (None, run.id): + page = await service.list_all( + case.actor, + case.scope, + run_id=run_id, + query=IncidentQuery(status=["active"]), + position=None, + context=AuditContext(), + ) + assert [item["id"] for item in page["items"]] == [str(incident["id"])] + view = await service.read(case.actor, case.scope, incident["id"], context=AuditContext()) + assert isinstance(view, IncidentView) and view.id == incident["id"] + + +async def test_real_incident_reads_filters_and_resolution(operations_case, task_service, worker_ids, services): + case = operations_case + run, incident = await open_incident(case, task_service, worker_ids) + service = incidents(case) + view = await service.read(case.actor, case.scope, incident["id"], context=AuditContext()) + assert isinstance(view, IncidentView) + assert (view.node_id, view.instance_key, view.status, view.revision, view.run_status) == ( + "work", + "", + "active", + incident["revision"], + "suspended", + ) + assert view.workflow is not None and (view.workflow.name, view.workflow.version) == ("worker-flow", "1.0.0") + assert view.opened_at is not None and view.opened_at == view.updated_at + assert not view.opened_at_estimated and not view.test + dumped = view.model_dump(mode="json") + assert "instance_key" not in dumped and "test" not in dumped and "opened_at_estimated" not in dumped + opened = view.opened_at + + def at(delta=timedelta()): + return (opened + delta).isoformat() + + matching = [ + {"status": ["active"]}, + {"status": ["active", "closed"]}, + {"code": incident["code"]}, + {"workflow": "worker-flow"}, + {"node_id": "work"}, + {"opened_after": at()}, + {"opened_before": at(timedelta(microseconds=1))}, + {"opened_after": at(), "opened_before": at(timedelta(seconds=1))}, + {"include_test": True}, + ] + missing = [ + {"status": ["resolved"]}, + {"status": ["closed", "resolved"]}, + {"code": "WV-OTHER"}, + {"workflow": "other-flow"}, + {"node_id": "other"}, + {"opened_after": at(timedelta(microseconds=1))}, + {"opened_before": at()}, + ] + for values, expected in [(item, [str(incident["id"])]) for item in matching] + [(item, []) for item in missing]: + for order in ("opened_desc", "opened_asc", "id"): + assert await walk(case, {**values, "order": order}) == expected, (values, order) + assert await walk(case, {**values, "order": order}, run_id=run.id) == expected, (values, order) + + resolver = services(case.sessions).resolve(IncidentService) + request = IncidentResolution(kind="retry_safe", receipt_id=uuid4(), reason="reviewed") + + async def resolve(body, revision): + async with case.tx() as tx: + return await resolver.resolve( + tx, incident["id"], body, revision, actor=case.actor, scope=case.scope, context=AuditContext() + ) + + receipt = await resolve(request, incident["revision"]) + assert await resolve(request, incident["revision"]) == receipt + with pytest.raises(CatalogError) as conflict: + await resolve(request.model_copy(update={"receipt_id": uuid4()}), incident["revision"]) + assert (conflict.value.status, conflict.value.code) == (409, "WV-INCIDENT-REVISION") + with pytest.raises(CatalogError) as stale: + await resolve(request.model_copy(update={"receipt_id": uuid4()}), receipt.revision) + assert (stale.value.status, stale.value.code) == (409, "WV-RUNTIME-STATE") + resolved = await service.read(case.actor, case.scope, incident["id"], context=AuditContext()) + assert isinstance(resolved, IncidentView) + assert (resolved.status, resolved.revision, resolved.resolution) == ("resolved", receipt.revision, request) + assert resolved.opened_at == opened and resolved.updated_at is not None and resolved.updated_at > opened + assert resolved.updated_at == resolved.resolved_at + assert await walk(case, {"status": ["resolved"]}) == [str(incident["id"])] + assert await walk(case, {"status": ["active"]}) == [] + + +async def test_statement_count_does_not_depend_on_page_size(operations_case, task_service, worker_ids): + case = operations_case + run, incident = await open_incident(case, task_service, worker_ids) + await seed(case, run.id, 60) + service = incidents(case) + counts = {} + for run_id in (None, run.id): + for limit in (1, 50): + statements, stop = capture(case) + try: + page = await service.list_all( + case.actor, + case.scope, + run_id=run_id, + query=IncidentQuery(limit=limit, include_test=True), + position=None, + context=AuditContext(), + ) + finally: + stop() + assert len(page["items"]) == limit + assert not any("from runs " in sql and ("state" in sql or "artifact" in sql) for sql in statements) + assert not any("select runs.*" in sql or "r.state" in sql or "r.artifact" in sql for sql in statements) + counts[run_id, limit] = len(statements) + assert counts[None, 1] == counts[None, 50] + assert counts[run.id, 1] == counts[run.id, 50] + + +async def test_token_and_current_principal_both_need_incident_read( + monkeypatch, operations_case, task_service, worker_ids +): + from firefly_weave.access.models import Grant, Principal + from firefly_weave.definitions.service import DefinitionService + + case = operations_case + run, incident = await open_incident(case, task_service, worker_ids) + service = incidents(case) + opened = [] + original = DefinitionService.transaction + + def counting(self, *args, **kwargs): + opened.append(args) + return original(self, *args, **kwargs) + + monkeypatch.setattr(DefinitionService, "transaction", counting) + calls = ( + lambda actor: service.read(actor, case.scope, incident["id"], context=AuditContext()), + lambda actor: service.list_all(actor, case.scope, query=IncidentQuery(), position=None, context=AuditContext()), + lambda actor: service.list_all( + actor, case.scope, run_id=run.id, query=IncidentQuery(), position=None, context=AuditContext() + ), + ) + viewer = Principal(id=case.actor.id, kind=case.actor.kind, grants=(Grant(role="viewer", scope=case.scope),)) + for call in calls: + with pytest.raises(AccessDenied): + await call(viewer) + # A token without incident.read is refused before any read transaction opens. + assert opened == [] + async with case.owner.begin() as session: + await session.execute( + text("DELETE FROM role_bindings WHERE principal_id=:id AND role='operator'"), {"id": case.actor.id} + ) + for call in calls: + with pytest.raises(AccessDenied): + await call(case.actor) + # A token whose grant was removed meanwhile is refused inside the read transaction, by the stored principal. + assert len(opened) == len(calls) + + +async def test_other_scopes_never_find_the_incident_or_its_run(operations_case, provisioned, task_service, worker_ids): + from firefly_weave.access.models import Grant + + case = operations_case + run, incident = await open_incident(case, task_service, worker_ids) + other = provisioned[1][1] + await case.access.grant(case.admin, case.actor.id, Grant(role="operator", scope=other)) + actor = await case.access.load_principal(case.actor.id) + service = incidents(case) + with pytest.raises(CatalogError) as missing: + await service.read(actor, other, incident["id"], context=AuditContext()) + assert (missing.value.status, missing.value.code) == (404, "WV-NOT-FOUND") + with pytest.raises(CatalogError) as absent: + await service.list_all( + actor, other, run_id=run.id, query=IncidentQuery(), position=None, context=AuditContext() + ) + assert absent.value.status == 404 + page = await service.list_all(actor, other, query=IncidentQuery(), position=None, context=AuditContext()) + assert page == {"items": [], "next_cursor": None} + with pytest.raises(CatalogError) as unknown: + await service.read(actor, case.scope, uuid4(), context=AuditContext()) + assert unknown.value.status == 404 + + +async def test_test_runs_are_left_out_by_default_and_archived_runs_keep_their_incidents( + operations_case, task_service, worker_ids +): + case = operations_case + run, incident = await open_incident(case, task_service, worker_ids) + service = incidents(case) + async with case.owner.begin() as session: + await session.execute(text("UPDATE run_facts SET test=true WHERE run_id=:run"), {"run": run.id}) + await session.execute(text("UPDATE incident_facts SET test=true WHERE incident_id=:id"), {"id": incident["id"]}) + assert await walk(case, {}) == [] + assert await walk(case, {}, run_id=run.id) == [] + assert await walk(case, {"include_test": True}) == [str(incident["id"])] + view = await service.read(case.actor, case.scope, incident["id"], context=AuditContext()) + assert isinstance(view, IncidentView) and view.test and view.model_dump(mode="json")["test"] is True + async with case.owner.begin() as session: + await session.execute( + text("UPDATE incident_facts SET test=false WHERE incident_id=:id"), {"id": incident["id"]} + ) + await session.execute( + text( + "INSERT INTO run_archives(tenant_id,project_id,environment_id,run_id,archived,revision,archived_at) " + "SELECT tenant_id,project_id,environment_id,id,true,1,clock_timestamp() FROM runs WHERE id=:run" + ), + {"run": run.id}, + ) + # Archiving a run never hid its incidents and still does not. + assert await walk(case, {}) == [str(incident["id"])] + assert await walk(case, {"order": "id"}, run_id=run.id) == [str(incident["id"])] + assert isinstance(await service.read(case.actor, case.scope, incident["id"], context=AuditContext()), IncidentView) + + +@pytest.mark.parametrize("order", ["opened_desc", "opened_asc", "id"]) +async def test_pages_follow_sql_order_with_ties_and_unknown_times_last(operations_case, order): + case = operations_case + first, second = await case.start("first"), await case.start("second") + await seed(case, first.id, 120, prefix="a") + await seed(case, second.id, 37, prefix="b") + everything = await expected_order(case, order) + assert len(everything) == 157 + if order != "id": + unknown = { + str(row["id"]) + for row in await case.rows("SELECT incident_id AS id FROM incident_facts WHERE opened_at IS NULL") + } + assert set(everything[-len(unknown) :]) == unknown + for limit in (1, 7, 50, 100): + values = {"order": order, "limit": limit} + assert await walk(case, values) == everything + assert await walk(case, values, run_id=second.id) == await expected_order(case, order, run_id=second.id) + statuses = ["active", "resolved"] + assert await walk(case, {**values, "status": statuses}) == await expected_order(case, order, statuses=statuses) + + +async def test_newer_incidents_do_not_displace_a_continuation(operations_case): + case = operations_case + run = await case.start() + await seed(case, run.id, 20, prefix="a") + service = incidents(case) + query = IncidentQuery(limit=5) + first = await service.list_all(case.actor, case.scope, query=query, position=None, context=AuditContext()) + expected = await expected_order(case, "opened_desc") + await seed(case, run.id, 9, prefix="later") + async with case.owner.begin() as session: + await session.execute( + text( + "UPDATE incident_facts SET opened_at=timestamptz '2026-10-09T00:00:00Z' WHERE incident_id IN " + "(SELECT id FROM incidents WHERE incident_key LIKE 'later:%')" + ) + ) + position = position_of(case.scope, query, None, first["next_cursor"]) + rest = [] + while position is not None: + page = await service.list_all(case.actor, case.scope, query=query, position=position, context=AuditContext()) + rest += [item["id"] for item in page["items"]] + position = position_of(case.scope, query, None, page["next_cursor"]) if page["next_cursor"] else None + assert [item["id"] for item in first["items"]] + rest == expected + + +async def test_classification_comes_from_run_facts_and_policy_blocks( + monkeypatch, operations_case, task_service, worker_ids +): + from firefly_weave.contracts import language_features + + case = operations_case + run, incident = await open_incident(case, task_service, worker_ids) + service = incidents(case) + forbid_run_payloads(monkeypatch) + + async def answers(): + page = await service.list_all( + case.actor, case.scope, query=IncidentQuery(), position=None, context=AuditContext() + ) + direct = await service.read(case.actor, case.scope, incident["id"], context=AuditContext()) + return page["items"], direct + + withheld = ([{"id": str(incident["id"]), "unavailable": True, "omissions": UNAVAILABLE}], UnavailableResource) + # The run pins a language feature, as a workflow that builds text with concat does. + async with case.owner.begin() as session: + await session.execute( + text( + "UPDATE run_facts SET pinned_ir_version='weave/ir-v1alpha4'," + "pinned_features='[\"text.concat\"]'::jsonb WHERE run_id=:run" + ), + {"run": run.id}, + ) + items, direct = await answers() + assert isinstance(direct, IncidentView) and items[0]["id"] == str(incident["id"]) and "code" in items[0] + # A server that no longer runs the feature withholds the incident at once, from metadata alone. + monkeypatch.setattr(language_features, "ADVERTISED_FEATURES", ()) + items, direct = await answers() + assert (items, type(direct)) == withheld + monkeypatch.undo() + forbid_run_payloads(monkeypatch) + items, direct = await answers() + assert isinstance(direct, IncidentView) and items[0]["id"] == str(incident["id"]) and "code" in items[0] + for change in ( + "UPDATE run_facts SET classification_state='unsupported' WHERE run_id=:run", + "UPDATE run_facts SET classification_state='available',classification_policy='retired' WHERE run_id=:run", + ): + async with case.owner.begin() as session: + snapshot = ( + await session.execute( + text("SELECT classification_state,classification_policy FROM run_facts WHERE run_id=:run"), + {"run": run.id}, + ) + ).one() + await session.execute(text(change), {"run": run.id}) + items, direct = await answers() + assert (items, type(direct)) == withheld, change + async with case.owner.begin() as session: + await session.execute( + text( + "UPDATE run_facts SET classification_state=:state,classification_policy=:policy WHERE run_id=:run" + ), + {"state": snapshot[0], "policy": snapshot[1], "run": run.id}, + ) + async with case.owner.begin() as session: + await session.execute( + text( + "INSERT INTO run_policy_blocks(tenant_id,project_id,environment_id,run_id,reason_code) " + "SELECT tenant_id,project_id,environment_id,id,'WV-LEGACY-UNAVAILABLE' FROM runs WHERE id=:run" + ), + {"run": run.id}, + ) + items, direct = await answers() + assert (items, type(direct)) == withheld + + +@pytest.mark.parametrize( + "change", + [ + # A malformed historical identity cannot be normalized into a static step. + "UPDATE incident_facts SET node_id=NULL WHERE incident_id=:id", + # A missing projection does not mean the incident never existed. + "DELETE FROM incident_facts WHERE incident_id=:id", + ], +) +async def test_incidents_without_trustworthy_facts_stay_listed_as_unavailable( + operations_case, task_service, worker_ids, change +): + case = operations_case + run, incident = await open_incident(case, task_service, worker_ids) + async with case.owner.begin() as session: + await session.execute(text(change), {"id": incident["id"]}) + service = incidents(case) + for run_id in (None, run.id): + page = await service.list_all( + case.actor, case.scope, run_id=run_id, query=IncidentQuery(), position=None, context=AuditContext() + ) + assert page["items"] == [{"id": str(incident["id"]), "unavailable": True, "omissions": UNAVAILABLE}] + direct = await service.read(case.actor, case.scope, incident["id"], context=AuditContext()) + assert isinstance(direct, UnavailableResource) and direct.id == incident["id"] + + +async def test_instance_keys_and_run_level_incidents_keep_their_identity(operations_case): + case = operations_case + run = await case.start() + async with case.owner.begin() as session: + for key, source, node, instance in ( + ("loop", "work[2]", "work", "work[2]"), + ("agent", "work[1]#3.review", "work", "work[1]#3.review"), + ("@legacy", None, None, ""), + ): + await session.execute( + text( + "WITH incident AS (INSERT INTO incidents(id,tenant_id,project_id,environment_id,run_id," + "incident_key,node_id,generation,origin_code,code,status,revision) SELECT md5(:key)::uuid," + "tenant_id,project_id,environment_id,id,:key,:source,NULL,'WV-TASK-FAILED','WV-TASK-FAILED'," + "'active',1 FROM runs WHERE id=:run RETURNING *) INSERT INTO incident_facts(tenant_id," + "project_id,environment_id,incident_id,node_id,instance_key,opened_at,updated_at,workflow_name," + "workflow_version,test,opened_at_estimated) SELECT i.tenant_id,i.project_id,i.environment_id," + "i.id,:node,:instance,clock_timestamp(),clock_timestamp(),f.workflow_name,f.workflow_version," + "false,true FROM incident i JOIN run_facts f ON f.run_id=i.run_id" + ), + {"key": key, "source": source, "node": node, "instance": instance, "run": run.id}, + ) + service = incidents(case) + page = await service.list_all(case.actor, case.scope, query=IncidentQuery(), position=None, context=AuditContext()) + found = {item["incident_key"]: item for item in page["items"]} + assert (found["loop"]["node_id"], found["loop"]["instance_key"]) == ("work", "work[2]") + assert (found["agent"]["node_id"], found["agent"]["instance_key"]) == ("work", "work[1]#3.review") + assert found["@legacy"]["node_id"] is None and "instance_key" not in found["@legacy"] + assert all(item["opened_at_estimated"] is True for item in found.values()) + by_step = await service.list_all( + case.actor, case.scope, query=IncidentQuery(node_id="work"), position=None, context=AuditContext() + ) + assert {item["incident_key"] for item in by_step["items"]} == {"loop", "agent"} + async with case.owner.begin() as session: + await session.execute(text("UPDATE incident_facts SET opened_at=NULL WHERE incident_id=md5('loop')::uuid")) + unknown = await service.read(case.actor, case.scope, UUID(found["loop"]["id"]), context=AuditContext()) + # An unknown opening time is never labeled estimated. + assert isinstance(unknown, IncidentView) and unknown.opened_at is None and not unknown.opened_at_estimated + + +@pytest.mark.parametrize("order", ["id", "opened_desc"]) +async def test_incident_pages_keep_the_logical_byte_ceiling(operations_case, order): + from firefly_weave.persistence.paging import PAGE_BYTES + + case = operations_case + run = await case.start() + await seed(case, run.id, 2) + page = await expected_order(case, order) + + async def pad(characters): + # Only the last item of the page grows, so a one-item page never charges it. + async with case.owner.begin() as session: + await session.execute( + text( + "UPDATE incidents SET resolution=jsonb_build_object('receipt_id',cast(:receipt AS text)," + "'kind','accept_reconciled_result','reason','padding','output',repeat('x',:count)," + "'evidence_reference','padding') WHERE id=:id" + ), + {"receipt": str(uuid4()), "count": characters, "id": UUID(page[-1])}, + ) + measured = await case.rows( + "SELECT octet_length(to_jsonb(i)::text) AS bytes FROM incidents i WHERE id=ANY(:ids)", + ids=[UUID(identifier) for identifier in page], + ) + return sum(row["bytes"] for row in measured) + + async def listed(limit): + query = IncidentQuery(order=order, limit=limit) + return await incidents(case).list_all( + case.actor, case.scope, query=query, position=None, context=AuditContext() + ) + + baseline = await pad(0) + assert await pad(PAGE_BYTES - baseline) == PAGE_BYTES + full = await listed(2) + assert [item["id"] for item in full["items"]] == page + assert len(full["items"][-1]["resolution"]["output"]) == PAGE_BYTES - baseline + assert await pad(PAGE_BYTES - baseline + 1) == PAGE_BYTES + 1 + with pytest.raises(CatalogError) as rejected: + await listed(2) + assert (rejected.value.status, rejected.value.code) == (429, "WV-PAGE-LIMIT") + assert [item["id"] for item in (await listed(1))["items"]] == page[:1] + + +async def test_keyword_arguments_keep_id_order_and_raw_cursors(operations_case): + case = operations_case + run = await case.start() + await seed(case, run.id, 3) + service = incidents(case) + ordered = await expected_order(case, "id") + first = await service.list_all(case.actor, case.scope, limit=2, context=AuditContext()) + assert [item["id"] for item in first["items"]] == ordered[:2] and first["next_cursor"] == ordered[1] + rest = await service.list_all( + case.actor, case.scope, limit=2, cursor=UUID(first["next_cursor"]), context=AuditContext() + ) + assert [item["id"] for item in rest["items"]] == ordered[2:] and rest["next_cursor"] is None + by_run = await service.list_all(case.actor, case.scope, run_id=run.id, limit=3, context=AuditContext()) + assert [item["id"] for item in by_run["items"]] == ordered + with pytest.raises(ValueError): + await service.list_all( + case.actor, case.scope, cursor=UUID(ordered[0]), query=IncidentQuery(order="id"), context=AuditContext() + ) + with pytest.raises(ValueError): + await service.list_all( + case.actor, case.scope, query=IncidentQuery(), position=UUID(ordered[0]), context=AuditContext() + ) + with pytest.raises(CatalogError) as unbounded: + await service.list_all(case.actor, case.scope, limit=101, context=AuditContext()) + assert (unbounded.value.status, unbounded.value.code) == (422, "WV-PAGE") + + +async def test_routes_sdk_etags_and_scope_bound_cursors( + operations_case, authenticated_client, access_db, provisioned, task_service, worker_ids +): + from firefly_weave.access.models import Grant + from firefly_weave.api.transport import encode_cursor, encode_cursor_v2 + from firefly_weave.sdk.client import WeaveClient + + case = operations_case + run, incident = await open_incident(case, task_service, worker_ids) + other_run = await case.start("other") + await seed(case, other_run.id, 3) + client, tokens, scopes = authenticated_client + async with access_db[1]() as session: + subjects = { + row.subject: row.principal_id + for row in (await session.execute(text("SELECT subject,principal_id FROM identity_links"))).all() + } + await access_db[2].grant(provisioned[0], subjects["0"], Grant(role="operator", scope=scopes[0])) + await access_db[2].grant(provisioned[0], subjects["1"], Grant(role="operator", scope=scopes[1])) + headers = {"Authorization": "Bearer " + tokens[0]} + legacy = ( + f"/tenants/{case.scope.tenant_id}/projects/{case.scope.project_id}/environments/{case.scope.environment_id}" + ) + prefix = "/api/v1" + legacy + other = ( + f"/api/v1/tenants/{scopes[1].tenant_id}/projects/{scopes[1].project_id}/environments/{scopes[1].environment_id}" + ) + identifier = str(incident["id"]) + + listed = await client.get(prefix + "/incidents?status=active&status=closed&limit=1", headers=headers) + assert listed.status_code == 200, listed.text + assert listed.json()["items"][0]["id"] == identifier + read = await client.get(prefix + "/incidents/" + identifier, headers=headers) + assert read.status_code == 200 and read.headers["ETag"] == f'"{incident["revision"]}"' + assert read.json()["workflow"] == {"name": "worker-flow", "version": "1.0.0"} + assert read.json()["run_status"] == "suspended" and read.json()["node_id"] == "work" + assert (await client.get(legacy + "/incidents/" + identifier, headers=headers)).headers["ETag"] == str( + incident["revision"] + ) + run_page = await client.get(prefix + f"/runs/{run.id}/incidents", headers=headers) + assert [item["id"] for item in run_page.json()["items"]] == [identifier] + # The earlier route keeps its plain array. + array = await client.get(legacy + f"/runs/{run.id}/incidents", headers=headers) + assert isinstance(array.json(), list) and array.json()[0]["id"] == identifier + + cursor = (await client.get(prefix + "/incidents?limit=1", headers=headers)).json()["next_cursor"] + assert cursor is not None + for path, params in ( + ("/incidents", {"cursor": cursor, "status": "active"}), + ("/incidents", {"cursor": cursor, "order": "opened_asc"}), + (f"/runs/{run.id}/incidents", {"cursor": cursor}), + ("/incidents", {"colour": "red"}), + ("/incidents", {"opened_after": "2026-10-07T12:00:00Z", "opened_before": "2026-10-07T12:00:00Z"}), + ): + response = await client.get(prefix + path, params=params, headers=headers) + assert response.status_code == 422, (path, params, response.text) + run_cursor = (await client.get(prefix + f"/runs/{other_run.id}/incidents?limit=1", headers=headers)).json()[ + "next_cursor" + ] + assert ( + await client.get(prefix + f"/runs/{run.id}/incidents", params={"cursor": run_cursor}, headers=headers) + ).status_code == 422 + stolen = encode_cursor_v2(scopes[1], IncidentQuery().cursor_collection(), None, identifier) + assert (await client.get(prefix + "/incidents", params={"cursor": stolen}, headers=headers)).status_code == 422 + + ordered = await expected_order(case, "id") + old = encode_cursor(case.scope, "incidents", UUID(ordered[0])) + continued = await client.get(prefix + "/incidents", params={"order": "id", "cursor": old}, headers=headers) + assert [item["id"] for item in continued.json()["items"]] == ordered[1:] + raw = await client.get(legacy + "/incidents", params={"order": "id", "limit": 1}, headers=headers) + assert raw.json()["next_cursor"] == ordered[0] + + other_headers = {"Authorization": "Bearer " + tokens[1]} + assert (await client.get(other + "/incidents/" + identifier, headers=other_headers)).status_code == 404 + assert (await client.get(other + f"/runs/{run.id}/incidents", headers=other_headers)).status_code == 404 + + async with case.owner.begin() as session: + await session.execute( + text( + "INSERT INTO run_policy_blocks(tenant_id,project_id,environment_id,run_id,reason_code) " + "SELECT tenant_id,project_id,environment_id,id,'WV-LEGACY-UNAVAILABLE' FROM runs WHERE id=:run" + ), + {"run": run.id}, + ) + blocked = await client.get(prefix + "/incidents/" + identifier, headers=headers) + assert blocked.status_code == 200 and "ETag" not in blocked.headers + assert blocked.json() == {"id": identifier, "unavailable": True, "omissions": UNAVAILABLE} + + async with WeaveClient( + "https://platform.example", lambda: tokens[0], case.scope, transport=client._transport + ) as sdk: + # Repeated statuses reach the server as repeated parameters, not only the last value. + found = await sdk.list_incidents(query=IncidentQuery(status=["active", "closed"], include_test=True)) + assert {item.id for item in found.items} >= {UUID(identifier)} + hidden = await sdk.list_incidents(query=IncidentQuery(status=["closed", "resolved"])) + assert UUID(identifier) not in {item.id for item in hidden.items} + direct = await sdk.read_incident(UUID(identifier)) + assert isinstance(direct, UnavailableResource) + expected = await expected_order(case, "id", run_id=other_run.id) + seeded = await sdk.run_incidents(other_run.id, query=IncidentQuery(order="id", limit=2)) + assert [str(item.id) for item in seeded.items] == expected[:2] and seeded.next_cursor is not None + with pytest.raises(ValueError): + await sdk.list_incidents(limit=10, query=IncidentQuery()) From e5e80da19c0cf90104943f96b9148cea87b10ab7 Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Sun, 11 Oct 2026 19:44:08 -0700 Subject: [PATCH 10/63] Keep run history replayable with the recorded start origin New runs record how they started as `origin` on their `started` event, but history pages and exports treated that field as unclassified. They withheld it with a `classification_unavailable` omission, so replay stopped at the first event with WV-REPLAY-REDACTED and reported the history of every new run as incomplete. Disclose `origin` on `started` events when it is one of the run origins and keep withholding any other value, so replay verifies these runs again. --- CHANGELOG.md | 3 ++ src/firefly_weave/operations/exports.py | 14 ++++++- tests/integration/test_history.py | 3 ++ tests/unit/operations/test_replay.py | 49 +++++++++++++++++++++++++ 4 files changed, 67 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2b43042b..756c58ad 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -38,6 +38,9 @@ SPDX-License-Identifier: Apache-2.0 - The Python SDK's `list_runs` rejects a `query` combined with keyword filters. - Request telemetry records authorized `501 WV-UNAVAILABLE` answers with the status `unavailable` instead of `failed`. +- A new run's `started` history event records how the run started as `origin`, + the value run summaries report. History pages and exports include it, and + replay accepts it. - Serve incident lists from incident facts without loading runs, newest opened first. They filter by status, code, workflow, step, opening time, and test runs (left out unless `include_test=true`). Explicit `order=id` continues diff --git a/src/firefly_weave/operations/exports.py b/src/firefly_weave/operations/exports.py index 362ce1a3..2b1b88d8 100644 --- a/src/firefly_weave/operations/exports.py +++ b/src/firefly_weave/operations/exports.py @@ -17,18 +17,21 @@ """Pure safe evidence projection and stable comparison; no storage or executors.""" import json -from typing import cast +from typing import Final, cast, get_args from firefly_weave.compiler.api import CompiledArtifact from firefly_weave.compiler.canonical import canonical_digest from firefly_weave.compiler.ir import ActionNode, HumanTaskNode, SignalNode from firefly_weave.contracts.definitions import ActionDefinition, load_definition +from firefly_weave.contracts.run_views import RunOrigin from firefly_weave.contracts.values import JsonObject, JsonValue from firefly_weave.contracts.workers import TaskError from firefly_weave.operations.redaction import Omission, project from firefly_weave.runtime.kernel import action_schemas, workflow from firefly_weave.runtime.models import RuntimeEvent, Transition +RUN_ORIGINS: Final = frozenset(get_args(RunOrigin.__value__)) + def lock_digest(artifact: CompiledArtifact) -> str: return canonical_digest({"dependencies": artifact.executable["dependencies"]}) @@ -54,7 +57,7 @@ def safe_event(event: RuntimeEvent, artifact: CompiledArtifact | None, *, unavai data = dict(event.data) omissions: list[Omission] = [] fields = { - "started": {"admission_policy"}, + "started": {"admission_policy", "origin"}, "human_completed": {"node_id", "task_id", "actor_id", "output"}, "paused": {"actor_id", "reason", "control_revision"}, "resumed": {"actor_id", "reason", "control_revision"}, @@ -78,6 +81,13 @@ def safe_event(event: RuntimeEvent, artifact: CompiledArtifact | None, *, unavai "wait_elapsed": {"node_id", "deadline", "wait_id"}, "signal_received": {"node_id", "output", "accepted_at", "deadline"}, }[event.type] + if ( + event.type == "started" + and "origin" in data + and not (type(data["origin"]) is str and data["origin"] in RUN_ORIGINS) + ): + # Only the server-recorded closed set is disclosed; any other retained value stays withheld. + fields = fields - {"origin"} if any(key not in fields for key in data): data = {key: value for key, value in data.items() if key in fields} omissions.append(Omission(path="/data/*", reason="classification_unavailable")) diff --git a/tests/integration/test_history.py b/tests/integration/test_history.py index 1bed084f..cafd51e9 100644 --- a/tests/integration/test_history.py +++ b/tests/integration/test_history.py @@ -34,6 +34,9 @@ async def test_history_export_and_concurrent_append( first = await client.get(url + "/history?limit=1", headers=headers) assert first.status_code == 200, first.text assert first.json()["high_water_sequence"] == 1 + started = first.json()["events"][0] + assert started["data"] == {"admission_policy": "classified-v1", "origin": "manual"} + assert started["evidence"]["omissions"] == [] async with transaction_factory() as tx: lease = (await task_service.claim(tx, worker_ids[0], 1))[0] await task_service.complete(tx, lease.proof, uuid4(), 9) diff --git a/tests/unit/operations/test_replay.py b/tests/unit/operations/test_replay.py index dd74fe24..d12d84fc 100644 --- a/tests/unit/operations/test_replay.py +++ b/tests/unit/operations/test_replay.py @@ -19,10 +19,13 @@ import json import socket from datetime import UTC, datetime +from typing import get_args from uuid import uuid4 import pytest +from firefly_weave.contracts.run_views import RunOrigin + NOW = datetime(2026, 1, 1, tzinfo=UTC) @@ -128,6 +131,52 @@ def test_required_redaction_stops_at_prefix(worker_runtime_fixture, recorded_eve assert report.status == "incomplete" and report.last_verified_sequence == 1 +def recorded_start(artifact, recorded_events, origin): + from firefly_weave.operations.exports import transition_digest + from firefly_weave.runtime.kernel import transition + from firefly_weave.runtime.models import RunState + + # The runtime records how a run started next to its admission policy. + started = recorded_events[0].model_copy(deep=True) + started.data["origin"] = origin + change = transition(RunState(input=3), started.model_copy(update={"evidence": None}), artifact) + started.evidence["expected_transition_digest"] = transition_digest(change) + return started + + +@pytest.mark.parametrize("origin", sorted(get_args(RunOrigin.__value__))) +def test_recorded_start_origin_is_disclosed_and_replays(worker_runtime_fixture, recorded_events, origin): + from firefly_weave.operations.exports import safe_event + from firefly_weave.operations.replay import replay + + artifact = worker_runtime_fixture["artifact"] + events = (recorded_start(artifact, recorded_events, origin), recorded_events[1]) + disclosed = tuple(safe_event(event, artifact) for event in events) + assert disclosed[0].data == {"admission_policy": "classified-v1", "origin": origin} + assert disclosed[0].evidence["omissions"] == [] + for supplied in (events, disclosed): + report = replay(artifact, supplied) + assert (report.status, report.last_verified_sequence) == ("consistent", 2), report + + +@pytest.mark.parametrize( + "origin", ["Manual", "private operator note", "", None, 1, True, ["manual"], {"origin": "manual"}] +) +def test_unrecognized_start_origin_stays_withheld(worker_runtime_fixture, recorded_events, origin): + from firefly_weave.operations.exports import safe_event + from firefly_weave.operations.replay import replay + + artifact = worker_runtime_fixture["artifact"] + events = (recorded_start(artifact, recorded_events, origin), recorded_events[1]) + withheld = safe_event(events[0], artifact) + assert withheld.data == {"admission_policy": "classified-v1"} + assert withheld.evidence["omissions"] == [{"path": "/data/*", "reason": "classification_unavailable"}] + for supplied in (events, (withheld, events[1])): + report = replay(artifact, supplied) + assert (report.status, report.last_verified_sequence) == ("incomplete", 0), report + assert [(d.code, d.sequence) for d in report.diagnostics] == [("WV-REPLAY-REDACTED", 1)] + + @pytest.mark.parametrize( "mutation", ["artifact", "lock", "duplicate", "transition", "task", "generation", "boolean-generation", "node"] ) From 933e7b58867e6e8a0fdb13f2a89d387a13446389 Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Sun, 11 Oct 2026 21:39:16 -0700 Subject: [PATCH 11/63] Serve control-step incidents and withhold mismatched identities Incidents opened on a switch or parallel branch output or on a join come from control steps, which incident facts deliberately record without a step ID. Incident lists and reads treated that as an identity that could not be normalized and served these incidents as unavailable placeholders without an ETag, so they could not be shown or resolved. Serve them with the control-step ID stored on the incident and no instance key. Keep withholding every other disagreement, including facts that name a different step or instance than the incident's own key. Document that CLIs and SDKs must be upgraded with the server, which payloads keep their bytes, and that a resolve answer is the stored incident, so clients read it again. --- CHANGELOG.md | 9 +- docs/operations/upgrades.md | 25 ++-- docs/reference/api.md | 14 ++- src/firefly_weave/contracts/operations.py | 1 + src/firefly_weave/operations/fact_reads.py | 26 +++- tests/integration/test_incident_fact_reads.py | 118 ++++++++++++++++++ .../unit/operations/test_incident_identity.py | 118 ++++++++++++++++++ 7 files changed, 293 insertions(+), 18 deletions(-) create mode 100644 tests/unit/operations/test_incident_identity.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 756c58ad..a49564c3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -50,7 +50,14 @@ SPDX-License-Identifier: Apache-2.0 `read_incident`) serves direct incident links and sends the revision as `ETag`. - Incident views add optional `opened_at`, `opened_at_estimated`, `updated_at`, `workflow`, `run_status`, `test`, and `instance_key`; for a repeated step - instance, `node_id` is the static step ID. Earlier payloads keep their bytes. + instance, `node_id` is the static step ID. Upgrade CLIs and SDKs with the + server: earlier releases reject the new fields and fail with `502 WV-WIRE` on + incident lists. Resolution receipts, `incidents.resolve` answers, and the + earlier plain array of a run's incidents keep their bytes. A resolve answer is + the incident as stored, so read it again with `incidents.read`. +- An incident on a switch or parallel branch output or a join keeps its + control-step `node_id` in incident lists and reads. An incident whose step + does not match its incident facts is withheld as unavailable. - The Python SDK's `list_incidents` and `run_incidents` accept a typed `query` and reject one combined with keyword arguments. diff --git a/docs/operations/upgrades.md b/docs/operations/upgrades.md index 22432e2e..4683f287 100644 --- a/docs/operations/upgrades.md +++ b/docs/operations/upgrades.md @@ -45,7 +45,7 @@ the desktop app never migrates a platform. | alpha13 | `0030_worker_presence` | No new schema migration. Workers retry explicit capacity rejections while they read task context or credentials, and an Agentic preparation timeout before provider execution counts as not started. Upgrade the Agentic and Files workers to 0.1.5 together with the server. | | alpha14 | `0030_worker_presence` | No new schema migration. Adds the detached Docker development platform (`weave platform up`); existing foreground installations keep working and are never converted. The Agentic and Files workers 0.1.6 pin this server version. | | alpha15 | `0030_worker_presence` | No new schema migration and no role change. Upgrade CLIs and SDKs together with the server, review native executor `capacity` and the legacy private-network settings before restarting, and finish runs that use text templates before any rollback to alpha14; see [what changes in alpha15](#what-changes-in-alpha15). The Agentic and Files workers 0.1.7 pin this server version. | -| Unreleased | `0031_operations_facts` | Adds [scoped operational fact storage](#operational-fact-storage), a bounded historical backfill, and reserved capacity for terminal fact updates. | +| Unreleased | `0031_operations_facts` | Adds [scoped operational fact storage](#operational-fact-storage), a bounded historical backfill, and reserved capacity for terminal fact updates. Upgrade CLIs and SDKs together with the server; see [incident list compatibility](#incident-list-compatibility). | ![Schema, compatibility and execution acceptance gates](../diagrams/operations-upgrade.svg) @@ -214,14 +214,23 @@ The incident lists also change how they read requests: ID order. Upgrade servers before relying on them. Incident views gain optional fields that are left out while unknown, false, or -empty, so earlier responses and stored resolution receipts keep their bytes. +empty. **Upgrade CLIs and SDKs with the server.** Earlier releases reject the +new fields: `weave runs incidents list` and `run-list`, and the Python SDK's +`list_incidents` and `run_incidents`, fail with `502 WV-WIRE` on the first page +that lists an available incident. Stored resolution receipts and +`incidents.resolve` answers keep their bytes, and so does the earlier plain +array of a run's incidents. A resolve answer is the incident as stored, so read +it again with `incidents.read` to show it as the lists do. + For an incident of a repeated step instance, `node_id` becomes the static step -ID and `instance_key` carries the full key; other incidents keep the same -`node_id`. Whether an incident is withheld now comes from its run's metadata -and policy blocks, so a list never loads runs. The new `incidents.read` -operation serves direct links and sends the revision as `ETag`. The Python -SDK adds `read_incident`, and `list_incidents` and `run_incidents` take either -a `query` or the keyword arguments; passing both raises `ValueError`. +ID and `instance_key` carries the full key; other incidents, including those on +a switch or parallel branch output or a join, keep the same `node_id`. Whether +an incident is withheld now comes from its run's metadata, its policy blocks, +and whether its step matches its incident facts, so a list never loads runs. +The new `incidents.read` operation serves direct links and sends the revision +as `ETag`. The Python SDK adds `read_incident`, and `list_incidents` and +`run_incidents` take either a `query` or the keyword arguments; passing both +raises `ValueError`. Historical opening times come from each incident's opening event. When that event was not retained, the upgrade used the run's last recorded event and diff --git a/docs/reference/api.md b/docs/reference/api.md index bc74575b..c7591c73 100644 --- a/docs/reference/api.md +++ b/docs/reference/api.md @@ -485,9 +485,17 @@ incident for a direct link and sends its revision as the `ETag`, ready for opening time taken from the run's last recorded event because the incident's own opening event was not retained; an unknown opening time is never estimated. -- An incident whose run metadata cannot be classified, or whose run has a - recorded policy block, is an unavailable placeholder. `incidents.read` then - sends no `ETag`. +- An incident on a control step, a switch or parallel branch output + (`@branch:…`) or a join (`@join:…`), keeps that control-step ID as `node_id` + and has no `instance_key`. The `node_id` filter takes author step IDs, so it + does not select these incidents. +- An incident whose run metadata cannot be classified, whose run has a recorded + policy block, or whose step does not match its incident facts is an + unavailable placeholder. `incidents.read` then sends no `ETag`. +- `incidents.resolve` answers with the incident as stored, without the added + fields; for a repeated step instance, its `node_id` is the full instance key. + Read the incident again with `incidents.read` after resolving to show it as + the lists do. - A page is limited to 8 MiB of retained incident data; a larger page answers `429` with `WV-PAGE-LIMIT`. Unknown query parameters are rejected, and a `limit` outside 1 to 100 answers `422` with `WV-VALIDATION`. diff --git a/src/firefly_weave/contracts/operations.py b/src/firefly_weave/contracts/operations.py index 1025894f..48b789eb 100644 --- a/src/firefly_weave/contracts/operations.py +++ b/src/firefly_weave/contracts/operations.py @@ -55,6 +55,7 @@ class IncidentWorkflow(ContractModel): class IncidentView(ContractModel): """An incident. Read from operational facts, `node_id` is the static step ID and `instance_key` the full key. + An incident on a control step (a branch output or a join) keeps its stored `@` step ID and has no instance key. The fields after `external_effects_may_continue` are additive: each is left out while it is unknown, false or empty, so earlier payloads keep their bytes. `opened_at_estimated` marks an opening time inferred from the run's last recorded event because the incident's own opening event was not retained. diff --git a/src/firefly_weave/operations/fact_reads.py b/src/firefly_weave/operations/fact_reads.py index 7f86714e..c87bea55 100644 --- a/src/firefly_weave/operations/fact_reads.py +++ b/src/firefly_weave/operations/fact_reads.py @@ -253,22 +253,36 @@ def _json(value: object) -> str: return value.isoformat() if isinstance(value, datetime) else str(value) +def _step_identity(source: str | None, static: str | None, instance: str | None) -> tuple[str | None, str] | None: + """The served ``node_id`` and ``instance_key``, or None when the incident and its facts disagree.""" + if source is None: + # A run-level incident has no step facts. + return (None, "") if static is None and not instance else None + if static is None: + # Facts never attribute a control step (a branch output or join) to an author step; it keeps its own ID. + return (source, "") if source.startswith("@") and not instance else None + # The facts normalized the incident's own key: its full instance key, or the static step ID for a plain step. + return (static, instance or "") if source == (instance or static) else None + + def incident_of(row: Mapping[str, Any]) -> IncidentView | UnavailableResource: """An incident from its own columns and its facts; withheld unless its run's metadata is available. - The facts carry the static step ID and full instance key normalized when they were written. An incident whose - facts are missing, or whose step identity could not be normalized, is unavailable rather than guessed. + The facts carry the static step ID and full instance key normalized when they were written; a control step keeps + the step ID stored on the incident. An incident whose facts are missing, or whose step identity disagrees with its + facts, is unavailable rather than guessed. """ identifier = row["id"] - source = row["node_id"] - if not row["projected"] or not available(row) or (source is None) != (row["static_node_id"] is None): + identity = _step_identity(row["node_id"], row["static_node_id"], row["instance_key"]) + if not row["projected"] or not available(row) or identity is None: return unavailable(identifier) + node_id, instance_key = identity resolution = row["resolution"] values = dict( id=identifier, run_id=row["run_id"], incident_key=row["incident_key"], - node_id=row["static_node_id"], + node_id=node_id, generation=row["generation"], origin_code=row["origin_code"], code=row["code"], @@ -278,7 +292,7 @@ def incident_of(row: Mapping[str, Any]) -> IncidentView | UnavailableResource: resolved_at=row["resolved_at"], resolution=resolution, external_effects_may_continue=isinstance(resolution, dict) and resolution.get("kind") == "terminate", - instance_key=row["instance_key"], + instance_key=instance_key, opened_at=row["opened_at"], # A time is only estimated when it is known; an unknown opening time stays unknown. opened_at_estimated=row["opened_at_estimated"] is True and row["opened_at"] is not None, diff --git a/tests/integration/test_incident_fact_reads.py b/tests/integration/test_incident_fact_reads.py index 92c280ad..f501e7fe 100644 --- a/tests/integration/test_incident_fact_reads.py +++ b/tests/integration/test_incident_fact_reads.py @@ -500,6 +500,9 @@ async def answers(): [ # A malformed historical identity cannot be normalized into a static step. "UPDATE incident_facts SET node_id=NULL WHERE incident_id=:id", + # Facts that name another step or instance disagree with the incident's own step. + "UPDATE incident_facts SET node_id='other' WHERE incident_id=:id", + "UPDATE incident_facts SET instance_key='work[2]' WHERE incident_id=:id", # A missing projection does not mean the incident never existed. "DELETE FROM incident_facts WHERE incident_id=:id", ], @@ -561,6 +564,121 @@ async def test_instance_keys_and_run_level_incidents_keep_their_identity(operati assert isinstance(unknown, IncidentView) and unknown.opened_at is None and not unknown.opened_at_estimated +async def start_branch_failure(case): + """A run whose parallel branch output reads a missing input field: the incident opens on a control step.""" + import json + + from firefly_weave.contracts.catalog import ActivationRequest + from firefly_weave.contracts.runtime import StartRunRequest + + branch = { + "steps": [{"id": "copy", "kind": "transform", "value": {"literal": 3}}], + "output": {"ref": "/input/missing"}, + } + document = { + "apiVersion": "weave/v1alpha1", + "kind": "Workflow", + "metadata": {"name": "branch-flow", "version": "1.0.0"}, + "spec": { + "inputSchema": {}, + "outputSchema": {}, + "steps": [{"id": "fork", "kind": "parallel", "concurrency": 1, "branches": {"one": branch}}], + "output": {"literal": None}, + }, + } + definitions = case.runtime.definitions + published = await definitions.publish( + case.actor, case.scope, "Workflow", json.dumps(document), "json", str(uuid4()), context=AuditContext() + ) + activation = await definitions.activate( + case.actor, + case.scope, + ActivationRequest(scope=case.scope, version_id=published.id, artifact_digest=published.digest), + str(uuid4()), + context=AuditContext(), + ) + return await case.runtime.start( + case.actor, + case.scope, + StartRunRequest(activation_id=activation.id, input={}), + str(uuid4()), + context=AuditContext(), + ) + + +async def test_control_step_incidents_are_listed_read_and_resolved( + operations_case, authenticated_client, access_db, provisioned +): + from firefly_weave.access.models import Grant + + case = operations_case + run = await start_branch_failure(case) + assert run.state.status == "suspended" + [incident] = await case.rows("SELECT * FROM incidents WHERE run_id=:run", run=run.id) + assert (incident["incident_key"], incident["node_id"], incident["code"]) == ( + "@branch:fork:0:0", + "@branch:fork:0", + "WV-EXPR-MISSING", + ) + # The projection never attributes a control step to an author step. + assert await case.rows( + "SELECT node_id,instance_key FROM incident_facts WHERE incident_id=:id", id=incident["id"] + ) == [{"node_id": None, "instance_key": ""}] + service = incidents(case) + for run_id in (None, run.id): + page = await service.list_all( + case.actor, case.scope, run_id=run_id, query=IncidentQuery(), position=None, context=AuditContext() + ) + [item] = page["items"] + assert "unavailable" not in item, item + assert (item["id"], item["node_id"], item["code"], item["revision"], item["run_status"]) == ( + str(incident["id"]), + "@branch:fork:0", + "WV-EXPR-MISSING", + incident["revision"], + "suspended", + ) + assert "instance_key" not in item and item["workflow"] == {"name": "branch-flow", "version": "1.0.0"} + # The step filter takes static author step IDs, which a control-step incident does not have. + assert await walk(case, {"node_id": "fork"}) == [] + + client, tokens, scopes = authenticated_client + async with access_db[1]() as session: + subjects = { + row.subject: row.principal_id + for row in (await session.execute(text("SELECT subject,principal_id FROM identity_links"))).all() + } + await access_db[2].grant(provisioned[0], subjects["0"], Grant(role="operator", scope=scopes[0])) + headers = {"Authorization": "Bearer " + tokens[0]} + path = ( + f"/api/v1/tenants/{case.scope.tenant_id}/projects/{case.scope.project_id}" + f"/environments/{case.scope.environment_id}/incidents/{incident['id']}" + ) + read = await client.get(path, headers=headers) + assert read.status_code == 200, read.text + assert read.headers["ETag"] == f'"{incident["revision"]}"' + assert (read.json()["node_id"], read.json()["code"]) == ("@branch:fork:0", "WV-EXPR-MISSING") + assert "instance_key" not in read.json() + + resolution = IncidentResolution(kind="terminate", receipt_id=uuid4(), reason="branch output cannot be mapped") + resolved = await client.post( + path + "/resolve", + content=resolution.model_dump_json(exclude_unset=True), + headers={**headers, "If-Match": read.headers["ETag"], "Content-Type": "application/json"}, + ) + assert resolved.status_code == 200, resolved.text + revision = resolved.json()["revision"] + assert resolved.json()["status"] == "resolved" and revision > incident["revision"] + after = await client.get(path, headers=headers) + assert after.status_code == 200 and after.headers["ETag"] == f'"{revision}"' + assert (after.json()["node_id"], after.json()["status"], after.json()["revision"]) == ( + "@branch:fork:0", + "resolved", + revision, + ) + assert after.json()["resolution"]["kind"] == "terminate" and after.json()["external_effects_may_continue"] + + @pytest.mark.parametrize("order", ["id", "opened_desc"]) async def test_incident_pages_keep_the_logical_byte_ceiling(operations_case, order): from firefly_weave.persistence.paging import PAGE_BYTES diff --git a/tests/unit/operations/test_incident_identity.py b/tests/unit/operations/test_incident_identity.py new file mode 100644 index 00000000..f2e12207 --- /dev/null +++ b/tests/unit/operations/test_incident_identity.py @@ -0,0 +1,118 @@ +# Copyright 2026 Firefly Software Foundation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# Author: Firefly Software Foundation +# SPDX-License-Identifier: Apache-2.0 + +"""An incident's step identity comes from the incident and its facts; a disagreement is withheld, never guessed.""" + +from datetime import UTC, datetime +from uuid import uuid4 + +import pytest + +from firefly_weave.compiler.ir import accepted_ir_versions +from firefly_weave.contracts import language_features +from firefly_weave.contracts.operations import IncidentView +from firefly_weave.contracts.public import UnavailableResource +from firefly_weave.operations.fact_reads import incident_of +from firefly_weave.runtime import admission + +AT = datetime(2026, 10, 1, tzinfo=UTC) + + +def detail(source, static, instance="", **changes): + """A selected incident as the detail statement returns it, on a run whose metadata is available.""" + return { + "id": uuid4(), + "run_id": uuid4(), + "incident_key": source or "@legacy", + "node_id": source, + "generation": 0, + "origin_code": "WV-EXPR-MISSING", + "code": "WV-EXPR-MISSING", + "status": "active", + "revision": 1, + "actor_id": None, + "resolved_at": None, + "resolution": None, + "oversized": False, + "projected": True, + "static_node_id": static, + "instance_key": instance, + "opened_at": AT, + "updated_at": AT, + "opened_at_estimated": False, + "workflow_name": "flow", + "workflow_version": "1.0.0", + "test": False, + "run_status": "suspended", + "classification_state": "available", + "classification_policy": admission.POLICY, + "pinned_ir_version": accepted_ir_versions(language_features.ADVERTISED_FEATURES)[0], + "pinned_features": [], + "policy_blocked": False, + **changes, + } + + +@pytest.mark.parametrize( + "source,static,instance,served", + [ + ("work", "work", "", ("work", "")), + ("work[2]", "work", "work[2]", ("work", "work[2]")), + ("work[1]#3.review", "work", "work[1]#3.review", ("work", "work[1]#3.review")), + (None, None, "", (None, "")), + # Switch and parallel branch outputs and joins are control steps: their facts carry no step ID. + ("@branch:fork:0", None, "", ("@branch:fork:0", "")), + ("@join:fork", None, "", ("@join:fork", "")), + ], +) +def test_incident_identity_is_served_from_matching_facts(source, static, instance, served): + view = incident_of(detail(source, static, instance)) + assert isinstance(view, IncidentView), view + assert (view.node_id, view.instance_key) == served + assert view.code == "WV-EXPR-MISSING" and view.revision == 1 + if not served[1]: + assert "instance_key" not in view.model_dump(mode="json") + + +@pytest.mark.parametrize( + "source,static,instance", + [ + # An author step whose facts lost the step ID could not be normalized. + ("work", None, ""), + ("work[2]", None, "work[2]"), + # A run-level incident never has step facts. + (None, "work", ""), + (None, None, "work[2]"), + # A control step is never attributed to an author step or an instance. + ("@branch:fork:0", "fork", ""), + ("@branch:fork:0", None, "@branch:fork:0"), + # Both identities are present but disagree. + ("work[2]", "work", ""), + ("work", "other", ""), + ("work[2]", "work", "work[3]"), + ("work", "work", "work[2]"), + ], +) +def test_incident_identity_that_disagrees_with_its_facts_is_withheld(source, static, instance): + row = detail(source, static, instance) + view = incident_of(row) + assert isinstance(view, UnavailableResource) and view.id == row["id"] + assert [omission.reason for omission in view.omissions] == ["classification_unavailable"] + + +@pytest.mark.parametrize("changes", [{"projected": False}, {"policy_blocked": True}]) +def test_control_step_incident_still_needs_facts_and_available_run_metadata(changes): + assert isinstance(incident_of(detail("@branch:fork:0", None, **changes)), UnavailableResource) From a1f7543a5391d47617c482e31b585cb6013f304c Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Sun, 11 Oct 2026 22:30:56 -0700 Subject: [PATCH 12/63] Report bounded worker identity and observed placement members --- CHANGELOG.md | 18 + docs/guides/workers.md | 29 ++ docs/operations/operations-runner.md | 13 +- docs/operations/upgrades.md | 40 ++- docs/reference/api.md | 31 ++ docs/reference/sdk.md | 11 + src/firefly_weave/connectors/dispatcher.py | 10 +- src/firefly_weave/contracts/deployments.py | 21 ++ src/firefly_weave/contracts/schema_export.py | 4 + src/firefly_weave/contracts/workers.py | 58 ++- .../deployment_runner/compose.py | 52 ++- .../deployment_runner/kubernetes.py | 103 +++++- .../deployment_runner/runtime.py | 24 ++ src/firefly_weave/sdk/client.py | 9 +- src/firefly_weave/sdk/worker_identity.py | 127 +++++++ src/firefly_weave/sdk/worker_registration.py | 74 ++++ src/firefly_weave/workers/models.py | 21 +- src/firefly_weave/workers/repository.py | 30 +- src/firefly_weave/workers/service.py | 4 +- .../test_worker_identity_contract.py | 335 ++++++++++++++++++ tests/integration/test_compose_operations.py | 18 + .../test_operations_fact_writes.py | 235 ++++++++++++ tests/unit/connectors/test_local_build.py | 21 ++ tests/unit/sdk/test_worker_identity.py | 221 ++++++++++++ tests/unit/sdk/test_worker_registration.py | 268 ++++++++++++++ tests/unit/test_deployment_compose.py | 185 ++++++++++ tests/unit/test_deployment_kubernetes.py | 249 ++++++++++++- .../agentic/src/weave_agentic_worker/main.py | 16 +- workers/agentic/tests/test_main.py | 153 ++++++++ workers/files/src/weave_files_worker/main.py | 16 +- workers/files/tests/test_files_main.py | 157 ++++++++ 31 files changed, 2520 insertions(+), 33 deletions(-) create mode 100644 src/firefly_weave/sdk/worker_identity.py create mode 100644 src/firefly_weave/sdk/worker_registration.py create mode 100644 tests/contracts/test_worker_identity_contract.py create mode 100644 tests/unit/sdk/test_worker_identity.py create mode 100644 tests/unit/sdk/test_worker_registration.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 25066bc5..1271b7ca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -60,6 +60,24 @@ SPDX-License-Identifier: Apache-2.0 does not match its incident facts is withheld as unavailable. - The Python SDK's `list_incidents` and `run_incidents` accept a typed `query` and reject one combined with keyword arguments. +- Workers report where they run when they register: an optional, bounded + `identity` with host, runtime, versions, platform, Kubernetes namespace and + node, and up to 8 labels from `WEAVE_WORKER_LABELS` + (`WEAVE_WORKER_REPORT_HOST=false` leaves out the host). The Agentic and Files + workers, native executors and the SDK's `register_worker` send it, and retry + once without it when an older server answers `422 WV-VALIDATION`. New SDK + helpers: `worker_identity` and `register_worker_http`. +- Worker reads, lists, and drain or resume answers add optional `identity` and + `registered_at`. Upgrade CLIs and SDKs with the server: earlier releases fail + with `502 WV-WIRE` on workers registered after the upgrade. The registration + answer and stored worker records keep their bytes, and a worker cannot report + a placement. +- Deployment observations list up to 100 `members` of each worker component: + Compose containers and Kubernetes pods, with host, ordinal, state, readiness, + start time, restarts and node. Observations without members keep their bytes + and digests. Kubernetes runners need `list` on the namespace's pods to + observe worker components; a worker component with more than 100 members + fails the observation instead of being truncated. - Keep typing in the selected form field when deferred row autofocus completes. Rapid row additions focus the newest item without overriding a later choice. diff --git a/docs/guides/workers.md b/docs/guides/workers.md index 5525eace..68cfe76b 100644 --- a/docs/guides/workers.md +++ b/docs/guides/workers.md @@ -230,6 +230,35 @@ and stops claiming early enough to drain its declared 180-second tasks; a supervisor can restart it with fresh credentials. It is not an indefinitely refreshing token client. +### Report where a worker runs + +A worker can describe where it runs when it registers. Operators then see the +host, runtime, versions and labels on the worker, and the server can match the +host to a container or pod of an observed deployment. The identity is display +metadata: it never grants a worker more authority, and a worker cannot report +its own placement. + +The packaged Agentic and Files workers and the API's native executors send it +automatically. A worker you write sends it by registering with +`register_worker_http(client, environment_url, body)`, with +`"identity": worker_identity().model_dump(mode="json")` in the body, or with +`WeaveClient.register_worker`. Both retry once without the identity when an +older server refuses it. The checked-in example registers without one. + +| Variable | Effect | +| --- | --- | +| `WEAVE_WORKER_REPORT_HOST` | `false` leaves out the hostname (the pod name in Kubernetes); default `true` | +| `WEAVE_WORKER_LABELS` | Up to 8 `key=value` pairs separated by commas, for example `team=payments,tier=gold` | +| `WEAVE_POD_NAMESPACE`, `WEAVE_NODE_NAME` | The pod's namespace and node, set from the Kubernetes downward API | +| `KUBERNETES_SERVICE_HOST` | Set by Kubernetes; the worker then reports a `container` runtime and the `kubernetes` platform | + +Labels are visible to everyone who can read workers. Never put credentials in +them: keys that name a token, password, secret or authorization are refused, +and an invalid label stops the worker before it registers (for native +executors, it stops the API's executor start), with a message that does not +repeat the value. An invalid hostname or platform value is left out +instead. + ## 4. Observe one task through completion Start a workflow that calls the published Action. Inspect its run history and diff --git a/docs/operations/operations-runner.md b/docs/operations/operations-runner.md index 89e9a0bf..60377579 100644 --- a/docs/operations/operations-runner.md +++ b/docs/operations/operations-runner.md @@ -57,7 +57,9 @@ substantial host authority: isolate the runner and keep its provider permissions within the destination it operates. For Kubernetes, grant only the namespace and resource permissions needed by the allowed deployments. API-component updates also require reading that namespace's horizontal pod autoscalers so the runner -can reject a conflicting scaling policy. +can reject a conflicting scaling policy. Observing a worker component also lists +the namespace's pods (`list` on `pods`), so grant that read before upgrading +the runner; without it, observing a target with a worker component fails. ## 2. Copy the destination's actual identity @@ -236,6 +238,15 @@ The adapter changes only the selected container's image/resources, replicas, and its ownership label. A worker scaling plan does not change the image. Objects already labeled as belonging to another Weave target are rejected. +For a worker component, the observation also lists the deployment's pods as +members, read with the deployment's own `matchLabels` selector inside the +pinned namespace. A pod counts only when its controlling ReplicaSet is named +after the deployment and its pod template hash. More than 100 member pods, or a +selector label that is not valid Kubernetes label syntax, fails the observation +instead of reporting part of it. A Compose worker service reports its containers the same way, +using each container's ID prefix and Compose number; it also fails with more +than 100 containers. + ### Azure Container Apps Set `context` to the subscription UUID, `boundary` to the resource group, diff --git a/docs/operations/upgrades.md b/docs/operations/upgrades.md index aef919de..0d694d0f 100644 --- a/docs/operations/upgrades.md +++ b/docs/operations/upgrades.md @@ -45,7 +45,7 @@ the desktop app never migrates a platform. | alpha13 | `0030_worker_presence` | No new schema migration. Workers retry explicit capacity rejections while they read task context or credentials, and an Agentic preparation timeout before provider execution counts as not started. Upgrade the Agentic and Files workers to 0.1.5 together with the server. | | alpha14 | `0030_worker_presence` | No new schema migration. Adds the detached Docker development platform (`weave platform up`); existing foreground installations keep working and are never converted. The Agentic and Files workers 0.1.6 pin this server version. | | alpha15 | `0030_worker_presence` | No new schema migration and no role change. Upgrade CLIs and SDKs together with the server, review native executor `capacity` and the legacy private-network settings before restarting, and finish runs that use text templates before any rollback to alpha14; see [what changes in alpha15](#what-changes-in-alpha15). The Agentic and Files workers 0.1.7 pin this server version. | -| Unreleased | `0031_operations_facts` | Adds [scoped operational fact storage](#operational-fact-storage), a bounded historical backfill, and reserved capacity for terminal fact updates. Upgrade CLIs and SDKs together with the server; see [incident list compatibility](#incident-list-compatibility). | +| Unreleased | `0031_operations_facts` | Adds [scoped operational fact storage](#operational-fact-storage), a bounded historical backfill, and reserved capacity for terminal fact updates. Upgrade CLIs and SDKs together with the server; see [incident list compatibility](#incident-list-compatibility) and [worker identity compatibility](#worker-identity-compatibility). | ![Schema, compatibility and execution acceptance gates](../diagrams/operations-upgrade.svg) @@ -237,6 +237,37 @@ event was not retained, the upgrade used the run's last recorded event and marks the time `opened_at_estimated`; without either, the opening time stays unknown. +## Worker identity compatibility + +Workers can now report where they run when they register, and worker reads +return it. The change crosses versions in both directions: + +- **New workers, older server.** The Agentic and Files workers, native + executors, and the Python SDK's `register_worker` send an `identity`. A server + without it refuses the field with `422 WV-VALIDATION`; the worker then + registers exactly once more without it, on the same connection and + credentials. Other failures are not retried. Upgrade servers first to keep the + identity. +- **Older clients, new server.** Worker reads, lists, and drain or resume + answers add `identity` and `registered_at` when they are known. **Upgrade + CLIs and SDKs with the server.** Earlier releases reject the new fields: + `weave workers list`, `read`, `drain`, and `resume`, and the Python SDK's + `read_worker`, `list_workers`, `drain_worker`, and `resume_worker`, fail with + `502 WV-WIRE` for a worker registered after the upgrade. A drain or resume is + still applied; read the worker again with an upgraded client. Workers + registered before the upgrade, the registration answer, and stored worker + records keep their earlier shape. +- **Labels.** `WEAVE_WORKER_LABELS` is read only from the worker's own + environment. An invalid label stops the worker before it registers, so check + the setting before rolling out new worker images. + +Deployment observations add `members` to worker components, left out when +empty, so existing observations, their digests, and plans reviewed against them +are unchanged. Compose and Kubernetes resource versions are also unchanged. A +Kubernetes runner now lists the namespace's pods to observe a worker component: +grant `list` on `pods` in that namespace before upgrading the runner. See +[the runner guide](operations-runner.md#kubernetes). + ## Operational fact storage The `0031_operations_facts` migration adds five scoped operational projections @@ -257,9 +288,10 @@ instead of being labeled as queued. A run requiring an unsupported IR version or language feature keeps that separate classification. Historical origins remain unknown unless an explicit start, retry or matching start receipt proves them. Signal delivery receipts do not establish how a run started. Historical step -and task times are not invented, and older workers have an unknown registration -time and empty identity until observed. Each run projection records when its -coverage began. +and task times are not invented, and worker instances registered before the +upgrade keep an unknown registration time and no identity; a worker reports its +identity when it next registers. Each run projection records when its coverage +began. The new facts count toward ordinary storage usage. Active executions reserve additional capacity for bounded terminal fact updates, including allocation diff --git a/docs/reference/api.md b/docs/reference/api.md index f021bb52..c7d06996 100644 --- a/docs/reference/api.md +++ b/docs/reference/api.md @@ -684,6 +684,30 @@ incident for a direct link and sends its revision as the `ETag`, ready for `429` with `WV-PAGE-LIMIT`. Unknown query parameters are rejected, and a `limit` outside 1 to 100 answers `422` with `WV-VALIDATION`. +**Worker identity.** `workers.create` accepts an optional `identity` describing +where the worker runs: `host`, `runtime` (`embedded`, `container`, or +`process`), `sdk_version`, `language_version`, `os`, `arch`, `platform_hint` +(`kubernetes`, `docker`, or `none`), `namespace`, `node`, and up to 8 +`labels`. It is display metadata and never grants authority or placement. + +- `host`, `namespace`, and `node` are host or DNS names of at most 253 + characters. Versions, `os`, and `arch` have at most 64 letters, digits, `.`, + `_`, `+`, or `-`. +- Label keys have at most 64 characters and must not name a token, password, + secret, or authorization; values have at most 256 characters without control + characters. The whole identity is at most 4 KiB as stored JSON. +- Unknown fields, a `placement` among them, are refused with `422` + `WV-VALIDATION`; placement is derived by the server, never reported. +- The registration answer keeps its earlier shape and does not repeat the + identity. `workers.read`, `workers.list`, `workers.drain`, and + `workers.resume` return `identity` and `registered_at` (the database time of + registration) when they are known. Workers registered before the server + recorded them have neither. + +An older server refuses `identity` with `422` `WV-VALIDATION`. Current workers +and the Python SDK then register exactly once more without it, using the same +client and credentials; any other failure is not retried. + **Answers that say "unavailable".** Cancellation and task completion can return an unavailable acknowledgment when the platform cannot classify the historical data or compare payloads; task completion then includes @@ -932,6 +956,13 @@ and runner registrations belong to one environment. Child-resource reads check the target grant. Lists filter authorized target IDs before pagination; use `target_id` to select a target, and `deployment_id` to narrow plan history. An observation is a dated snapshot, not a promise of current health. +A worker component's observed resource lists up to 100 `members`, one per +container or pod: `host`, `ordinal`, `state`, `ready`, `started_at`, +`restarts`, and `node`. Compose reports the container ID prefix, which is the +container's default hostname, and its Compose number as `ordinal`; Kubernetes +reports the pod name and node. A runner refuses to report a worker component +with more than 100 members rather than truncate it. Resources without members +leave the field out, so their observations keep their digests. The server creates an immutable plan from the desired revision and a fresh, complete observation. Approval records its exact digest. Applying checks the diff --git a/docs/reference/sdk.md b/docs/reference/sdk.md index bf4ff611..2f14cf51 100644 --- a/docs/reference/sdk.md +++ b/docs/reference/sdk.md @@ -378,6 +378,17 @@ code. Paginated methods return a typed `Page`, except run history, whose belong to the client's scope. Secret leases use the `CredentialLease` model, whose ordinary serialization leaves out the resolved value. +**Worker registration.** `register_worker` sends the request's optional +`identity`, usually from `worker_identity()` in `firefly_weave.sdk.worker_identity`. +When an older server refuses it with `422` `WV-VALIDATION`, the method registers +exactly once more without it; other errors are raised unchanged. A worker that +registers with its own HTTP client calls +`register_worker_http(client, environment_url, body)` from +`firefly_weave.sdk.worker_registration` for the same behavior on that client. +`worker_identity()` reads only `WEAVE_WORKER_REPORT_HOST`, `WEAVE_WORKER_LABELS`, +`WEAVE_POD_NAMESPACE`, `WEAVE_NODE_NAME`, and `KUBERNETES_SERVICE_HOST`; see +[worker identity](../guides/workers.md#report-where-a-worker-runs). + ## Deployment Operations through the SDK Use `WeaveClient.invoke` with models from diff --git a/src/firefly_weave/connectors/dispatcher.py b/src/firefly_weave/connectors/dispatcher.py index 93b7cff0..2df5034e 100644 --- a/src/firefly_weave/connectors/dispatcher.py +++ b/src/firefly_weave/connectors/dispatcher.py @@ -36,6 +36,7 @@ from firefly_weave.contracts.workers import InstanceRequest from firefly_weave.operations.execution import ReservedSlots from firefly_weave.sdk.worker import Worker +from firefly_weave.sdk.worker_identity import worker_identity from firefly_weave.workers.repository import WorkerRepository LOCAL_BUILD_FORMAT = "weave/local-build-v1" @@ -147,6 +148,8 @@ async def open(self) -> None: raise RuntimeError("Native executors must share one build attestation") if self.image_digest is None: raise RuntimeError("Native dispatcher requires its operator-provisioned immutable image identity") + # Native executors run inside the API process; each registration reports that once. + identity = worker_identity(runtime="embedded") for entry in self.entries: async with self.service.uow.open(entry.scope) as tx: actor = await self.service.access.load_principal(entry.principal_id, tx=tx) @@ -170,7 +173,12 @@ async def open(self) -> None: instance = await self.service.tasks.workers.register_instance( actor, entry.scope, - InstanceRequest(release_id=entry.release_id, task_types=entry.task_types, capacity=entry.capacity), + InstanceRequest( + release_id=entry.release_id, + task_types=entry.task_types, + capacity=entry.capacity, + identity=identity, + ), context=AuditContext(), tx=tx, ) diff --git a/src/firefly_weave/contracts/deployments.py b/src/firefly_weave/contracts/deployments.py index b49552c6..669959f3 100644 --- a/src/firefly_weave/contracts/deployments.py +++ b/src/firefly_weave/contracts/deployments.py @@ -24,6 +24,7 @@ from firefly_weave.contracts.access import Scope from firefly_weave.contracts.catalog import Digest from firefly_weave.contracts.definitions import ContractModel +from firefly_weave.contracts.workers import WorkerHost # Names are data identifiers, never paths, shell fragments, or executable arguments. type DeploymentName = Annotated[str, Field(pattern=r"^[a-zA-Z0-9][a-zA-Z0-9_.-]{0,99}$")] @@ -110,6 +111,24 @@ class Deployment(DeploymentRequest): created_at: AwareDatetime +class ObservedMember(ContractModel): + """One container or pod of an observed worker component, as the runner saw it. + + `host` is the hostname the member's worker reports: the container ID prefix for Compose + and the pod name for Kubernetes. Members describe placement only; they never identify or + authorize a worker. + """ + + host: WorkerHost + ordinal: int | None = Field(ge=0, le=1000000) + # The adapter's own lowercase state, such as running, exited, pending or terminating. + state: Annotated[str, Field(pattern=r"^[a-z][a-z0-9_-]{0,31}$")] + ready: bool + started_at: AwareDatetime | None + restarts: int = Field(ge=0, le=2147483647) + node: WorkerHost | None + + class ObservedResource(ContractModel): name: DeploymentName external_identity: ExternalIdentity @@ -120,6 +139,8 @@ class ObservedResource(ContractModel): version: ExternalIdentity ownership: Literal["managed", "imported", "unknown"] state: Literal["ready", "progressing", "stopped", "failed", "unknown"] + # Left out when empty, so observations without members keep their bytes and digests. + members: list[ObservedMember] = Field(default_factory=list, max_length=100, exclude_if=lambda v: not v) class ObservationReport(ContractModel): diff --git a/src/firefly_weave/contracts/schema_export.py b/src/firefly_weave/contracts/schema_export.py index dbe9c89f..0c34f672 100644 --- a/src/firefly_weave/contracts/schema_export.py +++ b/src/firefly_weave/contracts/schema_export.py @@ -104,6 +104,7 @@ def contract_models() -> dict[str, SchemaModel]: DeploymentTarget, JobCancelRequest, ObservationReport, + ObservedMember, ObservedResource, ObserveRequest, PlanApproval, @@ -232,6 +233,7 @@ def contract_models() -> dict[str, SchemaModel]: TaskLease, UnavailableCompletionReceipt, WorkerControlRequest, + WorkerIdentity, WorkerInstance, WorkerRelease, WorkerStatus, @@ -248,6 +250,7 @@ def contract_models() -> dict[str, SchemaModel]: "operations-component-spec": ComponentSpec, "operations-deployment-request": DeploymentRequest, "operations-deployment": Deployment, + "operations-observed-member": ObservedMember, "operations-observed-resource": ObservedResource, "operations-observation-report": ObservationReport, "operations-deployment-observation": DeploymentObservation, @@ -391,6 +394,7 @@ def contract_models() -> dict[str, SchemaModel]: "start-run-request": StartRunRequest, "worker-release-request": ReleaseRequest, "worker-release": WorkerRelease, + "worker-identity": WorkerIdentity, "worker-instance-request": InstanceRequest, "worker-instance": WorkerInstance, "worker-status": WorkerStatus, diff --git a/src/firefly_weave/contracts/workers.py b/src/firefly_weave/contracts/workers.py index a9dec108..227cf42f 100644 --- a/src/firefly_weave/contracts/workers.py +++ b/src/firefly_weave/contracts/workers.py @@ -16,7 +16,8 @@ """Remote worker protocol. No persistence, startup or secret-provider dependencies.""" -from typing import Literal +import json +from typing import Annotated, Literal, Self from uuid import UUID from pydantic import AwareDatetime, Field, model_validator @@ -63,13 +64,61 @@ class WorkerRelease(ReleaseRequest): id: UUID +# A hostname, pod name, or Kubernetes namespace or node name: never a path, URL or shell text. +type WorkerHost = Annotated[str, Field(pattern=r"^[A-Za-z0-9][A-Za-z0-9.-]{0,252}$")] +# A reported software or platform value, such as 0.1.0a15, 3.12.4, Linux or x86_64. +type WorkerVersion = Annotated[str, Field(pattern=r"^[A-Za-z0-9][A-Za-z0-9._+-]{0,63}$")] +type WorkerLabelKey = Annotated[str, Field(pattern=r"^[A-Za-z0-9][A-Za-z0-9._/-]{0,63}$")] +type WorkerLabelValue = Annotated[str, Field(max_length=256, pattern=r"^[^\x00-\x1f\x7f-\x9f]*$")] + +# Label keys that would invite operators to put credentials into worker metadata. +CREDENTIAL_LABEL_WORDS = ("token", "password", "secret", "authorization") +# The worker_facts.identity column stores at most this many bytes of jsonb text. +WORKER_IDENTITY_BYTES = 4096 + + +def identity_text_bytes(value: object) -> int: + """The size of a JSON value as PostgreSQL renders it from jsonb, in UTF-8 bytes.""" + return len(json.dumps(value, ensure_ascii=False, separators=(", ", ": ")).encode()) + + +class WorkerIdentity(ContractModel): + """Where a worker says it runs. Display metadata only: it never grants authority or placement.""" + + host: WorkerHost | None = Field(default=None, exclude_if=lambda v: v is None) + runtime: Literal["embedded", "container", "process"] + sdk_version: WorkerVersion | None = Field(default=None, exclude_if=lambda v: v is None) + language_version: WorkerVersion | None = Field(default=None, exclude_if=lambda v: v is None) + os: WorkerVersion | None = Field(default=None, exclude_if=lambda v: v is None) + arch: WorkerVersion | None = Field(default=None, exclude_if=lambda v: v is None) + platform_hint: Literal["kubernetes", "docker", "none"] = "none" + namespace: WorkerHost | None = Field(default=None, exclude_if=lambda v: v is None) + node: WorkerHost | None = Field(default=None, exclude_if=lambda v: v is None) + labels: dict[WorkerLabelKey, WorkerLabelValue] = Field( + default_factory=dict, max_length=8, exclude_if=lambda v: not v + ) + + @model_validator(mode="after") + def bounded(self) -> Self: + if any(word in key.lower() for key in self.labels for word in CREDENTIAL_LABEL_WORDS): + raise ValueError("Worker labels cannot name credentials") + if identity_text_bytes(self.model_dump(mode="json")) > WORKER_IDENTITY_BYTES: + raise ValueError("Worker identity exceeds 4 KiB") + return self + + class InstanceRequest(ContractModel): release_id: UUID task_types: list[str] = Field(min_length=1, max_length=100) capacity: int = Field(ge=1, le=100) + # Sent by current workers; a server that predates it refuses the field, and workers then + # register once more without it. + identity: WorkerIdentity | None = Field(default=None, exclude_if=lambda v: v is None) class WorkerInstance(InstanceRequest): + """The stable registration answer and stored instance. Identity lives in the worker's facts.""" + id: UUID principal_id: UUID revoked: bool = False @@ -80,7 +129,11 @@ class WorkerControlRequest(ContractModel): class WorkerStatus(WorkerInstance): - """A scoped observation, separate from the stable worker registration protocol.""" + """A scoped observation, separate from the stable worker registration protocol. + + `identity` and `registered_at` come from the worker's facts and are left out when unknown, + as they are for workers registered before they were recorded. + """ revision: int = Field(ge=1) draining: bool @@ -91,6 +144,7 @@ class WorkerStatus(WorkerInstance): observed_at: AwareDatetime active_leases: int = Field(ge=0) available_capacity: int | None = Field(ge=0, le=100) + registered_at: AwareDatetime | None = Field(default=None, exclude_if=lambda v: v is None) class LeaseProof(ContractModel): diff --git a/src/firefly_weave/deployment_runner/compose.py b/src/firefly_weave/deployment_runner/compose.py index 48da3386..f6bf1980 100644 --- a/src/firefly_weave/deployment_runner/compose.py +++ b/src/firefly_weave/deployment_runner/compose.py @@ -27,23 +27,63 @@ from pathlib import Path from typing import Any, cast +from pydantic import ValidationError + from firefly_weave.compiler.canonical import canonical_digest -from firefly_weave.contracts.deployments import DeploymentPlan, ObservationReport, ObservedResource +from firefly_weave.contracts.deployments import DeploymentPlan, ObservationReport, ObservedMember, ObservedResource from firefly_weave.contracts.values import JsonObject from firefly_weave.deployment_runner.command import run_command from firefly_weave.deployment_runner.policy import DestinationPolicy -from firefly_weave.deployment_runner.runtime import RunnerPolicyError +from firefly_weave.deployment_runner.runtime import RunnerPolicyError, observed_time _INSPECT = ( '{"id":{{json .Id}},"image":{{json .Config.Image}},"state":{{json .State.Status}},' '"started":{{json .State.StartedAt}},"restart":{{json .RestartCount}},' '"project":{{json (index .Config.Labels "com.docker.compose.project")}},' '"service":{{json (index .Config.Labels "com.docker.compose.service")}},' + '"number":{{json (index .Config.Labels "com.docker.compose.container-number")}},' '"target":{{json (index .Config.Labels "firefly.weave.target")}},' '"health":{{with (index .State "Health")}}{{json .Status}}{{else}}""{{end}}}' ) +def _ready(fact: dict[str, Any]) -> bool: + return fact.get("state") == "running" and fact.get("health") in {"", "healthy"} + + +def _versioned(fact: dict[str, Any]) -> dict[str, Any]: + # The container number is read for members only. Leaving it out keeps each resource + # version, and the plans reviewed against it, as they were before members existed. + return {key: value for key, value in fact.items() if key != "number"} + + +def _members(facts: list[dict[str, Any]]) -> list[ObservedMember]: + """Worker containers as members; more than the contract holds refuses instead of truncating.""" + if len(facts) > 100: + raise RunnerPolicyError() + members = [] + for fact in facts: + number = fact.get("number") + if number not in (None, "") and not (isinstance(number, str) and number.isdigit() and len(number) <= 7): + raise RunnerPolicyError() + try: + members.append( + ObservedMember( + # Docker's default hostname, which the worker reports, is the ID prefix. + host=fact["id"][:12], + ordinal=int(number) if number else None, + state=fact["state"], + ready=_ready(fact), + started_at=observed_time(fact["started"]), + restarts=fact["restart"], + node=None, + ) + ) + except (KeyError, ValidationError): + raise RunnerPolicyError() from None + return members + + class ComposeAdapter: def __init__(self, policy: DestinationPolicy, *, command: Callable[..., Awaitable[bytes]] = run_command) -> None: if policy.adapter != "docker-compose" or not policy.compose_file or not policy.lock_file: @@ -132,7 +172,7 @@ async def observe(self) -> ObservationReport: image = None if not isinstance(image, str) or not self.policy.allows_image(image): image = None - ready = sum(item.get("state") == "running" and item.get("health") in {"", "healthy"} for item in members) + ready = sum(_ready(item) for item in members) desired = spec.get("deploy", {}).get("replicas", 1) stopped = not members and desired == 0 stable = stopped or (ready == len(members) == desired) @@ -148,9 +188,13 @@ async def observe(self) -> ObservationReport: image=image, replicas=len(members), ready_replicas=ready, - version=canonical_digest(cast(JsonObject, {"configuration": configuration, "members": members})), + version=canonical_digest( + cast(JsonObject, {"configuration": configuration, "members": [_versioned(m) for m in members]}) + ), ownership="managed" if managed else "imported", state="stopped" if stopped else "ready" if stable else "progressing", + # Placement only concerns workers; other components keep their observation bytes. + members=_members(members) if binding.kind == "worker" else [], ) ) return ObservationReport(resources=resources, complete=True, settled=settled) diff --git a/src/firefly_weave/deployment_runner/kubernetes.py b/src/firefly_weave/deployment_runner/kubernetes.py index 8efd12ce..780976e4 100644 --- a/src/firefly_weave/deployment_runner/kubernetes.py +++ b/src/firefly_weave/deployment_runner/kubernetes.py @@ -25,13 +25,24 @@ from decimal import Decimal from typing import Any -from firefly_weave.contracts.deployments import ComponentSpec, DeploymentPlan, ObservationReport, ObservedResource +from pydantic import ValidationError + +from firefly_weave.contracts.deployments import ( + ComponentSpec, + DeploymentPlan, + ObservationReport, + ObservedMember, + ObservedResource, +) from firefly_weave.deployment_runner.command import run_command from firefly_weave.deployment_runner.policy import ComponentBinding, DestinationPolicy -from firefly_weave.deployment_runner.runtime import RunnerPolicyError +from firefly_weave.deployment_runner.runtime import RunnerPolicyError, observed_time TARGET_LABEL = "firefly.weave/target" CONFIGURATION_ANNOTATION = "firefly.weave/configuration" +# Kubernetes label syntax, so a selector built from a deployment is only ever `key=value` pairs. +_LABEL_KEY = re.compile(r"(?:[a-z0-9](?:[-a-z0-9.]{0,251}[a-z0-9])?/)?[A-Za-z0-9](?:[-A-Za-z0-9_.]{0,61}[A-Za-z0-9])?") +_LABEL_VALUE = re.compile(r"(?:[A-Za-z0-9](?:[-A-Za-z0-9_.]{0,61}[A-Za-z0-9])?)?") def _quantity(value: Any) -> Decimal: @@ -108,7 +119,88 @@ def _container(self, value: dict[str, Any], binding: ComponentBinding) -> tuple[ raise RunnerPolicyError() return found[0] - def _fact(self, value: dict[str, Any], binding: ComponentBinding) -> ObservedResource: + def _selector(self, value: dict[str, Any]) -> dict[str, str]: + selector = value.get("spec", {}).get("selector") + labels = selector.get("matchLabels", {}) if isinstance(selector, dict) else None + if ( + not isinstance(labels, dict) + or not (labels or selector.get("matchExpressions")) + or not all( + isinstance(key, str) + and isinstance(item, str) + and _LABEL_KEY.fullmatch(key) + and _LABEL_VALUE.fullmatch(item) + for key, item in labels.items() + ) + ): + raise RunnerPolicyError() + return labels + + def _member( + self, pod: Any, deployment: str, selector: dict[str, str], binding: ComponentBinding + ) -> ObservedMember | None: + """A pod of this deployment, or None for a pod that only shares its labels. + + Pod data that does not have the shape Kubernetes reports refuses the observation. + """ + try: + metadata = pod["metadata"] + labels = metadata.get("labels") or {} + owners = [item for item in metadata.get("ownerReferences") or [] if item.get("controller") is True] + template = labels.get("pod-template-hash") + # The deployment controller names each ReplicaSet `-`. + if ( + metadata.get("namespace") != self.policy.boundary + or len(owners) != 1 + or owners[0].get("apiVersion") != "apps/v1" + or owners[0].get("kind") != "ReplicaSet" + or not isinstance(template, str) + or owners[0].get("name") != f"{deployment}-{template}" + or any(labels.get(key) != item for key, item in selector.items()) + ): + return None + status = pod.get("status") or {} + container: dict[str, Any] = next( + (item for item in status.get("containerStatuses") or [] if item.get("name") == binding.container), {} + ) + phase = status.get("phase", "Unknown") + running = (container.get("state") or {}).get("running") or {} + return ObservedMember( + host=metadata.get("name"), + ordinal=None, + state="terminating" + if metadata.get("deletionTimestamp") + else (phase.lower() if isinstance(phase, str) else ""), + ready=any( + item.get("type") == "Ready" and item.get("status") == "True" + for item in status.get("conditions") or [] + ), + started_at=observed_time(running.get("startedAt")), + restarts=container.get("restartCount", 0), + node=(pod.get("spec") or {}).get("nodeName"), + ) + except (AttributeError, KeyError, TypeError, ValidationError): + raise RunnerPolicyError() from None + + async def _members(self, value: dict[str, Any], binding: ComponentBinding) -> list[ObservedMember]: + """Pods the deployment owns, read through its own selector inside the pinned namespace.""" + selector = self._selector(value) + arguments = ["get", "pods", "-o=json"] + if selector: + arguments.insert(2, "--selector=" + ",".join(f"{key}={item}" for key, item in sorted(selector.items()))) + pods = (await self._json(*arguments)).get("items") + if not isinstance(pods, list): + raise RunnerPolicyError() + found = [self._member(pod, value["metadata"]["name"], selector, binding) for pod in pods] + members = [member for member in found if member is not None] + # The contract holds 100 members; refuse rather than report a truncated observation. + if len(members) > 100: + raise RunnerPolicyError() + return sorted(members, key=lambda member: member.host) + + def _fact( + self, value: dict[str, Any], binding: ComponentBinding, members: list[ObservedMember] | None = None + ) -> ObservedResource: _, container = self._container(value, binding) metadata, spec, status = value["metadata"], value["spec"], value.get("status", {}) desired = spec.get("replicas", 1) @@ -138,6 +230,7 @@ def _fact(self, value: dict[str, Any], binding: ComponentBinding) -> ObservedRes if metadata.get("labels", {}).get(TARGET_LABEL) == str(self.policy.target_id) else "imported", state=("stopped" if desired == 0 else "ready") if settled else "progressing", + members=members or [], ) async def observe(self) -> ObservationReport: @@ -146,7 +239,9 @@ async def observe(self) -> ObservationReport: for binding in self.policy.components: value = await self._deployment(binding) if value: - resources.append(self._fact(value, binding)) + # Placement only concerns workers; other components need no pod reads. + members = await self._members(value, binding) if binding.kind == "worker" else [] + resources.append(self._fact(value, binding, members)) return ObservationReport( resources=resources, complete=True, settled=all(item.state in {"ready", "stopped"} for item in resources) ) diff --git a/src/firefly_weave/deployment_runner/runtime.py b/src/firefly_weave/deployment_runner/runtime.py index d1c998cb..e623a716 100644 --- a/src/firefly_weave/deployment_runner/runtime.py +++ b/src/firefly_weave/deployment_runner/runtime.py @@ -19,6 +19,7 @@ from __future__ import annotations import asyncio +import re from collections.abc import Awaitable, Callable from datetime import UTC, datetime from typing import Protocol, cast @@ -41,6 +42,29 @@ def __init__(self) -> None: super().__init__("The operation does not match the runner's local authority or current observation.") +_OBSERVED_TIME = re.compile(r"(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2})(?:\.(\d{1,9}))?(Z|[+-]\d{2}:\d{2})") + + +def observed_time(value: object) -> datetime | None: + """An RFC 3339 time a provider reported, to the microsecond; its zero time means never. + + Docker reports nanoseconds and `0001-01-01T00:00:00Z` for a container that never started. + Anything else that is not such a time refuses the observation instead of guessing. + """ + if value is None: + return None + match = _OBSERVED_TIME.fullmatch(value) if isinstance(value, str) else None + if match is None: + raise RunnerPolicyError() + if match[1].startswith("0001-01-01T"): + return None + fraction = "." + match[2][:6].ljust(6, "0") if match[2] else "" + try: + return datetime.fromisoformat(match[1] + fraction + ("+00:00" if match[3] == "Z" else match[3])) + except ValueError: + raise RunnerPolicyError() from None + + class InfrastructureAdapter(Protocol): async def observe(self) -> ObservationReport: ... async def apply(self, plan: DeploymentPlan, before_write: Callable[[], Awaitable[None]]) -> ObservationReport: ... diff --git a/src/firefly_weave/sdk/client.py b/src/firefly_weave/sdk/client.py index 183b3397..16df4581 100644 --- a/src/firefly_weave/sdk/client.py +++ b/src/firefly_weave/sdk/client.py @@ -136,6 +136,7 @@ ) from firefly_weave.operations.debug.models import DebugCommand, DebugCreate, DebugSession from firefly_weave.sdk.errors import ContractError, PreconditionFailed, TransportError, WeaveError +from firefly_weave.sdk.worker_registration import negotiate_worker_registration from firefly_weave.triggers.models import Trigger, TriggerRequest if TYPE_CHECKING: @@ -567,7 +568,13 @@ async def register_release(self, request: ReleaseRequest) -> WorkerRelease: return cast(WorkerRelease, await self.invoke("releases.create", body=request)) async def register_worker(self, request: InstanceRequest) -> WorkerInstance: - return cast(WorkerInstance, await self.invoke("workers.create", body=request)) + """Register a worker; a server that predates `identity` gets one retry without it.""" + + async def send(include_identity: bool) -> WorkerInstance: + value = request if include_identity else request.model_copy(update={"identity": None}) + return cast(WorkerInstance, await self.invoke("workers.create", body=value)) + + return await negotiate_worker_registration(send, has_identity=request.identity is not None) async def read_release(self, identifier: UUID) -> WorkerRelease: return cast(WorkerRelease, await self.invoke("releases.read", identifier=identifier)) diff --git a/src/firefly_weave/sdk/worker_identity.py b/src/firefly_weave/sdk/worker_identity.py new file mode 100644 index 00000000..1dc169b6 --- /dev/null +++ b/src/firefly_weave/sdk/worker_identity.py @@ -0,0 +1,127 @@ +# Copyright 2026 Firefly Software Foundation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# Author: Firefly Software Foundation +# SPDX-License-Identifier: Apache-2.0 + +"""The identity a worker reports when it registers. + +It reads exactly five settings and never lists, logs or echoes the environment. Optional +values that are not valid (an unusual hostname, platform or Kubernetes name) are left out +instead of stopping the worker; labels the operator set explicitly fail clearly. +""" + +from __future__ import annotations + +import os +import platform +import socket +from collections.abc import Mapping +from importlib import metadata +from typing import Literal + +from pydantic import TypeAdapter, ValidationError + +from firefly_weave.contracts.workers import WorkerHost, WorkerIdentity, WorkerVersion + +REPORT_HOST = "WEAVE_WORKER_REPORT_HOST" +LABELS = "WEAVE_WORKER_LABELS" +NAMESPACE = "WEAVE_POD_NAMESPACE" +NODE = "WEAVE_NODE_NAME" +KUBERNETES = "KUBERNETES_SERVICE_HOST" + +_HOST: TypeAdapter[str] = TypeAdapter(WorkerHost) +_VERSION: TypeAdapter[str] = TypeAdapter(WorkerVersion) + + +def _bounded(adapter: TypeAdapter[str], value: object) -> str | None: + if not isinstance(value, str): + return None + try: + return adapter.validate_python(value, strict=True) + except ValidationError: + return None + + +def _report_host(raw: str | None) -> bool: + if raw is None: + return True + value = raw.strip().lower() + if value in {"true", "1", "yes"}: + return True + if value in {"false", "0", "no"}: + return False + raise ValueError(f"{REPORT_HOST} must be true or false") + + +def _labels(raw: str | None) -> dict[str, str]: + labels: dict[str, str] = {} + for item in (raw or "").split(","): + if not item.strip(): + continue + key, separator, value = item.partition("=") + key, value = key.strip(), value.strip() + if not separator or not key or key in labels: + raise ValueError(f"{LABELS} needs distinct key=value pairs separated by commas") + labels[key] = value + return labels + + +def _sdk_version() -> str | None: + try: + return metadata.version("firefly-weave") + except metadata.PackageNotFoundError: + return None + + +def worker_identity( + *, + runtime: Literal["embedded", "container", "process"] | None = None, + environ: Mapping[str, str] | None = None, +) -> WorkerIdentity: + """Describe this worker for its registration. + + `runtime` is what the caller knows about how it runs; without it a worker in a + Kubernetes pod reports `container` and any other worker `process`. The host is the + machine's hostname (the pod name in Kubernetes) unless `WEAVE_WORKER_REPORT_HOST=false`. + """ + settings = os.environ if environ is None else environ + kubernetes = bool(settings.get(KUBERNETES)) + chosen: Literal["embedded", "container", "process"] = runtime or ("container" if kubernetes else "process") + hint: Literal["kubernetes", "docker", "none"] = ( + "kubernetes" if kubernetes else "docker" if chosen == "container" else "none" + ) + host = _bounded(_HOST, socket.gethostname()) if _report_host(settings.get(REPORT_HOST)) else None + labels = _labels(settings.get(LABELS)) + try: + return WorkerIdentity( + host=host, + runtime=chosen, + sdk_version=_bounded(_VERSION, _sdk_version()), + language_version=_bounded(_VERSION, platform.python_version()), + os=_bounded(_VERSION, platform.system()), + arch=_bounded(_VERSION, platform.machine()), + platform_hint=hint, + namespace=_bounded(_HOST, settings.get(NAMESPACE)), + node=_bounded(_HOST, settings.get(NODE)), + labels=labels, + ) + except ValidationError: + pass + # Raised outside the handler so no chained validation error repeats the rejected input, + # which may be a value the operator meant to keep private. + raise ValueError( + f"{LABELS} accepts up to 8 labels. Keys have up to 64 letters, digits, '.', '_', '/' or '-' " + "and do not name a token, password, secret or authorization. Values have up to 256 " + "characters without control characters, and the whole identity stays within 4 KiB." + ) diff --git a/src/firefly_weave/sdk/worker_registration.py b/src/firefly_weave/sdk/worker_registration.py new file mode 100644 index 00000000..028f8867 --- /dev/null +++ b/src/firefly_weave/sdk/worker_registration.py @@ -0,0 +1,74 @@ +# Copyright 2026 Firefly Software Foundation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# Author: Firefly Software Foundation +# SPDX-License-Identifier: Apache-2.0 + +"""Worker registration that stays compatible with servers that predate worker identity. + +An older server refuses the unknown `identity` field with 422 `WV-VALIDATION`. Registration +then retries exactly once without it, on the same client, authentication and transport. Any +other failure, including a second refusal, a transport error or cancellation, is raised as is. +""" + +from __future__ import annotations + +from collections.abc import Awaitable, Callable, Mapping +from typing import TYPE_CHECKING + +from firefly_weave.contracts.values import JsonValue +from firefly_weave.sdk.errors import WeaveError + +if TYPE_CHECKING: + import httpx + + +def identity_rejected(error: WeaveError | httpx.HTTPStatusError) -> bool: + """Whether a registration failed the way a server without worker identity refuses it.""" + if isinstance(error, WeaveError): + return error.status == 422 and error.code == "WV-VALIDATION" + if error.response.status_code != 422: + return False + try: + body = error.response.json() + except ValueError: + return False + return isinstance(body, dict) and body.get("code") == "WV-VALIDATION" + + +async def negotiate_worker_registration[T](send: Callable[[bool], Awaitable[T]], *, has_identity: bool) -> T: + """Call `send(True)` when there is an identity, and `send(False)` once if it is refused.""" + import httpx + + try: + return await send(has_identity) + except (WeaveError, httpx.HTTPStatusError) as error: + if not has_identity or not identity_rejected(error): + raise + return await send(False) + + +async def register_worker_http( + client: httpx.AsyncClient, environment_url: str, body: Mapping[str, JsonValue] +) -> httpx.Response: + """Register through a worker's existing HTTP client; `body` itself is never changed.""" + + async def send(include_identity: bool) -> httpx.Response: + payload = dict(body) + if not include_identity: + payload.pop("identity", None) + response = await client.post(environment_url.rstrip("/") + "/workers", json=payload) + response.raise_for_status() + return response + + return await negotiate_worker_registration(send, has_identity=body.get("identity") is not None) diff --git a/src/firefly_weave/workers/models.py b/src/firefly_weave/workers/models.py index 8a047778..0142f12b 100644 --- a/src/firefly_weave/workers/models.py +++ b/src/firefly_weave/workers/models.py @@ -17,12 +17,15 @@ """Lease identities and shared safe failures.""" import hashlib +import json import secrets from dataclasses import dataclass from datetime import datetime, timedelta from typing import Any, Literal -from firefly_weave.contracts.workers import WorkerInstance, WorkerStatus +from pydantic import ValidationError + +from firefly_weave.contracts.workers import WorkerIdentity, WorkerInstance, WorkerStatus from firefly_weave.definitions.models import CatalogError from firefly_weave.persistence.uow import Transaction from firefly_weave.runtime.kernel import checked_deadline @@ -44,6 +47,16 @@ def unavailable() -> CatalogError: return CatalogError(409, "WV-LEASE", "Lease or worker authority unavailable") +def stored_identity(value: object) -> WorkerIdentity | None: + """A worker's recorded identity; empty (never reported) or unreadable reads as unknown.""" + if not value: + return None + try: + return WorkerIdentity.model_validate_json(json.dumps(value)) + except ValidationError: + return None + + def observed_worker( instance: WorkerInstance, *, @@ -52,6 +65,8 @@ def observed_worker( revision: int, active_leases: int, observed_at: datetime, + identity: WorkerIdentity | None = None, + registered_at: datetime | None = None, ) -> WorkerStatus: expires = last_seen_at + timedelta(seconds=60) if last_seen_at else None presence: Literal["unknown", "recent", "stale"] = ( @@ -61,7 +76,9 @@ def observed_worker( if draining or instance.revoked: available = 0 return WorkerStatus( - **instance.model_dump(), + **instance.model_dump(exclude={"identity"}), + identity=identity, + registered_at=registered_at, revision=revision, draining=draining, presence=presence, diff --git a/src/firefly_weave/workers/repository.py b/src/firefly_weave/workers/repository.py index 959d08c9..0da1f040 100644 --- a/src/firefly_weave/workers/repository.py +++ b/src/firefly_weave/workers/repository.py @@ -23,10 +23,16 @@ from sqlalchemy import text -from firefly_weave.contracts.workers import CompletionReceipt, WorkerInstance, WorkerRelease, WorkerStatus +from firefly_weave.contracts.workers import ( + CompletionReceipt, + WorkerIdentity, + WorkerInstance, + WorkerRelease, + WorkerStatus, +) from firefly_weave.definitions.models import CatalogError from firefly_weave.persistence.uow import Transaction -from firefly_weave.workers.models import observed_worker, unavailable +from firefly_weave.workers.models import observed_worker, stored_identity, unavailable SCOPE = "tenant_id=:tenant AND project_id=:project AND environment_id=:environment" @@ -76,9 +82,25 @@ async def observe(self, identifier: UUID) -> None: id=identifier, ) + async def record_registration(self, identifier: UUID, identity: WorkerIdentity | None) -> None: + """Record a new instance's facts in its registration transaction, never in its payload.""" + await self.execute( + "INSERT INTO worker_facts(tenant_id,project_id,environment_id,worker_id,registered_at,identity) " + "VALUES(:tenant,:project,:environment,:id,clock_timestamp(),cast(:identity AS jsonb))", + id=identifier, + identity=json.dumps({} if identity is None else identity.model_dump(mode="json")), + ) + async def status(self, identifier: UUID, observed_at: datetime | None = None) -> WorkerStatus: now = observed_at or await self.now() - rows = await self.rows(f"SELECT * FROM worker_instances WHERE {SCOPE} AND id=:id", id=identifier) + rows = await self.rows( + "SELECT w.payload,w.revoked,w.draining,w.control_revision,w.last_seen_at," + "f.identity,f.registered_at FROM worker_instances w LEFT JOIN worker_facts f " + "ON f.worker_id=w.id AND f.tenant_id=w.tenant_id AND f.project_id=w.project_id " + "AND f.environment_id=w.environment_id WHERE w.tenant_id=:tenant AND w.project_id=:project " + "AND w.environment_id=:environment AND w.id=:id", + id=identifier, + ) if not rows: raise CatalogError(404, "WV-NOT-FOUND", "Worker resource not found") row = rows[0] @@ -95,6 +117,8 @@ async def status(self, identifier: UUID, observed_at: datetime | None = None) -> revision=row["control_revision"], active_leases=active[0]["count"], observed_at=now, + identity=stored_identity(row["identity"]), + registered_at=row["registered_at"], ) async def lease(self, task: UUID, generation: int) -> dict[str, Any]: diff --git a/src/firefly_weave/workers/service.py b/src/firefly_weave/workers/service.py index a2db6405..38a109c4 100644 --- a/src/firefly_weave/workers/service.py +++ b/src/firefly_weave/workers/service.py @@ -159,7 +159,8 @@ async def register_instance( raise AccessDenied() for capability in request.task_types: await self.require(actor, scope, "worker.register", context, tx, capability) - instance = WorkerInstance(id=uuid4(), principal_id=actor.id, **request.model_dump()) + # The stored instance and the registration answer keep their bytes; identity is a fact. + instance = WorkerInstance(id=uuid4(), principal_id=actor.id, **request.model_dump(exclude={"identity"})) await repository.execute( "INSERT INTO worker_instances " "(id,tenant_id,project_id,environment_id,release_id,principal_id,payload,revoked) " @@ -170,6 +171,7 @@ async def register_instance( principal=actor.id, payload=instance.model_dump_json(), ) + await repository.record_registration(instance.id, request.identity) await audit( tx.session, actor, diff --git a/tests/contracts/test_worker_identity_contract.py b/tests/contracts/test_worker_identity_contract.py new file mode 100644 index 00000000..228bf1ea --- /dev/null +++ b/tests/contracts/test_worker_identity_contract.py @@ -0,0 +1,335 @@ +# Copyright 2026 Firefly Software Foundation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# Author: Firefly Software Foundation +# SPDX-License-Identifier: Apache-2.0 + +"""Worker identity and observed members are bounded, optional and byte-stable when absent.""" + +import json +from datetime import UTC, datetime +from uuid import uuid4 + +import pytest +from pydantic import ValidationError + +from firefly_weave.compiler.canonical import canonical_digest +from firefly_weave.contracts.deployments import ( + DeploymentObservation, + ObservationReport, + ObservedMember, + ObservedResource, +) +from firefly_weave.contracts.workers import InstanceRequest, WorkerIdentity, WorkerInstance, WorkerStatus + + +@pytest.mark.parametrize("host", ["-host", "host/path", "host\nother", "host\n", "a" * 254, "", "host name"]) +def test_worker_host_is_bounded(host): + with pytest.raises(ValidationError): + WorkerIdentity(host=host, runtime="process") + + +def test_worker_host_accepts_names_and_pod_names(): + for host in ("a" * 253, "worker-7f9c5d", "weave-worker-5d8f7c9b6-x2kq4", "host.example.com", "0a1b2c3d4e5f"): + assert WorkerIdentity(host=host, runtime="container").host == host + + +def test_identity_dump_leaves_out_unknown_values(): + identity = WorkerIdentity(runtime="process") + assert identity.model_dump(mode="json") == {"runtime": "process", "platform_hint": "none"} + assert identity.model_dump(mode="json") == identity.model_dump(mode="json", exclude_none=True) + + +@pytest.mark.parametrize("field", ["sdk_version", "language_version", "os", "arch"]) +@pytest.mark.parametrize("value", ["a" * 65, "", "3.12\n", "-x", "x y", "x\x00"]) +def test_reported_versions_are_bounded(field, value): + with pytest.raises(ValidationError): + WorkerIdentity(runtime="process", **{field: value}) + + +@pytest.mark.parametrize("field", ["namespace", "node"]) +def test_kubernetes_names_are_bounded(field): + assert getattr(WorkerIdentity(runtime="container", **{field: "a" * 253}), field) == "a" * 253 + for value in ("a" * 254, "ns/other", "-ns", ""): + with pytest.raises(ValidationError): + WorkerIdentity(runtime="container", **{field: value}) + + +def test_runtime_and_platform_hint_are_closed_vocabularies(): + for runtime in ("embedded", "container", "process"): + WorkerIdentity(runtime=runtime) + for hint in ("kubernetes", "docker", "none"): + WorkerIdentity(runtime="container", platform_hint=hint) + with pytest.raises(ValidationError): + WorkerIdentity(runtime="vm") + with pytest.raises(ValidationError): + WorkerIdentity(runtime="process", platform_hint="nomad") + with pytest.raises(ValidationError): + WorkerIdentity() + + +@pytest.mark.parametrize( + "labels", + [ + {str(index): "x" for index in range(9)}, + {"": "x"}, + {" team": "x"}, + {"k" * 65: "x"}, + {"team": "x" * 257}, + {"team": "line\nbreak"}, + {"team": "tab\there"}, + {"team": "bell\x07"}, + {"team": "c1\x85"}, + {"api_token": "x"}, + {"DB-Password": "x"}, + {"client.secret": "x"}, + {"Authorization": "x"}, + {"team=a": "x"}, + {"team,a": "x"}, + ], +) +def test_labels_are_bounded_and_never_name_credentials(labels): + with pytest.raises(ValidationError): + WorkerIdentity(runtime="process", labels=labels) + + +def test_labels_accept_eight_bounded_pairs(): + labels = {f"app.example/key-{index}": "v" * 256 for index in range(8)} + labels["app.example/key-0"] = "" + identity = WorkerIdentity(runtime="process", labels=labels) + assert identity.labels == labels + assert "labels" not in WorkerIdentity(runtime="process").model_dump(mode="json") + + +def test_identity_is_limited_to_what_its_database_column_stores(): + def identity(value): + return WorkerIdentity( + host="h" * 253, + runtime="container", + sdk_version="s" * 64, + language_version="l" * 64, + os="o" * 64, + arch="a" * 64, + platform_hint="kubernetes", + namespace="n" * 253, + node="d" * 253, + labels={f"key-{index}": value for index in range(8)}, + ) + + with pytest.raises(ValidationError, match="4 KiB"): + identity("é" * 256) + small = identity("v" * 100) + text = json.dumps(small.model_dump(mode="json"), ensure_ascii=False, separators=(", ", ": ")) + assert len(text.encode()) <= 4096 + + +def test_unknown_identity_and_registration_fields_are_refused(): + base = {"release_id": str(uuid4()), "task_types": ["echo@1.0.0"], "capacity": 1} + placement = {"target_id": str(uuid4()), "target_name": "t", "deployment_id": str(uuid4()), "component": "w"} + with pytest.raises(ValidationError): + InstanceRequest.model_validate_json(json.dumps({**base, "placement": placement})) + with pytest.raises(ValidationError): + InstanceRequest.model_validate_json( + json.dumps({**base, "identity": {"runtime": "process", "placement": placement}}) + ) + with pytest.raises(ValidationError): + InstanceRequest.model_validate_json(json.dumps({**base, "registered_at": "2026-10-01T00:00:00Z"})) + + +def test_request_without_identity_keeps_its_bytes(): + request = InstanceRequest(release_id=uuid4(), task_types=["echo@1.0.0"], capacity=1) + assert set(request.model_dump(mode="json")) == {"release_id", "task_types", "capacity"} + with_identity = request.model_copy(update={"identity": WorkerIdentity(host="worker-1", runtime="process")}) + assert with_identity.model_dump(mode="json")["identity"] == { + "host": "worker-1", + "runtime": "process", + "platform_hint": "none", + } + parsed = InstanceRequest.model_validate_json(with_identity.model_dump_json()) + assert parsed == with_identity + + +def test_old_worker_instance_and_status_answers_still_deserialize(): + instance = { + "release_id": str(uuid4()), + "task_types": ["echo@1.0.0"], + "capacity": 2, + "id": str(uuid4()), + "principal_id": str(uuid4()), + "revoked": False, + } + parsed = WorkerInstance.model_validate_json(json.dumps(instance)) + assert parsed.identity is None and json.loads(parsed.model_dump_json()) == instance + now = datetime.now(UTC).isoformat() + status = { + **instance, + "revision": 1, + "draining": False, + "presence": "unknown", + "last_seen_at": None, + "presence_expires_at": None, + "presence_ttl_seconds": 60, + "observed_at": now, + "active_leases": 0, + "available_capacity": None, + } + old = WorkerStatus.model_validate_json(json.dumps(status)) + assert old.identity is None and old.registered_at is None + assert set(json.loads(old.model_dump_json())) == set(status) + + +def test_status_reports_identity_and_registration_time_when_known(): + registered = datetime(2026, 10, 1, tzinfo=UTC) + status = WorkerStatus( + release_id=uuid4(), + task_types=["echo@1.0.0"], + capacity=1, + id=uuid4(), + principal_id=uuid4(), + revision=1, + draining=False, + presence="unknown", + last_seen_at=None, + presence_expires_at=None, + observed_at=registered, + active_leases=0, + available_capacity=None, + identity=WorkerIdentity(host="worker-1", runtime="process"), + registered_at=registered, + ) + dumped = status.model_dump(mode="json") + assert dumped["identity"]["host"] == "worker-1" + assert dumped["registered_at"] == "2026-10-01T00:00:00Z" + assert WorkerStatus.model_validate_json(status.model_dump_json()) == status + + +def resource(**changes): + values = dict( + name="worker", + external_identity="compose/test/worker", + kind="worker", + image=None, + replicas=0, + ready_replicas=0, + version="stopped", + ownership="managed", + state="stopped", + ) + return ObservedResource(**{**values, **changes}) + + +def member(host="0a1b2c3d4e5f", **changes): + values = dict( + host=host, + ordinal=1, + state="running", + ready=True, + started_at=datetime(2026, 10, 1, tzinfo=UTC), + restarts=0, + node=None, + ) + return ObservedMember(**{**values, **changes}) + + +def test_observation_without_members_preserves_digest(): + report = ObservationReport(resources=[resource()], complete=True, settled=True) + assert "members" not in report.model_dump(mode="json")["resources"][0] + assert report.model_dump(mode="json") == report.model_copy( + update={"resources": [resource().model_copy(update={"members": []})]} + ).model_dump(mode="json") + legacy = { + "settled": True, + "complete": True, + "resources": [ + { + "name": "worker", + "external_identity": "compose/test/worker", + "kind": "worker", + "image": None, + "replicas": 0, + "ready_replicas": 0, + "version": "stopped", + "ownership": "managed", + "state": "stopped", + } + ], + } + assert report.model_dump(mode="json") == legacy + stored = { + "id": str(uuid4()), + "target_id": str(uuid4()), + "target_revision": 1, + "observed_at": "2026-10-01T00:00:00Z", + "expires_at": "2026-10-01T00:15:00Z", + "digest": "0" * 64, + } + # The server stores a runner report as it does in deployments/leases.py. + observation = DeploymentObservation.model_validate_json(json.dumps({**report.model_dump(mode="json"), **stored})) + legacy_observation = DeploymentObservation.model_validate_json(json.dumps({**legacy, **stored})) + assert canonical_digest(observation.model_dump(mode="json", exclude={"digest"})) == canonical_digest( + legacy_observation.model_dump(mode="json", exclude={"digest"}) + ) + + +def test_members_are_reported_and_bounded(): + observed = resource(replicas=2, ready_replicas=1, members=[member(), member("1b2c3d4e5f6a", ordinal=2)]) + dumped = observed.model_dump(mode="json")["members"] + assert dumped[0] == { + "host": "0a1b2c3d4e5f", + "ordinal": 1, + "state": "running", + "ready": True, + "started_at": "2026-10-01T00:00:00Z", + "restarts": 0, + "node": None, + } + assert ObservedResource.model_validate_json(observed.model_dump_json()) == observed + resource(members=[member(f"host-{index}") for index in range(100)]) + with pytest.raises(ValidationError): + resource(members=[member(f"host-{index}") for index in range(101)]) + + +@pytest.mark.parametrize( + "changes", + [ + {"host": "host/path"}, + {"host": "-host"}, + {"host": "a" * 254}, + {"ordinal": -1}, + {"state": ""}, + {"state": "Running now"}, + {"state": "s" * 33}, + {"restarts": -1}, + {"node": "node/other"}, + {"started_at": datetime(2026, 10, 1)}, + ], +) +def test_member_fields_are_bounded(changes): + with pytest.raises(ValidationError): + member(**changes) + + +def test_members_carry_no_worker_authority_fields(): + payload = member().model_dump(mode="json") + for forged in ("worker_id", "placement", "principal_id", "target_id"): + with pytest.raises(ValidationError): + ObservedMember.model_validate_json(json.dumps({**payload, forged: str(uuid4())})) + + +def test_identity_and_member_contracts_are_exported(): + from firefly_weave.contracts.schema_export import export_schemas + + schemas = export_schemas() + assert "worker-identity" in schemas and "operations-observed-member" in schemas + request = schemas["worker-instance-request"] + assert "identity" in request["properties"] and "identity" not in request.get("required", []) diff --git a/tests/integration/test_compose_operations.py b/tests/integration/test_compose_operations.py index cc29c7c6..ca026feb 100644 --- a/tests/integration/test_compose_operations.py +++ b/tests/integration/test_compose_operations.py @@ -120,6 +120,23 @@ def plan(observed, replicas, action): running = await adapter.apply(first, before_write) assert running.settled and running.resources[0].ready_replicas == 1 assert running.resources[0].ownership == "managed" + # The member host is the hostname a worker in the container reports. + [member] = running.resources[0].members + container = (await run_command([*adapter._files(), "ps", "--quiet", "--no-trunc", "worker"])).decode().strip() + hostname = json.loads( + await run_command( + [executable, "--context", context, "inspect", "--format", "{{json .Config.Hostname}}", container] + ) + ) + assert member.host == container[:12] == hostname + assert (member.ordinal, member.state, member.ready, member.restarts, member.node) == ( + 1, + "running", + True, + 0, + None, + ) + assert member.started_at is not None and member.started_at <= datetime.now(UTC) # The stale first observation cannot be reused after creating a container. with pytest.raises(RunnerPolicyError): await adapter.apply(first, before_write) @@ -132,6 +149,7 @@ def plan(observed, replicas, action): assert running.settled and running.resources[0].ready_replicas == 1 stopped = await adapter.apply(plan(running, 0, "scale_workers"), before_write) assert stopped.settled and stopped.resources[0].replicas == 0 + assert stopped.resources[0].members == [] assert stopped.resources[0].image == image assert effects == 3 finally: diff --git a/tests/integration/test_operations_fact_writes.py b/tests/integration/test_operations_fact_writes.py index 291e95a8..5b6e5921 100644 --- a/tests/integration/test_operations_fact_writes.py +++ b/tests/integration/test_operations_fact_writes.py @@ -705,6 +705,15 @@ async def test_listing_cache_cannot_veto_admitted_completion( if cache == "backfilled": for table in ("run_event_evidence", "run_events", "run_facts"): await owner.execute(text(f"DELETE FROM {table} WHERE run_id=:id"), {"id": run.id}) + # Registration already recorded these workers' facts. The frozen backfill runs once on an + # empty worker_facts table during the upgrade, so restore that state before replaying it. + await owner.execute( + text( + "DELETE FROM worker_facts WHERE tenant_id=:tenant AND project_id=:project " + "AND environment_id=:environment" + ), + params, + ) frozen = runpy.run_path( str(Path(__file__).resolve().parents[2] / "migrations/versions/0031_operations_facts.py") ) @@ -854,3 +863,229 @@ async def test_join_progress_preserves_first_observed_times( assert (fact["ended_at"] is not None) is (status == "succeeded") async with case.owner() as owner: await assert_usage(owner, {"tenant": case.scope.tenant_id, "project": case.scope.project_id}) + + +def worker_identity_value(): + from firefly_weave.contracts.workers import WorkerIdentity + + return WorkerIdentity(host="worker-1", runtime="process") + + +def registration_request(case, identity): + from firefly_weave.contracts.workers import InstanceRequest + + existing = case.instances[0] + return InstanceRequest( + release_id=existing.release_id, task_types=existing.task_types, capacity=1, identity=identity + ) + + +WORKER_FACTS = ( + "SELECT w.payload,f.identity,f.registered_at FROM worker_instances w " + "JOIN worker_facts f ON f.worker_id=w.id AND f.tenant_id=w.tenant_id " + "AND f.project_id=w.project_id AND f.environment_id=w.environment_id " + "WHERE w.tenant_id=:tenant AND w.project_id=:project AND w.environment_id=:environment AND w.id=:id" +) + + +@pytest.mark.parametrize("identity", [None, "bounded"]) +async def test_worker_registration_facts_match_source(operations_case, identity): + case = operations_case + identity = worker_identity_value() if identity else None + request = registration_request(case, identity) + async with case.tx() as tx: + before = await tx.session.scalar(text("SELECT clock_timestamp()")) + registered = await case.workers.register_instance(case.actor, case.scope, request, context=AuditContext()) + rows = await case.rows(WORKER_FACTS, id=registered.id) + assert len(rows) == 1 and rows[0]["registered_at"] is not None and rows[0]["registered_at"] >= before + assert "identity" not in rows[0]["payload"] + assert rows[0]["identity"] == ({} if identity is None else identity.model_dump(mode="json", exclude_none=True)) + # The registration answer keeps its bytes; identity is read from the worker's facts. + assert registered.identity is None and "identity" not in registered.model_dump(mode="json") + status = await case.workers.read(case.actor, case.scope, registered.id, context=AuditContext()) + assert status.identity == identity and status.registered_at == rows[0]["registered_at"] + from operations_support import assert_usage + + async with case.owner() as owner: + await assert_usage(owner, {"tenant": case.scope.tenant_id, "project": case.scope.project_id}) + + +async def test_workers_without_facts_or_identity_read_as_unknown(operations_case): + case = operations_case + legacy, backfilled = case.instances + async with case.owner.begin() as owner: + await owner.execute(text("DELETE FROM worker_facts WHERE worker_id=:id"), {"id": legacy.id}) + await owner.execute( + text("UPDATE worker_facts SET registered_at=NULL,identity='{}' WHERE worker_id=:id"), {"id": backfilled.id} + ) + for instance in (legacy, backfilled): + status = await case.workers.read(case.actor, case.scope, instance.id, context=AuditContext()) + assert status.identity is None and status.registered_at is None + assert not {"identity", "registered_at"} & set(status.model_dump(mode="json")) + page = await case.workers.list(case.actor, case.scope, context=AuditContext()) + assert all("identity" not in item and "registered_at" not in item for item in page["items"]) + + +async def test_largest_identity_fits_its_database_column(operations_case): + import json + + from firefly_weave.contracts.workers import WorkerIdentity + + case = operations_case + identity = WorkerIdentity( + host="h" * 253, + runtime="container", + sdk_version="s" * 64, + language_version="l" * 64, + os="o" * 64, + arch="a" * 64, + platform_hint="kubernetes", + namespace="n" * 253, + node="d" * 253, + labels={f"key-{index}": 'é\\"' + "v" * 150 for index in range(8)}, + ) + registered = await case.workers.register_instance( + case.actor, case.scope, registration_request(case, identity), context=AuditContext() + ) + measured = ( + await case.rows( + "SELECT octet_length(identity::text) AS size FROM worker_facts WHERE worker_id=:id", id=registered.id + ) + )[0]["size"] + text_form = json.dumps(identity.model_dump(mode="json"), ensure_ascii=False, separators=(", ", ": ")) + assert measured == len(text_form.encode()) <= 4096 + + +async def worker_rows(case): + async with case.owner() as owner: + return { + table: (await owner.execute(text(f"SELECT to_jsonb(t)::text AS row FROM {table} t ORDER BY 1"))) + .scalars() + .all() + for table in ("worker_instances", "worker_facts", "operation_usage", "access_audit") + } + + +@pytest.mark.parametrize("after", [False, True]) +async def test_failed_worker_fact_insert_rolls_back_the_registration(operations_case, monkeypatch, after): + from firefly_weave.workers.repository import WorkerRepository + + case = operations_case + before = await worker_rows(case) + original = WorkerRepository.record_registration + reached = [] + + async def fail(self, *args, **kwargs): + reached.append( + await self.tx.session.scalar(text("SELECT count(*) FROM worker_instances WHERE id=:id"), {"id": args[0]}) + ) + if after: + await original(self, *args, **kwargs) + raise RuntimeError("worker fact rollback") + + monkeypatch.setattr(WorkerRepository, "record_registration", fail) + with pytest.raises(RuntimeError, match="worker fact rollback"): + await case.workers.register_instance( + case.actor, case.scope, registration_request(case, worker_identity_value()), context=AuditContext() + ) + assert reached == [1] + assert await worker_rows(case) == before + + +@pytest.mark.parametrize("refused", ["before_registration", "at_fact_insert"]) +async def test_worker_registration_facts_roll_back_at_quota(operations_case, monkeypatch, refused): + from firefly_weave.definitions.models import CatalogError + from firefly_weave.workers.repository import WorkerRepository + + case = operations_case + saturate = text( + "UPDATE operation_policy SET limits=jsonb_set(limits,'{ordinary_bytes}',to_jsonb((SELECT value FROM " + "operation_usage WHERE tenant_id=:tenant AND project_id=:project " + "AND environment_id='00000000-0000-0000-0000-000000000000' AND metric='ordinary_bytes')))" + ) + scope = {"tenant": case.scope.tenant_id, "project": case.scope.project_id} + reached = [] + if refused == "before_registration": + async with case.owner.begin() as owner: + await owner.execute(saturate, scope) + else: + original = WorkerRepository.record_registration + + async def saturate_then_record(self, *args, **kwargs): + # The source row is already charged in this transaction; the limit now equals the + # committed usage, so only the fact insert can exceed it. + reached.append(args[0]) + async with case.owner.begin() as owner: + await owner.execute(saturate, scope) + await original(self, *args, **kwargs) + + monkeypatch.setattr(WorkerRepository, "record_registration", saturate_then_record) + before = await worker_rows(case) + with pytest.raises(CatalogError) as caught: + await case.workers.register_instance( + case.actor, case.scope, registration_request(case, worker_identity_value()), context=AuditContext() + ) + assert (caught.value.status, caught.value.code) == (429, "WV-OPERATION-CAPACITY") + assert len(reached) == (0 if refused == "before_registration" else 1) + assert await worker_rows(case) == before + from operations_support import assert_usage + + async with case.owner() as owner: + await assert_usage(owner, scope) + + +async def test_registration_over_http_reports_identity_and_refuses_forged_fields( + client, access_db, provisioned, headers, env_url, worker_runtime_fixture +): + from firefly_weave.access.models import Grant + from firefly_weave.sdk.worker_registration import register_worker_http + + sessions, owner, access, _ = access_db + scope = provisioned[1][0] + async with sessions() as tx: + principal = await tx.scalar(text("SELECT principal_id FROM identity_links WHERE subject='0'")) + for role in ("deployer", "operator", "worker"): + await access.grant(provisioned[0], principal, Grant(role=role, scope=scope)) + contract = next( + v.definition.value for k, v in worker_runtime_fixture["catalog"].resources.items() if k[0] == "TaskCapability" + ) + release = await client.post( + env_url + "/worker-releases", + headers=headers, + json={"image_digest": "sha256:" + "c" * 64, "capabilities": [contract]}, + ) + assert release.status_code == 201, release.text + body = {"release_id": release.json()["id"], "task_types": ["echo@1.2.0"], "capacity": 1} + identity = {"host": "worker-1", "runtime": "container", "platform_hint": "docker", "labels": {"team": "a"}} + placement = {"target_id": str(uuid4()), "target_name": "t", "deployment_id": str(uuid4()), "component": "w"} + for forged in ( + {**body, "placement": placement}, + {**body, "identity": {**identity, "placement": placement}}, + {**body, "identity": {**identity, "labels": {"api_token": "x"}}}, + {**body, "registered_at": "2026-10-01T00:00:00Z"}, + ): + refused = await client.post(env_url + "/workers", headers=headers, json=forged) + assert (refused.status_code, refused.json()["code"]) == (422, "WV-VALIDATION"), refused.text + sent = [] + + async def count(request): + sent.append(request) + + hooks = list(client.event_hooks["request"]) + client.event_hooks = {**client.event_hooks, "request": [*hooks, count]} + client.headers.update(headers) + try: + # A server that knows worker identity accepts it on the first request. + response = await register_worker_http(client, env_url, {**body, "identity": identity}) + finally: + client.event_hooks = {**client.event_hooks, "request": hooks} + assert response.status_code == 201 and len(sent) == 1 + assert set(response.json()) == {"release_id", "task_types", "capacity", "id", "principal_id", "revoked"} + read = await client.get(env_url + "/workers/" + response.json()["id"], headers=headers) + assert read.status_code == 200, read.text + assert read.json()["identity"] == identity and read.json()["registered_at"] + async with owner() as tx: + stored = ( + await tx.execute(text("SELECT payload FROM worker_instances WHERE id=:id"), {"id": response.json()["id"]}) + ).scalar_one() + assert "identity" not in stored diff --git a/tests/unit/connectors/test_local_build.py b/tests/unit/connectors/test_local_build.py index eebd98d5..c38c07b5 100644 --- a/tests/unit/connectors/test_local_build.py +++ b/tests/unit/connectors/test_local_build.py @@ -410,3 +410,24 @@ def test_environment_configuration_carries_the_local_build(monkeypatch): monkeypatch.setenv("WEAVE_NATIVE_IMAGE_DIGEST", "sha256:" + "a" * 64) monkeypatch.setenv("WEAVE_NATIVE_EXECUTORS", json.dumps([_executor().model_dump(mode="json")])) assert Settings.from_env().native_executors[0].build == "local-development" + + +async def test_native_executors_register_an_embedded_identity_once(admission, monkeypatch): + from firefly_weave.contracts.workers import WorkerIdentity + + calls = [] + + def identity(*, runtime=None, environ=None): + calls.append(runtime) + return WorkerIdentity(host="api-1", runtime=runtime) + + monkeypatch.setattr(dispatcher, "worker_identity", identity) + digest = "sha256:" + "e" * 64 + service = _Service(_release(digest)) + native = NativeDispatcher(service, (_executor(), _executor()), digest, 1) # type: ignore[arg-type] + await native.open() + await native.close() + assert calls == ["embedded"] + assert [request.identity for request in service.registered] == [ + WorkerIdentity(host="api-1", runtime="embedded") + ] * 2 diff --git a/tests/unit/sdk/test_worker_identity.py b/tests/unit/sdk/test_worker_identity.py new file mode 100644 index 00000000..83677f52 --- /dev/null +++ b/tests/unit/sdk/test_worker_identity.py @@ -0,0 +1,221 @@ +# Copyright 2026 Firefly Software Foundation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# Author: Firefly Software Foundation +# SPDX-License-Identifier: Apache-2.0 + +"""A worker reports bounded identity from five explicit settings and nothing else.""" + +import logging +from collections.abc import Iterator, Mapping + +import pytest + +from firefly_weave.sdk import worker_identity as module +from firefly_weave.sdk.worker_identity import worker_identity + +SETTINGS = { + "WEAVE_WORKER_REPORT_HOST", + "WEAVE_WORKER_LABELS", + "WEAVE_POD_NAMESPACE", + "WEAVE_NODE_NAME", + "KUBERNETES_SERVICE_HOST", +} + + +class Recorder(Mapping[str, str]): + """An environment that fails if it is listed and records each value read.""" + + def __init__(self, values: dict[str, str]) -> None: + self.values, self.read = values, [] + + def __getitem__(self, key: str) -> str: + self.read.append(key) + return self.values[key] + + def __iter__(self) -> Iterator[str]: + raise AssertionError("worker identity must not list the environment") + + def __len__(self) -> int: + raise AssertionError("worker identity must not measure the environment") + + +@pytest.fixture(autouse=True) +def fixed_host(monkeypatch): + monkeypatch.setattr(module.socket, "gethostname", lambda: "worker-1") + for name in SETTINGS: + monkeypatch.delenv(name, raising=False) + + +def test_identity_can_omit_host(monkeypatch): + monkeypatch.setenv("WEAVE_WORKER_REPORT_HOST", "false") + value = worker_identity() + assert value.host is None and "host" not in value.model_dump(mode="json", exclude_none=True) + + +@pytest.mark.parametrize( + "raw,expected", + [ + (None, "worker-1"), + ("true", "worker-1"), + ("TRUE", "worker-1"), + ("1", "worker-1"), + ("yes", "worker-1"), + ("False", None), + ("0", None), + ("no", None), + (" false ", None), + ], +) +def test_report_host_setting(raw, expected): + environ = {} if raw is None else {"WEAVE_WORKER_REPORT_HOST": raw} + assert worker_identity(environ=environ).host == expected + + +def test_unclear_report_host_setting_is_refused_without_echoing_it(): + with pytest.raises(ValueError, match="WEAVE_WORKER_REPORT_HOST") as caught: + worker_identity(environ={"WEAVE_WORKER_REPORT_HOST": "secret-ish"}) + assert "secret-ish" not in str(caught.value) + + +@pytest.mark.parametrize("host", ["-bad", "bad_host", "a" * 254, "", "host\nname"]) +def test_invalid_hostname_is_omitted_instead_of_stopping_the_worker(monkeypatch, host): + monkeypatch.setattr(module.socket, "gethostname", lambda: host) + assert worker_identity(environ={}).host is None + + +def test_reads_only_its_five_settings(): + environ = Recorder( + { + "WEAVE_WORKER_LABELS": "team=payments", + "KUBERNETES_SERVICE_HOST": "10.0.0.1", + "WEAVE_POD_NAMESPACE": "weave-workers", + "WEAVE_NODE_NAME": "node-1", + "DATABASE_URL": "postgresql://user:password@db/weave", + "WEAVE_WORKER_TOKEN": "token-value", + } + ) + identity = worker_identity(environ=environ) + assert set(environ.read) <= SETTINGS + dumped = identity.model_dump_json() + assert "password" not in dumped and "token-value" not in dumped + assert identity.labels == {"team": "payments"} + assert (identity.namespace, identity.node, identity.platform_hint) == ("weave-workers", "node-1", "kubernetes") + + +def test_process_defaults_and_reported_platform(monkeypatch): + monkeypatch.setattr(module.platform, "python_version", lambda: "3.12.4") + monkeypatch.setattr(module.platform, "system", lambda: "Linux") + monkeypatch.setattr(module.platform, "machine", lambda: "x86_64") + monkeypatch.setattr(module.metadata, "version", lambda name: "0.1.0a15") + identity = worker_identity(environ={}) + assert identity.model_dump(mode="json") == { + "host": "worker-1", + "runtime": "process", + "sdk_version": "0.1.0a15", + "language_version": "3.12.4", + "os": "Linux", + "arch": "x86_64", + "platform_hint": "none", + } + + +def test_unusual_platform_values_are_omitted(monkeypatch): + monkeypatch.setattr(module.platform, "machine", lambda: "") + monkeypatch.setattr(module.platform, "system", lambda: "x" * 65) + monkeypatch.setattr(module.platform, "python_version", lambda: "3.12 (custom)") + + def missing(name): + raise module.metadata.PackageNotFoundError(name) + + monkeypatch.setattr(module.metadata, "version", missing) + dumped = worker_identity(environ={}).model_dump(mode="json") + assert not {"arch", "os", "language_version", "sdk_version"} & set(dumped) + + +@pytest.mark.parametrize( + "runtime,environ,expected", + [ + (None, {}, ("process", "none")), + (None, {"KUBERNETES_SERVICE_HOST": "10.0.0.1"}, ("container", "kubernetes")), + ("container", {}, ("container", "docker")), + ("embedded", {}, ("embedded", "none")), + ("embedded", {"KUBERNETES_SERVICE_HOST": "10.0.0.1"}, ("embedded", "kubernetes")), + ("process", {"KUBERNETES_SERVICE_HOST": "10.0.0.1"}, ("process", "kubernetes")), + (None, {"KUBERNETES_SERVICE_HOST": ""}, ("process", "none")), + ], +) +def test_runtime_comes_from_the_caller_or_the_kubernetes_setting(runtime, environ, expected): + identity = worker_identity(runtime=runtime, environ=environ) + assert (identity.runtime, identity.platform_hint) == expected + + +def test_invalid_kubernetes_names_are_omitted(): + identity = worker_identity(environ={"WEAVE_POD_NAMESPACE": "ns/other", "WEAVE_NODE_NAME": "-node"}) + assert identity.namespace is None and identity.node is None + + +@pytest.mark.parametrize( + "raw,expected", + [ + ("", {}), + (" ", {}), + ("team=payments", {"team": "payments"}), + (" team = payments , tier=gold ,", {"team": "payments", "tier": "gold"}), + ("app.example/name=weave,empty=", {"app.example/name": "weave", "empty": ""}), + (",".join(f"k{index}=v" for index in range(8)), {f"k{index}": "v" for index in range(8)}), + ], +) +def test_labels_are_parsed(raw, expected): + assert worker_identity(environ={"WEAVE_WORKER_LABELS": raw}).labels == expected + + +@pytest.mark.parametrize( + "raw", + [ + ",".join(f"k{index}=v" for index in range(9)), + "team=a,team=b", + "=value", + "team", + "team payments=x", + "api_token=hunter2-value", + "db_password=hunter2-value", + "secret=hunter2-value", + "authorization=hunter2-value", + "team=" + "x" * 257, + "k" * 65 + "=hunter2-value", + "team=line\x07hunter2-value", + ], +) +def test_invalid_labels_fail_clearly_without_echoing_values(raw, caplog): + caplog.set_level(logging.DEBUG) + with pytest.raises(ValueError, match="WEAVE_WORKER_LABELS") as caught: + worker_identity(environ={"WEAVE_WORKER_LABELS": raw}) + message = str(caught.value) + repr(caught.value.__cause__) + repr(caught.value.__context__) + assert "hunter2" not in message and "x" * 257 not in message + assert caught.value.__cause__ is None and caught.value.__context__ is None + assert caplog.records == [] + + +def test_identity_that_exceeds_storage_fails_without_echoing_values(): + raw = ",".join(f"key-{index}=" + "é" * 256 for index in range(8)) + with pytest.raises(ValueError, match="WEAVE_WORKER_LABELS") as caught: + worker_identity(environ={"WEAVE_WORKER_LABELS": raw}) + assert "é" not in str(caught.value) and caught.value.__context__ is None + + +def test_identity_reads_the_process_environment_by_default(monkeypatch): + monkeypatch.setenv("WEAVE_WORKER_LABELS", "team=payments") + monkeypatch.setenv("WEAVE_POD_NAMESPACE", "weave-workers") + identity = worker_identity() + assert identity.labels == {"team": "payments"} and identity.namespace == "weave-workers" diff --git a/tests/unit/sdk/test_worker_registration.py b/tests/unit/sdk/test_worker_registration.py new file mode 100644 index 00000000..a77ecd0c --- /dev/null +++ b/tests/unit/sdk/test_worker_registration.py @@ -0,0 +1,268 @@ +# Copyright 2026 Firefly Software Foundation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# Author: Firefly Software Foundation +# SPDX-License-Identifier: Apache-2.0 + +"""Registration sends identity and retries exactly once without it when an older server refuses it.""" + +import asyncio +import json +from uuid import UUID, uuid4 + +import httpx +import pytest + +from firefly_weave.contracts.access import Scope +from firefly_weave.contracts.public import Problem +from firefly_weave.contracts.workers import InstanceRequest, WorkerIdentity, WorkerInstance +from firefly_weave.sdk.client import WeaveClient +from firefly_weave.sdk.errors import TransportError, WeaveError +from firefly_weave.sdk.worker_registration import ( + identity_rejected, + negotiate_worker_registration, + register_worker_http, +) + +RELEASE = uuid4() +IDENTITY = WorkerIdentity(host="worker-1", runtime="process") +VALIDATION = {"status": 422, "code": "WV-VALIDATION", "message": "Invalid request contract"} + + +def answer(status, body=None): + if body is None: + return httpx.Response(status) + return httpx.Response(status, json=body, headers={"content-type": "application/problem+json"}) + + +def registered(request): + body = json.loads(request.content) + instance = WorkerInstance( + id=uuid4(), + principal_id=uuid4(), + release_id=UUID(body["release_id"]), + task_types=body["task_types"], + capacity=body["capacity"], + ) + return httpx.Response(201, json=instance.model_dump(mode="json")) + + +FIRST = { + "accepted": [lambda request: registered(request)], + "old server": [lambda request: answer(422, VALIDATION), registered], + "refused twice": [lambda request: answer(422, VALIDATION), lambda request: answer(422, VALIDATION)], +} +ONCE = { + "other 422 code": lambda: answer(422, {**VALIDATION, "code": "WV-PAGE"}), + "malformed 422": lambda: httpx.Response(422, content=b"not json"), + "422 list": lambda: httpx.Response(422, json=[VALIDATION]), + "401": lambda: answer(401, {**VALIDATION, "status": 401, "code": "WV-UNAUTHENTICATED"}), + "403": lambda: answer(403, {**VALIDATION, "status": 403, "code": "WV-FORBIDDEN"}), + "409": lambda: answer(409, {**VALIDATION, "status": 409, "code": "WV-CONFLICT"}), + "500": lambda: answer(500, {**VALIDATION, "status": 500, "code": "WV-INTERNAL"}), +} + + +def recorder(responses): + requests = [] + + async def receive(request): + requests.append(request) + return responses[len(requests) - 1](request) + + return requests, receive + + +def http_body(identity=True): + body = {"release_id": str(RELEASE), "task_types": ["echo@1.0.0"], "capacity": 2} + if identity: + body["identity"] = IDENTITY.model_dump(mode="json") + return body + + +@pytest.mark.parametrize("case", FIRST) +async def test_http_registration_retries_only_an_identity_validation_refusal(case): + requests, receive = recorder(FIRST[case]) + body = http_body() + original = json.dumps(body) + async with httpx.AsyncClient( + base_url="https://weave.example", transport=httpx.MockTransport(receive), headers={"authorization": "Bearer t"} + ) as client: + if case == "refused twice": + with pytest.raises(httpx.HTTPStatusError): + await register_worker_http(client, "/environment/", body) + else: + response = await register_worker_http(client, "/environment/", body) + assert response.status_code == 201 and "id" in response.json() + assert json.dumps(body) == original + sent = [json.loads(request.content) for request in requests] + assert [str(request.url) for request in requests] == ["https://weave.example/environment/workers"] * len(sent) + assert [request.headers["authorization"] for request in requests] == ["Bearer t"] * len(sent) + assert sent[0] == body + assert len(sent) == (1 if case == "accepted" else 2) + if len(sent) == 2: + assert sent[1] == {key: value for key, value in body.items() if key != "identity"} + + +@pytest.mark.parametrize("case", ONCE) +async def test_http_registration_does_not_retry_other_failures(case): + requests, receive = recorder([lambda request: ONCE[case]()]) + async with httpx.AsyncClient(base_url="https://weave.example", transport=httpx.MockTransport(receive)) as client: + with pytest.raises(httpx.HTTPStatusError): + await register_worker_http(client, "/environment", http_body()) + assert len(requests) == 1 + + +@pytest.mark.parametrize("error", [httpx.ConnectError("refused"), httpx.ReadTimeout("slow"), asyncio.CancelledError()]) +async def test_http_registration_does_not_retry_transport_failures_or_cancellation(error): + requests = [] + + async def receive(request): + requests.append(request) + raise error + + async with httpx.AsyncClient(base_url="https://weave.example", transport=httpx.MockTransport(receive)) as client: + with pytest.raises(type(error)): + await register_worker_http(client, "/environment", http_body()) + assert len(requests) == 1 + + +async def test_http_registration_without_identity_never_retries(): + requests, receive = recorder([lambda request: answer(422, VALIDATION)]) + async with httpx.AsyncClient(base_url="https://weave.example", transport=httpx.MockTransport(receive)) as client: + with pytest.raises(httpx.HTTPStatusError): + await register_worker_http(client, "/environment", http_body(identity=False)) + assert len(requests) == 1 and "identity" not in json.loads(requests[0].content) + + +async def test_null_identity_counts_as_no_identity(): + requests, receive = recorder([lambda request: answer(422, VALIDATION)]) + async with httpx.AsyncClient(base_url="https://weave.example", transport=httpx.MockTransport(receive)) as client: + with pytest.raises(httpx.HTTPStatusError): + await register_worker_http(client, "/environment", {**http_body(identity=False), "identity": None}) + assert len(requests) == 1 + + +def client(receive): + scope = Scope(tenant_id=uuid4(), project_id=uuid4(), environment_id=uuid4()) + return WeaveClient("https://platform.example", lambda: "token", scope, transport=httpx.MockTransport(receive)) + + +def request(identity=True): + return InstanceRequest( + release_id=RELEASE, task_types=["echo@1.0.0"], capacity=2, identity=IDENTITY if identity else None + ) + + +@pytest.mark.parametrize("case", FIRST) +async def test_sdk_registration_negotiates_identity_once(case): + requests, receive = recorder(FIRST[case]) + value = request() + async with client(receive) as weave: + if case == "refused twice": + with pytest.raises(WeaveError) as caught: + await weave.register_worker(value) + assert caught.value.code == "WV-VALIDATION" + else: + assert type(await weave.register_worker(value)) is WorkerInstance + assert value.identity == IDENTITY + sent = [json.loads(item.content) for item in requests] + assert sent[0]["identity"] == IDENTITY.model_dump(mode="json") + assert len(sent) == (1 if case == "accepted" else 2) + if len(sent) == 2: + assert sent[1] == {key: item for key, item in sent[0].items() if key != "identity"} + assert [item.headers["authorization"] for item in requests] == ["Bearer token"] * 2 + + +@pytest.mark.parametrize("case", ONCE) +async def test_sdk_registration_does_not_retry_other_failures(case): + requests, receive = recorder([lambda request: ONCE[case]()]) + async with client(receive) as weave: + with pytest.raises(WeaveError): + await weave.register_worker(request()) + assert len(requests) == 1 + + +async def test_sdk_registration_does_not_retry_transport_failures_or_without_identity(): + async def unreachable(request): + unreachable.calls += 1 + raise httpx.ConnectError("refused") + + unreachable.calls = 0 + async with client(unreachable) as weave: + with pytest.raises(TransportError): + await weave.register_worker(request()) + assert unreachable.calls == 1 + requests, receive = recorder([lambda request: answer(422, VALIDATION)]) + async with client(receive) as weave: + with pytest.raises(WeaveError): + await weave.register_worker(request(identity=False)) + assert len(requests) == 1 and "identity" not in json.loads(requests[0].content) + + +def problem(status, code): + return WeaveError(Problem(status=status, code=code, message="m")) + + +def http_error(status, body): + request = httpx.Request("POST", "https://weave.example/workers") + response = httpx.Response(status, json=body, request=request) + return httpx.HTTPStatusError("failed", request=request, response=response) + + +def test_identity_rejection_is_exactly_validation_422(): + assert identity_rejected(problem(422, "WV-VALIDATION")) + assert not identity_rejected(problem(422, "WV-PAGE")) + assert not identity_rejected(problem(400, "WV-VALIDATION")) + assert identity_rejected(http_error(422, {"code": "WV-VALIDATION"})) + assert not identity_rejected(http_error(422, {"code": "WV-FILTER"})) + assert not identity_rejected(http_error(409, {"code": "WV-VALIDATION"})) + assert not identity_rejected(http_error(422, ["WV-VALIDATION"])) + + +async def test_negotiation_passes_the_identity_choice_and_keeps_the_first_failure_when_not_retrying(): + seen = [] + + async def send(include_identity): + seen.append(include_identity) + raise problem(403, "WV-FORBIDDEN") + + with pytest.raises(WeaveError, match="WV-FORBIDDEN"): + await negotiate_worker_registration(send, has_identity=True) + assert seen == [True] + + async def fallback(include_identity): + seen.append(include_identity) + if include_identity: + raise problem(422, "WV-VALIDATION") + return "registered" + + seen.clear() + assert await negotiate_worker_registration(fallback, has_identity=True) == "registered" + assert seen == [True, False] + seen.clear() + with pytest.raises(WeaveError): + await negotiate_worker_registration(lambda include: fallback(True), has_identity=False) + assert seen == [True] + + +async def test_negotiation_does_not_hide_programming_errors(): + calls = [] + + async def send(include_identity): + calls.append(include_identity) + raise RuntimeError("bug") + + with pytest.raises(RuntimeError): + await negotiate_worker_registration(send, has_identity=True) + assert calls == [True] diff --git a/tests/unit/test_deployment_compose.py b/tests/unit/test_deployment_compose.py index 8b65c9e2..d8494ca4 100644 --- a/tests/unit/test_deployment_compose.py +++ b/tests/unit/test_deployment_compose.py @@ -276,3 +276,188 @@ async def command(argv, **kwargs): assert len(calls) == 2 assert "--dry-run" in calls[0][0] and "--wait" not in calls[0][0] assert all(kwargs["timeout"] == 120 for _, kwargs in calls) + + +def test_inspect_reads_only_whitelisted_metadata(): + from firefly_weave.deployment_runner.compose import _INSPECT + + fields = json.loads( + _INSPECT.replace("{{json .Id}}", '""') + .replace("{{json .Config.Image}}", '""') + .replace("{{json .State.Status}}", '""') + .replace("{{json .State.StartedAt}}", '""') + .replace("{{json .RestartCount}}", "0") + .replace('{{json (index .Config.Labels "com.docker.compose.project")}}', '""') + .replace('{{json (index .Config.Labels "com.docker.compose.service")}}', '""') + .replace('{{json (index .Config.Labels "com.docker.compose.container-number")}}', '""') + .replace('{{json (index .Config.Labels "firefly.weave.target")}}', '""') + .replace('{{with (index .State "Health")}}{{json .Status}}{{else}}""{{end}}', '""') + ) + assert set(fields) == { + "id", + "image", + "state", + "started", + "restart", + "project", + "service", + "target", + "health", + "number", + } + for secret in ("Env", "Mounts", "Cmd", "Entrypoint", "Secrets", "HostConfig"): + assert secret not in _INSPECT + + +def worker_project(config, containers): + """A Compose project whose worker service has the given containers, keyed by full ID.""" + + async def command(argv, **kwargs): + if "info" in argv: + return b'"daemon-id"' + if "config" in argv: + return json.dumps( + { + "services": { + "worker": { + "image": "example.test/worker@sha256:" + "b" * 64, + "labels": {"firefly.weave.target": str(config.target_id)}, + "deploy": {"replicas": len(containers)}, + } + } + } + ).encode() + if "ps" in argv: + return json.dumps([{"ID": item["id"], "Service": "worker"} for item in containers]).encode() + assert "inspect" in argv, argv + return json.dumps(next(item for item in containers if item["id"] == argv[-1])).encode() + + return command + + +def container(identifier, number, **changes): + return { + "id": identifier, + "image": "example.test/worker@sha256:" + "b" * 64, + "state": "running", + "started": "2026-10-03T08:15:30.123456789Z", + "restart": 0, + "project": "weave-test", + "service": "worker", + "target": None, + "health": "healthy", + "number": number, + **changes, + } + + +async def test_worker_members_report_hostname_number_and_state(tmp_path): + config = policy(tmp_path) + containers = [ + container("b" * 64, "2", state="exited", started="0001-01-01T00:00:00Z", restart=3, health=""), + container("a" * 64, "1"), + container("c" * 64, "3", health="unhealthy", started="2026-10-03T09:00:00Z"), + ] + report = await ComposeAdapter(config, command=worker_project(config, containers)).observe() + members = [member.model_dump(mode="json") for member in report.resources[0].members] + assert members == [ + { + "host": "a" * 12, + "ordinal": 1, + "state": "running", + "ready": True, + "started_at": "2026-10-03T08:15:30.123456Z", + "restarts": 0, + "node": None, + }, + { + "host": "b" * 12, + "ordinal": 2, + "state": "exited", + "ready": False, + "started_at": None, + "restarts": 3, + "node": None, + }, + { + "host": "c" * 12, + "ordinal": 3, + "state": "running", + "ready": False, + "started_at": "2026-10-03T09:00:00Z", + "restarts": 0, + "node": None, + }, + ] + assert report.resources[0].ready_replicas == 1 and report.resources[0].replicas == 3 + + +async def test_member_number_does_not_change_the_reviewed_resource_version(tmp_path): + config = policy(tmp_path) + containers = [container("a" * 64, "1")] + report = await ComposeAdapter(config, command=worker_project(config, containers)).observe() + configuration = json.loads(await worker_project(config, containers)(["config"])) + before_members = [{key: value for key, value in containers[0].items() if key != "number"}] + assert report.resources[0].version == canonical_digest({"configuration": configuration, "members": before_members}) + + +async def test_container_without_a_compose_number_has_no_ordinal(tmp_path): + config = policy(tmp_path) + report = await ComposeAdapter(config, command=worker_project(config, [container("a" * 64, "")])).observe() + assert report.resources[0].members[0].ordinal is None + + +@pytest.mark.parametrize( + "changes", + [{"number": "1x"}, {"number": 1}, {"started": "yesterday"}, {"restart": -1}, {"state": "Running Now"}], +) +async def test_malformed_member_facts_refuse_the_observation(tmp_path, changes): + config = policy(tmp_path) + with pytest.raises(RunnerPolicyError): + await ComposeAdapter( + config, command=worker_project(config, [{**container("a" * 64, "1"), **changes}]) + ).observe() + + +async def test_more_than_one_hundred_worker_containers_refuse_instead_of_truncating(tmp_path): + config = policy(tmp_path) + containers = [container(f"{index:064x}", str(index + 1)) for index in range(101)] + with pytest.raises(RunnerPolicyError): + await ComposeAdapter(config, command=worker_project(config, containers)).observe() + report = await ComposeAdapter(config, command=worker_project(config, containers[:100])).observe() + assert len(report.resources[0].members) == 100 + + +async def test_only_worker_components_report_members(tmp_path): + source = tmp_path / "compose.yaml" + source.write_text("services: {}") + config = DestinationPolicy( + target_id=uuid4(), + adapter="docker-compose", + external_identity="daemon-id", + boundary="weave-test", + context="owned-test", + executable="/usr/bin/docker", + capabilities=["observe", "update"], + compose_file=str(source), + lock_file=str(tmp_path / "runner.lock"), + components=[{"name": "worker", "kind": "api", "configuration": "current"}], + image_repositories=["example.test/worker"], + ) + report = await ComposeAdapter(config, command=worker_project(config, [container("a" * 64, "1")])).observe() + assert report.resources[0].members == [] + assert "members" not in report.model_dump(mode="json")["resources"][0] + + +async def test_member_observation_stays_read_only(tmp_path): + config = policy(tmp_path) + calls = [] + command = worker_project(config, [container("a" * 64, "1")]) + + async def recorded(argv, **kwargs): + calls.append(argv) + return await command(argv, **kwargs) + + await ComposeAdapter(config, command=recorded).observe() + verbs = [next(word for word in argv[3:] if word in {"info", "config", "ps", "inspect"}) for argv in calls] + assert verbs == ["info", "config", "ps", "inspect"] diff --git a/tests/unit/test_deployment_kubernetes.py b/tests/unit/test_deployment_kubernetes.py index dd8fffdd..5e21ce81 100644 --- a/tests/unit/test_deployment_kubernetes.py +++ b/tests/unit/test_deployment_kubernetes.py @@ -83,6 +83,7 @@ def fixture(kind="worker", action="update"): }, "spec": { "replicas": 1, + "selector": {"matchLabels": {"app": "engine"}}, "strategy": {"type": "Recreate"}, "template": { "metadata": {"annotations": {"firefly.weave/configuration": "production"}}, @@ -100,11 +101,47 @@ def fixture(kind="worker", action="update"): return policy, plan, resource +def pod(name, *, template="7d9f8c6b5", owner=None, namespace="owned", labels=None, deleting=None, **status): + """A pod as `kubectl get pods -o json` returns it, owned by the engine deployment's ReplicaSet.""" + return { + "metadata": { + "name": name, + "namespace": namespace, + "labels": {"app": "engine", "pod-template-hash": template} if labels is None else labels, + "ownerReferences": [ + owner + or { + "apiVersion": "apps/v1", + "kind": "ReplicaSet", + "name": f"engine-{template}", + "uid": "replicaset-uid", + "controller": True, + } + ], + **({"deletionTimestamp": deleting} if deleting else {}), + }, + "spec": {"nodeName": status.pop("node", "node-1")}, + "status": { + "phase": status.pop("phase", "Running"), + "conditions": [{"type": "Ready", "status": status.pop("ready", "True")}], + "containerStatuses": status.pop( + "containers", + [ + {"name": "sidecar", "restartCount": 9, "state": {"running": {"startedAt": "2026-10-01T00:00:00Z"}}}, + {"name": "main", "restartCount": 2, "state": {"running": {"startedAt": "2026-10-03T08:15:30Z"}}}, + ], + ), + **status, + }, + } + + class Server: def __init__(self, policy, resource): self.policy, self.resource, self.calls, self.events = policy, resource, [], [] self.namespace_uid = policy.external_identity self.fail_dry = False + self.pods = [pod("engine-7d9f8c6b5-x2kq4")] async def command(self, argv, **kwargs): self.calls.append((argv, kwargs)) @@ -120,6 +157,9 @@ async def command(self, argv, **kwargs): return json.dumps({"metadata": {"name": self.policy.boundary, "uid": self.namespace_uid}}).encode() if "horizontalpodautoscalers.autoscaling" in argv: return b'{"items":[]}' + if "pods" in argv: + assert argv[6:] == ["get", "pods", "--selector=app=engine", "-o=json"] + return json.dumps({"apiVersion": "v1", "kind": "List", "items": self.pods}).encode() assert "deployments.apps" in argv and "engine" in argv if "get" in argv: return json.dumps(self.resource).encode() @@ -160,11 +200,12 @@ async def test_observe_only_fixed_deployments_and_pinned_namespace(): result = await KubernetesAdapter(policy, command=server.command).observe() assert result.complete and result.settled and result.resources[0].ownership == "imported" assert result.resources[0].version == "8" - assert len(server.calls) == 2 + # Namespace identity, the deployment, then its pods through the deployment's own selector. + assert len(server.calls) == 3 and "pods" in server.calls[2][0] server.namespace_uid = "other" with pytest.raises(RunnerPolicyError): await KubernetesAdapter(policy, command=server.command).observe() - assert len(server.calls) == 3 + assert len(server.calls) == 4 @pytest.mark.parametrize("action", ["update", "scale_workers"]) @@ -322,3 +363,207 @@ async def command(argv, **kwargs): with pytest.raises(RunnerPolicyError): await KubernetesAdapter(policy, command=command).apply(plan, server.before_write) assert server.events == ["dry"] + + +async def test_worker_members_come_from_the_deployment_pods(): + policy, _, resource = fixture() + server = Server(policy, resource) + server.pods = [ + pod("engine-7d9f8c6b5-zz9", phase="Pending", ready="False", containers=[], node=None), + pod("engine-7d9f8c6b5-x2kq4"), + pod("engine-5c4b3a2-old1", template="5c4b3a2", deleting="2026-10-03T09:00:00Z"), + ] + result = await KubernetesAdapter(policy, command=server.command).observe() + assert [member.model_dump(mode="json") for member in result.resources[0].members] == [ + { + "host": "engine-5c4b3a2-old1", + "ordinal": None, + "state": "terminating", + "ready": True, + "started_at": "2026-10-03T08:15:30Z", + "restarts": 2, + "node": "node-1", + }, + { + "host": "engine-7d9f8c6b5-x2kq4", + "ordinal": None, + "state": "running", + "ready": True, + "started_at": "2026-10-03T08:15:30Z", + "restarts": 2, + "node": "node-1", + }, + { + "host": "engine-7d9f8c6b5-zz9", + "ordinal": None, + "state": "pending", + "ready": False, + "started_at": None, + "restarts": 0, + "node": None, + }, + ] + assert all(argv[6] == "get" for argv, _ in server.calls) + + +@pytest.mark.parametrize( + "foreign", + [ + pod( + "other-abc12", + template="abc12", + owner={ + "apiVersion": "apps/v1", + "kind": "ReplicaSet", + "name": "other-abc12", + "uid": "u", + "controller": True, + }, + ), + pod( + "engine-7d9f8c6b5-a", + owner={ + "apiVersion": "apps/v1", + "kind": "ReplicaSet", + "name": "engine-7d9f8c6b5", + "uid": "u", + "controller": False, + }, + ), + pod( + "engine-7d9f8c6b5-b", + owner={"apiVersion": "batch/v1", "kind": "Job", "name": "engine-7d9f8c6b5", "uid": "u", "controller": True}, + ), + pod("engine-7d9f8c6b5-c", namespace="elsewhere"), + pod("engine-7d9f8c6b5-d", labels={"app": "engine"}), + pod("engine-7d9f8c6b5-e", labels={"app": "other", "pod-template-hash": "7d9f8c6b5"}), + ], +) +async def test_pods_the_deployment_does_not_own_are_not_members(foreign): + policy, _, resource = fixture() + server = Server(policy, resource) + server.pods = [foreign, pod("engine-7d9f8c6b5-x2kq4")] + result = await KubernetesAdapter(policy, command=server.command).observe() + assert [member.host for member in result.resources[0].members] == ["engine-7d9f8c6b5-x2kq4"] + + +async def test_more_than_one_hundred_member_pods_refuse_instead_of_truncating(): + policy, _, resource = fixture() + server = Server(policy, resource) + server.pods = [pod(f"engine-7d9f8c6b5-{index:05d}") for index in range(101)] + with pytest.raises(RunnerPolicyError): + await KubernetesAdapter(policy, command=server.command).observe() + server.pods = server.pods[:100] + result = await KubernetesAdapter(policy, command=server.command).observe() + assert len(result.resources[0].members) == 100 + + +@pytest.mark.parametrize( + "selector", + [ + None, + {}, + {"matchLabels": {"app": "engine,tier=x"}}, + {"matchLabels": {"app in (a)": "engine"}}, + {"matchLabels": {"app": "engine", "bad key": "x"}}, + {"matchLabels": {"app": "x" * 64}}, + {"matchLabels": ["app=engine"]}, + {"matchLabels": {"app": 1}}, + ], +) +async def test_a_selector_that_is_not_plain_labels_is_refused_before_listing_pods(selector): + policy, _, resource = fixture() + if selector is None: + del resource["spec"]["selector"] + else: + resource["spec"]["selector"] = selector + server = Server(policy, resource) + with pytest.raises(RunnerPolicyError): + await KubernetesAdapter(policy, command=server.command).observe() + assert not any("pods" in argv for argv, _ in server.calls) + + +async def test_selector_uses_every_match_label_in_a_stable_order(): + policy, _, resource = fixture() + resource["spec"]["selector"] = { + "matchLabels": {"tier": "workers", "app.kubernetes.io/name": "engine", "app": "engine"}, + "matchExpressions": [{"key": "track", "operator": "In", "values": ["stable"]}], + } + server = Server(policy, resource) + server.pods = [] + calls = [] + + async def command(argv, **kwargs): + calls.append(argv) + if "pods" in argv: + return b'{"items":[]}' + return await server.command(argv, **kwargs) + + result = await KubernetesAdapter(policy, command=command).observe() + assert result.resources[0].members == [] + assert calls[-1][6:] == [ + "get", + "pods", + "--selector=app=engine,app.kubernetes.io/name=engine,tier=workers", + "-o=json", + ] + + +@pytest.mark.parametrize( + "changes", + [ + {"phase": "Running Now"}, + {"node": "node/1"}, + {"containers": [{"name": "main", "restartCount": -1, "state": {}}]}, + {"containers": [{"name": "main", "restartCount": 0, "state": {"running": {"startedAt": "today"}}}]}, + ], +) +async def test_malformed_pod_facts_refuse_the_observation(changes): + policy, _, resource = fixture() + server = Server(policy, resource) + server.pods = [pod("engine-7d9f8c6b5-x2kq4", **changes)] + with pytest.raises(RunnerPolicyError): + await KubernetesAdapter(policy, command=server.command).observe() + + +async def test_api_components_list_no_pods(): + policy, _, resource = fixture(kind="api") + server = Server(policy, resource) + result = await KubernetesAdapter(policy, command=server.command).observe() + assert result.resources[0].members == [] and not any("pods" in argv for argv, _ in server.calls) + + +async def test_expression_only_selector_lists_the_namespace_and_keeps_owned_pods(): + policy, _, resource = fixture() + resource["spec"]["selector"] = {"matchExpressions": [{"key": "app", "operator": "In", "values": ["engine"]}]} + server = Server(policy, resource) + calls = [] + + async def command(argv, **kwargs): + calls.append(argv) + if "pods" in argv: + other = pod("other-abc12", template="abc12", labels={"app": "other", "pod-template-hash": "abc12"}) + other["metadata"]["ownerReferences"][0]["name"] = "other-abc12" + return json.dumps({"items": [other, pod("engine-7d9f8c6b5-x2kq4")]}).encode() + return await server.command(argv, **kwargs) + + result = await KubernetesAdapter(policy, command=command).observe() + assert calls[-1][6:] == ["get", "pods", "-o=json"] + assert [member.host for member in result.resources[0].members] == ["engine-7d9f8c6b5-x2kq4"] + + +@pytest.mark.parametrize( + "shape", + [ + "not a pod", + {"spec": {}}, + {"metadata": {"name": "engine-7d9f8c6b5-x", "labels": ["app=engine"]}}, + {"metadata": {"name": "engine-7d9f8c6b5-x", "namespace": "owned", "ownerReferences": ["engine"]}}, + ], +) +async def test_pod_data_without_the_kubernetes_shape_refuses_the_observation(shape): + policy, _, resource = fixture() + server = Server(policy, resource) + server.pods = [shape] + with pytest.raises(RunnerPolicyError): + await KubernetesAdapter(policy, command=server.command).observe() diff --git a/workers/agentic/src/weave_agentic_worker/main.py b/workers/agentic/src/weave_agentic_worker/main.py index 07ea9d9e..c5cfa8da 100644 --- a/workers/agentic/src/weave_agentic_worker/main.py +++ b/workers/agentic/src/weave_agentic_worker/main.py @@ -37,6 +37,8 @@ from firefly_weave.sdk.transport import WorkerTransport from firefly_weave.sdk.worker import Worker from firefly_weave.sdk.worker_auth import ClientCredentialsTokenProvider, WorkerTokenAuth, private_transport +from firefly_weave.sdk.worker_identity import worker_identity +from firefly_weave.sdk.worker_registration import register_worker_http from weave_agentic_worker.handler import AgenticTaskHandler, WorkerPolicy @@ -101,11 +103,17 @@ async def main() -> None: # Plain HTTP to the API only through this process's platform-api entry, pinned and peer checked. transport=private_transport(api, "platform-api", max_connections=4), ) as client: - response = await client.post( - prefix + "/workers", - json={"release_id": os.environ["WEAVE_WORKER_RELEASE_ID"], "task_types": [CAPABILITY], "capacity": slots}, + # A server that predates worker identity gets one retry without it, on this same client. + response = await register_worker_http( + client, + prefix, + { + "release_id": os.environ["WEAVE_WORKER_RELEASE_ID"], + "task_types": [CAPABILITY], + "capacity": slots, + "identity": worker_identity().model_dump(mode="json"), + }, ) - response.raise_for_status() transport = WorkerTransport(client, prefix, UUID(response.json()["id"])) worker = Worker(transport, {CAPABILITY: AgenticTaskHandler(transport, policy)}, slots) loop = asyncio.get_running_loop() diff --git a/workers/agentic/tests/test_main.py b/workers/agentic/tests/test_main.py index 45e9d085..246f7f21 100644 --- a/workers/agentic/tests/test_main.py +++ b/workers/agentic/tests/test_main.py @@ -234,3 +234,156 @@ def test_worker_token_mode_remains_available(tmp_path, monkeypatch): monkeypatch.delenv("WEAVE_WORKER_OAUTH_CONFIG_FILE", raising=False) monkeypatch.setenv("WEAVE_WORKER_TOKEN_FILE", str(tmp_path / "token")) assert isinstance(main.worker_auth("https://weave.example"), TokenFileAuth) + + +IDENTITY_SETTINGS = ( + "WEAVE_WORKER_REPORT_HOST", + "WEAVE_WORKER_LABELS", + "WEAVE_POD_NAMESPACE", + "WEAVE_NODE_NAME", + "KUBERNETES_SERVICE_HOST", +) +VALIDATION = {"status": 422, "code": "WV-VALIDATION", "message": "Invalid request contract"} + + +def registration_environment(tmp_path, monkeypatch): + boot_environment(tmp_path, monkeypatch, "http://api.weave.test:8080") + for name in IDENTITY_SETTINGS: + monkeypatch.delenv(name, raising=False) + + +async def test_main_retries_identity_validation_once_on_the_pinned_client(tmp_path, monkeypatch): + import json + + import weave_agentic_worker.main as main + + registration_environment(tmp_path, monkeypatch) + requests = [] + + async def receive(request): + requests.append(request) + return httpx.Response(422, json={"code": "WV-VALIDATION"}) if len(requests) == 1 else httpx.Response(503) + + pinned = [] + + def transport(url, purpose, *, max_connections): + pinned.append((url, purpose, max_connections)) + return httpx.MockTransport(receive) + + monkeypatch.setattr(main, "private_transport", transport) + with po.installed(po.PrivateOrigins.empty()), pytest.raises(httpx.HTTPStatusError): + await main.main() + assert pinned == [("http://api.weave.test:8080", "platform-api", 4)] + assert len(requests) == 2 + bodies = [json.loads(request.content) for request in requests] + assert bodies[0]["identity"] and "identity" not in bodies[1] + assert {k: v for k, v in bodies[0].items() if k != "identity"} == bodies[1] + assert bodies[1]["capacity"] == 2 + assert [request.headers["authorization"] for request in requests] == ["Bearer worker-token"] * 2 + assert [str(request.url) for request in requests] == ["http://api.weave.test:8080/environment/workers"] * 2 + + +def agentic_registration(monkeypatch, responses): + import weave_agentic_worker.main as main + + requests = [] + + async def receive(request): + requests.append(request) + result = responses[len(requests) - 1] + if isinstance(result, BaseException): + raise result + return result + + def transport(url, purpose, *, max_connections): + return httpx.MockTransport(receive) + + monkeypatch.setattr(main, "private_transport", transport) + return main, requests + + +@pytest.mark.parametrize("first", ["accepted", "old server"]) +async def test_main_serves_with_the_registered_id_after_at_most_one_fallback(tmp_path, monkeypatch, first): + import asyncio + import json + from uuid import uuid4 + + from firefly_weave.sdk.transport import WorkerTransport + + registration_environment(tmp_path, monkeypatch) + worker_id = uuid4() + accepted = httpx.Response(201, json={"id": str(worker_id)}) + responses = [accepted] if first == "accepted" else [httpx.Response(422, json=VALIDATION), accepted] + main, requests = agentic_registration(monkeypatch, responses) + served = [] + + async def run(self): + served.append(self) + + monkeypatch.setattr(main.Worker, "run", run) + monkeypatch.setattr(type(asyncio.get_running_loop()), "add_signal_handler", lambda *args: None) + with po.installed(po.PrivateOrigins.empty()): + await main.main() + assert len(requests) == len(responses) + assert "identity" in json.loads(requests[0].content) + assert len(served) == 1 and type(served[0].client) is WorkerTransport + assert served[0].client.worker_id == worker_id and served[0].concurrency == 2 + + +@pytest.mark.parametrize( + "first,raised", + [ + (httpx.Response(401, json={"status": 401, "code": "WV-UNAUTHENTICATED"}), httpx.HTTPStatusError), + (httpx.Response(403, json={"status": 403, "code": "WV-FORBIDDEN"}), httpx.HTTPStatusError), + (httpx.Response(409, json={"status": 409, "code": "WV-CONFLICT"}), httpx.HTTPStatusError), + (httpx.Response(500, json={"status": 500, "code": "WV-INTERNAL"}), httpx.HTTPStatusError), + (httpx.Response(422, json={"status": 422, "code": "WV-PAGE"}), httpx.HTTPStatusError), + (httpx.Response(422, content=b"not json"), httpx.HTTPStatusError), + (httpx.ConnectError("refused"), httpx.ConnectError), + ("cancelled", BaseException), + ], +) +async def test_main_registers_once_on_any_other_failure(tmp_path, monkeypatch, first, raised): + import asyncio + + registration_environment(tmp_path, monkeypatch) + failure = asyncio.CancelledError() if first == "cancelled" else first + main, requests = agentic_registration(monkeypatch, [failure, httpx.Response(201, json={"id": "unused"})]) + with po.installed(po.PrivateOrigins.empty()), pytest.raises(raised) as caught: + await main.main() + if first == "cancelled": + assert isinstance(caught.value, asyncio.CancelledError) + assert len(requests) == 1 + + +async def test_main_stops_after_a_second_identity_refusal(tmp_path, monkeypatch): + registration_environment(tmp_path, monkeypatch) + main, requests = agentic_registration( + monkeypatch, [httpx.Response(422, json=VALIDATION), httpx.Response(422, json=VALIDATION)] + ) + with po.installed(po.PrivateOrigins.empty()), pytest.raises(httpx.HTTPStatusError): + await main.main() + assert len(requests) == 2 + + +async def test_main_reports_its_bounded_identity(tmp_path, monkeypatch): + import json + + registration_environment(tmp_path, monkeypatch) + monkeypatch.setenv("WEAVE_WORKER_LABELS", "team=payments") + monkeypatch.setenv("WEAVE_WORKER_REPORT_HOST", "false") + main, requests = agentic_registration(monkeypatch, [httpx.Response(503)]) + with po.installed(po.PrivateOrigins.empty()), pytest.raises(httpx.HTTPStatusError): + await main.main() + identity = json.loads(requests[0].content)["identity"] + assert "host" not in identity and identity["labels"] == {"team": "payments"} + assert identity["runtime"] == "process" and identity["platform_hint"] == "none" + + +async def test_main_refuses_invalid_labels_before_any_request(tmp_path, monkeypatch): + registration_environment(tmp_path, monkeypatch) + monkeypatch.setenv("WEAVE_WORKER_LABELS", "api_token=hunter2") + main, requests = agentic_registration(monkeypatch, [httpx.Response(201, json={"id": "unused"})]) + with po.installed(po.PrivateOrigins.empty()), pytest.raises(ValueError, match="WEAVE_WORKER_LABELS") as caught: + await main.main() + assert requests == [] and "hunter2" not in str(caught.value) diff --git a/workers/files/src/weave_files_worker/main.py b/workers/files/src/weave_files_worker/main.py index 3982aab8..867f83ed 100644 --- a/workers/files/src/weave_files_worker/main.py +++ b/workers/files/src/weave_files_worker/main.py @@ -42,6 +42,8 @@ ) from firefly_weave.sdk.transport import WorkerTransport from firefly_weave.sdk.worker import Worker +from firefly_weave.sdk.worker_identity import worker_identity +from firefly_weave.sdk.worker_registration import register_worker_http from pydantic import BaseModel, ConfigDict, Field from weave_files_worker.handler import FileTaskHandler @@ -122,11 +124,17 @@ async def main() -> None: trust_env=False, follow_redirects=False, ) as client: - response = await client.post( - prefix + "/workers", - json={"release_id": os.environ["WEAVE_WORKER_RELEASE_ID"], "task_types": CAPABILITIES, "capacity": 1}, + # A server that predates worker identity gets one retry without it, on this same client. + response = await register_worker_http( + client, + prefix, + { + "release_id": os.environ["WEAVE_WORKER_RELEASE_ID"], + "task_types": [*CAPABILITIES], + "capacity": 1, + "identity": worker_identity().model_dump(mode="json"), + }, ) - response.raise_for_status() transport = WorkerTransport(client, prefix, UUID(response.json()["id"])) handler = FileTaskHandler(transport, policy) worker = Worker(transport, {capability: handler for capability in CAPABILITIES}, 1) diff --git a/workers/files/tests/test_files_main.py b/workers/files/tests/test_files_main.py index 04289461..d013965f 100644 --- a/workers/files/tests/test_files_main.py +++ b/workers/files/tests/test_files_main.py @@ -43,3 +43,160 @@ def test_worker_policy_requires_specific_origins(tmp_path): file.write_text('{"origins":["https://*.example.com"]}') with pytest.raises(ValueError): read_policy(file) + + +IDENTITY_SETTINGS = ( + "WEAVE_WORKER_REPORT_HOST", + "WEAVE_WORKER_LABELS", + "WEAVE_POD_NAMESPACE", + "WEAVE_NODE_NAME", + "KUBERNETES_SERVICE_HOST", +) +VALIDATION = {"status": 422, "code": "WV-VALIDATION", "message": "Invalid request contract"} + + +def registration_environment(tmp_path, monkeypatch): + policy = tmp_path / "policy.json" + policy.write_text('{"origins":["https://graph.microsoft.com"]}') + token = tmp_path / "token" + token.write_text("worker-token") + for key, value in { + "WEAVE_FILES_POLICY_FILE": str(policy), + "WEAVE_WORKER_TOKEN_FILE": str(token), + "WEAVE_API_URL": "https://weave.example", + "WEAVE_ENVIRONMENT_URL": "/environment", + "WEAVE_WORKER_RELEASE_ID": "release-1", + }.items(): + monkeypatch.setenv(key, value) + for name in IDENTITY_SETTINGS: + monkeypatch.delenv(name, raising=False) + + +def files_registration(monkeypatch, responses): + import httpx + + import weave_files_worker.main as main + import weave_files_worker.providers # noqa: F401 Imported before AsyncClient is replaced below. + + requests, configurations = [], [] + + async def receive(request): + requests.append(request) + result = responses[len(requests) - 1] + if isinstance(result, BaseException): + raise result + return result + + original = httpx.AsyncClient + + def client(**kwargs): + configurations.append(kwargs) + return original(**kwargs, transport=httpx.MockTransport(receive)) + + monkeypatch.setattr(main.httpx, "AsyncClient", client) + return main, requests, configurations + + +async def test_main_retries_identity_validation_once_on_the_existing_client(tmp_path, monkeypatch): + import httpx + + registration_environment(tmp_path, monkeypatch) + main, requests, configurations = files_registration( + monkeypatch, [httpx.Response(422, json={"code": "WV-VALIDATION"}), httpx.Response(503)] + ) + with pytest.raises(httpx.HTTPStatusError): + await main.main() + assert len(requests) == 2 and len(configurations) == 1 + assert configurations[0]["timeout"] == 15 and configurations[0]["trust_env"] is False + assert configurations[0]["follow_redirects"] is False + bodies = [json.loads(request.content) for request in requests] + assert bodies[0]["identity"] and "identity" not in bodies[1] + assert {k: v for k, v in bodies[0].items() if k != "identity"} == bodies[1] + assert bodies[1]["capacity"] == 1 + assert [request.headers["authorization"] for request in requests] == ["Bearer worker-token"] * 2 + assert [str(request.url) for request in requests] == ["https://weave.example/environment/workers"] * 2 + + +@pytest.mark.parametrize("first", ["accepted", "old server"]) +async def test_main_serves_with_the_registered_id_after_at_most_one_fallback(tmp_path, monkeypatch, first): + import asyncio + from uuid import uuid4 + + import httpx + from firefly_weave.sdk.transport import WorkerTransport + + registration_environment(tmp_path, monkeypatch) + worker_id = uuid4() + accepted = httpx.Response(201, json={"id": str(worker_id)}) + responses = [accepted] if first == "accepted" else [httpx.Response(422, json=VALIDATION), accepted] + main, requests, configurations = files_registration(monkeypatch, responses) + served = [] + + async def run(self): + served.append(self) + + monkeypatch.setattr(main.Worker, "run", run) + monkeypatch.setattr(type(asyncio.get_running_loop()), "add_signal_handler", lambda *args: None) + await main.main() + assert len(requests) == len(responses) and len(configurations) == 1 + assert "identity" in json.loads(requests[0].content) + assert len(served) == 1 and type(served[0].client) is WorkerTransport + assert served[0].client.worker_id == worker_id and served[0].concurrency == 1 + + +@pytest.mark.parametrize( + "status,body,network", + [ + (401, {"status": 401, "code": "WV-UNAUTHENTICATED"}, None), + (403, {"status": 403, "code": "WV-FORBIDDEN"}, None), + (409, {"status": 409, "code": "WV-CONFLICT"}, None), + (500, {"status": 500, "code": "WV-INTERNAL"}, None), + (422, {"status": 422, "code": "WV-PAGE"}, None), + (422, None, None), + (None, None, "connect"), + (None, None, "cancelled"), + ], +) +async def test_main_registers_once_on_any_other_failure(tmp_path, monkeypatch, status, body, network): + import asyncio + + import httpx + + registration_environment(tmp_path, monkeypatch) + if network == "connect": + first, raised = httpx.ConnectError("refused"), httpx.ConnectError + elif network == "cancelled": + first, raised = asyncio.CancelledError(), asyncio.CancelledError + else: + content = {"json": body} if body is not None else {"content": b"not json"} + first, raised = httpx.Response(status, **content), httpx.HTTPStatusError + main, requests, _ = files_registration(monkeypatch, [first, httpx.Response(201, json={"id": "unused"})]) + with pytest.raises(raised): + await main.main() + assert len(requests) == 1 + + +async def test_main_stops_after_a_second_identity_refusal(tmp_path, monkeypatch): + import httpx + + registration_environment(tmp_path, monkeypatch) + main, requests, _ = files_registration( + monkeypatch, [httpx.Response(422, json=VALIDATION), httpx.Response(422, json=VALIDATION)] + ) + with pytest.raises(httpx.HTTPStatusError): + await main.main() + assert len(requests) == 2 + + +async def test_main_reports_its_bounded_identity(tmp_path, monkeypatch): + import httpx + + registration_environment(tmp_path, monkeypatch) + monkeypatch.setenv("WEAVE_POD_NAMESPACE", "weave-files") + monkeypatch.setenv("KUBERNETES_SERVICE_HOST", "10.0.0.1") + main, requests, _ = files_registration(monkeypatch, [httpx.Response(503)]) + with pytest.raises(httpx.HTTPStatusError): + await main.main() + identity = json.loads(requests[0].content)["identity"] + assert identity["namespace"] == "weave-files" and identity["platform_hint"] == "kubernetes" + assert identity["runtime"] == "container" From 27451976ab5bbfea68af3f3d6c83184a6b49f981 Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Mon, 12 Oct 2026 00:43:19 -0700 Subject: [PATCH 13/63] Report only live placement members and never fail an observation on them --- CHANGELOG.md | 27 +- docs/guides/workers.md | 6 +- docs/operations/operations-runner.md | 36 ++- docs/operations/upgrades.md | 54 +++- docs/reference/api.md | 26 +- src/firefly_weave/contracts/workers.py | 10 +- .../deployment_runner/compose.py | 62 ++-- .../deployment_runner/kubernetes.py | 199 +++++++----- src/firefly_weave/workers/models.py | 2 +- .../test_worker_identity_contract.py | 49 ++- .../test_operations_fact_writes.py | 45 ++- tests/unit/test_deployment_compose.py | 78 +++-- tests/unit/test_deployment_kubernetes.py | 292 ++++++++++++------ 13 files changed, 598 insertions(+), 288 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1271b7ca..cf0fbefb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -64,20 +64,29 @@ SPDX-License-Identifier: Apache-2.0 `identity` with host, runtime, versions, platform, Kubernetes namespace and node, and up to 8 labels from `WEAVE_WORKER_LABELS` (`WEAVE_WORKER_REPORT_HOST=false` leaves out the host). The Agentic and Files - workers, native executors and the SDK's `register_worker` send it, and retry - once without it when an older server answers `422 WV-VALIDATION`. New SDK - helpers: `worker_identity` and `register_worker_http`. + workers and the SDK's `register_worker` send it and retry once without it when + an older server answers `422 WV-VALIDATION`; native executors send it inside + the API process. With native executors configured, an invalid + `WEAVE_WORKER_LABELS` stops the API from starting. New SDK helpers: + `worker_identity` and `register_worker_http`. - Worker reads, lists, and drain or resume answers add optional `identity` and `registered_at`. Upgrade CLIs and SDKs with the server: earlier releases fail with `502 WV-WIRE` on workers registered after the upgrade. The registration answer and stored worker records keep their bytes, and a worker cannot report a placement. -- Deployment observations list up to 100 `members` of each worker component: - Compose containers and Kubernetes pods, with host, ordinal, state, readiness, - start time, restarts and node. Observations without members keep their bytes - and digests. Kubernetes runners need `list` on the namespace's pods to - observe worker components; a worker component with more than 100 members - fails the observation instead of being truncated. +- Deployment observations list up to 100 live `members` of each worker + component: running or starting Compose containers and Kubernetes pods, with + host, ordinal, state, readiness, start time, restarts and node. Stopped, + terminating, finished, failed and evicted ones are left out. With more than 100 + live members, or when the runner cannot list them all, `members` is left out + (placement unknown) and the observation still succeeds. Kubernetes runners + read members through the deployment's selector, including set-based + `matchExpressions`, and need `list` on the namespace's pods. Observations + without members keep their bytes and digests. +- Upgrade the server before its deployment runners: an older server refuses the + `members` an upgraded runner reports, so observe jobs fail and applies end + `reconciliation_required`. Earlier CLIs and SDKs fail with `502 WV-WIRE` on + observation reads and lists that carry members. - Keep typing in the selected form field when deferred row autofocus completes. Rapid row additions focus the newest item without overriding a later choice. diff --git a/docs/guides/workers.md b/docs/guides/workers.md index 68cfe76b..148f7208 100644 --- a/docs/guides/workers.md +++ b/docs/guides/workers.md @@ -254,9 +254,9 @@ older server refuses it. The checked-in example registers without one. Labels are visible to everyone who can read workers. Never put credentials in them: keys that name a token, password, secret or authorization are refused, -and an invalid label stops the worker before it registers (for native -executors, it stops the API's executor start), with a message that does not -repeat the value. An invalid hostname or platform value is left out +and an invalid label stops the worker before it registers, with a message that +does not repeat the value. On the API host, with native executors configured, +an invalid label stops the API from starting. An invalid hostname or platform value is left out instead. ## 4. Observe one task through completion diff --git a/docs/operations/operations-runner.md b/docs/operations/operations-runner.md index 60377579..dec4037f 100644 --- a/docs/operations/operations-runner.md +++ b/docs/operations/operations-runner.md @@ -57,9 +57,12 @@ substantial host authority: isolate the runner and keep its provider permissions within the destination it operates. For Kubernetes, grant only the namespace and resource permissions needed by the allowed deployments. API-component updates also require reading that namespace's horizontal pod autoscalers so the runner -can reject a conflicting scaling policy. Observing a worker component also lists -the namespace's pods (`list` on `pods`), so grant that read before upgrading -the runner; without it, observing a target with a worker component fails. +can reject a conflicting scaling policy. To report where worker pods run, the +runner also lists the namespace's pods (`list` on `pods`); without that read, +observations leave the worker's members out. Upgrade the API server before its +runners: an older server refuses the members a newer runner reports, so its +observe jobs fail and its applies end in reconciliation. Grant the read after +the server upgrade and before the runner upgrade. ## 2. Copy the destination's actual identity @@ -213,6 +216,13 @@ then applies only the named services and waits for container readiness. Add meaningful health checks to your Compose services; without one, a running process is the only readiness evidence Docker provides. +For a worker service, the observation also lists its live containers as +members: the container ID prefix, which is the container's default hostname, +and its Compose number. Exited, dead, and removing containers are left out. With +more than 100 live containers, or a container whose details cannot be read, the +observation leaves the members out (placement unknown) and still succeeds. A +service that sets its own `hostname` gets no placement. + ### Kubernetes The adapter manages **existing deployments**, not arbitrary resources. Each @@ -238,14 +248,18 @@ The adapter changes only the selected container's image/resources, replicas, and its ownership label. A worker scaling plan does not change the image. Objects already labeled as belonging to another Weave target are rejected. -For a worker component, the observation also lists the deployment's pods as -members, read with the deployment's own `matchLabels` selector inside the -pinned namespace. A pod counts only when its controlling ReplicaSet is named -after the deployment and its pod template hash. More than 100 member pods, or a -selector label that is not valid Kubernetes label syntax, fails the observation -instead of reporting part of it. A Compose worker service reports its containers the same way, -using each container's ID prefix and Compose number; it also fails with more -than 100 containers. +For a worker component, the observation also lists the deployment's live pods +as members, inside the pinned namespace and through the deployment's own +selector: `matchLabels` become `key=value` and `matchExpressions` become +set-based requirements (`key in (a,b)`, `key notin (a,b)`, `key`, `!key`). The +runner reads only the few pod fields it needs. A pod counts only when its +controlling ReplicaSet is named after the deployment and its pod template hash; +terminating, finished, failed, and evicted pods are left out. Members are +placement only and never fail an observation or an apply: with more than 100 +live pods, a selector kubectl cannot express exactly, a label that is not valid +Kubernetes syntax, or a listing that fails, the observation leaves the members +out (placement unknown) instead of reporting part of them, and it never lists +the whole namespace. ### Azure Container Apps diff --git a/docs/operations/upgrades.md b/docs/operations/upgrades.md index 0d694d0f..a74e0513 100644 --- a/docs/operations/upgrades.md +++ b/docs/operations/upgrades.md @@ -45,7 +45,7 @@ the desktop app never migrates a platform. | alpha13 | `0030_worker_presence` | No new schema migration. Workers retry explicit capacity rejections while they read task context or credentials, and an Agentic preparation timeout before provider execution counts as not started. Upgrade the Agentic and Files workers to 0.1.5 together with the server. | | alpha14 | `0030_worker_presence` | No new schema migration. Adds the detached Docker development platform (`weave platform up`); existing foreground installations keep working and are never converted. The Agentic and Files workers 0.1.6 pin this server version. | | alpha15 | `0030_worker_presence` | No new schema migration and no role change. Upgrade CLIs and SDKs together with the server, review native executor `capacity` and the legacy private-network settings before restarting, and finish runs that use text templates before any rollback to alpha14; see [what changes in alpha15](#what-changes-in-alpha15). The Agentic and Files workers 0.1.7 pin this server version. | -| Unreleased | `0031_operations_facts` | Adds [scoped operational fact storage](#operational-fact-storage), a bounded historical backfill, and reserved capacity for terminal fact updates. Upgrade CLIs and SDKs together with the server; see [incident list compatibility](#incident-list-compatibility) and [worker identity compatibility](#worker-identity-compatibility). | +| Unreleased | `0031_operations_facts` | Adds [scoped operational fact storage](#operational-fact-storage), a bounded historical backfill, and reserved capacity for terminal fact updates. Upgrade CLIs and SDKs together with the server, and upgrade the server before its deployment runners; see [incident list compatibility](#incident-list-compatibility) and [worker identity compatibility](#worker-identity-compatibility). | ![Schema, compatibility and execution acceptance gates](../diagrams/operations-upgrade.svg) @@ -240,14 +240,15 @@ unknown. ## Worker identity compatibility Workers can now report where they run when they register, and worker reads -return it. The change crosses versions in both directions: - -- **New workers, older server.** The Agentic and Files workers, native - executors, and the Python SDK's `register_worker` send an `identity`. A server - without it refuses the field with `422 WV-VALIDATION`; the worker then - registers exactly once more without it, on the same connection and - credentials. Other failures are not retried. Upgrade servers first to keep the - identity. +return it. Deployment observations can also list the containers or pods of a +worker component. Both changes cross versions: + +- **New workers, older server.** The Agentic and Files workers and the Python + SDK's `register_worker` send an `identity`. A server without it refuses the + field with `422 WV-VALIDATION`; the worker then registers exactly once more + without it, on the same connection and credentials. Other failures are not + retried. Upgrade servers first to keep the identity. Native executors register + inside the API process, so they always use a server of their own version. - **Older clients, new server.** Worker reads, lists, and drain or resume answers add `identity` and `registered_at` when they are known. **Upgrade CLIs and SDKs with the server.** Earlier releases reject the new fields: @@ -259,14 +260,33 @@ return it. The change crosses versions in both directions: records keep their earlier shape. - **Labels.** `WEAVE_WORKER_LABELS` is read only from the worker's own environment. An invalid label stops the worker before it registers, so check - the setting before rolling out new worker images. - -Deployment observations add `members` to worker components, left out when -empty, so existing observations, their digests, and plans reviewed against them -are unchanged. Compose and Kubernetes resource versions are also unchanged. A -Kubernetes runner now lists the namespace's pods to observe a worker component: -grant `list` on `pods` in that namespace before upgrading the runner. See -[the runner guide](operations-runner.md#kubernetes). + the setting before rolling out new worker images. With native executors + configured, an invalid label on the API's own environment stops the API from + starting. + +**Upgrade the server before its runners.** An upgraded runner reports +`members` for worker components, and an older server refuses them: the runner's +observe jobs fail, and an apply that touches a worker component ends +`reconciliation_required` with an ambiguous receipt, even though the change +was applied. Upgrade the API first, then each runner. + +- **Older clients, observations with members.** Observations that an upgraded + runner recorded with `members` fail with `502 WV-WIRE` in earlier releases: + `weave operations observations read` and `list`, and the Python SDK's + `invoke("deployment_observations.read")` and + `invoke("deployment_observations.list")`. Upgrade CLIs and SDKs with the + server before upgrading runners. +- **Unchanged.** Observations without members, their digests, plans reviewed + against them, and Compose and Kubernetes resource versions keep their bytes. +- **Members are placement only.** They list live containers or pods: stopped, + terminating, finished, failed, and evicted ones are left out. When a worker + component has more than 100 live members, or the runner cannot list them all, + `members` is left out and placement is unknown; the observation itself still + succeeds. +- **Kubernetes.** After upgrading the server, and before upgrading a Kubernetes + runner, grant it `list` on `pods` in the namespace. Without that permission + the runner leaves `members` out. See + [the runner guide](operations-runner.md#kubernetes). ## Operational fact storage diff --git a/docs/reference/api.md b/docs/reference/api.md index c7d06996..973f9de6 100644 --- a/docs/reference/api.md +++ b/docs/reference/api.md @@ -704,9 +704,11 @@ where the worker runs: `host`, `runtime` (`embedded`, `container`, or registration) when they are known. Workers registered before the server recorded them have neither. -An older server refuses `identity` with `422` `WV-VALIDATION`. Current workers -and the Python SDK then register exactly once more without it, using the same -client and credentials; any other failure is not retried. +An older server refuses `identity` with `422` `WV-VALIDATION`. Current packaged +workers and the Python SDK then register exactly once more without it, using the +same client and credentials; any other failure is not retried. Native executors +register inside the API process and always use a server of their own version. +The registration answer's schema (`WorkerInstance`) has no `identity` field. **Answers that say "unavailable".** Cancellation and task completion can return an unavailable acknowledgment when the platform cannot classify the historical @@ -956,13 +958,17 @@ and runner registrations belong to one environment. Child-resource reads check the target grant. Lists filter authorized target IDs before pagination; use `target_id` to select a target, and `deployment_id` to narrow plan history. An observation is a dated snapshot, not a promise of current health. -A worker component's observed resource lists up to 100 `members`, one per -container or pod: `host`, `ordinal`, `state`, `ready`, `started_at`, -`restarts`, and `node`. Compose reports the container ID prefix, which is the -container's default hostname, and its Compose number as `ordinal`; Kubernetes -reports the pod name and node. A runner refuses to report a worker component -with more than 100 members rather than truncate it. Resources without members -leave the field out, so their observations keep their digests. +A worker component's observed resource lists up to 100 live `members`, one per +running or starting container or pod: `host`, `ordinal`, `state`, `ready`, +`started_at`, `restarts`, and `node`. Compose reports the container ID prefix, +which is the container's default hostname, and its Compose number as +`ordinal`; Kubernetes reports the pod name and node. Stopped, terminating, +finished, failed, and evicted containers or pods are left out. Members are +placement only: with more than 100 live members, or when the runner cannot list +them all, the field is left out (placement unknown) rather than truncated, and +the observation still succeeds. A resource without the field therefore has no +members or unknown ones, and its observation keeps its digest. Upgrade the +server before its runners; an older server refuses a report with `members`. The server creates an immutable plan from the desired revision and a fresh, complete observation. Approval records its exact digest. Applying checks the diff --git a/src/firefly_weave/contracts/workers.py b/src/firefly_weave/contracts/workers.py index 227cf42f..0b21e141 100644 --- a/src/firefly_weave/contracts/workers.py +++ b/src/firefly_weave/contracts/workers.py @@ -107,16 +107,21 @@ def bounded(self) -> Self: return self -class InstanceRequest(ContractModel): +class InstanceFields(ContractModel): + """What a registration request and the stored instance share.""" + release_id: UUID task_types: list[str] = Field(min_length=1, max_length=100) capacity: int = Field(ge=1, le=100) + + +class InstanceRequest(InstanceFields): # Sent by current workers; a server that predates it refuses the field, and workers then # register once more without it. identity: WorkerIdentity | None = Field(default=None, exclude_if=lambda v: v is None) -class WorkerInstance(InstanceRequest): +class WorkerInstance(InstanceFields): """The stable registration answer and stored instance. Identity lives in the worker's facts.""" id: UUID @@ -144,6 +149,7 @@ class WorkerStatus(WorkerInstance): observed_at: AwareDatetime active_leases: int = Field(ge=0) available_capacity: int | None = Field(ge=0, le=100) + identity: WorkerIdentity | None = Field(default=None, exclude_if=lambda v: v is None) registered_at: AwareDatetime | None = Field(default=None, exclude_if=lambda v: v is None) diff --git a/src/firefly_weave/deployment_runner/compose.py b/src/firefly_weave/deployment_runner/compose.py index f6bf1980..7f020e12 100644 --- a/src/firefly_weave/deployment_runner/compose.py +++ b/src/firefly_weave/deployment_runner/compose.py @@ -21,6 +21,7 @@ import fcntl import json import os +import re import stat import tempfile from collections.abc import Awaitable, Callable @@ -57,31 +58,44 @@ def _versioned(fact: dict[str, Any]) -> dict[str, Any]: return {key: value for key, value in fact.items() if key != "number"} -def _members(facts: list[dict[str, Any]]) -> list[ObservedMember]: - """Worker containers as members; more than the contract holds refuses instead of truncating.""" - if len(facts) > 100: +def _ordinal(number: object) -> int | None: + """The Compose container number; only ASCII digits, as Compose writes it.""" + if number in (None, ""): + return None + if not isinstance(number, str) or not re.fullmatch(r"[0-9]{1,7}", number): raise RunnerPolicyError() - members = [] - for fact in facts: - number = fact.get("number") - if number not in (None, "") and not (isinstance(number, str) and number.isdigit() and len(number) <= 7): - raise RunnerPolicyError() - try: - members.append( - ObservedMember( - # Docker's default hostname, which the worker reports, is the ID prefix. - host=fact["id"][:12], - ordinal=int(number) if number else None, - state=fact["state"], - ready=_ready(fact), - started_at=observed_time(fact["started"]), - restarts=fact["restart"], - node=None, - ) + return int(number) + + +# Containers in these states run no worker that placement could describe. +_STOPPED = frozenset({"exited", "dead", "removing"}) + + +def _members(facts: list[dict[str, Any]]) -> list[ObservedMember] | None: + """Live worker containers as members, or None when they cannot all be reported. + + Members are placement only, so they never fail an observation or an apply: more than 100 live + containers or one unreadable container leaves them out (unknown) rather than reporting part of them. + """ + live = [fact for fact in facts if fact.get("state") not in _STOPPED] + if len(live) > 100: + return None + try: + return [ + ObservedMember( + # Docker's default hostname, which the worker reports, is the ID prefix. + host=fact["id"][:12], + ordinal=_ordinal(fact.get("number")), + state=fact["state"], + ready=_ready(fact), + started_at=observed_time(fact["started"]), + restarts=fact["restart"], + node=None, ) - except (KeyError, ValidationError): - raise RunnerPolicyError() from None - return members + for fact in live + ] + except (KeyError, RunnerPolicyError, ValidationError): + return None class ComposeAdapter: @@ -194,7 +208,7 @@ async def observe(self) -> ObservationReport: ownership="managed" if managed else "imported", state="stopped" if stopped else "ready" if stable else "progressing", # Placement only concerns workers; other components keep their observation bytes. - members=_members(members) if binding.kind == "worker" else [], + members=(_members(members) or []) if binding.kind == "worker" else [], ) ) return ObservationReport(resources=resources, complete=True, settled=settled) diff --git a/src/firefly_weave/deployment_runner/kubernetes.py b/src/firefly_weave/deployment_runner/kubernetes.py index 780976e4..06ceed2b 100644 --- a/src/firefly_weave/deployment_runner/kubernetes.py +++ b/src/firefly_weave/deployment_runner/kubernetes.py @@ -25,8 +25,6 @@ from decimal import Decimal from typing import Any -from pydantic import ValidationError - from firefly_weave.contracts.deployments import ( ComponentSpec, DeploymentPlan, @@ -40,9 +38,71 @@ TARGET_LABEL = "firefly.weave/target" CONFIGURATION_ANNOTATION = "firefly.weave/configuration" -# Kubernetes label syntax, so a selector built from a deployment is only ever `key=value` pairs. +# Kubernetes label syntax, so a selector built from a deployment never carries selector operators of its own. _LABEL_KEY = re.compile(r"(?:[a-z0-9](?:[-a-z0-9.]{0,251}[a-z0-9])?/)?[A-Za-z0-9](?:[-A-Za-z0-9_.]{0,61}[A-Za-z0-9])?") _LABEL_VALUE = re.compile(r"(?:[A-Za-z0-9](?:[-A-Za-z0-9_.]{0,61}[A-Za-z0-9])?)?") +# The only pod fields read for members, so the listing stays small: about 250 bytes a pod. +_POD_FIELDS = ( + ".metadata.name", + ".metadata.namespace", + ".metadata.deletionTimestamp", + ".metadata.labels.pod-template-hash", + ".metadata.ownerReferences[?(@.controller==true)].apiVersion", + ".metadata.ownerReferences[?(@.controller==true)].kind", + ".metadata.ownerReferences[?(@.controller==true)].name", + ".status.phase", + ".status.reason", + '.status.conditions[?(@.type=="Ready")].status', + '.status.containerStatuses[?(@.name=="{container}")].restartCount', + '.status.containerStatuses[?(@.name=="{container}")].state.running.startedAt', + ".spec.nodeName", +) + + +def _pod_fields(container: str) -> str: + """A kubectl JSONPath printing one tab-separated line per pod; `container` is a policy name.""" + fields = '{"\\t"}'.join("{" + field.replace("{container}", container) + "}" for field in _POD_FIELDS) + return "{range .items[*]}" + fields + '{"\\n"}{end}' + + +def _label(key: object, value: object = "") -> bool: + return ( + isinstance(key, str) + and isinstance(value, str) + and _LABEL_KEY.fullmatch(key) is not None + and _LABEL_VALUE.fullmatch(value) is not None + ) + + +def _selector(selector: object) -> str | None: + """A deployment selector as a kubectl label selector, or None when kubectl cannot express it exactly. + + `matchLabels` become `key=value`; `matchExpressions` become set-based requirements (`key in (a,b)`, + `key notin (a,b)`, `key`, `!key`). Anything else, including an empty selector, lists no pods. + """ + if not isinstance(selector, dict) or not set(selector) <= {"matchLabels", "matchExpressions"}: + return None + labels, expressions = selector.get("matchLabels") or {}, selector.get("matchExpressions") or [] + if not isinstance(labels, dict) or not isinstance(expressions, list): + return None + if not all(_label(key, value) for key, value in labels.items()): + return None + requirements = [f"{key}={value}" for key, value in sorted(labels.items())] + for expression in expressions: + if not isinstance(expression, dict) or not set(expression) <= {"key", "operator", "values"}: + return None + key, operator, values = expression.get("key"), expression.get("operator"), expression.get("values") or [] + if not _label(key) or not isinstance(values, list) or not all(_label(key, item) and item for item in values): + return None + if operator in {"In", "NotIn"} and values: + requirements.append(f"{key} {operator.lower()} ({','.join(sorted(values))})") + elif operator == "Exists" and not values: + requirements.append(f"{key}") + elif operator == "DoesNotExist" and not values: + requirements.append(f"!{key}") + else: + return None + return ",".join(requirements) or None def _quantity(value: Any) -> Decimal: @@ -68,8 +128,8 @@ def __init__(self, policy: DestinationPolicy, *, command: Callable[..., Awaitabl raise RunnerPolicyError() self.policy, self.command = policy, command - async def _json(self, *args: str, stdin: bytes | None = None) -> dict[str, Any]: - raw = await self.command( + async def _run(self, *args: str, stdin: bytes | None = None) -> bytes: + return await self.command( [ self.policy.executable, "--context", @@ -83,6 +143,9 @@ async def _json(self, *args: str, stdin: bytes | None = None) -> dict[str, Any]: timeout=30, limit=1024 * 1024, ) + + async def _json(self, *args: str, stdin: bytes | None = None) -> dict[str, Any]: + raw = await self._run(*args, stdin=stdin) try: value = json.loads(raw) if raw.strip() else {} if not isinstance(value, dict): @@ -119,87 +182,63 @@ def _container(self, value: dict[str, Any], binding: ComponentBinding) -> tuple[ raise RunnerPolicyError() return found[0] - def _selector(self, value: dict[str, Any]) -> dict[str, str]: - selector = value.get("spec", {}).get("selector") - labels = selector.get("matchLabels", {}) if isinstance(selector, dict) else None - if ( - not isinstance(labels, dict) - or not (labels or selector.get("matchExpressions")) - or not all( - isinstance(key, str) - and isinstance(item, str) - and _LABEL_KEY.fullmatch(key) - and _LABEL_VALUE.fullmatch(item) - for key, item in labels.items() - ) - ): - raise RunnerPolicyError() - return labels - - def _member( - self, pod: Any, deployment: str, selector: dict[str, str], binding: ComponentBinding - ) -> ObservedMember | None: - """A pod of this deployment, or None for a pod that only shares its labels. + async def _members(self, value: dict[str, Any], binding: ComponentBinding) -> list[ObservedMember] | None: + """The deployment's live pods as members, or None when they cannot all be listed. - Pod data that does not have the shape Kubernetes reports refuses the observation. + Members are placement only, so listing them never fails an observation or an apply: a selector kubectl + cannot express, a failed or unreadable listing, a malformed pod, or more than 100 live pods leaves them + out (unknown) rather than reporting part of them. """ + selector = _selector(value.get("spec", {}).get("selector")) + if selector is None or binding.container is None: + return None try: - metadata = pod["metadata"] - labels = metadata.get("labels") or {} - owners = [item for item in metadata.get("ownerReferences") or [] if item.get("controller") is True] - template = labels.get("pod-template-hash") - # The deployment controller names each ReplicaSet `-`. - if ( - metadata.get("namespace") != self.policy.boundary - or len(owners) != 1 - or owners[0].get("apiVersion") != "apps/v1" - or owners[0].get("kind") != "ReplicaSet" - or not isinstance(template, str) - or owners[0].get("name") != f"{deployment}-{template}" - or any(labels.get(key) != item for key, item in selector.items()) - ): - return None - status = pod.get("status") or {} - container: dict[str, Any] = next( - (item for item in status.get("containerStatuses") or [] if item.get("name") == binding.container), {} - ) - phase = status.get("phase", "Unknown") - running = (container.get("state") or {}).get("running") or {} - return ObservedMember( - host=metadata.get("name"), - ordinal=None, - state="terminating" - if metadata.get("deletionTimestamp") - else (phase.lower() if isinstance(phase, str) else ""), - ready=any( - item.get("type") == "Ready" and item.get("status") == "True" - for item in status.get("conditions") or [] - ), - started_at=observed_time(running.get("startedAt")), - restarts=container.get("restartCount", 0), - node=(pod.get("spec") or {}).get("nodeName"), + raw = await self._run( + "get", "pods", "--selector=" + selector, "-o=jsonpath=" + _pod_fields(binding.container) ) - except (AttributeError, KeyError, TypeError, ValidationError): - raise RunnerPolicyError() from None + text = raw.decode() + if text and not text.endswith("\n"): + raise ValueError("Incomplete pod listing") + members = [] + for line in text.split("\n")[:-1]: + member = self._member(line.split("\t"), value["metadata"]["name"]) + if member is not None: + members.append(member) + if len(members) > 100: + return None + return sorted(members, key=lambda member: member.host) + except Exception: + # Placement stays unknown; cancellation is not an Exception and still propagates. + return None - async def _members(self, value: dict[str, Any], binding: ComponentBinding) -> list[ObservedMember]: - """Pods the deployment owns, read through its own selector inside the pinned namespace.""" - selector = self._selector(value) - arguments = ["get", "pods", "-o=json"] - if selector: - arguments.insert(2, "--selector=" + ",".join(f"{key}={item}" for key, item in sorted(selector.items()))) - pods = (await self._json(*arguments)).get("items") - if not isinstance(pods, list): - raise RunnerPolicyError() - found = [self._member(pod, value["metadata"]["name"], selector, binding) for pod in pods] - members = [member for member in found if member is not None] - # The contract holds 100 members; refuse rather than report a truncated observation. - if len(members) > 100: - raise RunnerPolicyError() - return sorted(members, key=lambda member: member.host) + def _member(self, fields: list[str], deployment: str) -> ObservedMember | None: + """A live pod of this deployment, None for any other pod, or an error for an unreadable one.""" + if len(fields) != len(_POD_FIELDS): + raise ValueError("Unexpected pod fields") + name, namespace, deleting, template, api, kind, owner, phase, reason, ready, restarts, started, node = fields + # The deployment controller names each ReplicaSet `-`; exactly one + # controller owner is listed, so several controllers never match. + if namespace != self.policy.boundary or (api, kind) != ("apps/v1", "ReplicaSet") or not template: + return None + if owner != f"{deployment}-{template}": + return None + # Terminating, finished, failed and evicted pods run no worker that placement could describe. + if deleting or phase in {"Succeeded", "Failed"} or reason == "Evicted": + return None + if ready not in {"", "True", "False"} or not re.fullmatch(r"[0-9]{0,10}", restarts): + raise ValueError("Unexpected pod status") + return ObservedMember( + host=name, + ordinal=None, + state=phase.lower() if phase else "unknown", + ready=ready == "True", + started_at=observed_time(started or None), + restarts=int(restarts or "0"), + node=node or None, + ) def _fact( - self, value: dict[str, Any], binding: ComponentBinding, members: list[ObservedMember] | None = None + self, value: dict[str, Any], binding: ComponentBinding, members: list[ObservedMember] | None ) -> ObservedResource: _, container = self._container(value, binding) metadata, spec, status = value["metadata"], value["spec"], value.get("status", {}) @@ -240,7 +279,7 @@ async def observe(self) -> ObservationReport: value = await self._deployment(binding) if value: # Placement only concerns workers; other components need no pod reads. - members = await self._members(value, binding) if binding.kind == "worker" else [] + members = await self._members(value, binding) if binding.kind == "worker" else None resources.append(self._fact(value, binding, members)) return ObservationReport( resources=resources, complete=True, settled=all(item.state in {"ready", "stopped"} for item in resources) diff --git a/src/firefly_weave/workers/models.py b/src/firefly_weave/workers/models.py index 0142f12b..cba9a272 100644 --- a/src/firefly_weave/workers/models.py +++ b/src/firefly_weave/workers/models.py @@ -76,7 +76,7 @@ def observed_worker( if draining or instance.revoked: available = 0 return WorkerStatus( - **instance.model_dump(exclude={"identity"}), + **instance.model_dump(), identity=identity, registered_at=registered_at, revision=revision, diff --git a/tests/contracts/test_worker_identity_contract.py b/tests/contracts/test_worker_identity_contract.py index 228bf1ea..e52d15f7 100644 --- a/tests/contracts/test_worker_identity_contract.py +++ b/tests/contracts/test_worker_identity_contract.py @@ -169,7 +169,7 @@ def test_old_worker_instance_and_status_answers_still_deserialize(): "revoked": False, } parsed = WorkerInstance.model_validate_json(json.dumps(instance)) - assert parsed.identity is None and json.loads(parsed.model_dump_json()) == instance + assert "identity" not in WorkerInstance.model_fields and json.loads(parsed.model_dump_json()) == instance now = datetime.now(UTC).isoformat() status = { **instance, @@ -333,3 +333,50 @@ def test_identity_and_member_contracts_are_exported(): assert "worker-identity" in schemas and "operations-observed-member" in schemas request = schemas["worker-instance-request"] assert "identity" in request["properties"] and "identity" not in request.get("required", []) + + +def boundary_identity(over: bool = False) -> WorkerIdentity: + """An identity of exactly 4,096 bytes as PostgreSQL renders it from jsonb, or one byte more. + + `{"labels": {…}, "runtime": "process", "platform_hint": "none"}` takes 139 bytes of keys, quotes and + `", "`/`": "` separators. Seven labels of 256 "é" add 512 UTF-8 bytes each (3,584), and the eighth holds + 117 double quotes, escaped to 2 bytes each, plus 139 letters: 373 bytes. 139 + 3,584 + 373 = 4,096. + """ + labels = {f"k{index}": "é" * 256 for index in range(7)} + labels["k7"] = '"' * (118 if over else 117) + "a" * (138 if over else 139) + return WorkerIdentity(runtime="process", labels=labels) + + +def test_identity_of_exactly_4096_stored_bytes_is_accepted_and_one_more_is_refused(): + from firefly_weave.contracts.workers import identity_text_bytes + + identity = boundary_identity() + assert identity_text_bytes(identity.model_dump(mode="json")) == 4096 + with pytest.raises(ValidationError, match="4 KiB"): + boundary_identity(over=True) + + +def test_registration_answer_schema_has_only_what_the_answer_sends(): + from firefly_weave.contracts.openapi import export_openapi + from firefly_weave.contracts.schema_export import export_schemas + + schemas = export_schemas() + assert "identity" not in schemas["worker-instance"]["properties"] + assert {"identity", "registered_at"} <= set(schemas["worker-status"]["properties"]) + assert "identity" in schemas["worker-instance-request"]["properties"] + components = export_openapi()["components"]["schemas"] + assert "identity" not in components["WorkerInstance"]["properties"] + assert "identity" in components["WorkerStatus"]["properties"] + with pytest.raises(ValidationError): + WorkerInstance.model_validate_json( + json.dumps( + { + "release_id": str(uuid4()), + "task_types": ["echo@1.0.0"], + "capacity": 1, + "id": str(uuid4()), + "principal_id": str(uuid4()), + "identity": {"runtime": "process"}, + } + ) + ) diff --git a/tests/integration/test_operations_fact_writes.py b/tests/integration/test_operations_fact_writes.py index 5b6e5921..b5a72f6a 100644 --- a/tests/integration/test_operations_fact_writes.py +++ b/tests/integration/test_operations_fact_writes.py @@ -901,7 +901,7 @@ async def test_worker_registration_facts_match_source(operations_case, identity) assert "identity" not in rows[0]["payload"] assert rows[0]["identity"] == ({} if identity is None else identity.model_dump(mode="json", exclude_none=True)) # The registration answer keeps its bytes; identity is read from the worker's facts. - assert registered.identity is None and "identity" not in registered.model_dump(mode="json") + assert "identity" not in registered.model_dump(mode="json") status = await case.workers.read(case.actor, case.scope, registered.id, context=AuditContext()) assert status.identity == identity and status.registered_at == rows[0]["registered_at"] from operations_support import assert_usage @@ -926,24 +926,22 @@ async def test_workers_without_facts_or_identity_read_as_unknown(operations_case assert all("identity" not in item and "registered_at" not in item for item in page["items"]) -async def test_largest_identity_fits_its_database_column(operations_case): +def boundary_labels(over=False): + """Labels that make a process identity exactly 4,096 bytes as jsonb text, or one byte more.""" + labels = {f"k{index}": "é" * 256 for index in range(7)} + labels["k7"] = '"' * (118 if over else 117) + "a" * (138 if over else 139) + return labels + + +async def test_identity_at_the_size_limit_fits_its_database_column(operations_case): import json - from firefly_weave.contracts.workers import WorkerIdentity + from sqlalchemy.exc import DBAPIError + + from firefly_weave.contracts.workers import WorkerIdentity, identity_text_bytes case = operations_case - identity = WorkerIdentity( - host="h" * 253, - runtime="container", - sdk_version="s" * 64, - language_version="l" * 64, - os="o" * 64, - arch="a" * 64, - platform_hint="kubernetes", - namespace="n" * 253, - node="d" * 253, - labels={f"key-{index}": 'é\\"' + "v" * 150 for index in range(8)}, - ) + identity = WorkerIdentity(runtime="process", labels=boundary_labels()) registered = await case.workers.register_instance( case.actor, case.scope, registration_request(case, identity), context=AuditContext() ) @@ -952,8 +950,20 @@ async def test_largest_identity_fits_its_database_column(operations_case): "SELECT octet_length(identity::text) AS size FROM worker_facts WHERE worker_id=:id", id=registered.id ) )[0]["size"] - text_form = json.dumps(identity.model_dump(mode="json"), ensure_ascii=False, separators=(", ", ": ")) - assert measured == len(text_form.encode()) <= 4096 + assert measured == identity_text_bytes(identity.model_dump(mode="json")) == 4096 + over = {"runtime": "process", "platform_hint": "none", "labels": boundary_labels(over=True)} + async with case.owner.begin() as owner: + assert ( + await owner.scalar(text("SELECT octet_length(cast(:value AS jsonb)::text)"), {"value": json.dumps(over)}) + == 4097 + ) + with pytest.raises(DBAPIError) as caught: + async with owner.begin_nested(): + await owner.execute( + text("UPDATE worker_facts SET identity=cast(:value AS jsonb) WHERE worker_id=:id"), + {"value": json.dumps(over), "id": registered.id}, + ) + assert caught.value.orig.sqlstate == "23514" async def worker_rows(case): @@ -1062,6 +1072,7 @@ async def test_registration_over_http_reports_identity_and_refuses_forged_fields {**body, "placement": placement}, {**body, "identity": {**identity, "placement": placement}}, {**body, "identity": {**identity, "labels": {"api_token": "x"}}}, + {**body, "identity": {"runtime": "process", "labels": boundary_labels(over=True)}}, {**body, "registered_at": "2026-10-01T00:00:00Z"}, ): refused = await client.post(env_url + "/workers", headers=headers, json=forged) diff --git a/tests/unit/test_deployment_compose.py b/tests/unit/test_deployment_compose.py index d8494ca4..097ac463 100644 --- a/tests/unit/test_deployment_compose.py +++ b/tests/unit/test_deployment_compose.py @@ -351,16 +351,29 @@ def container(identifier, number, **changes): } -async def test_worker_members_report_hostname_number_and_state(tmp_path): +async def test_worker_members_are_the_live_containers(tmp_path): config = policy(tmp_path) containers = [ container("b" * 64, "2", state="exited", started="0001-01-01T00:00:00Z", restart=3, health=""), container("a" * 64, "1"), container("c" * 64, "3", health="unhealthy", started="2026-10-03T09:00:00Z"), + container("d" * 64, "4", state="created", started="0001-01-01T00:00:00Z", health=""), + container("e" * 64, "5", state="dead", health=""), + container("f" * 64, "6", state="removing", health=""), + container("0" * 64, "7", state="restarting", restart=5, health=""), ] report = await ComposeAdapter(config, command=worker_project(config, containers)).observe() members = [member.model_dump(mode="json") for member in report.resources[0].members] assert members == [ + { + "host": "0" * 12, + "ordinal": 7, + "state": "restarting", + "ready": False, + "started_at": "2026-10-03T08:15:30.123456Z", + "restarts": 5, + "node": None, + }, { "host": "a" * 12, "ordinal": 1, @@ -370,15 +383,6 @@ async def test_worker_members_report_hostname_number_and_state(tmp_path): "restarts": 0, "node": None, }, - { - "host": "b" * 12, - "ordinal": 2, - "state": "exited", - "ready": False, - "started_at": None, - "restarts": 3, - "node": None, - }, { "host": "c" * 12, "ordinal": 3, @@ -388,8 +392,18 @@ async def test_worker_members_report_hostname_number_and_state(tmp_path): "restarts": 0, "node": None, }, + { + "host": "d" * 12, + "ordinal": 4, + "state": "created", + "ready": False, + "started_at": None, + "restarts": 0, + "node": None, + }, ] - assert report.resources[0].ready_replicas == 1 and report.resources[0].replicas == 3 + # Replica counts still describe every container; only placement leaves stopped ones out. + assert report.resources[0].ready_replicas == 1 and report.resources[0].replicas == 7 async def test_member_number_does_not_change_the_reviewed_resource_version(tmp_path): @@ -409,22 +423,44 @@ async def test_container_without_a_compose_number_has_no_ordinal(tmp_path): @pytest.mark.parametrize( "changes", - [{"number": "1x"}, {"number": 1}, {"started": "yesterday"}, {"restart": -1}, {"state": "Running Now"}], + [ + {"number": "1x"}, + {"number": 1}, + {"number": "\u00b2"}, + {"number": "\u0661"}, + {"number": "12345678"}, + {"started": "yesterday"}, + {"restart": -1}, + {"state": "Running Now"}, + ], ) -async def test_malformed_member_facts_refuse_the_observation(tmp_path, changes): +async def test_one_malformed_live_container_leaves_all_members_out(tmp_path, changes): config = policy(tmp_path) - with pytest.raises(RunnerPolicyError): - await ComposeAdapter( - config, command=worker_project(config, [{**container("a" * 64, "1"), **changes}]) - ).observe() + containers = [container("a" * 64, "1"), {**container("b" * 64, "2"), **changes}] + report = await ComposeAdapter(config, command=worker_project(config, containers)).observe() + assert report.complete and report.resources[0].members == [] + assert "members" not in report.model_dump(mode="json")["resources"][0] + + +@pytest.mark.parametrize("number", ["\u00b2", "\u0661", "\uff11", "1x", "", "12345678"]) +def test_compose_numbers_are_ascii_digits(number): + from firefly_weave.deployment_runner.compose import _ordinal + if number == "": + assert _ordinal(number) is None + else: + with pytest.raises(RunnerPolicyError): + _ordinal(number) + assert _ordinal("1234567") == 1234567 and _ordinal(None) is None -async def test_more_than_one_hundred_worker_containers_refuse_instead_of_truncating(tmp_path): + +async def test_more_than_one_hundred_live_worker_containers_leave_members_out(tmp_path): config = policy(tmp_path) containers = [container(f"{index:064x}", str(index + 1)) for index in range(101)] - with pytest.raises(RunnerPolicyError): - await ComposeAdapter(config, command=worker_project(config, containers)).observe() - report = await ComposeAdapter(config, command=worker_project(config, containers[:100])).observe() + report = await ComposeAdapter(config, command=worker_project(config, containers)).observe() + assert report.complete and report.resources[0].members == [] and report.resources[0].replicas == 101 + stopped = [container(f"{index + 200:064x}", str(index + 200), state="exited", health="") for index in range(5)] + report = await ComposeAdapter(config, command=worker_project(config, containers[:100] + stopped)).observe() assert len(report.resources[0].members) == 100 diff --git a/tests/unit/test_deployment_kubernetes.py b/tests/unit/test_deployment_kubernetes.py index 5e21ce81..917a5aa1 100644 --- a/tests/unit/test_deployment_kubernetes.py +++ b/tests/unit/test_deployment_kubernetes.py @@ -136,6 +136,55 @@ def pod(name, *, template="7d9f8c6b5", owner=None, namespace="owned", labels=Non } +# Exactly the pod fields the runner reads: one tab-separated line per pod, never whole pod objects. +POD_FIELDS = ( + "{range .items[*]}{.metadata.name}" + '{"\\t"}{.metadata.namespace}' + '{"\\t"}{.metadata.deletionTimestamp}' + '{"\\t"}{.metadata.labels.pod-template-hash}' + '{"\\t"}{.metadata.ownerReferences[?(@.controller==true)].apiVersion}' + '{"\\t"}{.metadata.ownerReferences[?(@.controller==true)].kind}' + '{"\\t"}{.metadata.ownerReferences[?(@.controller==true)].name}' + '{"\\t"}{.status.phase}' + '{"\\t"}{.status.reason}' + '{"\\t"}{.status.conditions[?(@.type=="Ready")].status}' + '{"\\t"}{.status.containerStatuses[?(@.name=="main")].restartCount}' + '{"\\t"}{.status.containerStatuses[?(@.name=="main")].state.running.startedAt}' + '{"\\t"}{.spec.nodeName}{"\\n"}{end}' +) + + +def render(pods, container="main"): + """What kubectl prints for POD_FIELDS: missing values are empty and several matches join with spaces.""" + + def text(value): + return "" if value is None else str(value) + + lines = [] + for item in pods: + metadata, status = item["metadata"], item.get("status", {}) + controllers = [ref for ref in metadata.get("ownerReferences", []) if ref.get("controller") is True] + statuses = [entry for entry in status.get("containerStatuses", []) if entry.get("name") == container] + ready = [entry for entry in status.get("conditions", []) if entry.get("type") == "Ready"] + values = [ + metadata.get("name"), + metadata.get("namespace"), + metadata.get("deletionTimestamp"), + metadata.get("labels", {}).get("pod-template-hash"), + " ".join(text(ref.get("apiVersion")) for ref in controllers), + " ".join(text(ref.get("kind")) for ref in controllers), + " ".join(text(ref.get("name")) for ref in controllers), + status.get("phase"), + status.get("reason"), + " ".join(text(entry.get("status")) for entry in ready), + " ".join(text(entry.get("restartCount")) for entry in statuses), + " ".join(text(entry.get("state", {}).get("running", {}).get("startedAt")) for entry in statuses), + item.get("spec", {}).get("nodeName"), + ] + lines.append("\t".join(text(value) for value in values) + "\n") + return "".join(lines).encode() + + class Server: def __init__(self, policy, resource): self.policy, self.resource, self.calls, self.events = policy, resource, [], [] @@ -158,8 +207,8 @@ async def command(self, argv, **kwargs): if "horizontalpodautoscalers.autoscaling" in argv: return b'{"items":[]}' if "pods" in argv: - assert argv[6:] == ["get", "pods", "--selector=app=engine", "-o=json"] - return json.dumps({"apiVersion": "v1", "kind": "List", "items": self.pods}).encode() + assert argv[6:] == ["get", "pods", "--selector=app=engine", "-o=jsonpath=" + POD_FIELDS] + return render(self.pods) assert "deployments.apps" in argv and "engine" in argv if "get" in argv: return json.dumps(self.resource).encode() @@ -365,25 +414,40 @@ async def command(argv, **kwargs): assert server.events == ["dry"] -async def test_worker_members_come_from_the_deployment_pods(): +def members_of(result): + return [member.model_dump(mode="json") for member in result.resources[0].members] + + +def pods_command(server, *, pods=None, failure=None, output=None): + """Wrap the fake server so the pods listing can answer differently.""" + calls = [] + + async def command(argv, **kwargs): + calls.append(argv) + if "pods" in argv: + if failure is not None: + raise failure + if output is not None: + return output + return render(server.pods if pods is None else pods) + return await server.command(argv, **kwargs) + + return command, calls + + +async def test_worker_members_are_the_deployment_live_pods(): policy, _, resource = fixture() server = Server(policy, resource) server.pods = [ pod("engine-7d9f8c6b5-zz9", phase="Pending", ready="False", containers=[], node=None), pod("engine-7d9f8c6b5-x2kq4"), pod("engine-5c4b3a2-old1", template="5c4b3a2", deleting="2026-10-03T09:00:00Z"), + pod("engine-7d9f8c6b5-done", phase="Succeeded", ready="False"), + pod("engine-7d9f8c6b5-evicted", phase="Failed", reason="Evicted", ready="False", containers=[]), + pod("engine-7d9f8c6b5-crashed", phase="Failed", ready="False"), ] result = await KubernetesAdapter(policy, command=server.command).observe() - assert [member.model_dump(mode="json") for member in result.resources[0].members] == [ - { - "host": "engine-5c4b3a2-old1", - "ordinal": None, - "state": "terminating", - "ready": True, - "started_at": "2026-10-03T08:15:30Z", - "restarts": 2, - "node": "node-1", - }, + assert members_of(result) == [ { "host": "engine-7d9f8c6b5-x2kq4", "ordinal": None, @@ -422,21 +486,14 @@ async def test_worker_members_come_from_the_deployment_pods(): ), pod( "engine-7d9f8c6b5-a", - owner={ - "apiVersion": "apps/v1", - "kind": "ReplicaSet", - "name": "engine-7d9f8c6b5", - "uid": "u", - "controller": False, - }, + owner={"apiVersion": "apps/v1", "kind": "ReplicaSet", "name": "engine-7d9f8c6b5", "controller": False}, ), pod( "engine-7d9f8c6b5-b", - owner={"apiVersion": "batch/v1", "kind": "Job", "name": "engine-7d9f8c6b5", "uid": "u", "controller": True}, + owner={"apiVersion": "batch/v1", "kind": "Job", "name": "engine-7d9f8c6b5", "controller": True}, ), pod("engine-7d9f8c6b5-c", namespace="elsewhere"), pod("engine-7d9f8c6b5-d", labels={"app": "engine"}), - pod("engine-7d9f8c6b5-e", labels={"app": "other", "pod-template-hash": "7d9f8c6b5"}), ], ) async def test_pods_the_deployment_does_not_own_are_not_members(foreign): @@ -444,18 +501,84 @@ async def test_pods_the_deployment_does_not_own_are_not_members(foreign): server = Server(policy, resource) server.pods = [foreign, pod("engine-7d9f8c6b5-x2kq4")] result = await KubernetesAdapter(policy, command=server.command).observe() - assert [member.host for member in result.resources[0].members] == ["engine-7d9f8c6b5-x2kq4"] + assert [member["host"] for member in members_of(result)] == ["engine-7d9f8c6b5-x2kq4"] -async def test_more_than_one_hundred_member_pods_refuse_instead_of_truncating(): +async def test_two_controllers_are_not_one_owner(): policy, _, resource = fixture() server = Server(policy, resource) - server.pods = [pod(f"engine-7d9f8c6b5-{index:05d}") for index in range(101)] - with pytest.raises(RunnerPolicyError): - await KubernetesAdapter(policy, command=server.command).observe() + owned = pod("engine-7d9f8c6b5-twice") + owned["metadata"]["ownerReferences"].append(dict(owned["metadata"]["ownerReferences"][0])) + server.pods = [owned] + assert members_of(await KubernetesAdapter(policy, command=server.command).observe()) == [] + + +def surge(policy, plan, resource, *, live_old): + """A worker at 100 replicas during a rolling update: 100 new pods and 25 old ones.""" + component = plan.steps[0].component.model_copy(update={"replicas": 100}) + plan = plan.model_copy(update={"steps": [plan.steps[0].model_copy(update={"component": component})]}) + resource["spec"]["replicas"] = 100 + resource["status"].update(replicas=100, readyReplicas=100, updatedReplicas=100, availableReplicas=100) + server = Server(policy, resource) + server.pods = [pod(f"engine-7d9f8c6b5-{index:05d}") for index in range(100)] + [ + pod( + f"engine-5c4b3a2-{index:05d}", template="5c4b3a2", **({} if live_old else {"deleting": "2026-10-03T09:00Z"}) + ) + for index in range(25) + ] + return plan, server + + +async def test_update_at_the_replica_limit_completes_while_old_pods_drain(): + policy, plan, resource = fixture() + plan, server = surge(policy, plan, resource, live_old=False) + for item in server.pods[100:]: + item["metadata"]["deletionTimestamp"] = "2026-10-03T09:00:00Z" + result = await KubernetesAdapter(policy, command=server.command).apply(plan, server.before_write) + assert result.settled and len(result.resources[0].members) == 100 + assert {member.host[:16] for member in result.resources[0].members} == {"engine-7d9f8c6b5"} + + +async def test_more_than_one_hundred_live_members_are_left_out_without_failing(): + policy, plan, resource = fixture() + plan, server = surge(policy, plan, resource, live_old=True) + result = await KubernetesAdapter(policy, command=server.command).apply(plan, server.before_write) + assert result.settled and result.resources[0].members == [] + assert "members" not in result.model_dump(mode="json")["resources"][0] + observed = await KubernetesAdapter(policy, command=server.command).observe() + assert observed.complete and observed.resources[0].members == [] server.pods = server.pods[:100] - result = await KubernetesAdapter(policy, command=server.command).observe() - assert len(result.resources[0].members) == 100 + assert len((await KubernetesAdapter(policy, command=server.command).observe()).resources[0].members) == 100 + + +@pytest.mark.parametrize( + "selector,expected", + [ + ({"matchLabels": {"tier": "workers", "app": "engine"}}, "app=engine,tier=workers"), + ( + { + "matchLabels": {"app": "engine"}, + "matchExpressions": [ + {"key": "track", "operator": "In", "values": ["stable", "canary"]}, + {"key": "app.kubernetes.io/part-of", "operator": "NotIn", "values": ["legacy"]}, + {"key": "weave.example/worker", "operator": "Exists"}, + {"key": "paused", "operator": "DoesNotExist", "values": []}, + ], + }, + "app=engine,track in (canary,stable),app.kubernetes.io/part-of notin (legacy),weave.example/worker,!paused", + ), + ({"matchExpressions": [{"key": "app", "operator": "In", "values": ["engine"]}]}, "app in (engine)"), + ], +) +async def test_selectors_become_kubectl_label_selectors(selector, expected): + policy, _, resource = fixture() + resource["spec"]["selector"] = selector + server = Server(policy, resource) + command, calls = pods_command(server) + result = await KubernetesAdapter(policy, command=command).observe() + [listing] = [argv for argv in calls if "pods" in argv] + assert listing[6:] == ["get", "pods", "--selector=" + expected, "-o=jsonpath=" + POD_FIELDS] + assert [member["host"] for member in members_of(result)] == ["engine-7d9f8c6b5-x2kq4"] @pytest.mark.parametrize( @@ -463,50 +586,53 @@ async def test_more_than_one_hundred_member_pods_refuse_instead_of_truncating(): [ None, {}, + {"matchLabels": {}}, {"matchLabels": {"app": "engine,tier=x"}}, {"matchLabels": {"app in (a)": "engine"}}, {"matchLabels": {"app": "engine", "bad key": "x"}}, {"matchLabels": {"app": "x" * 64}}, {"matchLabels": ["app=engine"]}, {"matchLabels": {"app": 1}}, + {"matchLabels": {"app": "engine"}, "other": {}}, + {"matchExpressions": [{"key": "app", "operator": "In", "values": []}]}, + {"matchExpressions": [{"key": "app", "operator": "In", "values": [""]}]}, + {"matchExpressions": [{"key": "app", "operator": "In", "values": ["a,b"]}]}, + {"matchExpressions": [{"key": "app", "operator": "NotIn"}]}, + {"matchExpressions": [{"key": "app", "operator": "Exists", "values": ["x"]}]}, + {"matchExpressions": [{"key": "app", "operator": "DoesNotExist", "values": ["x"]}]}, + {"matchExpressions": [{"key": "app", "operator": "Gt", "values": ["1"]}]}, + {"matchExpressions": [{"key": "bad key", "operator": "Exists"}]}, + {"matchExpressions": [{"key": "app", "operator": "Exists", "extra": True}]}, + {"matchExpressions": {"key": "app", "operator": "Exists"}}, + {"matchExpressions": ["app"]}, ], ) -async def test_a_selector_that_is_not_plain_labels_is_refused_before_listing_pods(selector): +async def test_a_selector_kubectl_cannot_express_lists_no_pods_and_leaves_members_out(selector): policy, _, resource = fixture() if selector is None: del resource["spec"]["selector"] else: resource["spec"]["selector"] = selector server = Server(policy, resource) - with pytest.raises(RunnerPolicyError): - await KubernetesAdapter(policy, command=server.command).observe() + result = await KubernetesAdapter(policy, command=server.command).observe() + assert result.complete and result.resources[0].members == [] assert not any("pods" in argv for argv, _ in server.calls) -async def test_selector_uses_every_match_label_in_a_stable_order(): +@pytest.mark.parametrize( + "output", + [ + b"engine-7d9f8c6b5-x2kq4\towned\n", + b"not\ttab\tseparated\tenough\n", + b"\xff\xfe\n", + render([pod("engine-7d9f8c6b5-x2kq4"), pod("engine-7d9f8c6b5-other")])[:-1], + ], +) +async def test_unreadable_pod_output_leaves_members_out(output): policy, _, resource = fixture() - resource["spec"]["selector"] = { - "matchLabels": {"tier": "workers", "app.kubernetes.io/name": "engine", "app": "engine"}, - "matchExpressions": [{"key": "track", "operator": "In", "values": ["stable"]}], - } - server = Server(policy, resource) - server.pods = [] - calls = [] - - async def command(argv, **kwargs): - calls.append(argv) - if "pods" in argv: - return b'{"items":[]}' - return await server.command(argv, **kwargs) - + command, _ = pods_command(Server(policy, resource), output=output) result = await KubernetesAdapter(policy, command=command).observe() - assert result.resources[0].members == [] - assert calls[-1][6:] == [ - "get", - "pods", - "--selector=app=engine,app.kubernetes.io/name=engine,tier=workers", - "-o=json", - ] + assert result.complete and result.resources[0].members == [] @pytest.mark.parametrize( @@ -515,55 +641,37 @@ async def command(argv, **kwargs): {"phase": "Running Now"}, {"node": "node/1"}, {"containers": [{"name": "main", "restartCount": -1, "state": {}}]}, + {"containers": [{"name": "main", "restartCount": "1x", "state": {}}]}, {"containers": [{"name": "main", "restartCount": 0, "state": {"running": {"startedAt": "today"}}}]}, + {"ready": "Maybe"}, ], ) -async def test_malformed_pod_facts_refuse_the_observation(changes): +async def test_one_malformed_live_pod_leaves_all_members_out(changes): policy, _, resource = fixture() server = Server(policy, resource) - server.pods = [pod("engine-7d9f8c6b5-x2kq4", **changes)] - with pytest.raises(RunnerPolicyError): - await KubernetesAdapter(policy, command=server.command).observe() - - -async def test_api_components_list_no_pods(): - policy, _, resource = fixture(kind="api") - server = Server(policy, resource) + server.pods = [pod("engine-7d9f8c6b5-good"), pod("engine-7d9f8c6b5-x2kq4", **changes)] result = await KubernetesAdapter(policy, command=server.command).observe() - assert result.resources[0].members == [] and not any("pods" in argv for argv, _ in server.calls) + assert result.complete and result.resources[0].members == [] -async def test_expression_only_selector_lists_the_namespace_and_keeps_owned_pods(): - policy, _, resource = fixture() - resource["spec"]["selector"] = {"matchExpressions": [{"key": "app", "operator": "In", "values": ["engine"]}]} - server = Server(policy, resource) - calls = [] - - async def command(argv, **kwargs): - calls.append(argv) - if "pods" in argv: - other = pod("other-abc12", template="abc12", labels={"app": "other", "pod-template-hash": "abc12"}) - other["metadata"]["ownerReferences"][0]["name"] = "other-abc12" - return json.dumps({"items": [other, pod("engine-7d9f8c6b5-x2kq4")]}).encode() - return await server.command(argv, **kwargs) +async def test_a_failed_pod_listing_never_fails_the_observation(): + from firefly_weave.deployment_runner.command import CommandError + policy, _, resource = fixture() + command, _ = pods_command(Server(policy, resource), failure=CommandError()) result = await KubernetesAdapter(policy, command=command).observe() - assert calls[-1][6:] == ["get", "pods", "-o=json"] - assert [member.host for member in result.resources[0].members] == ["engine-7d9f8c6b5-x2kq4"] + assert result.complete and result.settled and result.resources[0].members == [] -@pytest.mark.parametrize( - "shape", - [ - "not a pod", - {"spec": {}}, - {"metadata": {"name": "engine-7d9f8c6b5-x", "labels": ["app=engine"]}}, - {"metadata": {"name": "engine-7d9f8c6b5-x", "namespace": "owned", "ownerReferences": ["engine"]}}, - ], -) -async def test_pod_data_without_the_kubernetes_shape_refuses_the_observation(shape): +async def test_a_cancelled_pod_listing_still_cancels(): policy, _, resource = fixture() + command, _ = pods_command(Server(policy, resource), failure=asyncio.CancelledError()) + with pytest.raises(asyncio.CancelledError): + await KubernetesAdapter(policy, command=command).observe() + + +async def test_api_components_list_no_pods(): + policy, _, resource = fixture(kind="api") server = Server(policy, resource) - server.pods = [shape] - with pytest.raises(RunnerPolicyError): - await KubernetesAdapter(policy, command=server.command).observe() + result = await KubernetesAdapter(policy, command=server.command).observe() + assert result.resources[0].members == [] and not any("pods" in argv for argv, _ in server.calls) From f8fec2ef722821e532a4fe4344ecdbc786fd6ece Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Mon, 12 Oct 2026 01:35:36 -0700 Subject: [PATCH 14/63] Read no free-text pod status and keep unknown readiness among live members Kubernetes member listings no longer print the pod status reason, so no value can carry a line break into the listing; evicted pods are already left out by their Failed phase. A Ready condition of Unknown now counts as not ready instead of leaving every member out. Compose checks each container state inside the guarded block, so a state that is not text leaves members out instead of failing the observation. Adds a boundary test at exactly 101 live pods. --- docs/operations/operations-runner.md | 13 +-- .../deployment_runner/compose.py | 14 ++- .../deployment_runner/kubernetes.py | 11 ++- tests/unit/test_deployment_compose.py | 9 ++ tests/unit/test_deployment_kubernetes.py | 95 +++++++++++++++---- 5 files changed, 109 insertions(+), 33 deletions(-) diff --git a/docs/operations/operations-runner.md b/docs/operations/operations-runner.md index dec4037f..89447d53 100644 --- a/docs/operations/operations-runner.md +++ b/docs/operations/operations-runner.md @@ -254,12 +254,13 @@ selector: `matchLabels` become `key=value` and `matchExpressions` become set-based requirements (`key in (a,b)`, `key notin (a,b)`, `key`, `!key`). The runner reads only the few pod fields it needs. A pod counts only when its controlling ReplicaSet is named after the deployment and its pod template hash; -terminating, finished, failed, and evicted pods are left out. Members are -placement only and never fail an observation or an apply: with more than 100 -live pods, a selector kubectl cannot express exactly, a label that is not valid -Kubernetes syntax, or a listing that fails, the observation leaves the members -out (placement unknown) instead of reporting part of them, and it never lists -the whole namespace. +terminating, finished, failed, and evicted pods are left out. A pod whose +readiness is unknown, for example on an unreachable node, is listed as not +ready. Members are placement only and never fail an observation or an apply: +with more than 100 live pods, a selector kubectl cannot express exactly, a label +that is not valid Kubernetes syntax, or a listing that fails, the observation +leaves the members out (placement unknown) instead of reporting part of them, +and it never lists the whole namespace. ### Azure Container Apps diff --git a/src/firefly_weave/deployment_runner/compose.py b/src/firefly_weave/deployment_runner/compose.py index 7f020e12..38e2d819 100644 --- a/src/firefly_weave/deployment_runner/compose.py +++ b/src/firefly_weave/deployment_runner/compose.py @@ -71,16 +71,24 @@ def _ordinal(number: object) -> int | None: _STOPPED = frozenset({"exited", "dead", "removing"}) +def _state(fact: dict[str, Any]) -> str: + """The container state as Docker writes it; anything but text makes the container unreadable.""" + state = fact["state"] + if not isinstance(state, str): + raise RunnerPolicyError() + return state + + def _members(facts: list[dict[str, Any]]) -> list[ObservedMember] | None: """Live worker containers as members, or None when they cannot all be reported. Members are placement only, so they never fail an observation or an apply: more than 100 live containers or one unreadable container leaves them out (unknown) rather than reporting part of them. """ - live = [fact for fact in facts if fact.get("state") not in _STOPPED] - if len(live) > 100: - return None try: + live = [fact for fact in facts if _state(fact) not in _STOPPED] + if len(live) > 100: + return None return [ ObservedMember( # Docker's default hostname, which the worker reports, is the ID prefix. diff --git a/src/firefly_weave/deployment_runner/kubernetes.py b/src/firefly_weave/deployment_runner/kubernetes.py index 06ceed2b..9bd594e2 100644 --- a/src/firefly_weave/deployment_runner/kubernetes.py +++ b/src/firefly_weave/deployment_runner/kubernetes.py @@ -41,7 +41,8 @@ # Kubernetes label syntax, so a selector built from a deployment never carries selector operators of its own. _LABEL_KEY = re.compile(r"(?:[a-z0-9](?:[-a-z0-9.]{0,251}[a-z0-9])?/)?[A-Za-z0-9](?:[-A-Za-z0-9_.]{0,61}[A-Za-z0-9])?") _LABEL_VALUE = re.compile(r"(?:[A-Za-z0-9](?:[-A-Za-z0-9_.]{0,61}[A-Za-z0-9])?)?") -# The only pod fields read for members, so the listing stays small: about 250 bytes a pod. +# The only pod fields read for members, so the listing stays small: about 250 bytes a pod. Free-text status +# fields stay out, so no value can carry a line break into the listing; evicted pods always have phase Failed. _POD_FIELDS = ( ".metadata.name", ".metadata.namespace", @@ -51,7 +52,6 @@ ".metadata.ownerReferences[?(@.controller==true)].kind", ".metadata.ownerReferences[?(@.controller==true)].name", ".status.phase", - ".status.reason", '.status.conditions[?(@.type=="Ready")].status', '.status.containerStatuses[?(@.name=="{container}")].restartCount', '.status.containerStatuses[?(@.name=="{container}")].state.running.startedAt', @@ -215,7 +215,7 @@ def _member(self, fields: list[str], deployment: str) -> ObservedMember | None: """A live pod of this deployment, None for any other pod, or an error for an unreadable one.""" if len(fields) != len(_POD_FIELDS): raise ValueError("Unexpected pod fields") - name, namespace, deleting, template, api, kind, owner, phase, reason, ready, restarts, started, node = fields + name, namespace, deleting, template, api, kind, owner, phase, ready, restarts, started, node = fields # The deployment controller names each ReplicaSet `-`; exactly one # controller owner is listed, so several controllers never match. if namespace != self.policy.boundary or (api, kind) != ("apps/v1", "ReplicaSet") or not template: @@ -223,9 +223,10 @@ def _member(self, fields: list[str], deployment: str) -> ObservedMember | None: if owner != f"{deployment}-{template}": return None # Terminating, finished, failed and evicted pods run no worker that placement could describe. - if deleting or phase in {"Succeeded", "Failed"} or reason == "Evicted": + if deleting or phase in {"Succeeded", "Failed"}: return None - if ready not in {"", "True", "False"} or not re.fullmatch(r"[0-9]{0,10}", restarts): + # A Ready status of Unknown (for example on a lost node) is not ready. + if ready not in {"", "True", "False", "Unknown"} or not re.fullmatch(r"[0-9]{0,10}", restarts): raise ValueError("Unexpected pod status") return ObservedMember( host=name, diff --git a/tests/unit/test_deployment_compose.py b/tests/unit/test_deployment_compose.py index 097ac463..7797c3b6 100644 --- a/tests/unit/test_deployment_compose.py +++ b/tests/unit/test_deployment_compose.py @@ -442,6 +442,15 @@ async def test_one_malformed_live_container_leaves_all_members_out(tmp_path, cha assert "members" not in report.model_dump(mode="json")["resources"][0] +@pytest.mark.parametrize("state", [["running"], {"status": "running"}, 1, None]) +async def test_a_container_state_that_is_not_text_leaves_members_out(tmp_path, state): + config = policy(tmp_path) + containers = [container("a" * 64, "1"), container("b" * 64, "2", state=state)] + report = await ComposeAdapter(config, command=worker_project(config, containers)).observe() + assert report.complete and report.resources[0].members == [] + assert report.resources[0].replicas == 2 + + @pytest.mark.parametrize("number", ["\u00b2", "\u0661", "\uff11", "1x", "", "12345678"]) def test_compose_numbers_are_ascii_digits(number): from firefly_weave.deployment_runner.compose import _ordinal diff --git a/tests/unit/test_deployment_kubernetes.py b/tests/unit/test_deployment_kubernetes.py index 917a5aa1..4d65e3f0 100644 --- a/tests/unit/test_deployment_kubernetes.py +++ b/tests/unit/test_deployment_kubernetes.py @@ -19,6 +19,7 @@ import asyncio import copy import json +import re from datetime import UTC, datetime, timedelta from uuid import uuid4 @@ -146,7 +147,6 @@ def pod(name, *, template="7d9f8c6b5", owner=None, namespace="owned", labels=Non '{"\\t"}{.metadata.ownerReferences[?(@.controller==true)].kind}' '{"\\t"}{.metadata.ownerReferences[?(@.controller==true)].name}' '{"\\t"}{.status.phase}' - '{"\\t"}{.status.reason}' '{"\\t"}{.status.conditions[?(@.type=="Ready")].status}' '{"\\t"}{.status.containerStatuses[?(@.name=="main")].restartCount}' '{"\\t"}{.status.containerStatuses[?(@.name=="main")].state.running.startedAt}' @@ -154,8 +154,11 @@ def pod(name, *, template="7d9f8c6b5", owner=None, namespace="owned", labels=Non ) -def render(pods, container="main"): - """What kubectl prints for POD_FIELDS: missing values are empty and several matches join with spaces.""" +FIELD = re.compile(r"\{(\.[^{}]+)\}") + + +def render(pods, container="main", template=POD_FIELDS): + """What kubectl prints for the requested JSONPath fields: missing values are empty and matches join with spaces.""" def text(value): return "" if value is None else str(value) @@ -166,22 +169,32 @@ def text(value): controllers = [ref for ref in metadata.get("ownerReferences", []) if ref.get("controller") is True] statuses = [entry for entry in status.get("containerStatuses", []) if entry.get("name") == container] ready = [entry for entry in status.get("conditions", []) if entry.get("type") == "Ready"] - values = [ - metadata.get("name"), - metadata.get("namespace"), - metadata.get("deletionTimestamp"), - metadata.get("labels", {}).get("pod-template-hash"), - " ".join(text(ref.get("apiVersion")) for ref in controllers), - " ".join(text(ref.get("kind")) for ref in controllers), - " ".join(text(ref.get("name")) for ref in controllers), - status.get("phase"), - status.get("reason"), - " ".join(text(entry.get("status")) for entry in ready), - " ".join(text(entry.get("restartCount")) for entry in statuses), - " ".join(text(entry.get("state", {}).get("running", {}).get("startedAt")) for entry in statuses), - item.get("spec", {}).get("nodeName"), - ] - lines.append("\t".join(text(value) for value in values) + "\n") + values = { + ".metadata.name": metadata.get("name"), + ".metadata.namespace": metadata.get("namespace"), + ".metadata.deletionTimestamp": metadata.get("deletionTimestamp"), + ".metadata.labels.pod-template-hash": metadata.get("labels", {}).get("pod-template-hash"), + ".metadata.ownerReferences[?(@.controller==true)].apiVersion": " ".join( + text(ref.get("apiVersion")) for ref in controllers + ), + ".metadata.ownerReferences[?(@.controller==true)].kind": " ".join( + text(ref.get("kind")) for ref in controllers + ), + ".metadata.ownerReferences[?(@.controller==true)].name": " ".join( + text(ref.get("name")) for ref in controllers + ), + ".status.phase": status.get("phase"), + ".status.reason": status.get("reason"), + '.status.conditions[?(@.type=="Ready")].status': " ".join(text(entry.get("status")) for entry in ready), + f'.status.containerStatuses[?(@.name=="{container}")].restartCount': " ".join( + text(entry.get("restartCount")) for entry in statuses + ), + f'.status.containerStatuses[?(@.name=="{container}")].state.running.startedAt': " ".join( + text(entry.get("state", {}).get("running", {}).get("startedAt")) for entry in statuses + ), + ".spec.nodeName": item.get("spec", {}).get("nodeName"), + } + lines.append("\t".join(text(values[field]) for field in FIELD.findall(template)) + "\n") return "".join(lines).encode() @@ -513,6 +526,39 @@ async def test_two_controllers_are_not_one_owner(): assert members_of(await KubernetesAdapter(policy, command=server.command).observe()) == [] +async def test_free_text_pod_status_cannot_forge_a_member(): + policy, _, resource = fixture() + server = Server(policy, resource) + # A status reason holding a newline would complete one listing line and start a crafted one. + forged = ( + "\tTrue\t0\t\tnode-1\nengine-7d9f8c6b5-forged\towned\t\t7d9f8c6b5\tapps/v1\tReplicaSet\t" + "engine-7d9f8c6b5\tRunning\t" + ) + pods = [pod("engine-7d9f8c6b5-x2kq4", reason=forged)] + requested = [] + + async def command(argv, **kwargs): + if "pods" in argv: + requested.append(argv[-1]) + return render(pods, template=argv[-1].removeprefix("-o=jsonpath=")) + return await server.command(argv, **kwargs) + + result = await KubernetesAdapter(policy, command=command).observe() + assert [member["host"] for member in members_of(result)] == ["engine-7d9f8c6b5-x2kq4"] + assert requested and all(".status.reason" not in template for template in requested) + + +async def test_a_ready_status_of_unknown_counts_as_not_ready(): + policy, _, resource = fixture() + server = Server(policy, resource) + server.pods = [pod("engine-7d9f8c6b5-good"), pod("engine-7d9f8c6b5-lost", ready="Unknown")] + result = await KubernetesAdapter(policy, command=server.command).observe() + assert [(member["host"], member["ready"]) for member in members_of(result)] == [ + ("engine-7d9f8c6b5-good", True), + ("engine-7d9f8c6b5-lost", False), + ] + + def surge(policy, plan, resource, *, live_old): """A worker at 100 replicas during a rolling update: 100 new pods and 25 old ones.""" component = plan.steps[0].component.model_copy(update={"replicas": 100}) @@ -551,6 +597,17 @@ async def test_more_than_one_hundred_live_members_are_left_out_without_failing() assert len((await KubernetesAdapter(policy, command=server.command).observe()).resources[0].members) == 100 +async def test_exactly_one_hundred_and_one_live_pods_leave_members_out(): + policy, _, resource = fixture() + server = Server(policy, resource) + server.pods = [pod(f"engine-7d9f8c6b5-{index:05d}") for index in range(101)] + observed = await KubernetesAdapter(policy, command=server.command).observe() + assert observed.complete and observed.resources[0].members == [] + assert "members" not in observed.model_dump(mode="json")["resources"][0] + server.pods = server.pods[:100] + assert len((await KubernetesAdapter(policy, command=server.command).observe()).resources[0].members) == 100 + + @pytest.mark.parametrize( "selector,expected", [ From af3c667a6e6b2e69301729fada6745e0b696ce81 Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Mon, 12 Oct 2026 02:05:03 -0700 Subject: [PATCH 15/63] Query the worker fleet in one statement, retire stale workers and report release usage workers.list reads each page with one query that groups active leases, filters by presence, draining, release, task type, target and a literal host prefix, lists current workers by default and orders by last contact or registration with keyset cursors bound to every filter. Worker reads and lists add placement, matched from the latest complete and unexpired observation of the targets both the token and the current principal may read, and the time a worker started draining. workers.retire_stale revokes at most 100 workers per command that hold no active lease and whose last contact is older than the threshold, oldest first, locking them with SKIP LOCKED and checking again before revoking; a dry run previews the same batch and both replay by Idempotency-Key. releases.usage counts a release's workers and pinning activations and pages the readable deployment components that run it. The SDK adds typed worker queries, retire_stale_workers and release_usage; the API reference, upgrade guide and changelog describe the new defaults, cursors and older-client effects. --- CHANGELOG.md | 27 + docs/operations/upgrades.md | 36 +- docs/reference/api.md | 61 + docs/reference/sdk.md | 12 +- src/firefly_weave/api/workers.py | 33 +- src/firefly_weave/contracts/schema_export.py | 14 + src/firefly_weave/contracts/surface.py | 34 +- src/firefly_weave/contracts/worker_queries.py | 136 +++ src/firefly_weave/contracts/workers.py | 23 +- src/firefly_weave/operations/permissions.py | 66 ++ src/firefly_weave/sdk/client.py | 43 +- src/firefly_weave/workers/models.py | 32 +- src/firefly_weave/workers/repository.py | 255 +++- src/firefly_weave/workers/service.py | 261 ++++- tests/contracts/test_worker_queries.py | 432 +++++++ tests/integration/test_worker_fleet_facts.py | 1041 +++++++++++++++++ .../operations/test_target_permissions.py | 127 ++ tests/unit/test_studio_bridge_families.py | 10 +- 18 files changed, 2603 insertions(+), 40 deletions(-) create mode 100644 src/firefly_weave/contracts/worker_queries.py create mode 100644 src/firefly_weave/operations/permissions.py create mode 100644 tests/contracts/test_worker_queries.py create mode 100644 tests/integration/test_worker_fleet_facts.py create mode 100644 tests/unit/operations/test_target_permissions.py diff --git a/CHANGELOG.md b/CHANGELOG.md index cf0fbefb..144d21f9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -87,6 +87,33 @@ SPDX-License-Identifier: Apache-2.0 `members` an upgraded runner reports, so observe jobs fail and applies end `reconciliation_required`. Earlier CLIs and SDKs fail with `502 WV-WIRE` on observation reads and lists that carry members. +- `workers.list` reads each page with one query and lists current workers by + default: not revoked, and seen (or, never seen, registered) in the last hour; + `state=all` lists every worker. It filters by `presence`, `draining`, + `release_id`, `task_type`, `target_id` and `host_prefix` (a literal, + case-sensitive prefix) and orders by `last_seen_desc` (default) or + `registered_desc`, unknown times last. Cursors issued before the upgrade are + refused, unknown query parameters are rejected, and an out-of-range `limit` + answers `WV-VALIDATION` instead of `WV-PAGE`. Pages stay limited to 8 MiB. +- Worker reads and lists add optional `placement` (target, deployment and + component), matched from the latest complete, unexpired observation of the + targets the reader may read, and `drained_at`, when the worker started + draining; drain and resume answers add `drained_at`. Upgrade CLIs and SDKs + with the server: earlier releases fail with `502 WV-WIRE` on placed or + draining workers. +- New operation `workers.retire_stale` (`weave workers retire-stale`, SDK + `retire_stale_workers`) revokes up to 100 workers per command that hold no + active lease and were last seen, or registered if never seen, longer ago than + `older_than_seconds` (one hour to 30 days), oldest first. `dry_run` previews + the same batch, and `matched` counts every eligible worker. +- New operation `releases.usage` (`weave workers releases usage`, SDK + `release_usage`) counts a release's online, offline, draining and revoked + workers and the activations that pin it, and pages the deployment components + that run it on targets the reader may read. +- The Python SDK's `list_workers` accepts a typed `query` and rejects one + combined with keyword arguments. +- Kubernetes runners no longer read a pod's free-text status reason, and a pod + whose readiness is unknown is listed as a member that is not ready. - Keep typing in the selected form field when deferred row autofocus completes. Rapid row additions focus the newest item without overriding a later choice. diff --git a/docs/operations/upgrades.md b/docs/operations/upgrades.md index a74e0513..18a1251e 100644 --- a/docs/operations/upgrades.md +++ b/docs/operations/upgrades.md @@ -45,7 +45,7 @@ the desktop app never migrates a platform. | alpha13 | `0030_worker_presence` | No new schema migration. Workers retry explicit capacity rejections while they read task context or credentials, and an Agentic preparation timeout before provider execution counts as not started. Upgrade the Agentic and Files workers to 0.1.5 together with the server. | | alpha14 | `0030_worker_presence` | No new schema migration. Adds the detached Docker development platform (`weave platform up`); existing foreground installations keep working and are never converted. The Agentic and Files workers 0.1.6 pin this server version. | | alpha15 | `0030_worker_presence` | No new schema migration and no role change. Upgrade CLIs and SDKs together with the server, review native executor `capacity` and the legacy private-network settings before restarting, and finish runs that use text templates before any rollback to alpha14; see [what changes in alpha15](#what-changes-in-alpha15). The Agentic and Files workers 0.1.7 pin this server version. | -| Unreleased | `0031_operations_facts` | Adds [scoped operational fact storage](#operational-fact-storage), a bounded historical backfill, and reserved capacity for terminal fact updates. Upgrade CLIs and SDKs together with the server, and upgrade the server before its deployment runners; see [incident list compatibility](#incident-list-compatibility) and [worker identity compatibility](#worker-identity-compatibility). | +| Unreleased | `0031_operations_facts` | Adds [scoped operational fact storage](#operational-fact-storage), a bounded historical backfill, and reserved capacity for terminal fact updates. Upgrade CLIs and SDKs together with the server, and upgrade the server before its deployment runners; see [incident list compatibility](#incident-list-compatibility), [worker identity compatibility](#worker-identity-compatibility) and [worker fleet compatibility](#worker-fleet-compatibility). | ![Schema, compatibility and execution acceptance gates](../diagrams/operations-upgrade.svg) @@ -288,6 +288,40 @@ was applied. Upgrade the API first, then each runner. the runner leaves `members` out. See [the runner guide](operations-runner.md#kubernetes). +## Worker fleet compatibility + +`workers.list` now reads each page with one query and changes what it lists by +default: + +- **Current workers only.** Without `state=all`, it leaves out revoked workers + and workers neither seen nor registered in the last hour. Workers registered + before the upgrade that were never seen are listed only with `state=all`. +- **Newest contact first.** The default order is `last_seen_desc`; + `registered_desc` is also available, and unknown times come last. +- **Cursors.** Cursors issued before the upgrade are refused with + `422 WV-VALIDATION`; start the listing again. `weave workers list` sends only + `limit` and `cursor`, so it lists current workers. +- **Requests.** Unknown query parameters are refused with `422 WV-VALIDATION` + instead of being ignored, and a `limit` outside 1 to 100 answers + `422 WV-VALIDATION` instead of `422 WV-PAGE`. An older server ignores the new + filters and lists every worker in ID order, so upgrade servers before relying + on them. + +Worker reads and lists add optional `placement` and `drained_at`, and drain and +resume answers add `drained_at`. **Upgrade CLIs and SDKs with the server.** +Earlier releases reject these fields: `weave workers list`, `read`, `drain`, and +`resume`, and the Python SDK's `read_worker`, `list_workers`, `drain_worker`, +and `resume_worker`, fail with `502 WV-WIRE` for a placed or draining worker. A +drain or resume is still applied. Stored worker records and the registration +answer keep their bytes, and workers drained before the upgrade show no drain +time. + +Two operations are new: `workers.retire_stale` (`weave workers retire-stale`, +SDK `retire_stale_workers`) revokes up to 100 offline workers without active +leases per command, and `releases.usage` (`weave workers releases usage`, SDK +`release_usage`) reports where a release runs. Neither changes an existing +answer. Earlier servers do not serve them and answer `404` or `405`. + ## Operational fact storage The `0031_operations_facts` migration adds five scoped operational projections diff --git a/docs/reference/api.md b/docs/reference/api.md index 973f9de6..339ce3df 100644 --- a/docs/reference/api.md +++ b/docs/reference/api.md @@ -710,6 +710,65 @@ same client and credentials; any other failure is not retried. Native executors register inside the API process and always use a server of their own version. The registration answer's schema (`WorkerInstance`) has no `identity` field. +**Worker fleet.** `workers.list` reads every page with one database query, +including each worker's active leases, and checks `status.read` on both the +token and the current principal. + +- By default it lists current workers: not revoked, and seen in the last hour + or, never seen, registered in the last hour. `state=all` lists every worker, + including revoked ones and workers registered before registration times were + recorded. +- Filters: `presence` (`recent`, `stale`, or `unknown`, with the 60-second + presence window), `draining`, `release_id`, `task_type` (a capability as + workers register it, such as `echo@1.2.0`), `target_id`, and `host_prefix`, a + literal, case-sensitive prefix of the reported host in which `%` and `_` + match only themselves. `target_id` needs `deployment.read` on that target. +- Orders: `last_seen_desc` (default) and `registered_desc`; unknown times come + last, and equal times are ordered by ID. Cursors are bound to every filter and + the order. A `limit` outside 1 to 100 or an unknown query parameter answers + `422 WV-VALIDATION`, and a page is limited to 8 MiB of worker data + (`429 WV-PAGE-LIMIT`). +- `placement` names the deployment component a worker runs in: + `target_id`, `target_name`, `deployment_id`, and `component`. The server + matches the worker's reported `host` against the members of the latest + complete observation of each target, taken at the target's current revision + and not yet expired. A host listed by more than one target or component, a + native executor (`runtime: embedded`), and a worker without a reported host + have no placement. Only targets that both the token and the current principal + may read with `deployment.read` are considered, so other targets' names never + appear. +- `drained_at` is the time the worker started draining. A repeated drain keeps + it and resuming clears it; workers drained before the time was recorded show + none. `workers.read` returns the same fields as the list, and `workers.drain` + and `workers.resume` answers include `drained_at` but no placement. + +`workers.retire_stale` (`POST …/workers/retire-stale`, `release.retire`, +Idempotency-Key) revokes workers that hold no active lease and whose last +contact (or registration, if never seen) is older than `older_than_seconds`, +from one hour to 30 days. A worker without either time is never retired. Each +command revokes at most 100 workers, oldest last contact first, and answers +`{matched, revoked, worker_ids}`: `matched` counts every eligible worker, and +`revoked` and `worker_ids` describe this batch. `dry_run: true` returns the same +batch without revoking it. Repeating an Idempotency-Key returns the saved +answer; send a new key for the next batch. Retirement never deletes or settles +leases, and a worker another command is holding is left for the next batch. + +`releases.usage` (`GET …/worker-releases/{identifier}/usage`, `catalog.read`) +answers `{workers, deployments, activations, next_cursor}`: + +- `workers` counts every instance of the release once: `revoked`, else + `draining`, else `online` (seen in the last 60 seconds) or `offline`. +- `activations` counts the activation revisions that pin the release and can + still start runs. +- `deployments` lists, at most `limit` (1 to 100, default 100) at a time, the + desired deployment components whose admitted release and image digest are + this release: `target_id`, `target_name`, `deployment_id`, `deployment_name`, + `component`, and `replicas`. Only targets that both the token and the current + principal may read with `deployment.read` are listed; without that capability + the list is empty. `next_cursor` continues the list for the same release and + readable targets. +- A release of another environment answers `404 WV-NOT-FOUND`. + **Answers that say "unavailable".** Cancellation and task completion can return an unavailable acknowledgment when the platform cannot classify the historical data or compare payloads; task completion then includes @@ -873,10 +932,12 @@ operation in the [full API reference](api-explorer.md); to generate a client, | `releases.create` | `POST /api/v1/tenants/{tenant}/projects/{project}/environments/{environment}/worker-releases` | release.activate | | `releases.list` | `GET /api/v1/tenants/{tenant}/projects/{project}/environments/{environment}/worker-releases` | catalog.read | | `releases.read` | `GET /api/v1/tenants/{tenant}/projects/{project}/environments/{environment}/worker-releases/{identifier}` | catalog.read | +| `releases.usage` | `GET /api/v1/tenants/{tenant}/projects/{project}/environments/{environment}/worker-releases/{identifier}/usage` | catalog.read | | `workers.create` | `POST /api/v1/tenants/{tenant}/projects/{project}/environments/{environment}/workers` | worker.register | | `workers.list` | `GET /api/v1/tenants/{tenant}/projects/{project}/environments/{environment}/workers` | status.read | | `workers.read` | `GET /api/v1/tenants/{tenant}/projects/{project}/environments/{environment}/workers/{identifier}` | status.read | | `workers.revoke` | `POST /api/v1/tenants/{tenant}/projects/{project}/environments/{environment}/workers/{identifier}/revoke` | release.retire | +| `workers.retire_stale` | `POST /api/v1/tenants/{tenant}/projects/{project}/environments/{environment}/workers/retire-stale` | release.retire | | `workers.grant` | `POST /api/v1/tenants/{tenant}/projects/{project}/environments/{environment}/worker-connection-grants` | connection.manage | | `tasks.claim` | `POST /api/v1/tenants/{tenant}/projects/{project}/environments/{environment}/tasks/claim` | task.claim | | `tasks.context` | `POST /api/v1/tenants/{tenant}/projects/{project}/environments/{environment}/tasks/context` | task.claim | diff --git a/docs/reference/sdk.md b/docs/reference/sdk.md index 2f14cf51..a6279fc4 100644 --- a/docs/reference/sdk.md +++ b/docs/reference/sdk.md @@ -355,7 +355,7 @@ capability. | `human_tasks`, `read_human_task`, `claim_human_task`, `release_human_task`, `complete_human_task`, `reassign_human_task` | Human task inbox and decisions | | `assignment_bindings`, `put_assignment_binding`, `put_human_group` | Who may work on which human task | | `register_release`, `read_release`, `list_releases`, `register_worker`, `read_worker`, `list_workers`, `revoke_worker`, `grant_connection` | Trusted worker releases and instances | -| `drain_worker`, `resume_worker` | Revision-checked worker controls; existing leases can settle while drained | +| `drain_worker`, `resume_worker`, `retire_stale_workers`, `release_usage` | Revision-checked worker controls, bounded retirement of offline workers, and where a release runs; existing leases can settle while drained | | `invoke("deployment_targets.…")`, `invoke("deployments.…")`, `invoke("deployment_observations.…")`, `invoke("deployment_plans.…")`, `invoke("deployment_jobs.…")`, `invoke("deployment_runners.…")` | Typed, environment-scoped Operations contracts; operation names and required authority are in the [API inventory](api.md#operation-inventory) | | `invoke("ai_endpoints.list")`, `invoke("ai_models.list", …)`, `invoke("ai_readiness.read")`, `invoke("ai_setup.publish", …)`, `invoke("ai_setup.grant", …)`, `invoke("ai_connections.test", …)`, `invoke("bindable_connections.list")` | AI model discovery, readiness, setup and connection choices; see [AI operations through the SDK](#ai-operations-through-the-sdk) | | `claim_tasks`, `heartbeat`, `complete_task`, `fail_task`, `request_credentials` | The typed [worker protocol](worker-protocol.md); `WorkerTransport` remains supported | @@ -389,6 +389,16 @@ registers with its own HTTP client calls `WEAVE_POD_NAMESPACE`, `WEAVE_NODE_NAME`, and `KUBERNETES_SERVICE_HOST`; see [worker identity](../guides/workers.md#report-where-a-worker-runs). +**Worker fleet.** `list_workers` takes either the page keywords or a typed +`query=WorkerListQuery(...)` from `firefly_weave.contracts.worker_queries`; +passing both raises `ValueError`. Without a query the server lists current +workers, most recently seen first. `retire_stale_workers(RetireStaleRequest(...), +idempotency_key=...)` revokes, or with `dry_run=True` previews, up to 100 +offline workers; repeat it with a new key for the next batch. +`release_usage(release_id, query=ReleaseUsageQuery(...))` returns worker counts, +pinning activations and a page of readable deployments. See +[Discovery, debugging, and history](api.md#discovery-debugging-and-history) in the API reference for the rules. + ## Deployment Operations through the SDK Use `WeaveClient.invoke` with models from diff --git a/src/firefly_weave/api/workers.py b/src/firefly_weave/api/workers.py index 2d0347dc..0f8d2391 100644 --- a/src/firefly_weave/api/workers.py +++ b/src/firefly_weave/api/workers.py @@ -26,7 +26,8 @@ from firefly_weave.api.access import request_scope from firefly_weave.api.surface import operation -from firefly_weave.api.transport import page_request, page_response +from firefly_weave.api.transport import decode_cursor_v2, page_request, page_response, parse_query +from firefly_weave.contracts.worker_queries import ReleaseUsageQuery, RetireStaleRequest, WorkerListQuery from firefly_weave.contracts.workers import ( ClaimRequest, CompleteRequest, @@ -38,6 +39,7 @@ ReleaseRequest, WorkerControlRequest, ) +from firefly_weave.operations.fact_positions import run_position from firefly_weave.workers.leases import TaskService from firefly_weave.workers.service import WorkerService @@ -87,6 +89,17 @@ async def revoke(self, request: Request) -> JSONResponse: ) return JSONResponse({"revoked": True}) + @operation("workers.retire_stale") + async def retire_stale(self, request: Request) -> JSONResponse: + result = await self.service.retire_stale( + request.state.principal, + request_scope(request, environment=True), + RetireStaleRequest.model_validate_json(await request.body()), + request.headers.get("Idempotency-Key", ""), + context=request.state.audit_context, + ) + return JSONResponse(result.model_dump(mode="json"), headers={"Cache-Control": "no-store"}) + @operation("workers.drain") async def drain(self, request: Request) -> JSONResponse: return await self.control(request, draining=True) @@ -171,11 +184,12 @@ async def credentials(self, request: Request) -> JSONResponse: @operation("workers.list") async def discover(self, request: Request) -> JSONResponse: scope = request_scope(request, environment=True) - limit, cursor = page_request(request, scope, "workers") + query = parse_query(request.query_params, WorkerListQuery) + position = run_position(decode_cursor_v2(query.cursor, scope, query.cursor_collection())) result = await self.service.list( - request.state.principal, scope, limit=limit, cursor=cursor, context=request.state.audit_context + request.state.principal, scope, query=query, position=position, context=request.state.audit_context ) - return JSONResponse(page_response(result, scope, "workers", request.url.path)) + return JSONResponse(result) @operation("releases.list") async def releases(self, request: Request) -> JSONResponse: @@ -191,6 +205,17 @@ async def releases(self, request: Request) -> JSONResponse: ) return JSONResponse(page_response(result, scope, "worker-releases", request.url.path)) + @operation("releases.usage") + async def release_usage(self, request: Request) -> JSONResponse: + result = await self.service.release_usage( + request.state.principal, + request_scope(request, environment=True), + UUID(request.path_params["identifier"]), + parse_query(request.query_params, ReleaseUsageQuery), + context=request.state.audit_context, + ) + return JSONResponse(result.model_dump(mode="json")) + @operation("workers.read") async def read(self, request: Request) -> JSONResponse: result = await self.service.read( diff --git a/src/firefly_weave/contracts/schema_export.py b/src/firefly_weave/contracts/schema_export.py index 0c34f672..dbc6916a 100644 --- a/src/firefly_weave/contracts/schema_export.py +++ b/src/firefly_weave/contracts/schema_export.py @@ -217,6 +217,13 @@ def contract_models() -> dict[str, SchemaModel]: from firefly_weave.contracts.schedules import ScheduleOccurrence, ScheduleRequest, ScheduleView from firefly_weave.contracts.teams import TeamsReactivateRequest, TeamsReference, TeamsRevokeRequest from firefly_weave.contracts.whatsapp import WhatsAppDeliveryState, WhatsAppStatusFact + from firefly_weave.contracts.worker_queries import ( + ReleaseUsage, + ReleaseUsageQuery, + RetireStaleRequest, + RetireStaleResult, + WorkerListQuery, + ) from firefly_weave.contracts.workers import ( ClaimRequest, CompleteRequest, @@ -235,6 +242,7 @@ def contract_models() -> dict[str, SchemaModel]: WorkerControlRequest, WorkerIdentity, WorkerInstance, + WorkerPlacement, WorkerRelease, WorkerStatus, ) @@ -399,6 +407,12 @@ def contract_models() -> dict[str, SchemaModel]: "worker-instance": WorkerInstance, "worker-status": WorkerStatus, "worker-control-request": WorkerControlRequest, + "worker-placement": WorkerPlacement, + "worker-list-query": WorkerListQuery, + "retire-stale-request": RetireStaleRequest, + "retire-stale-result": RetireStaleResult, + "release-usage-query": ReleaseUsageQuery, + "release-usage": ReleaseUsage, "lease-proof": LeaseProof, "task-lease": TaskLease, "completion-receipt": CompletionReceipt, diff --git a/src/firefly_weave/contracts/surface.py b/src/firefly_weave/contracts/surface.py index bc5b954a..ce8a21e9 100644 --- a/src/firefly_weave/contracts/surface.py +++ b/src/firefly_weave/contracts/surface.py @@ -189,6 +189,13 @@ from firefly_weave.contracts.teams import TeamsReactivateRequest, TeamsReference, TeamsRevokeRequest from firefly_weave.contracts.values import JsonObjectData from firefly_weave.contracts.whatsapp import MessageId, WhatsAppDeliveryState, WhatsAppStatusFact +from firefly_weave.contracts.worker_queries import ( + ReleaseUsage, + ReleaseUsageQuery, + RetireStaleRequest, + RetireStaleResult, + WorkerListQuery, +) from firefly_weave.contracts.workers import ( ClaimRequest, CompleteRequest, @@ -1528,6 +1535,14 @@ def native(self) -> OpenAPIOperation: "releases.list", ENVIRONMENT + "/worker-releases", "GET", Page[WorkerRelease], "catalog.read", page=True ), Operation("releases.read", ENVIRONMENT + "/worker-releases/{identifier}", "GET", WorkerRelease, "catalog.read"), + Operation( + "releases.usage", + ENVIRONMENT + "/worker-releases/{identifier}/usage", + "GET", + ReleaseUsage, + "catalog.read", + query=ReleaseUsageQuery, + ), Operation( "workers.create", ENVIRONMENT + "/workers", @@ -1537,7 +1552,15 @@ def native(self) -> OpenAPIOperation: InstanceRequest, (201,), ), - Operation("workers.list", ENVIRONMENT + "/workers", "GET", Page[WorkerStatus], "status.read", page=True), + Operation( + "workers.list", + ENVIRONMENT + "/workers", + "GET", + Page[WorkerStatus], + "status.read", + page=True, + query=WorkerListQuery, + ), Operation("workers.read", ENVIRONMENT + "/workers/{identifier}", "GET", WorkerStatus, "status.read"), *( Operation( @@ -1552,6 +1575,15 @@ def native(self) -> OpenAPIOperation: for action in ("drain", "resume") ), Operation("workers.revoke", ENVIRONMENT + "/workers/{identifier}/revoke", "POST", Revoked, "release.retire"), + Operation( + "workers.retire_stale", + ENVIRONMENT + "/workers/retire-stale", + "POST", + RetireStaleResult, + "release.retire", + RetireStaleRequest, + idempotency=True, + ), Operation( "workers.grant", ENVIRONMENT + "/worker-connection-grants", diff --git a/src/firefly_weave/contracts/worker_queries.py b/src/firefly_weave/contracts/worker_queries.py new file mode 100644 index 00000000..d5f38a3b --- /dev/null +++ b/src/firefly_weave/contracts/worker_queries.py @@ -0,0 +1,136 @@ +# Copyright 2026 Firefly Software Foundation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# Author: Firefly Software Foundation +# SPDX-License-Identifier: Apache-2.0 + +"""Worker fleet queries, stale-worker retirement and release usage. + +`workers.list` lists current workers by default: not revoked, and seen (or, never seen, registered) in the last +hour. Cursors are bound to every filter and the order. `releases.usage` pages the deployments that run a release, +bound to the release and to the targets the reader may see. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Annotated, Literal +from uuid import UUID + +from pydantic import Field, TypeAdapter + +from firefly_weave.contracts.definitions import ContractModel +from firefly_weave.contracts.deployments import DeploymentName +from firefly_weave.contracts.run_views import Count, Cursor, SummaryLimit, _digest +from firefly_weave.contracts.values import JsonValue + +type WorkerPresence = Literal["unknown", "recent", "stale"] +type WorkerListState = Literal["current", "all"] +type WorkerOrder = Literal["last_seen_desc", "registered_desc"] +# A task capability exactly as workers register it, such as echo@1.2.0. +type TaskTypeFilter = Annotated[str, Field(min_length=1, max_length=200, pattern=r"^[^\x00-\x1f\x7f-\x9f]+$")] +# A literal, case-sensitive prefix of a reported host; `%` and `_` match only themselves. +type HostPrefix = Annotated[str, Field(min_length=1, max_length=253, pattern=r"^[^\x00-\x1f\x7f-\x9f]+$")] +# One retirement command revokes at most this many workers. +RETIREMENT_BATCH = 100 + + +class WorkerListQuery(ContractModel): + """`workers.list` query parameters; every combination is a valid filter.""" + + presence: WorkerPresence | None = None + draining: bool | None = None + release_id: UUID | None = None + target_id: UUID | None = None + task_type: TaskTypeFilter | None = None + host_prefix: HostPrefix | None = None + state: WorkerListState = "current" + order: WorkerOrder = "last_seen_desc" + limit: SummaryLimit = 50 + cursor: Cursor | None = None + + def cursor_collection(self) -> str: + filters = self.model_dump(mode="json", exclude={"order", "limit", "cursor"}) + return f"workers:{_digest(filters)}:{self.order}" + + +class RetireStaleRequest(ContractModel): + """Revoke workers without an active lease whose last contact is older than the threshold.""" + + older_than_seconds: int = Field(ge=3600, le=2592000) + dry_run: bool + + +class RetireStaleResult(ContractModel): + """`matched` counts every eligible worker; `revoked` and `worker_ids` describe this command's batch.""" + + matched: Count + revoked: int = Field(ge=0, le=RETIREMENT_BATCH) + worker_ids: list[UUID] = Field(max_length=RETIREMENT_BATCH) + + +class ReleaseUsageQuery(ContractModel): + """`releases.usage` query parameters; a cursor continues only for the same release and readable targets.""" + + limit: SummaryLimit = 100 + cursor: Cursor | None = None + + def cursor_collection(self, release_id: UUID, authorized_targets: tuple[UUID, ...] | None) -> str: + targets = "*" if authorized_targets is None else sorted(str(target) for target in authorized_targets) + return f"releases.usage:{release_id}:{_digest({'targets': targets})}" + + +class ReleaseWorkerCounts(ContractModel): + """Every instance of the release once: revoked, else draining, else online (seen recently) or offline.""" + + online: Count + offline: Count + draining: Count + revoked: Count + + +class ReleaseDeploymentUsage(ContractModel): + target_id: UUID + target_name: DeploymentName + deployment_id: UUID + deployment_name: DeploymentName + component: DeploymentName + replicas: int = Field(ge=0, le=100) + + +class ReleaseUsage(ContractModel): + workers: ReleaseWorkerCounts + deployments: list[ReleaseDeploymentUsage] = Field(max_length=100) + activations: Count + next_cursor: Cursor | None = None + + +@dataclass(frozen=True) +class ReleaseUsagePosition: + """The last deployment component of a release usage page.""" + + deployment_id: UUID + component: str + + +_COMPONENT: TypeAdapter[str] = TypeAdapter(DeploymentName) + + +def release_usage_position(position: tuple[JsonValue, str] | None) -> ReleaseUsagePosition | None: + """Validate a decoded release usage cursor before using its values in SQL.""" + if position is None: + return None + component, deployment = position + if type(component) is not str or type(deployment) is not str: + raise ValueError("Invalid release usage position") + return ReleaseUsagePosition(UUID(deployment), _COMPONENT.validate_python(component, strict=True)) diff --git a/src/firefly_weave/contracts/workers.py b/src/firefly_weave/contracts/workers.py index 0b21e141..dd20148d 100644 --- a/src/firefly_weave/contracts/workers.py +++ b/src/firefly_weave/contracts/workers.py @@ -107,6 +107,22 @@ def bounded(self) -> Self: return self +# A target, deployment or component name as deployment records store it. +type PlacementName = Annotated[str, Field(pattern=r"^[a-zA-Z0-9][a-zA-Z0-9_.-]{0,99}$")] + + +class WorkerPlacement(ContractModel): + """Where the latest complete, current observation of a target places a worker, matched by its host. + + The server derives it for targets the reader may see; a worker never reports it. + """ + + target_id: UUID + target_name: PlacementName + deployment_id: UUID + component: PlacementName + + class InstanceFields(ContractModel): """What a registration request and the stored instance share.""" @@ -136,8 +152,9 @@ class WorkerControlRequest(ContractModel): class WorkerStatus(WorkerInstance): """A scoped observation, separate from the stable worker registration protocol. - `identity` and `registered_at` come from the worker's facts and are left out when unknown, - as they are for workers registered before they were recorded. + `identity`, `registered_at` and `drained_at` come from the worker's facts and are left out when unknown, + as they are for workers registered or drained before they were recorded. `placement` is left out when + no observation the reader may see places the worker. """ revision: int = Field(ge=1) @@ -151,6 +168,8 @@ class WorkerStatus(WorkerInstance): available_capacity: int | None = Field(ge=0, le=100) identity: WorkerIdentity | None = Field(default=None, exclude_if=lambda v: v is None) registered_at: AwareDatetime | None = Field(default=None, exclude_if=lambda v: v is None) + placement: WorkerPlacement | None = Field(default=None, exclude_if=lambda v: v is None) + drained_at: AwareDatetime | None = Field(default=None, exclude_if=lambda v: v is None) class LeaseProof(ContractModel): diff --git a/src/firefly_weave/operations/permissions.py b/src/firefly_weave/operations/permissions.py new file mode 100644 index 00000000..06f8ab1b --- /dev/null +++ b/src/firefly_weave/operations/permissions.py @@ -0,0 +1,66 @@ +# Copyright 2026 Firefly Software Foundation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# Author: Firefly Software Foundation +# SPDX-License-Identifier: Apache-2.0 + +"""Deployment targets a reader may see, from both the token and the principal stored now.""" + +from uuid import UUID + +from firefly_weave.access.audit import AuditContext +from firefly_weave.access.authorization import AuthorizationService, contains +from firefly_weave.access.models import Principal +from firefly_weave.access.roles import ROLE_CAPABILITIES +from firefly_weave.contracts.access import Scope + + +def target_permissions( + authorization: AuthorizationService, + actor: Principal, + current: Principal, + scope: Scope, + capability: str, + context: AuditContext, +) -> tuple[UUID, ...] | None: + """`None` when both principals hold `capability` for every target; otherwise the sorted finite intersection. + + The result is empty when either principal holds it for no target. Each granted target is checked through the + authorization service, so callers never collect every target and filter afterwards. + """ + + def allowed(principal: Principal) -> set[UUID] | None: + if not principal.active or principal.kind == "worker": + return set() + candidates: set[UUID] = set() + for grant in principal.grants: + if capability not in ROLE_CAPABILITIES[grant.role] or not contains(grant.scope, scope): + continue + if not grant.resources: + authorization.require(principal, scope, capability, context=context) + return None + for resource in grant.resources: + try: + identifier = UUID(resource) + except ValueError: + continue + # Target grants name the canonical ID, as single target reads check it. + if str(identifier) != resource: + continue + authorization.require(principal, scope, capability, resource=str(identifier), context=context) + candidates.add(identifier) + return candidates + + left, right = allowed(actor), allowed(current) + result = right if left is None else left if right is None else left & right + return None if result is None else tuple(sorted(result)) diff --git a/src/firefly_weave/sdk/client.py b/src/firefly_weave/sdk/client.py index 16df4581..1cce6af2 100644 --- a/src/firefly_weave/sdk/client.py +++ b/src/firefly_weave/sdk/client.py @@ -117,6 +117,13 @@ from firefly_weave.contracts.teams import TeamsReactivateRequest, TeamsReference, TeamsRevokeRequest from firefly_weave.contracts.values import JsonObject from firefly_weave.contracts.whatsapp import WhatsAppDeliveryState, WhatsAppStatusFact +from firefly_weave.contracts.worker_queries import ( + ReleaseUsage, + ReleaseUsageQuery, + RetireStaleRequest, + RetireStaleResult, + WorkerListQuery, +) from firefly_weave.contracts.workers import ( ClaimRequest, CompleteRequest, @@ -767,8 +774,40 @@ async def list_runs( ), ) - async def list_workers(self, *, limit: int = 50, cursor: str | None = None) -> Page[WorkerStatus]: - return cast(Page[WorkerStatus], await self.invoke("workers.list", query=self._page(limit, cursor))) + async def list_workers( + self, *, limit: int = 50, cursor: str | None = None, query: WorkerListQuery | None = None + ) -> Page[WorkerStatus]: + """Workers with either a typed ``query`` or the page keywords; mixing them raises ``ValueError``. + + Without a query the server lists current workers, most recently seen first. + """ + if query is not None and (limit, cursor) != (50, None): + raise ValueError("Use either a query or keyword list arguments") + values = self._page(limit, cursor) if query is None else self._query(query) + return cast(Page[WorkerStatus], await self.invoke("workers.list", query=values)) + + async def retire_stale_workers(self, request: RetireStaleRequest, *, idempotency_key: str) -> RetireStaleResult: + """Revoke (or, with ``dry_run``, preview) up to 100 workers offline longer than the threshold.""" + return cast( + RetireStaleResult, + await self.invoke("workers.retire_stale", body=request, idempotency_key=idempotency_key), + ) + + async def release_usage(self, identifier: UUID, *, query: ReleaseUsageQuery | None = None) -> ReleaseUsage: + """Worker counts, pinning activations and the readable deployments that run a release.""" + return cast( + ReleaseUsage, + await self.invoke("releases.usage", identifier=identifier, query=self._query(query or ReleaseUsageQuery())), + ) + + @staticmethod + def _query(query: BaseModel) -> dict[str, str | int | list[str]]: + # Booleans travel as true or false; unset filters and empty lists are left out. + return { + key: str(value).lower() if isinstance(value, bool) else value + for key, value in query.model_dump(mode="json", exclude_none=True).items() + if not isinstance(value, list) or value + } async def list_releases(self, *, limit: int = 50, cursor: str | None = None) -> Page[WorkerRelease]: return cast(Page[WorkerRelease], await self.invoke("releases.list", query=self._page(limit, cursor))) diff --git a/src/firefly_weave/workers/models.py b/src/firefly_weave/workers/models.py index cba9a272..9f3e9fe0 100644 --- a/src/firefly_weave/workers/models.py +++ b/src/firefly_weave/workers/models.py @@ -25,7 +25,7 @@ from pydantic import ValidationError -from firefly_weave.contracts.workers import WorkerIdentity, WorkerInstance, WorkerStatus +from firefly_weave.contracts.workers import WorkerIdentity, WorkerInstance, WorkerPlacement, WorkerStatus from firefly_weave.definitions.models import CatalogError from firefly_weave.persistence.uow import Transaction from firefly_weave.runtime.kernel import checked_deadline @@ -57,6 +57,32 @@ def stored_identity(value: object) -> WorkerIdentity | None: return None +def stored_placement(value: object) -> WorkerPlacement | None: + """A placement the fleet query derived; one that does not validate reads as unknown.""" + if not value: + return None + try: + return WorkerPlacement.model_validate_json(json.dumps(value)) + except ValidationError: + return None + + +def fleet_status(row: dict[str, Any], observed_at: datetime) -> WorkerStatus: + """A worker read from one fleet query row, without any per-worker query.""" + return observed_worker( + WorkerInstance.model_validate_json(json.dumps({**row["payload"], "revoked": row["revoked"]})), + last_seen_at=row["last_seen_at"], + draining=row["draining"], + revision=row["control_revision"], + active_leases=row["active_leases"], + observed_at=observed_at, + identity=stored_identity(row["identity"]), + registered_at=row["registered_at"], + placement=stored_placement(row["placement"]), + drained_at=row["drained_at"], + ) + + def observed_worker( instance: WorkerInstance, *, @@ -67,6 +93,8 @@ def observed_worker( observed_at: datetime, identity: WorkerIdentity | None = None, registered_at: datetime | None = None, + placement: WorkerPlacement | None = None, + drained_at: datetime | None = None, ) -> WorkerStatus: expires = last_seen_at + timedelta(seconds=60) if last_seen_at else None presence: Literal["unknown", "recent", "stale"] = ( @@ -79,6 +107,8 @@ def observed_worker( **instance.model_dump(), identity=identity, registered_at=registered_at, + placement=placement, + drained_at=drained_at, revision=revision, draining=draining, presence=presence, diff --git a/src/firefly_weave/workers/repository.py b/src/firefly_weave/workers/repository.py index 0da1f040..058db177 100644 --- a/src/firefly_weave/workers/repository.py +++ b/src/firefly_weave/workers/repository.py @@ -23,6 +23,7 @@ from sqlalchemy import text +from firefly_weave.contracts.worker_queries import RETIREMENT_BATCH, ReleaseUsagePosition, WorkerListQuery from firefly_weave.contracts.workers import ( CompletionReceipt, WorkerIdentity, @@ -31,10 +32,63 @@ WorkerStatus, ) from firefly_weave.definitions.models import CatalogError +from firefly_weave.operations.fact_positions import RunPosition +from firefly_weave.persistence.paging import PAGE_BYTES from firefly_weave.persistence.uow import Transaction from firefly_weave.workers.models import observed_worker, stored_identity, unavailable SCOPE = "tenant_id=:tenant AND project_id=:project AND environment_id=:environment" +NOW = "cast(:now AS timestamptz)" +_W_SCOPE = "w.tenant_id=:tenant AND w.project_id=:project AND w.environment_id=:environment" +_FACTS = ( + "LEFT JOIN worker_facts f ON f.worker_id=w.id AND f.tenant_id=w.tenant_id AND f.project_id=w.project_id " + "AND f.environment_id=w.environment_id" +) +# A lease that still holds its worker: active, and neither expired nor past its deadline. +_ACTIVE_LEASE = ( + "SELECT 1 FROM task_leases l WHERE l.tenant_id=w.tenant_id AND l.project_id=w.project_id " + f"AND l.environment_id=w.environment_id AND l.owner=w.id AND l.status='active' AND l.expires_at>{NOW} " + f"AND l.deadline>{NOW}" +) +# A worker's last contact; one never seen counts from its recorded registration, and one without either has none. +_CONTACT = "coalesce(w.last_seen_at,f.registered_at)" +_RETIRABLE = ( + f"FROM worker_instances w {_FACTS} WHERE {_W_SCOPE} AND NOT w.revoked " + f"AND {_CONTACT}>'name' AS target_name " + "FROM deployment_observations o JOIN deployment_targets t ON t.id=o.target_id AND t.tenant_id=o.tenant_id " + "AND t.project_id=o.project_id AND t.environment_id=o.environment_id " + "WHERE o.tenant_id=:tenant AND o.project_id=:project AND o.environment_id=:environment " + "AND (:all_targets OR o.target_id=ANY(cast(:target_ids AS uuid[]))) " + "AND (o.payload->>'complete')::boolean AND (o.payload->>'target_revision')::integer=t.revision " + "ORDER BY o.target_id,(o.payload->>'observed_at')::timestamptz DESC,o.id DESC), " + "placements AS (SELECT DISTINCT member->>'host' AS host,o.target_id,o.target_name,d.id AS deployment_id," + "component->>'name' AS component FROM latest_complete o " + "CROSS JOIN LATERAL jsonb_array_elements(o.payload->'resources') resource " + "CROSS JOIN LATERAL jsonb_array_elements(coalesce(resource->'members','[]'::jsonb)) member " + "JOIN deployment_deployments d ON d.target_id=o.target_id AND d.tenant_id=:tenant AND d.project_id=:project " + "AND d.environment_id=:environment CROSS JOIN LATERAL jsonb_array_elements(d.payload->'components') component " + f"WHERE (o.payload->>'expires_at')::timestamptz>{NOW} AND resource->>'name'=component->>'name' " + "AND resource->>'kind'='worker' AND component->>'kind'='worker'), " + "unique_placement AS (SELECT host,(jsonb_agg(jsonb_build_object('target_id',target_id,'target_name'," + "target_name,'deployment_id',deployment_id,'component',component))->0) AS placement FROM placements " + "GROUP BY host HAVING count(*)=1)" +) +# Filters bind as typed nulls, so every filter combination is the same statement. +_FLEET_FILTERS = ( + f"(:all_workers OR (NOT revoked AND coalesce(last_seen_at,registered_at)>{NOW}-interval '1 hour')) " + "AND (cast(:presence AS text) IS NULL OR presence=cast(:presence AS text)) " + "AND (cast(:draining AS boolean) IS NULL OR draining=cast(:draining AS boolean)) " + "AND (cast(:release AS uuid) IS NULL OR release_id=cast(:release AS uuid)) " + "AND (cast(:task AS text) IS NULL OR payload->'task_types' @> jsonb_build_array(cast(:task AS text))) " + "AND (cast(:host_prefix AS text) IS NULL OR starts_with(identity->>'host',cast(:host_prefix AS text))) " + "AND (cast(:target AS uuid) IS NULL OR (placement->>'target_id')::uuid=cast(:target AS uuid))" +) class WorkerRepository: @@ -91,11 +145,209 @@ async def record_registration(self, identifier: UUID, identity: WorkerIdentity | identity=json.dumps({} if identity is None else identity.model_dump(mode="json")), ) + async def record_drain(self, identifier: UUID, *, draining: bool, was_draining: bool) -> None: + """Record when a worker starts draining; a repeated drain keeps that time and resuming clears it.""" + if draining and not was_draining: + await self.execute( + f"UPDATE worker_facts SET drained_at=clock_timestamp() WHERE {SCOPE} AND worker_id=:id", id=identifier + ) + elif not draining: + await self.execute( + f"UPDATE worker_facts SET drained_at=NULL WHERE {SCOPE} AND worker_id=:id AND drained_at IS NOT NULL", + id=identifier, + ) + + async def fleet( + self, + query: WorkerListQuery, + position: RunPosition | None, + *, + observed_at: datetime, + authorized_targets: tuple[UUID, ...] | None, + worker_id: UUID | None = None, + ) -> list[dict[str, Any]]: + """One page of workers and its lookahead row, with grouped active leases and permitted placement. + + One statement serves every filter, order and position. `authorized_targets` is None for every target and + empty for none. A page above the logical byte limit is refused before its payloads are returned. + """ + column = "last_seen_at" if query.order == "last_seen_desc" else "registered_at" + keyset = "" + bounds: dict[str, Any] = {} + if position is not None: + bounds["position_id"] = position.id + if position.at is None: + keyset = f" AND {column} IS NULL AND id{NOW} AND deadline>{NOW}" + + (" AND owner=:worker" if one else "") + + " GROUP BY owner), " + + _PLACEMENT_CTES + + ", fleet AS (SELECT w.id,w.release_id,w.payload,w.revoked,w.draining,w.control_revision," + "w.last_seen_at,f.registered_at,f.identity,f.drained_at,coalesce(a.active_leases,0) AS active_leases," + f"CASE WHEN w.last_seen_at IS NULL THEN 'unknown' WHEN w.last_seen_at>{NOW}-interval '60 seconds' " + "THEN 'recent' ELSE 'stale' END AS presence,p.placement " + f"FROM worker_instances w {_FACTS} LEFT JOIN active a ON a.owner=w.id " + "LEFT JOIN unique_placement p ON p.host=f.identity->>'host' AND f.identity->>'runtime'<>'embedded' " + f"WHERE {_W_SCOPE}" + (" AND w.id=:worker" if one else "") + "), " + f"page AS (SELECT fleet.*,row_number() OVER (ORDER BY {order}) AS page_row FROM fleet " + f"WHERE {_FLEET_FILTERS}{keyset} ORDER BY {order} LIMIT :page_size), " + "measured AS (SELECT page.*,sum(CASE WHEN page_row<=:limit THEN octet_length(payload::text)" + "+octet_length(coalesce(identity,'{}'::jsonb)::text) ELSE 0 END) OVER () AS page_bytes FROM page) " + "SELECT id,release_id,revoked,draining,control_revision,last_seen_at,registered_at,drained_at," + "active_leases,presence,placement,CASE WHEN page_bytes<=:budget THEN payload END AS payload," + "CASE WHEN page_bytes<=:budget THEN identity END AS identity,page_bytes>:budget AS oversized " + "FROM measured ORDER BY page_row" + ) + rows = await self.rows( + sql, + now=observed_at, + worker=worker_id, + all_targets=authorized_targets is None, + target_ids=list(authorized_targets or ()), + all_workers=query.state == "all", + presence=query.presence, + draining=query.draining, + release=query.release_id, + task=query.task_type, + host_prefix=query.host_prefix, + target=query.target_id, + page_size=query.limit + 1, + limit=query.limit, + budget=PAGE_BYTES, + **bounds, + ) + if any(row["oversized"] for row in rows): + raise CatalogError(429, "WV-PAGE-LIMIT", "Selected page exceeds the logical byte limit") + return rows + + async def retirable(self, cutoff: datetime, now: datetime) -> int: + """Every worker a retirement at this cutoff would revoke, across all batches.""" + rows = await self.rows(f"SELECT count(*) AS count {_RETIRABLE}", cutoff=cutoff, now=now) + return int(rows[0]["count"]) + + async def retirement_batch(self, cutoff: datetime, now: datetime, *, lock: bool) -> list[UUID]: + """The oldest retirable workers, oldest last contact first; locking skips rows another command holds.""" + rows = await self.rows( + f"SELECT w.id {_RETIRABLE} ORDER BY {_CONTACT} ASC,w.id ASC LIMIT :batch" + + (" FOR UPDATE OF w SKIP LOCKED" if lock else ""), + cutoff=cutoff, + now=now, + batch=RETIREMENT_BATCH, + ) + return [row["id"] for row in rows] + + async def retire(self, identifiers: list[UUID], cutoff: datetime, now: datetime) -> list[UUID]: + """Revoke the locked workers that still qualify; the conditions are checked again on the locked rows.""" + if not identifiers: + return [] + rows = await self.rows( + "UPDATE worker_instances w SET revoked=true,control_revision=w.control_revision+1 " + f"WHERE {_W_SCOPE} AND w.id=ANY(cast(:ids AS uuid[])) AND NOT w.revoked " + "AND coalesce(w.last_seen_at,(SELECT f.registered_at FROM worker_facts f WHERE f.worker_id=w.id " + "AND f.tenant_id=w.tenant_id AND f.project_id=w.project_id AND f.environment_id=w.environment_id))" + f" str: + rows = await self.rows(f"SELECT image_digest FROM worker_releases WHERE {SCOPE} AND id=:id", id=identifier) + if not rows: + raise CatalogError(404, "WV-NOT-FOUND", "Worker resource not found") + return str(rows[0]["image_digest"]) + + async def release_workers(self, identifier: UUID, now: datetime) -> dict[str, int]: + rows = await self.rows( + "SELECT count(*) FILTER (WHERE revoked) AS revoked," + "count(*) FILTER (WHERE NOT revoked AND draining) AS draining," + f"count(*) FILTER (WHERE NOT revoked AND NOT draining AND last_seen_at>{NOW}-interval '60 seconds') " + "AS online,count(*) FILTER (WHERE NOT revoked AND NOT draining AND (last_seen_at IS NULL " + f"OR last_seen_at<={NOW}-interval '60 seconds')) AS offline " + f"FROM worker_instances WHERE {SCOPE} AND release_id=:release", + release=identifier, + now=now, + ) + return {key: int(value) for key, value in rows[0].items()} + + async def release_activations(self, identifier: UUID) -> int: + """Activation revisions pinning the release that can still start runs. + + A revision counts unless it, or the latest revision of the same activation, is retired. + """ + pinned = ( + "EXISTS(SELECT 1 FROM jsonb_each_text(coalesce(a.payload->'request'->'{key}','{{}}'::jsonb)) pin " + "WHERE pin.value=:release)" + ) + rows = await self.rows( + "SELECT count(*) AS count FROM activation_revisions a WHERE a.tenant_id=:tenant AND a.project_id=:project " + f"AND a.environment_id=:environment AND ({pinned.format(key='worker_release_ids')} " + f"OR {pinned.format(key='connector_release_ids')}) " + "AND NOT coalesce((a.payload->>'retired')::boolean,false) " + "AND NOT coalesce((SELECT (l.payload->>'retired')::boolean FROM activation_revisions l " + "WHERE l.tenant_id=a.tenant_id AND l.project_id=a.project_id AND l.environment_id=a.environment_id " + "AND l.name=a.name ORDER BY l.revision DESC LIMIT 1),false)", + release=str(identifier), + ) + return int(rows[0]["count"]) + + async def release_deployments( + self, + identifier: UUID, + digest: str, + *, + authorized_targets: tuple[UUID, ...] | None, + position: ReleaseUsagePosition | None, + limit: int, + ) -> list[dict[str, Any]]: + """Deployment components that run exactly this admitted release, in deployment and component order.""" + keyset = "" + bounds: dict[str, Any] = {} + if position is not None: + keyset = ( + " AND (d.id,(component->>'name') COLLATE \"C\")>" + '(cast(:deployment AS uuid),cast(:component AS text) COLLATE "C")' + ) + bounds = dict(deployment=position.deployment_id, component=position.component) + return await self.rows( + "SELECT d.target_id,t.payload->>'name' AS target_name,d.id AS deployment_id," + "d.payload->>'name' AS deployment_name,component->>'name' AS component," + "(component->>'replicas')::integer AS replicas FROM deployment_deployments d " + "JOIN deployment_targets t ON t.id=d.target_id AND t.tenant_id=d.tenant_id AND t.project_id=d.project_id " + "AND t.environment_id=d.environment_id " + "CROSS JOIN LATERAL jsonb_array_elements(d.payload->'components') component " + "WHERE d.tenant_id=:tenant AND d.project_id=:project AND d.environment_id=:environment " + "AND (:all_targets OR d.target_id=ANY(cast(:target_ids AS uuid[]))) " + "AND component->>'worker_release_id'=:release AND split_part(component->>'image','@',2)=:digest" + + keyset + + " ORDER BY d.id,(component->>'name') COLLATE \"C\" LIMIT :limit", + release=str(identifier), + digest=digest, + all_targets=authorized_targets is None, + target_ids=list(authorized_targets or ()), + limit=limit + 1, + **bounds, + ) + async def status(self, identifier: UUID, observed_at: datetime | None = None) -> WorkerStatus: now = observed_at or await self.now() rows = await self.rows( "SELECT w.payload,w.revoked,w.draining,w.control_revision,w.last_seen_at," - "f.identity,f.registered_at FROM worker_instances w LEFT JOIN worker_facts f " + "f.identity,f.registered_at,f.drained_at FROM worker_instances w LEFT JOIN worker_facts f " "ON f.worker_id=w.id AND f.tenant_id=w.tenant_id AND f.project_id=w.project_id " "AND f.environment_id=w.environment_id WHERE w.tenant_id=:tenant AND w.project_id=:project " "AND w.environment_id=:environment AND w.id=:id", @@ -119,6 +371,7 @@ async def status(self, identifier: UUID, observed_at: datetime | None = None) -> observed_at=now, identity=stored_identity(row["identity"]), registered_at=row["registered_at"], + drained_at=row["drained_at"], ) async def lease(self, task: UUID, generation: int) -> dict[str, Any]: diff --git a/src/firefly_weave/workers/service.py b/src/firefly_weave/workers/service.py index 38a109c4..2f81699c 100644 --- a/src/firefly_weave/workers/service.py +++ b/src/firefly_weave/workers/service.py @@ -17,7 +17,7 @@ """Immutable release admission and scoped worker identities.""" import json -from datetime import datetime +from datetime import datetime, timedelta from typing import Any from uuid import UUID, uuid4 @@ -28,11 +28,22 @@ from firefly_weave.access.models import Principal from firefly_weave.access.repository import load_principal from firefly_weave.access.service import audit +from firefly_weave.api.transport import decode_cursor_v2, encode_cursor_v2 from firefly_weave.compiler.api import CompiledArtifact from firefly_weave.compiler.catalog import CatalogSnapshot from firefly_weave.compiler.expressions import measure_value from firefly_weave.contracts.access import Scope from firefly_weave.contracts.catalog import ActivationRequest +from firefly_weave.contracts.worker_queries import ( + ReleaseDeploymentUsage, + ReleaseUsage, + ReleaseUsageQuery, + ReleaseWorkerCounts, + RetireStaleRequest, + RetireStaleResult, + WorkerListQuery, + release_usage_position, +) from firefly_weave.contracts.workers import ( ConnectorExecutionPin, CredentialGrantRequest, @@ -47,8 +58,11 @@ from firefly_weave.definitions.ports import ConnectionBindingPort, WorkerAdmissionPort from firefly_weave.definitions.repository import DefinitionRepository from firefly_weave.definitions.service import DefinitionService +from firefly_weave.operations.fact_positions import RunPosition +from firefly_weave.operations.permissions import target_permissions from firefly_weave.persistence.idempotency import Idempotency, lock from firefly_weave.persistence.uow import Transaction +from firefly_weave.workers.models import fleet_status from firefly_weave.workers.repository import SCOPE, WorkerRepository @@ -72,6 +86,28 @@ async def require( for current in (actor, await load_principal(tx.session, actor.id)): self.definitions.authorization.require(current, scope, capability, resource=resource, context=context) + def precheck( + self, actor: Principal, scope: Scope, capability: str, context: AuditContext, resource: str | None = None + ) -> None: + """`capability` on the token principal, before any unit of work opens.""" + if scope.environment_id is None: + raise AccessDenied() + self.definitions.authorization.require(actor, scope, capability, resource=resource, context=context) + + async def current( + self, + actor: Principal, + scope: Scope, + capability: str, + context: AuditContext, + tx: Transaction, + resource: str | None = None, + ) -> Principal: + """`capability` on the principal stored now, inside the caller's unit of work; `precheck` runs first.""" + current = await load_principal(tx.session, actor.id) + self.definitions.authorization.require(current, scope, capability, resource=resource, context=context) + return current + async def capabilities(self, tx: Transaction, base: CatalogSnapshot) -> CatalogSnapshot: rows = await WorkerRepository(tx).rows( "SELECT payload FROM worker_releases WHERE tenant_id=:tenant AND project_id=:project" @@ -229,7 +265,7 @@ async def control( return WorkerStatus.model_validate_json(json.dumps(prior)) repository = WorkerRepository(enlisted) rows = await repository.rows( - f"SELECT revoked,control_revision FROM worker_instances WHERE {SCOPE} AND id=:id FOR UPDATE", + f"SELECT revoked,draining,control_revision FROM worker_instances WHERE {SCOPE} AND id=:id FOR UPDATE", id=identifier, ) if not rows: @@ -244,6 +280,7 @@ async def control( id=identifier, draining=draining, ) + await repository.record_drain(identifier, draining=draining, was_draining=rows[0]["draining"]) result = await repository.status(identifier) await audit( enlisted.session, @@ -414,6 +451,112 @@ async def grant_connection( context=context, ) + async def retire_stale( + self, + actor: Principal, + scope: Scope, + request: RetireStaleRequest, + idempotency_key: str, + *, + context: AuditContext, + tx: Transaction | None = None, + ) -> RetireStaleResult: + """Revoke at most one batch of workers without an active lease whose last contact is older than the threshold. + + A dry run selects the same oldest-first batch without revoking it. Both save their answer for replay under + the same Idempotency-Key; a worker that was never seen counts from its registration, and one whose + registration time is unknown is never retired. + """ + self.precheck(actor, scope, "release.retire", context) + async with self.definitions.transaction(scope, tx) as enlisted: + await self.current(actor, scope, "release.retire", context, enlisted) + replay = Idempotency( + enlisted, + actor.id, + f"workers.retire_stale:{scope.environment_id}", + idempotency_key, + request.model_dump(mode="json"), + ) + if prior := await replay.replay(): + return RetireStaleResult.model_validate_json(json.dumps(prior)) + repository = WorkerRepository(enlisted) + now = await repository.now() + cutoff = now - timedelta(seconds=request.older_than_seconds) + matched = await repository.retirable(cutoff, now) + if request.dry_run: + result = RetireStaleResult( + matched=matched, + revoked=0, + worker_ids=await repository.retirement_batch(cutoff, now, lock=False), + ) + else: + locked = await repository.retirement_batch(cutoff, now, lock=True) + revoked = await repository.retire(locked, cutoff, now) + result = RetireStaleResult(matched=matched, revoked=len(revoked), worker_ids=revoked) + await audit( + enlisted.session, + actor, + "worker.retire_stale", + str(scope.environment_id), + scope=scope, + capability="release.retire", + context=context, + details={ + "dry_run": request.dry_run, + "older_than_seconds": request.older_than_seconds, + **result.model_dump(mode="json"), + }, + ) + await replay.save(result.model_dump(mode="json")) + return result + + async def release_usage( + self, + actor: Principal, + scope: Scope, + release_id: UUID, + query: ReleaseUsageQuery, + *, + context: AuditContext, + ) -> ReleaseUsage: + """Worker counts, pinning activations and a page of the readable deployments that run an admitted release. + + The cursor is bound to the release and to the targets this reader may see, which only the read unit knows. + """ + self.precheck(actor, scope, "catalog.read", context) + async with self.definitions.transaction(scope, None, mutation=False) as tx: + current = await self.current(actor, scope, "catalog.read", context, tx) + repository = WorkerRepository(tx) + digest = await repository.release_digest(release_id) + targets = target_permissions( + self.definitions.authorization, actor, current, scope, "deployment.read", context + ) + collection = query.cursor_collection(release_id, targets) + position = release_usage_position(decode_cursor_v2(query.cursor, scope, collection)) + now = await repository.now() + workers = ReleaseWorkerCounts(**await repository.release_workers(release_id, now)) + activations = await repository.release_activations(release_id) + rows = ( + [] + if targets == () + else await repository.release_deployments( + release_id, digest, authorized_targets=targets, position=position, limit=query.limit + ) + ) + visible = rows[: query.limit] + cursor = None + if len(rows) > query.limit: + last = visible[-1] + cursor = encode_cursor_v2(scope, collection, last["component"], str(last["deployment_id"])) + return ReleaseUsage( + workers=workers, + deployments=[ + ReleaseDeploymentUsage.model_validate_json(json.dumps(row, default=str)) for row in visible + ], + activations=activations, + next_cursor=cursor, + ) + async def list( self, actor: Principal, @@ -422,25 +565,73 @@ async def list( releases: bool = False, limit: int = 50, cursor: UUID | None = None, + query: WorkerListQuery | None = None, + position: RunPosition | None = None, context: AuditContext, ) -> dict[str, Any]: + """Releases in ID order, or one page of workers from a single fleet query. + + Workers take a typed query and the decoded position of its cursor; the keyword `limit` selects the first page + of current workers. + """ + if not releases: + if query is None: + if cursor is not None: + raise ValueError("Continue a worker list with its query and position") + query = WorkerListQuery(limit=limit) + elif cursor is not None or limit != 50: + raise ValueError("Use either a query or legacy list arguments") + return await self._fleet(actor, scope, query, position, context=context) + if query is not None or position is not None: + raise ValueError("Release lists take no worker query") from firefly_weave.persistence.paging import page_ids - capability = "catalog.read" if releases else "status.read" async with self.definitions.transaction(scope, None, mutation=False) as tx: - await self.require(actor, scope, capability, context, tx) - ids = await page_ids(tx, "worker_releases" if releases else "worker_instances", limit, cursor) - observed_at = await WorkerRepository(tx).now() + await self.require(actor, scope, "catalog.read", context, tx) + ids = await page_ids(tx, "worker_releases", limit, cursor) items = [ - ( - await self.read( - actor, scope, identifier, releases=releases, context=context, tx=tx, observed_at=observed_at - ) - ).model_dump(mode="json", by_alias=True) + (await self.read(actor, scope, identifier, releases=True, context=context, tx=tx)).model_dump( + mode="json", by_alias=True + ) for identifier in ids[:limit] ] return {"items": items, "next_cursor": str(ids[limit - 1]) if len(ids) > limit else None} + async def _fleet( + self, + actor: Principal, + scope: Scope, + query: WorkerListQuery, + position: RunPosition | None, + *, + context: AuditContext, + ) -> dict[str, Any]: + authorization = self.definitions.authorization + target = None if query.target_id is None else str(query.target_id) + self.precheck(actor, scope, "status.read", context) + if target is not None: + authorization.require(actor, scope, "deployment.read", resource=target, context=context) + async with self.definitions.transaction(scope, None, mutation=False) as tx: + current = await self.current(actor, scope, "status.read", context, tx) + if target is not None: + authorization.require(current, scope, "deployment.read", resource=target, context=context) + targets = target_permissions(authorization, actor, current, scope, "deployment.read", context) + repository = WorkerRepository(tx) + observed_at = await repository.now() + rows = await repository.fleet(query, position, observed_at=observed_at, authorized_targets=targets) + visible = rows[: query.limit] + next_cursor = None + if len(rows) > query.limit: + last = visible[-1] + at = last["last_seen_at"] if query.order == "last_seen_desc" else last["registered_at"] + next_cursor = encode_cursor_v2( + scope, query.cursor_collection(), at.isoformat() if at else None, str(last["id"]) + ) + return { + "items": [fleet_status(row, observed_at).model_dump(mode="json") for row in visible], + "next_cursor": next_cursor, + } + async def read( self, actor: Principal, @@ -452,21 +643,43 @@ async def read( tx: Transaction | None = None, observed_at: datetime | None = None, ) -> WorkerStatus | WorkerRelease: + if not releases: + return await self._worker(actor, scope, identifier, context=context, tx=tx, observed_at=observed_at) async with self.definitions.transaction(scope, tx, mutation=False) as enlisted: - await self.require( - actor, - scope, - "catalog.read" if releases else "status.read", - context, - enlisted, - None if releases else str(identifier), - ) - table = "worker_releases" if releases else "worker_instances" + await self.require(actor, scope, "catalog.read", context, enlisted) rows = await WorkerRepository(enlisted).rows( - f"SELECT * FROM {table} WHERE {SCOPE} AND id=:id", id=identifier + f"SELECT payload FROM worker_releases WHERE {SCOPE} AND id=:id", id=identifier + ) + if not rows: + raise CatalogError(404, "WV-NOT-FOUND", "Worker resource not found") + return WorkerRelease.model_validate_json(json.dumps(rows[0]["payload"])) + + async def _worker( + self, + actor: Principal, + scope: Scope, + identifier: UUID, + *, + context: AuditContext, + tx: Transaction | None, + observed_at: datetime | None, + ) -> WorkerStatus: + """One worker through the fleet statement, so a read shows what a list shows, placement included.""" + self.precheck(actor, scope, "status.read", context, str(identifier)) + async with self.definitions.transaction(scope, tx, mutation=False) as enlisted: + current = await self.current(actor, scope, "status.read", context, enlisted, str(identifier)) + targets = target_permissions( + self.definitions.authorization, actor, current, scope, "deployment.read", context + ) + repository = WorkerRepository(enlisted) + now = observed_at or await repository.now() + rows = await repository.fleet( + WorkerListQuery(state="all", limit=1), + None, + observed_at=now, + authorized_targets=targets, + worker_id=identifier, ) if not rows: raise CatalogError(404, "WV-NOT-FOUND", "Worker resource not found") - if releases: - return WorkerRelease.model_validate_json(json.dumps(rows[0]["payload"])) - return await WorkerRepository(enlisted).status(identifier, observed_at) + return fleet_status(rows[0], now) diff --git a/tests/contracts/test_worker_queries.py b/tests/contracts/test_worker_queries.py new file mode 100644 index 00000000..22a07e3a --- /dev/null +++ b/tests/contracts/test_worker_queries.py @@ -0,0 +1,432 @@ +# Copyright 2026 Firefly Software Foundation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# Author: Firefly Software Foundation +# SPDX-License-Identifier: Apache-2.0 + +"""Worker fleet queries, stale-worker retirement and release usage contracts; earlier worker bytes are kept.""" + +import json +from datetime import UTC, datetime +from uuid import UUID, uuid4 + +import httpx +import pytest +from pydantic import ValidationError +from starlette.datastructures import QueryParams + +from firefly_weave.contracts.access import Scope +from firefly_weave.contracts.workers import WorkerInstance, WorkerStatus +from firefly_weave.workers.models import observed_worker + +ENVIRONMENT = "/api/v1/tenants/{tenant}/projects/{project}/environments/{environment}" + + +def query(model, values): + return model.model_validate_json(json.dumps(values)) + + +def status(**changes): + instance = WorkerInstance( + id=UUID("6f1d2c3b-4a59-4e8f-9d7c-1b2a3c4d5e6f"), + principal_id=UUID("0a1b2c3d-4e5f-4a6b-8c7d-9e0f1a2b3c4d"), + release_id=UUID("1b2c3d4e-5f6a-4b7c-8d9e-0f1a2b3c4d5e"), + task_types=["echo@1.2.0"], + capacity=2, + ) + return observed_worker( + instance, + last_seen_at=None, + draining=False, + revision=1, + active_leases=0, + observed_at=datetime(2026, 10, 12, tzinfo=UTC), + **changes, + ) + + +def test_list_defaults_are_current_workers_most_recently_seen_first(): + from firefly_weave.contracts.worker_queries import WorkerListQuery + + default = WorkerListQuery() + assert (default.state, default.order, default.limit, default.cursor) == ("current", "last_seen_desc", 50, None) + assert all( + getattr(default, name) is None + for name in ("presence", "draining", "release_id", "target_id", "task_type", "host_prefix") + ) + + +@pytest.mark.parametrize( + "values", + [ + {"presence": "online"}, + {"state": "retired"}, + {"order": "id"}, + {"order": "registered_asc"}, + {"limit": 0}, + {"limit": 101}, + {"task_type": ""}, + {"task_type": "t" * 201}, + {"host_prefix": ""}, + {"host_prefix": "h" * 254}, + {"host_prefix": "worker\n1"}, + {"release_id": "not-a-uuid"}, + {"cursor": "not a cursor"}, + {"draining": "yes"}, + {"identity": {}}, + ], +) +def test_malformed_list_values_are_validation_errors(values): + from firefly_weave.contracts.worker_queries import WorkerListQuery + + with pytest.raises(ValidationError): + query(WorkerListQuery, values) + + +def test_every_filter_and_bound_is_accepted_together(): + from firefly_weave.contracts.worker_queries import WorkerListQuery + + values = { + "presence": "stale", + "draining": True, + "release_id": str(uuid4()), + "target_id": str(uuid4()), + "task_type": "t" * 200, + "host_prefix": "%_" + "h" * 251, + "state": "all", + "order": "registered_desc", + "limit": 100, + } + parsed = query(WorkerListQuery, values) + assert parsed.model_dump(mode="json", exclude={"cursor"}) == values + for limit in (1, 100): + assert query(WorkerListQuery, {"limit": limit}).limit == limit + + +def test_route_decoding_is_strict(): + from firefly_weave.api.transport import parse_query + from firefly_weave.contracts.worker_queries import WorkerListQuery + + parsed = parse_query(QueryParams("draining=false&presence=recent&state=all&order=registered_desc"), WorkerListQuery) + assert (parsed.draining, parsed.presence, parsed.state, parsed.order) == (False, "recent", "all", "registered_desc") + for raw in ("colour=red", "limit=1&limit=2", "draining=yes", "presence=online", "limit=0"): + with pytest.raises(ValueError): + parse_query(QueryParams(raw), WorkerListQuery) + + +def test_cursor_collection_binds_every_filter_and_the_order(): + from firefly_weave.contracts.worker_queries import WorkerListQuery + + release, target = str(uuid4()), str(uuid4()) + variants = [ + {}, + {"order": "registered_desc"}, + {"presence": "recent"}, + {"presence": "unknown"}, + {"draining": True}, + {"draining": False}, + {"release_id": release}, + {"target_id": target}, + {"task_type": "echo@1.2.0"}, + {"host_prefix": "worker-"}, + {"state": "all"}, + ] + collections = {query(WorkerListQuery, values).cursor_collection() for values in variants} + assert len(collections) == len(variants) + assert all(item.startswith("workers:") for item in collections) + # Page size and position never change what a cursor continues. + assert WorkerListQuery(limit=3).cursor_collection() == WorkerListQuery().cursor_collection() + + +@pytest.mark.parametrize( + "seconds,valid", [(3599, False), (3600, True), (86400, True), (2592000, True), (2592001, False)] +) +def test_retirement_threshold_is_one_hour_to_thirty_days(seconds, valid): + from firefly_weave.contracts.worker_queries import RetireStaleRequest + + values = {"older_than_seconds": seconds, "dry_run": True} + if valid: + assert query(RetireStaleRequest, values).older_than_seconds == seconds + else: + with pytest.raises(ValidationError): + query(RetireStaleRequest, values) + + +@pytest.mark.parametrize( + "values", + [ + {"older_than_seconds": 86400}, + {"dry_run": False}, + {"older_than_seconds": "86400", "dry_run": False}, + {"older_than_seconds": 86400, "dry_run": "false"}, + {"older_than_seconds": 86400, "dry_run": False, "worker_ids": []}, + ], +) +def test_retirement_states_both_its_threshold_and_whether_it_is_a_preview(values): + from firefly_weave.contracts.worker_queries import RetireStaleRequest + + with pytest.raises(ValidationError): + query(RetireStaleRequest, values) + + +def test_retirement_result_lists_at_most_one_batch(): + from firefly_weave.contracts.worker_queries import RetireStaleResult + + batch = [str(uuid4()) for _ in range(100)] + result = query(RetireStaleResult, {"matched": 250, "revoked": 100, "worker_ids": batch}) + assert result.model_dump(mode="json") == {"matched": 250, "revoked": 100, "worker_ids": batch} + for invalid in ( + {"matched": 250, "revoked": 100, "worker_ids": [*batch, str(uuid4())]}, + {"matched": -1, "revoked": 0, "worker_ids": []}, + {"matched": 0, "revoked": 101, "worker_ids": []}, + ): + with pytest.raises(ValidationError): + query(RetireStaleResult, invalid) + + +def test_release_usage_page_is_bounded_and_bound_to_release_and_target_scope(): + from firefly_weave.contracts.worker_queries import ReleaseUsageQuery + + assert ReleaseUsageQuery().limit == 100 + for limit in (0, 101): + with pytest.raises(ValidationError): + query(ReleaseUsageQuery, {"limit": limit}) + with pytest.raises(ValidationError): + query(ReleaseUsageQuery, {"target_id": str(uuid4())}) + release, first, second = uuid4(), uuid4(), uuid4() + usage = ReleaseUsageQuery() + assert usage.cursor_collection(release, (first, second)) == usage.cursor_collection(release, (second, first)) + assert ReleaseUsageQuery(limit=1).cursor_collection(release, (first,)) == usage.cursor_collection(release, (first,)) + collections = { + usage.cursor_collection(release, None), + usage.cursor_collection(release, ()), + usage.cursor_collection(release, (first,)), + usage.cursor_collection(release, (first, second)), + usage.cursor_collection(uuid4(), (first,)), + } + assert len(collections) == 5 + + +@pytest.mark.parametrize( + "position", + [ + ("worker", "not-a-uuid"), + (7, str(uuid4())), + (None, str(uuid4())), + ("bad name", str(uuid4())), + ("w" * 101, str(uuid4())), + ("worker", 7), + ], +) +def test_release_usage_positions_are_typed_before_sql(position): + from firefly_weave.contracts.worker_queries import release_usage_position + + with pytest.raises(ValueError): + release_usage_position(position) + + +def test_release_usage_position_keeps_the_component_and_deployment(): + from firefly_weave.contracts.worker_queries import ReleaseUsagePosition, release_usage_position + + deployment = uuid4() + assert release_usage_position(None) is None + assert release_usage_position(("engine.workers", str(deployment))) == ReleaseUsagePosition( + deployment, "engine.workers" + ) + + +def test_release_usage_shape_and_bounds(): + from firefly_weave.contracts.worker_queries import ReleaseUsage + + row = { + "target_id": str(uuid4()), + "target_name": "local", + "deployment_id": str(uuid4()), + "deployment_name": "weave", + "component": "engine", + "replicas": 3, + } + payload = { + "workers": {"online": 1, "offline": 2, "draining": 0, "revoked": 4}, + "deployments": [row], + "activations": 2, + "next_cursor": None, + } + assert query(ReleaseUsage, payload).model_dump(mode="json") == payload + for invalid in ( + {**payload, "deployments": [row] * 101}, + {**payload, "activations": -1}, + {**payload, "workers": {**payload["workers"], "online": -1}}, + {**payload, "deployments": [{**row, "replicas": 101}]}, + {**payload, "deployments": [{**row, "target_name": "../x"}]}, + ): + with pytest.raises(ValidationError): + query(ReleaseUsage, invalid) + + +def test_earlier_worker_status_bytes_are_kept_and_new_fields_are_optional(): + from firefly_weave.contracts.workers import WorkerPlacement + + plain = status() + assert not {"placement", "drained_at"} & set(plain.model_dump(mode="json")) + placement = WorkerPlacement(target_id=uuid4(), target_name="local", deployment_id=uuid4(), component="engine") + drained = datetime(2026, 10, 11, 8, tzinfo=UTC) + enriched = status(placement=placement, drained_at=drained) + dumped = enriched.model_dump(mode="json") + assert dumped["placement"] == placement.model_dump(mode="json") + assert dumped["drained_at"] == "2026-10-11T08:00:00Z" + assert WorkerStatus.model_validate_json(json.dumps(dumped)) == enriched + # The stored instance and the registration answer never carry derived placement. + stored = plain.model_dump(mode="json", include=set(WorkerInstance.model_fields)) + with pytest.raises(ValidationError): + WorkerInstance.model_validate_json(json.dumps({**stored, "placement": dumped["placement"]})) + + +def test_placement_names_follow_deployment_names(): + from firefly_weave.contracts.deployments import Deployment + from firefly_weave.contracts.workers import WorkerPlacement + + def pattern(schema, name): + reference = schema["properties"][name]["$ref"].rsplit("/", 1)[-1] + return schema["$defs"][reference]["pattern"] + + deployment_name = pattern(Deployment.model_json_schema(), "name") + placement = WorkerPlacement.model_json_schema() + assert pattern(placement, "target_name") == pattern(placement, "component") == deployment_name + for name in ("../etc", "", "a" * 101, "a b"): + with pytest.raises(ValidationError): + WorkerPlacement(target_id=uuid4(), target_name=name, deployment_id=uuid4(), component="engine") + + +def test_worker_operations_declare_queries_commands_and_capabilities(): + from firefly_weave.contracts.openapi import export_openapi + from firefly_weave.contracts.public import Page + from firefly_weave.contracts.surface import OPERATIONS + from firefly_weave.contracts.worker_queries import ( + ReleaseUsage, + ReleaseUsageQuery, + RetireStaleRequest, + RetireStaleResult, + WorkerListQuery, + ) + + spec = export_openapi() + listed = OPERATIONS["workers.list"] + assert (listed.query, listed.page, listed.response, listed.capability) == ( + WorkerListQuery, + True, + Page[WorkerStatus], + "status.read", + ) + parameters = spec["paths"][listed.canonical_path]["get"]["parameters"] + assert [item["name"] for item in parameters if item["in"] == "query"] == list(WorkerListQuery.model_fields) + retire = OPERATIONS["workers.retire_stale"] + assert (retire.method, retire.canonical_path, retire.capability, retire.request, retire.response) == ( + "POST", + ENVIRONMENT + "/workers/retire-stale", + "release.retire", + RetireStaleRequest, + RetireStaleResult, + ) + assert retire.idempotency and retire.query is None and not retire.page + usage = OPERATIONS["releases.usage"] + assert (usage.method, usage.canonical_path, usage.capability, usage.response, usage.query) == ( + "GET", + ENVIRONMENT + "/worker-releases/{identifier}/usage", + "catalog.read", + ReleaseUsage, + ReleaseUsageQuery, + ) + assert usage.request is None and not usage.page + parameters = spec["paths"][usage.canonical_path]["get"]["parameters"] + assert [item["name"] for item in parameters if item["in"] == "query"] == ["limit", "cursor"] + + +def test_worker_query_schemas_are_exported(): + from firefly_weave.contracts.schema_export import export_schemas + + schemas = export_schemas() + assert schemas["worker-list-query"]["properties"]["state"]["default"] == "current" + assert schemas["worker-list-query"]["properties"]["order"]["default"] == "last_seen_desc" + assert set(schemas["retire-stale-request"]["required"]) == {"older_than_seconds", "dry_run"} + assert schemas["retire-stale-result"]["properties"]["worker_ids"]["maxItems"] == 100 + assert schemas["release-usage"]["properties"]["deployments"]["maxItems"] == 100 + assert "worker-placement" in schemas and "release-usage-query" in schemas + assert not {"placement", "drained_at"} & set(schemas["worker-status"]["required"]) + assert {"placement", "drained_at"} <= set(schemas["worker-status"]["properties"]) + + +async def test_sdk_sends_typed_worker_queries_and_commands(): + from firefly_weave.contracts.worker_queries import ( + ReleaseUsageQuery, + RetireStaleRequest, + WorkerListQuery, + ) + from firefly_weave.sdk.client import WeaveClient + + calls = [] + release = uuid4() + worker = status().model_dump(mode="json") + + def receive(request): + calls.append(request) + if request.url.path.endswith("/retire-stale"): + assert json.loads(request.content) == {"older_than_seconds": 86400, "dry_run": True} + assert request.headers["idempotency-key"] == "preview" + return httpx.Response(200, json={"matched": 1, "revoked": 0, "worker_ids": [worker["id"]]}) + if request.url.path.endswith("/usage"): + return httpx.Response( + 200, + json={ + "workers": {"online": 0, "offline": 1, "draining": 0, "revoked": 0}, + "deployments": [], + "activations": 1, + "next_cursor": None, + }, + ) + return httpx.Response(200, json={"items": [worker], "next_cursor": None}) + + scope = Scope(tenant_id=uuid4(), project_id=uuid4(), environment_id=uuid4()) + async with WeaveClient( + "https://platform.example", lambda: "token", scope, transport=httpx.MockTransport(receive) + ) as client: + page = await client.list_workers( + query=WorkerListQuery(draining=False, presence="stale", state="all", host_prefix="w", limit=5) + ) + assert page.items[0].id == UUID(worker["id"]) + await client.list_workers(limit=7, cursor="abc") + with pytest.raises(ValueError): + await client.list_workers(limit=7, query=WorkerListQuery()) + result = await client.retire_stale_workers( + RetireStaleRequest(older_than_seconds=86400, dry_run=True), idempotency_key="preview" + ) + assert (result.matched, result.revoked) == (1, 0) + usage = await client.release_usage(release, query=ReleaseUsageQuery(limit=10)) + assert usage.activations == 1 + await client.release_usage(release) + first, second, retire, usage_call, default_usage = calls + assert dict(first.url.params) == { + "draining": "false", + "presence": "stale", + "state": "all", + "host_prefix": "w", + "order": "last_seen_desc", + "limit": "5", + } + assert dict(second.url.params) == {"limit": "7", "cursor": "abc"} + assert retire.method == "POST" and retire.url.path.endswith( + f"/environments/{scope.environment_id}/workers/retire-stale" + ) + assert usage_call.url.path.endswith(f"/worker-releases/{release}/usage") + assert dict(usage_call.url.params) == {"limit": "10"} + assert dict(default_usage.url.params) == {"limit": "100"} diff --git a/tests/integration/test_worker_fleet_facts.py b/tests/integration/test_worker_fleet_facts.py new file mode 100644 index 00000000..9591d889 --- /dev/null +++ b/tests/integration/test_worker_fleet_facts.py @@ -0,0 +1,1041 @@ +# Copyright 2026 Firefly Software Foundation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# Author: Firefly Software Foundation +# SPDX-License-Identifier: Apache-2.0 + +"""Real PostgreSQL worker fleet: one grouped query, authorized placement, bounded retirement and release usage.""" + +import asyncio +import json +from datetime import datetime, timedelta +from uuid import UUID, uuid4 + +import pytest +from sqlalchemy import event, text + +from firefly_weave.access.audit import AuditContext +from firefly_weave.access.authorization import AccessDenied +from firefly_weave.access.models import Grant, Principal +from firefly_weave.contracts.deployments import ( + ComponentSpec, + Deployment, + DeploymentObservation, + DeploymentTarget, + ObservedMember, + ObservedResource, +) +from firefly_weave.contracts.worker_queries import ( + ReleaseUsageQuery, + RetireStaleRequest, + WorkerListQuery, +) +from firefly_weave.contracts.workers import WorkerControlRequest +from firefly_weave.definitions.models import CatalogError +from firefly_weave.workers.repository import WorkerRepository + +pytestmark = pytest.mark.integration + +DIGEST = "sha256:" + "a" * 64 +OTHER_DIGEST = "sha256:" + "c" * 64 + + +async def clock(case) -> datetime: + async with case.owner() as session: + value: datetime = await session.scalar(text("SELECT clock_timestamp()")) + return value + + +def freeze(monkeypatch, at: datetime) -> None: + async def fixed(self): + return at + + monkeypatch.setattr(WorkerRepository, "now", fixed) + + +def identity(host: str, runtime: str = "process") -> dict: + return {"host": host, "runtime": runtime, "platform_hint": "none"} + + +async def add_workers( + case, + count: int = 1, + *, + seen=None, + registered=None, + host=None, + runtime="process", + revoked=False, + draining=False, + task_types=("echo@1.2.0",), + facts=True, + release=None, +) -> list[UUID]: + """Workers inserted as stored records; `seen` and `registered` may be one time or one per worker.""" + template = case.instances[0] + release = release or template.release_id + created = [] + async with case.owner.begin() as session: + for index in range(count): + identifier = uuid4() + payload = template.model_copy( + update={"id": identifier, "release_id": release, "task_types": list(task_types)} + ).model_dump_json() + last_seen = seen[index] if isinstance(seen, list) else seen + at = registered[index] if isinstance(registered, list) else registered + await session.execute( + text( + "INSERT INTO worker_instances(id,tenant_id,project_id,environment_id,release_id,principal_id," + "payload,revoked,last_seen_at,draining) VALUES(:id,:tenant,:project,:environment,:release," + ":principal,cast(:payload AS jsonb),:revoked,:seen,:draining)" + ), + { + "id": identifier, + "tenant": case.scope.tenant_id, + "project": case.scope.project_id, + "environment": case.scope.environment_id, + "release": release, + "principal": template.principal_id, + "payload": payload, + "revoked": revoked, + "seen": last_seen, + "draining": draining, + }, + ) + if facts: + reported = {} if host is None else identity(host if count == 1 else f"{host}{index:03d}", runtime) + await session.execute( + text( + "INSERT INTO worker_facts(tenant_id,project_id,environment_id,worker_id,registered_at," + "identity) VALUES(:tenant,:project,:environment,:id,:registered,cast(:identity AS jsonb))" + ), + { + "tenant": case.scope.tenant_id, + "project": case.scope.project_id, + "environment": case.scope.environment_id, + "id": identifier, + "registered": at, + "identity": json.dumps(reported), + }, + ) + created.append(identifier) + return created + + +async def listed(case, actor=None, position=None, **values) -> dict: + return await case.workers.list( + actor or case.actor, case.scope, query=WorkerListQuery(**values), position=position, context=AuditContext() + ) + + +async def walk(case, actor=None, **values) -> list[UUID]: + from firefly_weave.api.transport import decode_cursor_v2 + from firefly_weave.operations.fact_positions import run_position + + query = WorkerListQuery(**values) + position, found = None, [] + while True: + page = await case.workers.list( + actor or case.actor, case.scope, query=query, position=position, context=AuditContext() + ) + assert len(page["items"]) <= query.limit + found += [UUID(item["id"]) for item in page["items"]] + if page["next_cursor"] is None: + return found + position = run_position(decode_cursor_v2(page["next_cursor"], case.scope, query.cursor_collection())) + + +def ids(page) -> list[UUID]: + return [UUID(item["id"]) for item in page["items"]] + + +def capture(case): + statements: list[str] = [] + engine = case.sessions.kw["bind"].sync_engine + + def record(connection, cursor, statement, parameters, context, executemany): + statements.append(" ".join(statement.lower().split())) + + event.listen(engine, "before_cursor_execute", record) + return statements, lambda: event.remove(engine, "before_cursor_execute", record) + + +async def owner_insert(case, table: str, identifier: UUID, target: UUID, payload, *, revision: int = 1) -> None: + async with case.owner.begin() as session: + await session.execute( + text( + f"INSERT INTO {table}(id,tenant_id,project_id,environment_id,target_id,revision,payload) " + "VALUES(:id,:tenant,:project,:environment,:target,:revision,cast(:payload AS jsonb))" + ), + { + "id": identifier, + "tenant": case.scope.tenant_id, + "project": case.scope.project_id, + "environment": case.scope.environment_id, + "target": target, + "revision": revision, + "payload": payload.model_dump_json(), + }, + ) + + +async def add_target(case, name: str, *, revision: int = 1) -> UUID: + identifier = uuid4() + target = DeploymentTarget( + name=name, + adapter="docker-compose", + external_identity="daemon-" + name, + boundary="weave-" + name, + runner_principal_id=uuid4(), + capabilities=["observe"], + id=identifier, + scope=case.scope, + revision=revision, + created_at=await clock(case), + ) + await owner_insert(case, "deployment_targets", identifier, identifier, target, revision=revision) + return identifier + + +async def add_deployment(case, target: UUID, name: str = "weave", components=None) -> UUID: + release = case.instances[0].release_id + identifier = uuid4() + specs = [ + ComponentSpec( + name=component, + kind=kind, + image=f"registry.example/{component}@{digest}", + configuration="current", + replicas=2 if kind == "worker" else 1, + worker_release_id=(component_release or release) if kind == "worker" else None, + ) + for component, kind, digest, component_release in (components or [("engine", "worker", DIGEST, None)]) + ] + deployment = Deployment( + target_id=target, + name=name, + ownership="managed", + components=specs, + id=identifier, + scope=case.scope, + revision=1, + created_at=await clock(case), + ) + await owner_insert(case, "deployment_deployments", identifier, target, deployment) + return identifier + + +async def add_observation( + case, target: UUID, members: dict, *, complete=True, at=None, expires=None, revision=1, kind="worker" +) -> UUID: + at = at or await clock(case) + identifier = uuid4() + observation = DeploymentObservation( + resources=[ + ObservedResource( + name=component, + external_identity=f"compose/weave/{component}", + kind=kind, + replicas=len(hosts), + ready_replicas=len(hosts), + version="v1", + ownership="managed", + state="ready", + members=[ + ObservedMember( + host=host, + ordinal=index + 1, + state="running", + ready=True, + started_at=None, + restarts=0, + node=None, + ) + for index, host in enumerate(hosts) + ], + ) + for component, hosts in members.items() + ], + complete=complete, + settled=complete, + id=identifier, + target_id=target, + target_revision=revision, + observed_at=at, + expires_at=expires or at + timedelta(minutes=15), + digest="0" * 64, + ) + await owner_insert(case, "deployment_observations", identifier, target, observation) + return identifier + + +async def principal_with(case, *grants: Grant) -> Principal: + identifier = await case.access.create_principal(case.admin, "application") + for grant in grants: + await case.access.grant(case.admin, identifier, grant) + return await case.access.load_principal(identifier) + + +async def owner_rows(case, sql: str, **params) -> list[dict]: + async with case.owner() as session: + return [dict(row) for row in (await session.execute(text(sql), params)).mappings()] + + +def placement(target: UUID, name: str, deployment: UUID, component: str = "engine") -> dict: + return {"target_id": str(target), "target_name": name, "deployment_id": str(deployment), "component": component} + + +def by_id(page) -> dict[UUID, dict]: + return {UUID(item["id"]): item for item in page["items"]} + + +async def test_fleet_reads_no_per_worker_status(monkeypatch, operations_case): + async def reject(*args, **kwargs): + raise AssertionError("per-worker query") + + monkeypatch.setattr(WorkerRepository, "status", reject) + page = await listed(operations_case, state="all") + assert len(page["items"]) == 2 + + +async def test_one_fleet_statement_whatever_the_page_size_filter_or_position(operations_case): + case = operations_case + now = await clock(case) + await add_workers(case, 120, seen=[now - timedelta(seconds=index) for index in range(120)], host="bulk-") + counts = {} + for limit in (1, 50, 100): + statements, stop = capture(case) + try: + page = await listed(case, limit=limit) + finally: + stop() + assert len(page["items"]) == limit and page["next_cursor"] is not None + fleet = [statement for statement in statements if "worker_instances" in statement] + assert len(fleet) == 1 and "task_leases" in fleet[0] and "group by owner" in fleet[0] + assert not any("from runs" in statement for statement in statements) + counts[limit] = len(statements) + assert counts[1] == counts[50] == counts[100] + from firefly_weave.api.transport import decode_cursor_v2 + from firefly_weave.operations.fact_positions import run_position + + query = WorkerListQuery(host_prefix="bulk-", presence="recent", draining=False, order="registered_desc", limit=7) + first = await case.workers.list(case.actor, case.scope, query=query, position=None, context=AuditContext()) + position = run_position(decode_cursor_v2(first["next_cursor"], case.scope, query.cursor_collection())) + statements, stop = capture(case) + try: + await case.workers.list(case.actor, case.scope, query=query, position=position, context=AuditContext()) + finally: + stop() + assert sum("worker_instances" in statement for statement in statements) == 1 + assert len(statements) == counts[1] + + +async def test_presence_and_current_boundaries_are_exact(monkeypatch, operations_case): + case = operations_case + now = await clock(case) + edge = timedelta(microseconds=1) + recent = (await add_workers(case, seen=now - timedelta(seconds=60) + edge, registered=now, host="edge-recent"))[0] + sixty = (await add_workers(case, seen=now - timedelta(seconds=60), registered=now, host="edge-sixty"))[0] + inside = (await add_workers(case, seen=now - timedelta(hours=1) + edge, host="edge-inside"))[0] + hour = (await add_workers(case, seen=now - timedelta(hours=1), host="edge-hour"))[0] + new = (await add_workers(case, registered=now - timedelta(hours=1) + edge, host="edge-new"))[0] + old = (await add_workers(case, registered=now - timedelta(hours=1), host="edge-old"))[0] + revoked = (await add_workers(case, seen=now, registered=now, host="edge-revoked", revoked=True))[0] + legacy = (await add_workers(case, facts=False))[0] + freeze(monkeypatch, now) + current = by_id(await listed(case, host_prefix="edge-")) + assert set(current) == {recent, sixty, inside, new} + assert current[recent]["presence"] == "recent" and current[recent]["available_capacity"] == 1 + assert current[sixty]["presence"] == "stale" and current[sixty]["available_capacity"] is None + assert current[new]["presence"] == "unknown" and current[new]["registered_at"] is not None + assert current[recent]["observed_at"] == now.isoformat().replace("+00:00", "Z") + everything = by_id(await listed(case, state="all")) + assert {hour, old, revoked, legacy} <= set(everything) + assert legacy not in by_id(await listed(case)) + assert not {"identity", "registered_at", "placement", "drained_at"} & set(everything[legacy]) + assert everything[revoked]["revoked"] and everything[revoked]["available_capacity"] == 0 + for presence, expected in (("recent", {recent}), ("stale", {sixty, inside}), ("unknown", {new})): + assert set(ids(await listed(case, host_prefix="edge-", presence=presence))) == expected + + +async def test_orders_walk_every_worker_once_with_ties_and_unknown_times(operations_case): + case = operations_case + now = await clock(case) + times = [now - timedelta(minutes=index % 4) if index % 5 else None for index in range(23)] + registered = [now - timedelta(minutes=index % 3) if index % 7 else None for index in range(23)] + created = await add_workers(case, 23, seen=times, registered=registered, host="order-") + rows = {identifier: (times[index], registered[index]) for index, identifier in enumerate(created)} + + def oracle(position: int) -> list[UUID]: + known = sorted( + (key for key, value in rows.items() if value[position]), key=lambda key: (rows[key][position], key) + ) + unknown = sorted(key for key, value in rows.items() if not value[position]) + return [*reversed(known), *reversed(unknown)] + + for order, position in (("last_seen_desc", 0), ("registered_desc", 1)): + expected = oracle(position) + for limit in (1, 2, 7, 23, 100): + assert await walk(case, host_prefix="order-", state="all", order=order, limit=limit) == expected + + +async def test_a_cursor_belongs_to_its_filters_and_order(operations_case): + from firefly_weave.api.transport import decode_cursor_v2 + + case = operations_case + await add_workers(case, 3, seen=await clock(case), host="bound-") + query = WorkerListQuery(host_prefix="bound-", limit=1) + page = await case.workers.list(case.actor, case.scope, query=query, position=None, context=AuditContext()) + for other in ( + WorkerListQuery(host_prefix="bound-", limit=1, order="registered_desc"), + WorkerListQuery(host_prefix="bound", limit=1), + WorkerListQuery(host_prefix="bound-", limit=1, state="all"), + ): + with pytest.raises(ValueError): + decode_cursor_v2(page["next_cursor"], case.scope, other.cursor_collection()) + + +async def test_filters_select_on_the_server_and_prefixes_are_literal(operations_case): + case = operations_case + now = await clock(case) + async with case.owner.begin() as session: + other_release = uuid4() + payload = ( + await session.execute( + text("SELECT payload FROM worker_releases WHERE id=:id"), {"id": case.instances[0].release_id} + ) + ).scalar_one() + await session.execute( + text( + "INSERT INTO worker_releases VALUES(:id,:tenant,:project,:environment,:digest,cast(:payload AS jsonb))" + ), + { + "id": other_release, + "tenant": case.scope.tenant_id, + "project": case.scope.project_id, + "environment": case.scope.environment_id, + "digest": OTHER_DIGEST, + "payload": json.dumps({**payload, "id": str(other_release), "image_digest": OTHER_DIGEST}), + }, + ) + a1 = (await add_workers(case, seen=now, host="fleet-a1"))[0] + b1 = (await add_workers(case, seen=now, host="fleet-b1", draining=True))[0] + other = (await add_workers(case, seen=now, host="fleet-c1", release=other_release, task_types=("other@2.0.0",)))[0] + assert set(ids(await listed(case, host_prefix="fleet-"))) == {a1, b1, other} + assert ids(await listed(case, host_prefix="fleet-a")) == [a1] + for literal in ("fleet-%", "fleet-_1", "FLEET-", "fleet-a1x"): + assert ids(await listed(case, host_prefix=literal)) == [] + assert ids(await listed(case, host_prefix="fleet-", draining=True)) == [b1] + assert set(ids(await listed(case, host_prefix="fleet-", draining=False))) == {a1, other} + assert ids(await listed(case, host_prefix="fleet-", release_id=other_release)) == [other] + assert ids(await listed(case, task_type="other@2.0.0")) == [other] + assert ids(await listed(case, task_type="other")) == [] + assert ids(await listed(case, host_prefix="fleet-", presence="unknown")) == [] + + +async def test_placement_comes_from_the_latest_complete_unexpired_observation(operations_case): + case = operations_case + now = await clock(case) + alpha, bravo, charlie, delta, echo = ( + await add_target(case, "alpha"), + await add_target(case, "bravo"), + await add_target(case, "charlie"), + await add_target(case, "delta"), + await add_target(case, "echo", revision=2), + ) + deployments = {target: await add_deployment(case, target) for target in (alpha, bravo, charlie, delta, echo)} + await add_observation(case, alpha, {"engine": ["alpha-1", "shared"]}, at=now - timedelta(minutes=5)) + await add_observation(case, bravo, {"engine": ["bravo-1", "shared"]}) + # An older complete observation still places members when a newer one is incomplete. + await add_observation(case, charlie, {"engine": ["charlie-1"]}, at=now - timedelta(minutes=3)) + await add_observation(case, charlie, {"engine": []}, complete=False, at=now - timedelta(minutes=1)) + # The latest complete observation has expired: an older one is never presented as current. + await add_observation( + case, delta, {"engine": ["delta-1"]}, at=now - timedelta(minutes=30), expires=now + timedelta(minutes=10) + ) + await add_observation( + case, delta, {"engine": ["delta-1"]}, at=now - timedelta(minutes=16), expires=now - timedelta(minutes=1) + ) + # Observed before the target changed. + await add_observation(case, echo, {"engine": ["echo-1"]}) + # A member of a component that is not a worker places nothing. + api = await add_target(case, "foxtrot") + await add_deployment(case, api, components=[("engine", "api", DIGEST, None)]) + await add_observation(case, api, {"engine": ["api-1"]}, kind="api") + hosts = { + name: (await add_workers(case, seen=now, host=name))[0] + for name in ("alpha-1", "bravo-1", "shared", "charlie-1", "delta-1", "echo-1", "api-1", "nowhere") + } + embedded = (await add_workers(case, seen=now, host="alpha-1", runtime="embedded"))[0] + twin = (await add_workers(case, seen=now, host="alpha-1"))[0] + reader = await principal_with( + case, Grant(role="viewer", scope=case.scope), Grant(role="deployment_reader", scope=case.scope) + ) + page = by_id(await listed(case, reader, state="all")) + assert page[hosts["alpha-1"]]["placement"] == placement(alpha, "alpha", deployments[alpha]) + assert page[twin]["placement"] == placement(alpha, "alpha", deployments[alpha]) + assert page[hosts["bravo-1"]]["placement"] == placement(bravo, "bravo", deployments[bravo]) + assert page[hosts["charlie-1"]]["placement"] == placement(charlie, "charlie", deployments[charlie]) + for unplaced in ("shared", "delta-1", "echo-1", "api-1", "nowhere"): + assert "placement" not in page[hosts[unplaced]], unplaced + assert "placement" not in page[embedded] and page[embedded]["identity"]["runtime"] == "embedded" + assert set(ids(await listed(case, reader, target_id=alpha))) == {hosts["alpha-1"], twin} + assert ids(await listed(case, reader, target_id=delta)) == [] + # The fixture's actor reads workers but no deployment target, so it sees no placement. + assert not any("placement" in item for item in (await listed(case, state="all"))["items"]) + read = await case.workers.read(reader, case.scope, hosts["bravo-1"], context=AuditContext()) + assert read.placement is not None and read.placement.model_dump(mode="json") == placement( + bravo, "bravo", deployments[bravo] + ) + + +async def test_placement_names_only_targets_both_principals_may_read(operations_case): + case = operations_case + now = await clock(case) + alpha, bravo = await add_target(case, "alpha"), await add_target(case, "bravo") + first, second = await add_deployment(case, alpha), await add_deployment(case, bravo) + await add_observation(case, alpha, {"engine": ["host-a1"]}) + await add_observation(case, bravo, {"engine": ["host-b1"]}) + on_alpha = (await add_workers(case, seen=now, host="host-a1"))[0] + on_bravo = (await add_workers(case, seen=now, host="host-b1"))[0] + viewer = Grant(role="viewer", scope=case.scope) + reader = await principal_with( + case, viewer, Grant(role="deployment_reader", scope=case.scope, resources=(str(alpha), str(bravo))) + ) + unrestricted = await principal_with(case, viewer, Grant(role="deployment_reader", scope=case.scope)) + page = by_id(await listed(case, unrestricted, state="all")) + assert page[on_alpha]["placement"] == placement(alpha, "alpha", first) + assert page[on_bravo]["placement"] == placement(bravo, "bravo", second) + # The stored principal loses bravo: the token alone no longer shows it. + async with case.owner.begin() as session: + await session.execute( + text( + "UPDATE role_bindings SET resources=jsonb_build_array(cast(:target AS text)) " + "WHERE principal_id=:id AND role='deployment_reader'" + ), + {"id": reader.id, "target": str(alpha)}, + ) + page = await listed(case, reader, state="all") + assert by_id(page)[on_alpha]["placement"] == placement(alpha, "alpha", first) + assert "placement" not in by_id(page)[on_bravo] + assert "bravo" not in json.dumps(page) and str(second) not in json.dumps(page) + assert ids(await listed(case, reader, target_id=alpha)) == [on_alpha] + with pytest.raises(AccessDenied): + await listed(case, reader, target_id=bravo) + # A token without bravo is refused even when the stored principal could read it. + token = reader.model_copy( + update={"grants": (viewer, Grant(role="deployment_reader", scope=case.scope, resources=(str(alpha),)))} + ) + async with case.owner.begin() as session: + await session.execute( + text("UPDATE role_bindings SET resources='[]' WHERE principal_id=:id AND role='deployment_reader'"), + {"id": reader.id}, + ) + assert "placement" not in by_id(await listed(case, token, state="all"))[on_bravo] + with pytest.raises(AccessDenied): + await listed(case, token, target_id=bravo) + # Without any deployment grant the fleet is still listed, with no placement at all. + plain = await principal_with(case, viewer) + page = await listed(case, plain, state="all") + assert not any("placement" in item for item in page["items"]) and "alpha" not in json.dumps(page) + with pytest.raises(AccessDenied): + await listed(case, plain, target_id=alpha) + read = await case.workers.read(plain, case.scope, on_alpha, context=AuditContext()) + assert read.placement is None + + +async def test_token_and_stored_principal_both_need_status_read(monkeypatch, operations_case): + from firefly_weave.definitions.service import DefinitionService + + case = operations_case + opened = [] + original = DefinitionService.transaction + + def counting(self, *args, **kwargs): + opened.append(args) + return original(self, *args, **kwargs) + + monkeypatch.setattr(DefinitionService, "transaction", counting) + outsider = Principal(id=case.actor.id, kind="application", grants=(Grant(role="developer", scope=case.scope),)) + with pytest.raises(AccessDenied): + await listed(case, outsider) + with pytest.raises(AccessDenied): + await case.workers.read(outsider, case.scope, case.instances[0].id, context=AuditContext()) + assert opened == [] + async with case.owner.begin() as session: + await session.execute( + text("DELETE FROM role_bindings WHERE principal_id=:id AND role='viewer'"), {"id": case.actor.id} + ) + with pytest.raises(AccessDenied): + await listed(case) + assert len(opened) == 1 + + +async def test_a_page_above_the_byte_limit_is_refused_unread(monkeypatch, operations_case): + import firefly_weave.workers.repository as repository + + case = operations_case + created = await add_workers(case, 2, seen=await clock(case), host="bytes-") + async with case.owner() as session: + exact = await session.scalar( + text( + "SELECT sum(octet_length(w.payload::text)+octet_length(f.identity::text)) FROM worker_instances w " + "JOIN worker_facts f ON f.worker_id=w.id WHERE w.id=ANY(:ids)" + ), + {"ids": created}, + ) + monkeypatch.setattr(repository, "PAGE_BYTES", exact) + assert len((await listed(case, host_prefix="bytes-"))["items"]) == 2 + monkeypatch.setattr(repository, "PAGE_BYTES", exact - 1) + with pytest.raises(CatalogError) as refused: + await listed(case, host_prefix="bytes-") + assert (refused.value.status, refused.value.code) == (429, "WV-PAGE-LIMIT") + # The lookahead row is never measured. + monkeypatch.setattr(repository, "PAGE_BYTES", exact // 2 + 1) + assert len((await listed(case, host_prefix="bytes-", limit=1))["items"]) == 1 + + +async def test_drain_time_is_recorded_once_and_cleared_on_resume(operations_case): + case = operations_case + worker = case.instances[0].id + await case.access.grant(case.admin, case.actor.id, Grant(role="worker_operator", scope=case.scope)) + actor = await case.access.load_principal(case.actor.id) + before = await clock(case) + + async def control(key, revision, draining): + return await case.workers.control( + actor, + case.scope, + worker, + WorkerControlRequest(expected_revision=revision), + key, + draining=draining, + context=AuditContext(), + ) + + drained = await control("drain", 1, True) + assert drained.drained_at is not None and drained.drained_at >= before + facts = await case.rows("SELECT drained_at FROM worker_facts WHERE worker_id=:id", id=worker) + assert facts[0]["drained_at"] == drained.drained_at + again = await control("drain-again", 2, True) + assert again.drained_at == drained.drained_at and again.revision == 3 + assert (await control("drain", 1, True)) == drained + listed_item = by_id(await listed(case, state="all"))[worker] + assert listed_item["drained_at"] == drained.drained_at.isoformat().replace("+00:00", "Z") + resumed = await control("resume", 3, False) + assert resumed.drained_at is None and "drained_at" not in resumed.model_dump(mode="json") + assert (await case.rows("SELECT drained_at FROM worker_facts WHERE worker_id=:id", id=worker))[0][ + "drained_at" + ] is None + # A worker drained before drain times were recorded keeps an unknown time. + legacy = case.instances[1].id + async with case.owner.begin() as session: + await session.execute(text("UPDATE worker_instances SET draining=true WHERE id=:id"), {"id": legacy}) + repeated = await case.workers.control( + actor, + case.scope, + legacy, + WorkerControlRequest(expected_revision=1), + "legacy", + draining=True, + context=AuditContext(), + ) + assert repeated.draining and repeated.drained_at is None + + +async def test_dry_run_and_idempotent_retirement(operations_case): + case = operations_case + async with case.owner.begin() as session: + await session.execute( + text("UPDATE worker_instances SET last_seen_at=clock_timestamp()-interval '25 hours' WHERE id=:id"), + {"id": case.instances[0].id}, + ) + request = RetireStaleRequest(older_than_seconds=86400, dry_run=True) + dry = await case.workers.retire_stale(case.actor, case.scope, request, "preview", context=AuditContext()) + assert (dry.matched, dry.revoked, dry.worker_ids) == (1, 0, [case.instances[0].id]) + assert not (await case.rows("SELECT revoked FROM worker_instances WHERE id=:id", id=case.instances[0].id))[0][ + "revoked" + ] + assert await case.workers.retire_stale(case.actor, case.scope, request, "preview", context=AuditContext()) == dry + applied = await case.workers.retire_stale( + case.actor, + case.scope, + RetireStaleRequest(older_than_seconds=86400, dry_run=False), + "apply", + context=AuditContext(), + ) + assert (applied.matched, applied.revoked, applied.worker_ids) == (1, 1, [case.instances[0].id]) + rows = await case.rows( + "SELECT revoked,control_revision FROM worker_instances WHERE id=:id", id=case.instances[0].id + ) + assert rows[0]["revoked"] and rows[0]["control_revision"] == 2 + with pytest.raises(CatalogError) as conflict: + await case.workers.retire_stale(case.actor, case.scope, request, "apply", context=AuditContext()) + assert conflict.value.code == "WV-IDEMPOTENCY-CONFLICT" + audits = await owner_rows(case, "SELECT event FROM access_audit WHERE action='worker.retire_stale' ORDER BY id") + assert [item["event"]["details"]["dry_run"] for item in audits] == [True, False] + assert audits[1]["event"]["details"]["worker_ids"] == [str(case.instances[0].id)] + assert audits[1]["event"]["capability"] == "release.retire" + + +async def test_retirement_threshold_and_last_contact_are_exact(monkeypatch, operations_case): + case = operations_case + now = await clock(case) + hour, edge = timedelta(hours=1), timedelta(microseconds=1) + exactly = (await add_workers(case, seen=now - hour))[0] + older = (await add_workers(case, seen=now - hour - edge))[0] + never_seen_new = (await add_workers(case, registered=now - hour - edge))[0] + never_seen_recent = (await add_workers(case, registered=now - hour))[0] + never_seen_legacy = (await add_workers(case, registered=None))[0] + without_facts = (await add_workers(case, facts=False))[0] + seen_long_ago_registered_now = (await add_workers(case, seen=now - 2 * hour, registered=now))[0] + await add_workers(case, seen=now - 2 * hour, revoked=True) + freeze(monkeypatch, now) + request = RetireStaleRequest(older_than_seconds=3600, dry_run=True) + result = await case.workers.retire_stale(case.actor, case.scope, request, "edge", context=AuditContext()) + # Oldest last contact first: the last contact wins over registration, and the unknown never qualifies. + assert result.worker_ids == [seen_long_ago_registered_now, *sorted((older, never_seen_new))] + assert result.matched == 3 + assert not {exactly, never_seen_recent, never_seen_legacy, without_facts} & set(result.worker_ids) + + +async def test_active_leases_keep_their_worker_and_rows(operations_case, task_service): + case = operations_case + await case.start("lease") + worker = case.instances[0].id + async with case.tx() as tx: + lease = (await task_service.claim(tx, worker, 1))[0] + async with case.owner.begin() as session: + await session.execute( + text( + "UPDATE task_leases SET expires_at=clock_timestamp()+interval '1 hour'," + "deadline=clock_timestamp()+interval '1 hour' WHERE task_id=:task" + ), + {"task": lease.proof.task_id}, + ) + await session.execute( + text("UPDATE worker_instances SET last_seen_at=clock_timestamp()-interval '30 hours' WHERE id=:id"), + {"id": worker}, + ) + retire = RetireStaleRequest(older_than_seconds=86400, dry_run=False) + kept = await case.workers.retire_stale(case.actor, case.scope, retire, "kept", context=AuditContext()) + assert (kept.matched, kept.revoked) == (0, 0) + listed_item = by_id(await listed(case, state="all"))[worker] + assert listed_item["active_leases"] == 1 and listed_item["available_capacity"] is None + async with case.owner.begin() as session: + await session.execute( + text("UPDATE task_leases SET expires_at=clock_timestamp()-interval '1 second' WHERE task_id=:task"), + {"task": lease.proof.task_id}, + ) + expired = await case.workers.retire_stale(case.actor, case.scope, retire, "expired", context=AuditContext()) + assert expired.worker_ids == [worker] + # Retirement never deletes or settles the attempt; recovery owns it. + rows = await case.rows("SELECT status FROM task_leases WHERE task_id=:task", task=lease.proof.task_id) + assert [row["status"] for row in rows] == ["active"] + + +async def test_one_hundred_workers_per_command_oldest_first(operations_case): + case = operations_case + now = await clock(case) + stale = await add_workers(case, 105, seen=[now - timedelta(hours=30, minutes=index) for index in range(105)]) + oldest_first = list(reversed(stale)) + retire = RetireStaleRequest(older_than_seconds=86400, dry_run=False) + first = await case.workers.retire_stale(case.actor, case.scope, retire, "one", context=AuditContext()) + assert (first.matched, first.revoked, first.worker_ids) == (105, 100, oldest_first[:100]) + assert await case.workers.retire_stale(case.actor, case.scope, retire, "one", context=AuditContext()) == first + audits = await owner_rows(case, "SELECT count(*) AS count FROM access_audit WHERE action='worker.retire_stale'") + assert audits[0]["count"] == 1 + second = await case.workers.retire_stale(case.actor, case.scope, retire, "two", context=AuditContext()) + assert (second.matched, second.revoked, second.worker_ids) == (5, 5, oldest_first[100:]) + revoked = await case.rows( + "SELECT count(*) AS count FROM worker_instances WHERE id=ANY(:ids) AND revoked AND control_revision=2", + ids=stale, + ) + assert revoked[0]["count"] == 105 + + +async def test_a_worker_locked_elsewhere_is_skipped_and_counted(operations_case): + case = operations_case + now = await clock(case) + first, second = await add_workers(case, 2, seen=[now - timedelta(hours=40), now - timedelta(hours=30)]) + retire = RetireStaleRequest(older_than_seconds=86400, dry_run=False) + async with case.owner.begin() as session: + await session.execute(text("SELECT 1 FROM worker_instances WHERE id=:id FOR UPDATE"), {"id": first}) + skipped = await case.workers.retire_stale(case.actor, case.scope, retire, "busy", context=AuditContext()) + assert (skipped.matched, skipped.revoked, skipped.worker_ids) == (2, 1, [second]) + later = await case.workers.retire_stale(case.actor, case.scope, retire, "free", context=AuditContext()) + assert later.worker_ids == [first] + + +async def test_the_locked_update_rechecks_contact_leases_and_revocation(operations_case, task_service): + case = operations_case + now = await clock(case) + await case.start("recheck") + leased = case.instances[0].id + async with case.tx() as tx: + lease = (await task_service.claim(tx, leased, 1))[0] + fresh = (await add_workers(case, seen=now))[0] + gone = (await add_workers(case, seen=now - timedelta(hours=30), revoked=True))[0] + eligible = (await add_workers(case, seen=now - timedelta(hours=30)))[0] + async with case.owner.begin() as session: + await session.execute( + text("UPDATE worker_instances SET last_seen_at=:seen WHERE id=:id"), + {"id": leased, "seen": now - timedelta(hours=30)}, + ) + await session.execute( + text("UPDATE task_leases SET expires_at=:until,deadline=:until WHERE task_id=:task"), + {"task": lease.proof.task_id, "until": now + timedelta(hours=1)}, + ) + async with case.tx() as tx: + revoked = await WorkerRepository(tx).retire([leased, fresh, gone, eligible], now - timedelta(hours=24), now) + assert revoked == [eligible] + + +async def test_claims_and_retirement_never_leave_a_revoked_owner_with_a_live_lease(operations_case, task_service): + case = operations_case + await case.start("race") + worker = case.instances[0].id + async with case.owner.begin() as session: + await session.execute( + text("UPDATE worker_instances SET last_seen_at=clock_timestamp()-interval '30 hours' WHERE id=:id"), + {"id": worker}, + ) + + async def claim(): + async with case.tx() as tx: + return await task_service.claim(tx, worker, 1) + + retire = case.workers.retire_stale( + case.actor, + case.scope, + RetireStaleRequest(older_than_seconds=86400, dry_run=False), + "race", + context=AuditContext(), + ) + outcomes = await asyncio.gather(claim(), retire, return_exceptions=True) + for outcome in outcomes: + if isinstance(outcome, CatalogError): + assert outcome.code in {"WV-OPERATION-CAPACITY", "WV-LEASE"}, outcome.code + else: + assert not isinstance(outcome, BaseException), outcome + live = await case.rows( + "SELECT count(*) AS count FROM task_leases l JOIN worker_instances w ON w.id=l.owner " + "WHERE w.revoked AND l.status='active' AND l.expires_at>clock_timestamp()" + ) + assert live[0]["count"] == 0 + + +async def test_retirement_needs_release_retire_on_the_token_and_the_stored_principal(monkeypatch, operations_case): + from firefly_weave.definitions.service import DefinitionService + + case = operations_case + opened = [] + original = DefinitionService.transaction + + def counting(self, *args, **kwargs): + opened.append(args) + return original(self, *args, **kwargs) + + monkeypatch.setattr(DefinitionService, "transaction", counting) + request = RetireStaleRequest(older_than_seconds=86400, dry_run=True) + viewer = Principal(id=case.actor.id, kind="application", grants=(Grant(role="viewer", scope=case.scope),)) + with pytest.raises(AccessDenied): + await case.workers.retire_stale(viewer, case.scope, request, "denied", context=AuditContext()) + assert opened == [] + async with case.owner.begin() as session: + await session.execute( + text("DELETE FROM role_bindings WHERE principal_id=:id AND role='deployer'"), {"id": case.actor.id} + ) + with pytest.raises(AccessDenied): + await case.workers.retire_stale(case.actor, case.scope, request, "removed", context=AuditContext()) + assert len(opened) == 1 + assert not await case.rows("SELECT 1 FROM mutation_idempotency WHERE idempotency_key IN ('denied','removed')") + + +async def test_release_usage_counts_workers_activations_and_permitted_deployments(operations_case): + case = operations_case + now = await clock(case) + release = case.instances[0].release_id + await add_workers(case, seen=now) + await add_workers(case, seen=now, draining=True) + await add_workers(case, seen=now - timedelta(hours=2)) + await add_workers(case, seen=now, revoked=True) + alpha, bravo = await add_target(case, "alpha"), await add_target(case, "bravo") + first = await add_deployment( + case, + alpha, + components=[ + ("engine", "worker", DIGEST, None), + ("batch", "worker", OTHER_DIGEST, None), + ("api", "api", DIGEST, None), + ], + ) + other_release = uuid4() + second = await add_deployment( + case, bravo, components=[("engine", "worker", DIGEST, None), ("zeta", "worker", DIGEST, None)] + ) + await add_deployment(case, bravo, name="elsewhere", components=[("engine", "worker", DIGEST, other_release)]) + + async def usage(actor, **values): + return await case.workers.release_usage( + actor, case.scope, release, ReleaseUsageQuery(**values), context=AuditContext() + ) + + # The fixture's actor reads the catalog but no deployment target. + catalog_only = await usage(case.actor) + assert catalog_only.workers.model_dump() == {"online": 1, "offline": 3, "draining": 1, "revoked": 1} + assert catalog_only.activations == 1 and catalog_only.deployments == [] and catalog_only.next_cursor is None + viewer = Grant(role="viewer", scope=case.scope) + everything = await principal_with(case, viewer, Grant(role="deployment_reader", scope=case.scope)) + rows = (await usage(everything)).deployments + expected = sorted( + [(alpha, "alpha", first, "engine"), (bravo, "bravo", second, "engine"), (bravo, "bravo", second, "zeta")], + key=lambda row: (row[2], row[3]), + ) + assert [(row.target_id, row.target_name, row.deployment_id, row.component) for row in rows] == expected + assert all(row.replicas == 2 and row.deployment_name == "weave" for row in rows) + restricted = await principal_with( + case, viewer, Grant(role="deployment_reader", scope=case.scope, resources=(str(alpha),)) + ) + limited = await usage(restricted) + assert [(row.target_id, row.component) for row in limited.deployments] == [(alpha, "engine")] + assert "bravo" not in limited.model_dump_json() + walked, cursor = [], None + while True: + page = await usage(everything, limit=1, **({"cursor": cursor} if cursor else {})) + walked += [(row.deployment_id, row.component) for row in page.deployments] + cursor = page.next_cursor + if cursor is None: + break + assert walked == [(row[2], row[3]) for row in expected] + first_page = await usage(everything, limit=1) + with pytest.raises(ValueError): + await usage(restricted, limit=1, cursor=first_page.next_cursor) + + +async def test_release_usage_activations_skip_retired_ones(operations_case): + case = operations_case + release = case.instances[0].release_id + async with case.owner.begin() as session: + row = ( + (await session.execute(text("SELECT * FROM activation_revisions WHERE id=:id"), {"id": case.activation.id})) + .mappings() + .one() + ) + retired = {**row["payload"], "id": str(uuid4()), "revision": row["revision"] + 1, "retired": True} + await session.execute( + text( + "INSERT INTO activation_revisions(id,tenant_id,project_id,environment_id,name,revision,version_id," + "payload) VALUES(:id,:tenant,:project,:environment,:name,:revision,:version,cast(:payload AS jsonb))" + ), + { + "id": UUID(retired["id"]), + "tenant": row["tenant_id"], + "project": row["project_id"], + "environment": row["environment_id"], + "name": row["name"], + "revision": retired["revision"], + "version": row["version_id"], + "payload": json.dumps(retired), + }, + ) + usage = await case.workers.release_usage( + case.actor, case.scope, release, ReleaseUsageQuery(), context=AuditContext() + ) + assert usage.activations == 0 + + +async def test_release_usage_is_scoped_and_authorized(monkeypatch, operations_case, provisioned): + from firefly_weave.definitions.service import DefinitionService + + case = operations_case + with pytest.raises(CatalogError) as missing: + await case.workers.release_usage(case.actor, case.scope, uuid4(), ReleaseUsageQuery(), context=AuditContext()) + assert (missing.value.status, missing.value.code) == (404, "WV-NOT-FOUND") + other = provisioned[1][1] + owner = await principal_with(case, Grant(role="viewer", scope=other)) + with pytest.raises(CatalogError) as foreign: + await case.workers.release_usage( + owner, other, case.instances[0].release_id, ReleaseUsageQuery(), context=AuditContext() + ) + assert foreign.value.status == 404 + opened = [] + original = DefinitionService.transaction + + def counting(self, *args, **kwargs): + opened.append(args) + return original(self, *args, **kwargs) + + monkeypatch.setattr(DefinitionService, "transaction", counting) + outsider = Principal(id=case.actor.id, kind="application", grants=(Grant(role="worker", scope=case.scope),)) + with pytest.raises(AccessDenied): + await case.workers.release_usage( + outsider, case.scope, case.instances[0].release_id, ReleaseUsageQuery(), context=AuditContext() + ) + assert opened == [] + + +async def test_http_fleet_retirement_and_usage( + client, authenticated_client, access_db, provisioned, headers, env_url, operations_case +): + from firefly_weave.api.transport import encode_cursor + + case = operations_case + async with access_db[0]() as session: + principal = await session.scalar(text("SELECT principal_id FROM identity_links WHERE subject='0'")) + canonical = "/api/v1" + env_url + for base in (env_url, canonical): + first = await client.get(base + "/workers?state=all&order=registered_desc&limit=1", headers=headers) + assert first.status_code == 200, first.text + assert len(first.json()["items"]) == 1 and first.json()["next_cursor"] + second = await client.get( + base + "/workers", + params={"state": "all", "order": "registered_desc", "limit": 1, "cursor": first.json()["next_cursor"]}, + headers=headers, + ) + assert second.status_code == 200 and second.json()["items"][0]["id"] != first.json()["items"][0]["id"] + for params in ({"colour": "red"}, {"limit": "0"}, {"presence": "online"}, {"limit": "1", "cursor": "abc"}): + refused = await client.get(base + "/workers", params=params, headers=headers) + assert (refused.status_code, refused.json()["code"]) == (422, "WV-VALIDATION"), params + earlier = encode_cursor(case.scope, "workers", case.instances[0].id) + for base, cursor in ((canonical, earlier), (env_url, str(case.instances[0].id))): + refused = await client.get(base + "/workers", params={"cursor": cursor}, headers=headers) + assert refused.status_code == 422 + body = {"older_than_seconds": 86400, "dry_run": True} + denied = await client.post( + canonical + "/workers/retire-stale", headers={**headers, "Idempotency-Key": "k"}, json=body + ) + assert (denied.status_code, denied.json()["code"]) == (403, "WV-FORBIDDEN") + await case.access.grant(provisioned[0], principal, Grant(role="deployer", scope=case.scope)) + missing = await client.post(canonical + "/workers/retire-stale", headers=headers, json=body) + assert missing.status_code == 422 + invalid = await client.post( + canonical + "/workers/retire-stale", + headers={**headers, "Idempotency-Key": "short"}, + json={"older_than_seconds": 60, "dry_run": True}, + ) + assert (invalid.status_code, invalid.json()["code"]) == (422, "WV-VALIDATION") + preview = await client.post( + canonical + "/workers/retire-stale", headers={**headers, "Idempotency-Key": "k"}, json=body + ) + assert preview.status_code == 200, preview.text + assert preview.json() == {"matched": 0, "revoked": 0, "worker_ids": []} + assert preview.headers["cache-control"] == "no-store" + release = case.instances[0].release_id + usage = await client.get(f"{canonical}/worker-releases/{release}/usage", headers=headers) + assert usage.status_code == 200, usage.text + assert usage.json()["deployments"] == [] and usage.json()["activations"] == 1 + assert (await client.get(f"{canonical}/worker-releases/{uuid4()}/usage", headers=headers)).status_code == 404 + assert ( + await client.get(f"{canonical}/worker-releases/{release}/usage?limit=101", headers=headers) + ).status_code == 422 diff --git a/tests/unit/operations/test_target_permissions.py b/tests/unit/operations/test_target_permissions.py new file mode 100644 index 00000000..f5ed77b6 --- /dev/null +++ b/tests/unit/operations/test_target_permissions.py @@ -0,0 +1,127 @@ +# Copyright 2026 Firefly Software Foundation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# Author: Firefly Software Foundation +# SPDX-License-Identifier: Apache-2.0 + +"""Target permissions are the finite intersection of what the token and the stored principal may read.""" + +from uuid import UUID, uuid4 + +import pytest + +from firefly_weave.access.audit import AuditContext +from firefly_weave.access.authorization import AuthorizationService +from firefly_weave.access.models import Grant, Principal +from firefly_weave.contracts.access import Scope + +SCOPE = Scope(tenant_id=uuid4(), project_id=uuid4(), environment_id=uuid4()) +FIRST, SECOND, THIRD = sorted((uuid4(), uuid4(), uuid4())) + + +def principal(*grants, kind="application", active=True): + return Principal(id=uuid4(), kind=kind, active=active, grants=grants) + + +def reader(*targets, role="deployment_reader", scope=SCOPE): + return Grant(role=role, scope=scope, resources=tuple(str(target) for target in targets)) + + +def permissions(actor, current, capability="deployment.read"): + from firefly_weave.operations.permissions import target_permissions + + return target_permissions(AuthorizationService(), actor, current, SCOPE, capability, AuditContext()) + + +def test_unrestricted_grants_on_both_principals_read_every_target(): + assert permissions(principal(reader()), principal(reader())) is None + + +@pytest.mark.parametrize( + "actor,current,expected", + [ + (principal(reader()), principal(reader(FIRST, SECOND)), (FIRST, SECOND)), + (principal(reader(SECOND, FIRST)), principal(reader()), (FIRST, SECOND)), + (principal(reader(FIRST, SECOND)), principal(reader(SECOND, THIRD)), (SECOND,)), + (principal(reader(FIRST), reader(THIRD)), principal(reader(THIRD), reader(FIRST)), (FIRST, THIRD)), + (principal(reader(FIRST)), principal(reader(SECOND)), ()), + ], +) +def test_restricted_grants_intersect_in_sorted_order(actor, current, expected): + assert permissions(actor, current) == expected + + +@pytest.mark.parametrize( + "grants", + [ + (), + (Grant(role="viewer", scope=SCOPE),), + (reader(scope=Scope(tenant_id=SCOPE.tenant_id, project_id=uuid4(), environment_id=SCOPE.environment_id)),), + (reader(scope=SCOPE.model_copy(update={"environment_id": uuid4()})),), + ], +) +def test_no_matching_grant_reads_no_target_and_never_raises(grants): + assert permissions(principal(*grants), principal(reader())) == () + assert permissions(principal(reader()), principal(*grants)) == () + + +def test_inactive_principals_and_workers_read_no_target(): + assert permissions(principal(reader(), active=False), principal(reader())) == () + assert permissions(principal(reader()), principal(reader(), kind="worker")) == () + + +def test_resources_that_are_not_target_ids_are_ignored(): + grant = Grant( + role="deployment_reader", + scope=SCOPE, + resources=("not-a-target", str(FIRST), "../x", str(SECOND).upper(), "{" + str(THIRD) + "}"), + ) + assert permissions(principal(grant), principal(reader())) == (FIRST,) + + +def test_an_unrestricted_grant_alongside_restricted_ones_reads_every_target(): + assert permissions(principal(reader(FIRST), reader()), principal(reader())) is None + + +def test_project_wide_grants_cover_the_environment(): + project = Scope(tenant_id=SCOPE.tenant_id, project_id=SCOPE.project_id) + assert permissions(principal(reader(scope=project)), principal(reader(FIRST, scope=project))) == (FIRST,) + + +def test_only_the_named_capability_counts(): + planner = Grant(role="deployment_planner", scope=SCOPE, resources=(str(FIRST),)) + approver = Grant(role="deployment_approver", scope=SCOPE, resources=(str(SECOND),)) + both = principal(planner, approver) + assert permissions(both, both) == (FIRST, SECOND) + assert permissions(both, both, "deployment.approve") == (SECOND,) + assert permissions(both, both, "deployment.plan") == (FIRST,) + + +def test_every_resource_is_checked_through_the_authorization_service(): + from firefly_weave.operations.permissions import target_permissions + + checks: list[tuple[UUID, str | None]] = [] + + class Recording(AuthorizationService): + def require(self, principal, scope, capability, *, resource=None, context=None): + checks.append((principal.id, resource)) + super().require(principal, scope, capability, resource=resource, context=context) + + actor, current = principal(reader(FIRST, SECOND)), principal(reader()) + assert target_permissions(Recording(), actor, current, SCOPE, "deployment.read", AuditContext()) == ( + FIRST, + SECOND, + ) + assert sorted(checks, key=str) == sorted( + [(actor.id, str(FIRST)), (actor.id, str(SECOND)), (current.id, None)], key=str + ) diff --git a/tests/unit/test_studio_bridge_families.py b/tests/unit/test_studio_bridge_families.py index 9fb54086..07817b4d 100644 --- a/tests/unit/test_studio_bridge_families.py +++ b/tests/unit/test_studio_bridge_families.py @@ -215,8 +215,9 @@ def test_every_operation_crosses_the_bridge_exactly_when_its_family_or_id_is_adm if identifier.split(".")[0] in STUDIO_FAMILIES or identifier in STUDIO_OPERATIONS } assert {"releases.create", "admin.grant", "admin.tenant"}.isdisjoint(crossing) - # The AI setup operations add exactly these six, and direct incident reads join the - # incidents family; every earlier crossing stays as it was. + # The AI setup operations add exactly these six, direct incident reads join the incidents + # family and stale-worker retirement joins the workers family; every earlier crossing stays + # as it was. Release usage stays outside until the releases allowlist names it. approved_additions = { "ai_endpoints.list", "ai_models.list", @@ -225,6 +226,7 @@ def test_every_operation_crosses_the_bridge_exactly_when_its_family_or_id_is_adm "ai_setup.grant", "bindable_connections.list", "incidents.read", + "workers.retire_stale", } assert approved_additions <= crossing preserved = crossing - approved_additions @@ -233,7 +235,9 @@ def test_every_operation_crosses_the_bridge_exactly_when_its_family_or_id_is_adm "3ba280e1522358905e2cdb2e4bf66976fdc988018cdc08742c4956b87d1ab159" ) assert {"workers.grant", "ai_connections.test"} <= preserved - assert {"releases.create", "tasks.credentials", "admin.grant", "admin.tenant"}.isdisjoint(crossing) + assert {"releases.create", "tasks.credentials", "admin.grant", "admin.tenant", "releases.usage"}.isdisjoint( + crossing + ) @pytest.mark.parametrize( From 6554340f9dda1a94b4b5275df3e5345bee50f138 Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Mon, 12 Oct 2026 03:09:55 -0700 Subject: [PATCH 16/63] Keep Studio listing every worker and count only startable activations Studio's Workers page asks for every worker, newest registered first, so revoked and long-offline workers stay listed, its revoked toggle works against the new default and rows no longer move between refreshes. The browser mock now answers like the server. The AI status check reads every worker of its release, so one unseen for hours reads as stale. Release usage no longer counts activations whose workflow version is retired, since runs cannot start from them. The changelog and upgrade guide say that earlier Studio hosts and desktop apps get the reduced worker list from an upgraded server, that older clients fail only on placed workers or workers drained since the upgrade, and how placement uniqueness depends on the targets a reader may see. Adds a test for the worker list argument guards. --- CHANGELOG.md | 11 ++++- docs/guides/workers.md | 11 +++-- docs/operations/upgrades.md | 10 +++- docs/reference/api.md | 15 +++--- src/firefly_weave/sdk/platform_ai_setup.py | 3 +- src/firefly_weave/workers/repository.py | 7 ++- .../src/app/operate/workers/workers-page.ts | 11 ++++- .../tests/browser/worker-operations.spec.ts | 47 +++++++++++++++---- tests/contracts/test_worker_queries.py | 32 +++++++++++++ tests/integration/test_worker_fleet_facts.py | 25 ++++++++++ tests/unit/cli/test_platform_ai_setup.py | 10 ++++ 11 files changed, 154 insertions(+), 28 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 144d21f9..5ab796d2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -95,12 +95,19 @@ SPDX-License-Identifier: Apache-2.0 `registered_desc`, unknown times last. Cursors issued before the upgrade are refused, unknown query parameters are rejected, and an out-of-range `limit` answers `WV-VALIDATION` instead of `WV-PAGE`. Pages stay limited to 8 MiB. +- Studio's Workers page asks for every worker (`state=all`), newest registered + first, so it keeps listing revoked and long-offline workers and its rows no + longer move between refreshes. Earlier Studio hosts and desktop apps get the + reduced default list from an upgraded server, because the Studio bridge + forwards answers unchanged: their Workers page lists only current workers, and + **Show revoked workers** shows none. Upgrade Studio and the desktop app with + the server. - Worker reads and lists add optional `placement` (target, deployment and component), matched from the latest complete, unexpired observation of the targets the reader may read, and `drained_at`, when the worker started draining; drain and resume answers add `drained_at`. Upgrade CLIs and SDKs - with the server: earlier releases fail with `502 WV-WIRE` on placed or - draining workers. + with the server: earlier releases fail with `502 WV-WIRE` on placed workers + and on workers drained since the upgrade. - New operation `workers.retire_stale` (`weave workers retire-stale`, SDK `retire_stale_workers`) revokes up to 100 workers per command that hold no active lease and were last seen, or registered if never seen, longer ago than diff --git a/docs/guides/workers.md b/docs/guides/workers.md index 148f7208..44610188 100644 --- a/docs/guides/workers.md +++ b/docs/guides/workers.md @@ -262,11 +262,12 @@ instead. ## 4. Observe one task through completion Start a workflow that calls the published Action. Inspect its run history and -worker status through the API. In Studio, **Operate › Workers** lists the -workers in the selected environment with their status (**Online**, -**Offline** or **Not seen yet**, and **Draining**), their load against their -task slots, and their last contact. **Show revoked workers** adds the -**Revoked** ones. Open a worker to **Drain** or **Resume** it. Use the +worker status through the API. In Studio, **Operate › Workers** lists every +worker in the selected environment that is not revoked, newest registered +first, including workers offline for a long time, with their status +(**Online**, **Offline** or **Not seen yet**, and **Draining**), their load +against their task slots, and their last contact. **Show revoked workers** adds +the **Revoked** ones. Open a worker to **Drain** or **Resume** it. Use the [API playground](api-playground.md) to authenticate and select your environment, then the [full API reference](../reference/api-explorer.md) for worker registration and run/history reads. A worker token is for execution; diff --git a/docs/operations/upgrades.md b/docs/operations/upgrades.md index 18a1251e..5f27907e 100644 --- a/docs/operations/upgrades.md +++ b/docs/operations/upgrades.md @@ -306,13 +306,19 @@ default: `422 WV-VALIDATION` instead of `422 WV-PAGE`. An older server ignores the new filters and lists every worker in ID order, so upgrade servers before relying on them. +- **Earlier Studio hosts and desktop apps.** The Studio bridge forwards answers + unchanged, so an earlier Studio host or desktop app gets this reduced default + list from an upgraded server. Its Workers page lists only current workers, + **Show revoked workers** shows none, and rows reorder as workers check in. + Upgrade Studio and the desktop app with the server. Until then, the Python + SDK's `list_workers(query=WorkerListQuery(state="all"))` lists every worker. Worker reads and lists add optional `placement` and `drained_at`, and drain and resume answers add `drained_at`. **Upgrade CLIs and SDKs with the server.** Earlier releases reject these fields: `weave workers list`, `read`, `drain`, and `resume`, and the Python SDK's `read_worker`, `list_workers`, `drain_worker`, -and `resume_worker`, fail with `502 WV-WIRE` for a placed or draining worker. A -drain or resume is still applied. Stored worker records and the registration +and `resume_worker`, fail with `502 WV-WIRE` for a worker that is placed, or +drained since the upgrade. A drain or resume is still applied. Stored worker records and the registration answer keep their bytes, and workers drained before the upgrade show no drain time. diff --git a/docs/reference/api.md b/docs/reference/api.md index 339ce3df..fef43c1f 100644 --- a/docs/reference/api.md +++ b/docs/reference/api.md @@ -732,11 +732,13 @@ token and the current principal. `target_id`, `target_name`, `deployment_id`, and `component`. The server matches the worker's reported `host` against the members of the latest complete observation of each target, taken at the target's current revision - and not yet expired. A host listed by more than one target or component, a - native executor (`runtime: embedded`), and a worker without a reported host - have no placement. Only targets that both the token and the current principal - may read with `deployment.read` are considered, so other targets' names never - appear. + and not yet expired. Only targets that both the token and the current + principal may read with `deployment.read` are considered, so other targets' + names never appear. A host listed by more than one readable target or + component, a native executor (`runtime: embedded`), and a worker without a + reported host have no placement. Because uniqueness is decided among readable + targets, a reader restricted to some targets can see a placement for a host + that a reader of every target sees as ambiguous, and so without placement. - `drained_at` is the time the worker started draining. A repeated drain keeps it and resuming clears it; workers drained before the time was recorded show none. `workers.read` returns the same fields as the list, and `workers.drain` @@ -759,7 +761,8 @@ answers `{workers, deployments, activations, next_cursor}`: - `workers` counts every instance of the release once: `revoked`, else `draining`, else `online` (seen in the last 60 seconds) or `offline`. - `activations` counts the activation revisions that pin the release and can - still start runs. + still start runs: neither they, the latest revision of their activation, nor + their workflow version is retired. - `deployments` lists, at most `limit` (1 to 100, default 100) at a time, the desired deployment components whose admitted release and image digest are this release: `target_id`, `target_name`, `deployment_id`, `deployment_name`, diff --git a/src/firefly_weave/sdk/platform_ai_setup.py b/src/firefly_weave/sdk/platform_ai_setup.py index 09e8fba3..622503c7 100644 --- a/src/firefly_weave/sdk/platform_ai_setup.py +++ b/src/firefly_weave/sdk/platform_ai_setup.py @@ -288,7 +288,8 @@ async def ensure_connection(client: WeaveClient, request: ConnectionRequest) -> async def presence(client: WeaveClient, release_id: str) -> dict[str, Any]: - page = await client.invoke("workers.list", query={"limit": 100}) + # Every worker of the release, so one not seen for hours reads as stale rather than not seen yet. + page = await client.invoke("workers.list", query={"release_id": release_id, "state": "all", "limit": 100}) found = [item for item in page.items if str(item.release_id) == release_id and not item.revoked] best = next((item for item in found if item.presence == "recent"), found[0] if found else None) if best is None: diff --git a/src/firefly_weave/workers/repository.py b/src/firefly_weave/workers/repository.py index 058db177..004fe50b 100644 --- a/src/firefly_weave/workers/repository.py +++ b/src/firefly_weave/workers/repository.py @@ -287,7 +287,8 @@ async def release_workers(self, identifier: UUID, now: datetime) -> dict[str, in async def release_activations(self, identifier: UUID) -> int: """Activation revisions pinning the release that can still start runs. - A revision counts unless it, or the latest revision of the same activation, is retired. + A revision counts unless it, the latest revision of the same activation, or its workflow version is + retired, as the start check decides. """ pinned = ( "EXISTS(SELECT 1 FROM jsonb_each_text(coalesce(a.payload->'request'->'{key}','{{}}'::jsonb)) pin " @@ -300,7 +301,9 @@ async def release_activations(self, identifier: UUID) -> int: "AND NOT coalesce((a.payload->>'retired')::boolean,false) " "AND NOT coalesce((SELECT (l.payload->>'retired')::boolean FROM activation_revisions l " "WHERE l.tenant_id=a.tenant_id AND l.project_id=a.project_id AND l.environment_id=a.environment_id " - "AND l.name=a.name ORDER BY l.revision DESC LIMIT 1),false)", + "AND l.name=a.name ORDER BY l.revision DESC LIMIT 1),false) " + "AND NOT EXISTS(SELECT 1 FROM definition_retirements r WHERE r.tenant_id=a.tenant_id " + "AND r.project_id=a.project_id AND r.version_id=a.version_id)", release=str(identifier), ) return int(rows[0]["count"]) diff --git a/studio/src/app/operate/workers/workers-page.ts b/studio/src/app/operate/workers/workers-page.ts index 5f2bdf19..983479ef 100644 --- a/studio/src/app/operate/workers/workers-page.ts +++ b/studio/src/app/operate/workers/workers-page.ts @@ -66,6 +66,10 @@ import { type WorkerStatus, } from "./workers-model"; +// Every worker, revoked and long offline ones included, newest registered first: +// the page filters on its own, and this order does not move rows between polls. +const EVERY_WORKER = "state=all&order=registered_desc"; + @Component({ selector: "weave-workers-page", standalone: true, @@ -490,7 +494,9 @@ export class WorkersPage implements DoCheck, OnInit, OnDestroy { const { items, cursor } = await readPages((query) => { if (generation !== this.generation) throw Error("List scope changed."); - return this.host.api.request(`${scope}/workers?${query}`); + return this.host.api.request( + `${scope}/workers?${query}&${EVERY_WORKER}`, + ); }, this.pages); if (generation !== this.generation) return false; this.workers = items; @@ -528,7 +534,8 @@ export class WorkersPage implements DoCheck, OnInit, OnDestroy { await this.listReads.run(async () => { if (generation !== this.generation || !this.canLoadMore) return; const page = await readPages( - (query) => this.host.api.request(`${scope}/workers?${query}`), + (query) => + this.host.api.request(`${scope}/workers?${query}&${EVERY_WORKER}`), 1, this.nextCursor, ); diff --git a/studio/tests/browser/worker-operations.spec.ts b/studio/tests/browser/worker-operations.spec.ts index c10f8133..b2037536 100644 --- a/studio/tests/browser/worker-operations.spec.ts +++ b/studio/tests/browser/worker-operations.spec.ts @@ -70,16 +70,32 @@ async function workers(page: Page, capabilities: string[]) { worker(revoked, { revoked: true }), ]; let lists = 0; + const queries: URLSearchParams[] = []; await page.route(`${environment}/workers?*`, (r) => { lists++; - return r.fulfill({ json: { items, next_cursor: null } }); + const query = new URL(r.request().url()).searchParams; + queries.push(query); + // As the server does: without state=all only current workers, not revoked + // and seen (or registered) in the last hour. + const listed = + query.get("state") === "all" ? items : items.filter(isCurrent); + return r.fulfill({ json: { items: listed, next_cursor: null } }); }); for (const item of items) await page.route(`${environment}/workers/${item.id}`, (r) => r.fulfill({ json: items.find((w) => w.id === item.id) }), ); - return { items, lists: () => lists }; + return { items, lists: () => lists, queries: () => queries }; } +const isCurrent = (item: Record) => { + const contact = (item.last_seen_at ?? item.registered_at) as string | null; + return ( + !item.revoked && + contact !== null && + contact !== undefined && + Date.parse(contact) > Date.now() - 3600_000 + ); +}; const rows = (page: Page) => page.getByRole("table", { name: "Workers" }).locator(".resource-row"); @@ -87,9 +103,12 @@ test("filtered workers can reach matching rows on a later page", async ({ page, }) => { await workers(page, ["status.read"]); - await page.route(`${environment}/workers?*`, (r) => - r.fulfill({ - json: new URL(r.request().url()).searchParams.has("cursor") + const queries: URLSearchParams[] = []; + await page.route(`${environment}/workers?*`, (r) => { + const query = new URL(r.request().url()).searchParams; + queries.push(query); + return r.fulfill({ + json: query.has("cursor") ? { items: [worker(stale, { presence: "stale" })], next_cursor: null } : { items: Array.from({ length: 50 }, (_, i) => @@ -97,8 +116,8 @@ test("filtered workers can reach matching rows on a later page", async ({ ), next_cursor: "next", }, - }), - ); + }); + }); await page.goto("/operate/workers?presence=offline"); const more = page.getByRole("button", { name: "Load more workers" }); await expect(more).toBeVisible(); @@ -108,6 +127,12 @@ test("filtered workers can reach matching rows on a later page", async ({ await more.click(); await expect(rows(page)).toHaveCount(1); await expect(rows(page)).toContainText("Offline"); + // Load more continues the same query, so its cursor stays valid. + expect(queries.some((query) => query.has("cursor"))).toBe(true); + for (const query of queries) { + expect(query.get("state")).toBe("all"); + expect(query.get("order")).toBe("registered_desc"); + } }); test("workers coalesce repeated appends and refresh all loaded pages", async ({ @@ -270,7 +295,7 @@ for (const viewport of [ test("lists presence, claims and load, and hides revoked workers", async ({ page, }) => { - await workers(page, ["status.read"]); + const mocked = await workers(page, ["status.read"]); await page.getByRole("button", { name: "Workers", exact: true }).click(); await expect(page).toHaveURL(/\/operate\/workers$/); await expect(rows(page)).toHaveCount(3); @@ -283,6 +308,12 @@ for (const viewport of [ await page.getByLabel("Show revoked workers").check(); await expect(rows(page)).toHaveCount(4); await expect(rows(page).nth(3)).toContainText("Revoked"); + // The page reads every worker in a stable order and filters them itself. + expect(mocked.queries().length).toBeGreaterThan(0); + for (const query of mocked.queries()) { + expect(query.get("state")).toBe("all"); + expect(query.get("order")).toBe("registered_desc"); + } }); test("filters narrow the list and stay in the address", async ({ diff --git a/tests/contracts/test_worker_queries.py b/tests/contracts/test_worker_queries.py index 22a07e3a..552d10fb 100644 --- a/tests/contracts/test_worker_queries.py +++ b/tests/contracts/test_worker_queries.py @@ -352,6 +352,38 @@ def test_worker_operations_declare_queries_commands_and_capabilities(): assert [item["name"] for item in parameters if item["in"] == "query"] == ["limit", "cursor"] +@pytest.mark.parametrize( + "names", + [("cursor",), ("query", "cursor"), ("query", "limit"), ("releases", "query"), ("releases", "position")], +) +async def test_worker_lists_refuse_mixed_arguments_before_reading(names): + from firefly_weave.access.audit import AuditContext + from firefly_weave.access.models import Principal + from firefly_weave.contracts.worker_queries import WorkerListQuery + from firefly_weave.operations.fact_positions import RunPosition + from firefly_weave.workers.service import WorkerService + + class Unread: + def __getattr__(self, name): + raise AssertionError("read before the list arguments were checked") + + values = { + "cursor": UUID(int=1), + "limit": 7, + "releases": True, + "query": WorkerListQuery(), + "position": RunPosition(None, UUID(int=2)), + } + scope = Scope(tenant_id=uuid4(), project_id=uuid4(), environment_id=uuid4()) + with pytest.raises(ValueError): + await WorkerService(Unread(), Unread()).list( + Principal(id=uuid4(), kind="application"), + scope, + context=AuditContext(), + **{name: values[name] for name in names}, + ) + + def test_worker_query_schemas_are_exported(): from firefly_weave.contracts.schema_export import export_schemas diff --git a/tests/integration/test_worker_fleet_facts.py b/tests/integration/test_worker_fleet_facts.py index 9591d889..9f817f7d 100644 --- a/tests/integration/test_worker_fleet_facts.py +++ b/tests/integration/test_worker_fleet_facts.py @@ -955,6 +955,31 @@ async def test_release_usage_activations_skip_retired_ones(operations_case): assert usage.activations == 0 +async def test_release_usage_activations_skip_retired_workflow_versions(operations_case): + case = operations_case + release = case.instances[0].release_id + + async def activations(): + usage = await case.workers.release_usage( + case.actor, case.scope, release, ReleaseUsageQuery(), context=AuditContext() + ) + return usage.activations + + assert await activations() == 1 + # Runs cannot start from an activation whose workflow version is retired, so it no longer counts. + project = case.scope.model_copy(update={"environment_id": None}) + await case.access.grant(case.admin, case.actor.id, Grant(role="deployer", scope=project)) + await case.workers.definitions.retire( + await case.access.load_principal(case.actor.id), + case.scope, + case.activation.request.version_id, + "Workflow", + "retire-version", + context=AuditContext(), + ) + assert await activations() == 0 + + async def test_release_usage_is_scoped_and_authorized(monkeypatch, operations_case, provisioned): from firefly_weave.definitions.service import DefinitionService diff --git a/tests/unit/cli/test_platform_ai_setup.py b/tests/unit/cli/test_platform_ai_setup.py index 45810f47..a8d5f34f 100644 --- a/tests/unit/cli/test_platform_ai_setup.py +++ b/tests/unit/cli/test_platform_ai_setup.py @@ -46,6 +46,7 @@ class Client: def __init__(self, connections=(), presence="recent", answer=None): self.calls, self.connections, self.presence = [], list(connections), presence + self.worker_queries = [] self.answer = answer or { "ok": True, "code": "ok", @@ -70,6 +71,7 @@ async def invoke(self, operation, **kwargs): if operation == "connections.create": return SimpleNamespace(id=REVISION) if operation == "workers.list": + self.worker_queries.append(kwargs.get("query")) worker = SimpleNamespace( id=uuid4(), release_id=RELEASE, presence=self.presence, revoked=False, last_seen_at=datetime.now(UTC) ) @@ -272,6 +274,14 @@ async def test_the_worker_must_come_online_in_time(): await setup.wait_online(Client(presence="stale"), str(RELEASE), sleep=immediately, clock=lambda: next(ticks)) +async def test_presence_reads_every_worker_of_the_release(): + # A worker last seen hours ago is stale, not unknown: the list must not keep only current workers. + client = Client(presence="stale") + found = await setup.presence(client, str(RELEASE)) + assert found["presence"] == "stale" and found["last_seen_at"] is not None + assert client.worker_queries == [{"release_id": str(RELEASE), "state": "all", "limit": 100}] + + async def test_the_connection_test_returns_its_result_with_a_timestamp(): client = Client() result = await setup.test_connection(client, str(REVISION), "qwen2.5:1.5b") From e78f281960d51851d3fdafb4c83d2d1c604ae53d Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Mon, 12 Oct 2026 04:01:49 -0700 Subject: [PATCH 17/63] Test that AI status ignores workers of other releases --- tests/unit/cli/test_platform_ai_setup.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/tests/unit/cli/test_platform_ai_setup.py b/tests/unit/cli/test_platform_ai_setup.py index a8d5f34f..ceba6091 100644 --- a/tests/unit/cli/test_platform_ai_setup.py +++ b/tests/unit/cli/test_platform_ai_setup.py @@ -44,9 +44,9 @@ class Client: """The public operations the AI setup uses, recorded.""" - def __init__(self, connections=(), presence="recent", answer=None): + def __init__(self, connections=(), presence="recent", answer=None, workers=None): self.calls, self.connections, self.presence = [], list(connections), presence - self.worker_queries = [] + self.worker_queries, self.workers = [], workers self.answer = answer or { "ok": True, "code": "ok", @@ -72,6 +72,8 @@ async def invoke(self, operation, **kwargs): return SimpleNamespace(id=REVISION) if operation == "workers.list": self.worker_queries.append(kwargs.get("query")) + if self.workers is not None: + return SimpleNamespace(items=self.workers, next_cursor=None) worker = SimpleNamespace( id=uuid4(), release_id=RELEASE, presence=self.presence, revoked=False, last_seen_at=datetime.now(UTC) ) @@ -282,6 +284,17 @@ async def test_presence_reads_every_worker_of_the_release(): assert client.worker_queries == [{"release_id": str(RELEASE), "state": "all", "limit": 100}] +async def test_presence_ignores_workers_of_another_release(): + # An older server ignores the release filter, so the answer can list other releases' workers first. + other = SimpleNamespace( + id=uuid4(), release_id=uuid4(), presence="recent", revoked=False, last_seen_at=datetime.now(UTC) + ) + own = SimpleNamespace(id=uuid4(), release_id=RELEASE, presence="stale", revoked=False, last_seen_at=None) + found = await setup.presence(Client(workers=[other, own]), str(RELEASE)) + assert found == {"presence": "stale", "last_seen_at": None, "worker_id": str(own.id)} + assert await setup.presence(Client(workers=[other]), str(RELEASE)) == {"presence": "unknown", "last_seen_at": None} + + async def test_the_connection_test_returns_its_result_with_a_timestamp(): client = Client() result = await setup.test_connection(client, str(REVISION), "qwen2.5:1.5b") From 88a6d607a6bd31d37105a229abe2284a2a8d405e Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Mon, 12 Oct 2026 05:38:40 -0700 Subject: [PATCH 18/63] Give back an outbox delivery slot when a delivery turn is refused --- CHANGELOG.md | 4 ++++ .../operations/event_delivery.py | 11 +++++++--- tests/unit/operations/test_outbox_capacity.py | 20 +++++++++++++++++++ 3 files changed, 32 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5ab796d2..630b9cb3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -121,6 +121,10 @@ SPDX-License-Identifier: Apache-2.0 combined with keyword arguments. - Kubernetes runners no longer read a pod's free-text status reason, and a pod whose readiness is unknown is listed as a member that is not ready. +- An outbox delivery turn that starts while the platform is still restricted, + such as while it opens its background work after startup, gives back its + delivery slot. Before, each such turn kept one of the process's four slots + until a restart, and event delivery stopped once all four were lost. - Keep typing in the selected form field when deferred row autofocus completes. Rapid row additions focus the newest item without overriding a later choice. diff --git a/src/firefly_weave/operations/event_delivery.py b/src/firefly_weave/operations/event_delivery.py index 89d59a71..53f34413 100644 --- a/src/firefly_weave/operations/event_delivery.py +++ b/src/firefly_weave/operations/event_delivery.py @@ -134,9 +134,14 @@ async def _dispatch( *, reserved: int = 0, ) -> DeliveryReport: - self.subscriptions.bindings.connections.registry.require_operational() - if not 1 <= limit <= 10: - raise ValueError("One bounded outbox quantum required") + try: + self.subscriptions.bindings.connections.registry.require_operational() + if not 1 <= limit <= 10: + raise ValueError("One bounded outbox quantum required") + except BaseException: + # A traversal turn hands over the capacity it reserved; a refused turn gives it back. + release(reserved) + raise count = reserved or reserve(limit) if not count: return DeliveryReport() diff --git a/tests/unit/operations/test_outbox_capacity.py b/tests/unit/operations/test_outbox_capacity.py index bc166a50..6de4545f 100644 --- a/tests/unit/operations/test_outbox_capacity.py +++ b/tests/unit/operations/test_outbox_capacity.py @@ -169,3 +169,23 @@ async def checked(tx, identifier, token): assert await harness.attempt() == "incident" assert harness.settled == [("AUTHORITY_REVOKED", False, True)] + + +@pytest.mark.parametrize("limit", [1, 11]) +async def test_a_refused_traversal_turn_gives_back_the_capacity_it_was_handed(limit): + from firefly_weave.operations import event_delivery + + def restricted(): + raise CatalogError(503, "WV-COMPATIBILITY", "Execution compatibility unavailable") + + registry = SimpleNamespace(require_operational=restricted if limit == 1 else lambda: None) + subscriptions = SimpleNamespace(uow=None, bindings=SimpleNamespace(connections=SimpleNamespace(registry=registry))) + dispatcher = OutboxDispatcher(SimpleNamespace(), subscriptions, None, HttpPolicy()) # type: ignore[arg-type] + before = event_delivery.available() + # A traversal reserves its turn before handing it over, as while effects open on a restricted platform. + reserved = event_delivery.reserve(1) + assert reserved == 1 + scope = Scope(tenant_id=uuid4(), project_id=uuid4(), environment_id=uuid4()) + with pytest.raises((CatalogError, ValueError)): + await dispatcher._dispatch(scope, limit, None, None, None, reserved=reserved) # type: ignore[arg-type] + assert event_delivery.available() == before From d2f928d7b1bbddef42a0da4914aecc59fd3d3702 Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Mon, 12 Oct 2026 05:38:40 -0700 Subject: [PATCH 19/63] Report background loop health and re-check run listing decisions in bounded cycles --- CHANGELOG.md | 8 + docs/operations/upgrades.md | 25 +- docs/reference/api.md | 9 +- migrations/versions/0031_operations_facts.py | 34 + src/firefly_weave/access/scheduler.py | 24 +- src/firefly_weave/app.py | 47 +- src/firefly_weave/operations/facts.py | 108 +++ src/firefly_weave/operations/loop.py | 229 +++++ src/firefly_weave/operations/loop_health.py | 198 +++++ src/firefly_weave/operations/outbox_loop.py | 32 +- src/firefly_weave/providers/loop.py | 22 +- src/firefly_weave/runtime/scheduler.py | 46 +- src/firefly_weave/triggers/kafka_loop.py | 38 +- .../test_operations_cache_refresh.py | 812 ++++++++++++++++++ tests/unit/operations/test_loop_context.py | 12 +- tests/unit/operations/test_loop_health.py | 426 +++++++++ tests/unit/operations/test_operations_loop.py | 330 +++++++ 17 files changed, 2356 insertions(+), 44 deletions(-) create mode 100644 src/firefly_weave/operations/loop.py create mode 100644 src/firefly_weave/operations/loop_health.py create mode 100644 tests/integration/test_operations_cache_refresh.py create mode 100644 tests/unit/operations/test_loop_health.py create mode 100644 tests/unit/operations/test_operations_loop.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 630b9cb3..1af6b6c1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,6 +35,14 @@ SPDX-License-Identifier: Apache-2.0 Outputs of agent, tool or yield instance keys stay withheld until they have their own schemas. A run waiting for a platform that runs it answers `422 WV-IR-UNSUPPORTED` as `runs.read` does. +- The scheduler re-checks stored run listing decisions in the background. Each + minute it visits up to 16 tenants and gives each one's next environment up to + one second of database work and up to 100 runs: runs recorded while this + platform could not run them, which it now runs, and runs recorded under an + earlier classification policy. Until a run is re-checked, run summaries, step timelines and incident + lists keep withholding it even when `runs.read` serves it again, and its + timeline answers `422 WV-IR-UNSUPPORTED`. A retained run larger than 128 MiB + is recorded as unavailable without being read. - The Python SDK's `list_runs` rejects a `query` combined with keyword filters. - Request telemetry records authorized `501 WV-UNAVAILABLE` answers with the status `unavailable` instead of `failed`. diff --git a/docs/operations/upgrades.md b/docs/operations/upgrades.md index 5f27907e..1ac506fb 100644 --- a/docs/operations/upgrades.md +++ b/docs/operations/upgrades.md @@ -331,8 +331,12 @@ answer. Earlier servers do not serve them and answer `404` or `405`. ## Operational fact storage The `0031_operations_facts` migration adds five scoped operational projections -and two bounded traversal cursors. It does not add columns to stored runs or -events. Apply it explicitly with all writers stopped, using the procedure above. +and the operations loop's bounded traversal cursors: one for tenants, one per +tenant for environments, and one per environment for re-checking stored listing +decisions. The scheduler login reaches tenants through the new execute-only +`weave_operations_tenants(integer)`, granted to `weave_scheduler`, so it never +moves the recovery traversal's cursors. The migration does not add columns to +stored runs or events. Apply it explicitly with all writers stopped, using the procedure above. The migration holds schema locks in one transaction and reads retained runs in UUID order, at most 10,000 IDs per batch. Its history work is proportional to the @@ -359,8 +363,8 @@ provenance. Backfill inventories retained data without granting extra admission capacity: a platform already above a quota can still migrate, but new work must satisfy the unchanged quotas. Purging an authorized archived run also removes its run, step, task and incident facts; deleting a worker removes its facts. -The two cursor tables hold only one row per tenant or environment and have no -business payload or retained history. +The cursor tables hold one row in all, per tenant or per environment and have +no business payload or retained history. ## Start and inspect compatibility @@ -479,6 +483,19 @@ on the older server records only the cancellation, as for and its state stays withheld after you upgrade. To keep a run's full record, let it finish or cancel it before you roll back. +After the upgrade, `runs.read` serves the run at once, but run summaries, step +timelines and incident lists keep withholding it until the scheduler re-checks +its stored listing decision; its timeline answers `422 WV-IR-UNSUPPORTED` +meanwhile. The scheduler does this in the background: each minute it visits up +to 16 tenants, gives each one's next environment up to one second of database +work, and re-checks up to 100 runs there, so a large backlog takes several +minutes or more. Runs recorded +under an earlier classification policy are re-checked the same way. API +replicas that disable the scheduler rely on the scheduler replica for this. A +run whose re-check fails, for example because its record cannot be locked in +time or ordinary storage is full, stays withheld, is logged by error class, and +is tried again on a later pass while the others continue. + **A server whose capabilities do not list `language_features` predates language features.** It cannot recognize such a run, treats it as legacy evidence, and can block it permanently; a blocked run's tasks are never offered diff --git a/docs/reference/api.md b/docs/reference/api.md index fef43c1f..82387a8f 100644 --- a/docs/reference/api.md +++ b/docs/reference/api.md @@ -640,8 +640,13 @@ Their unknown timestamps remain null. Legacy metadata is streamed in batches of 500, up to 100,000 keys and two seconds per request; exceeding either budget answers `429 WV-RUNTIME-LIMIT`. A run this platform cannot run yet answers `422 WV-IR-UNSUPPORTED`, as `runs.read` does; a run with unavailable legacy -evidence answers `409 WV-LEGACY-UNAVAILABLE`. In `runs.list`, only a recorded -policy block withholds a run before its own read classifies it. +evidence answers `409 WV-LEGACY-UNAVAILABLE`. Timelines, summaries and incident +lists follow the run's stored listing decision. After an upgrade lets the +platform run such a run again, `runs.read` serves it at once, but its timeline +keeps answering `422 WV-IR-UNSUPPORTED` until the scheduler re-checks the +decision in the background, up to 100 runs per environment each minute. In +`runs.list`, only a recorded policy block withholds a run before its own read +classifies it. **Incidents.** `incidents.list` and the paged `incidents.run_list` read incident facts after checking both token and current permissions; they never diff --git a/migrations/versions/0031_operations_facts.py b/migrations/versions/0031_operations_facts.py index fb09b2a5..7b17b2ba 100644 --- a/migrations/versions/0031_operations_facts.py +++ b/migrations/versions/0031_operations_facts.py @@ -128,6 +128,34 @@ def tables() -> None: f"({tenant}) WITH CHECK ({tenant})" ) op.execute("GRANT SELECT,INSERT,UPDATE ON operations_environment_cursors TO weave_app") + # The operations traversal pages tenants with its own cursor, so it never moves the recovery traversal's. + op.execute("CREATE TABLE operations_tenant_cursor(singleton boolean PRIMARY KEY CHECK(singleton),tenant_id uuid)") + op.execute("INSERT INTO operations_tenant_cursor VALUES(true,NULL)") + op.execute("GRANT SELECT,UPDATE ON operations_tenant_cursor TO weave_catalog_reader") + op.execute("""CREATE FUNCTION weave_operations_tenants(page_size integer) RETURNS SETOF uuid + LANGUAGE plpgsql SECURITY DEFINER SET search_path=pg_catalog,public AS $$ + DECLARE prior uuid; candidate uuid; visited uuid[] := ARRAY[]::uuid[]; + BEGIN + IF page_size IS NULL OR page_size<1 OR page_size>16 THEN RAISE EXCEPTION 'Bounded catalog page required'; END IF; + SELECT tenant_id INTO prior FROM public.operations_tenant_cursor WHERE singleton FOR UPDATE; + FOR candidate IN SELECT id FROM public.tenants WHERE prior IS NULL OR id>prior ORDER BY id LIMIT page_size LOOP + visited := array_append(visited,candidate); RETURN NEXT candidate; + END LOOP; + IF array_length(visited,1) IS NULL OR array_length(visited,1)0 THEN + UPDATE public.operations_tenant_cursor SET tenant_id=visited[array_length(visited,1)] WHERE singleton; + END IF; + END $$""") + op.execute("GRANT CREATE ON SCHEMA public TO weave_catalog_reader") + op.execute("ALTER FUNCTION weave_operations_tenants(integer) OWNER TO weave_catalog_reader") + op.execute("REVOKE CREATE ON SCHEMA public FROM weave_catalog_reader") + op.execute("REVOKE ALL ON FUNCTION weave_operations_tenants(integer) FROM PUBLIC") + op.execute("GRANT EXECUTE ON FUNCTION weave_operations_tenants(integer) TO weave_scheduler") op.execute(f"""CREATE TABLE operations_fact_cursors({SCOPE_COLUMNS},after_run_id uuid, PRIMARY KEY({SCOPE_KEY}),FOREIGN KEY({SCOPE_KEY}) REFERENCES environments(tenant_id,project_id,id) ON DELETE CASCADE)""") @@ -152,6 +180,12 @@ def indexes() -> None: ("retry", "retried_from_run_id", "WHERE retried_from_run_id IS NOT NULL"), ("ended", "ended_at", "WHERE ended_at IS NOT NULL"), ("active_test", "", "WHERE test AND ended_at IS NULL"), + # Rows a listing-metadata refresh can need: another policy's decisions and transient unsupported ones. + ( + "revalidation", + "run_id", + "WHERE classification_state='unsupported' OR classification_policy<>'classified-v1'", + ), ): op.execute( f"CREATE INDEX run_facts_{name} ON run_facts({SCOPE_KEY}{',' + columns if columns else ''}) {condition}" diff --git a/src/firefly_weave/access/scheduler.py b/src/firefly_weave/access/scheduler.py index 3fc6b108..2e9a04c5 100644 --- a/src/firefly_weave/access/scheduler.py +++ b/src/firefly_weave/access/scheduler.py @@ -64,22 +64,32 @@ async def scopes(engine: AsyncEngine, uow: UnitOfWork) -> list[_SchedulerScope]: return result -async def tenant_page(engine: AsyncEngine, limit: int = 16) -> list[UUID]: - """Reserve a persistent bounded catalog turn; this grants no business capability.""" +async def tenant_page(engine: AsyncEngine, limit: int = 16, *, operations: bool = False) -> list[UUID]: + """Reserve a persistent bounded catalog turn; this grants no business capability. + + The operations traversal keeps its own tenant cursor, so it never moves the recovery traversal's. + """ + function = "weave_operations_tenants" if operations else "weave_scheduler_tenants" async with engine.begin() as connection: await connection.execute(text("SET LOCAL statement_timeout='4000ms'")) await connection.execute(text("SET LOCAL lock_timeout='1000ms'")) - return list( - (await connection.execute(text("SELECT weave_scheduler_tenants(:limit)"), {"limit": limit})).scalars() - ) + return list((await connection.execute(text(f"SELECT {function}(:limit)"), {"limit": limit})).scalars()) async def next_scope( - uow: UnitOfWork, tenant: UUID, *, broker: bool = False, outbox: bool = False, provider: bool = False + uow: UnitOfWork, + tenant: UUID, + *, + broker: bool = False, + outbox: bool = False, + provider: bool = False, + operations: bool = False, ) -> _SchedulerScope | None: """One environment per tenant turn, advanced before work so failures cannot pin traversal.""" table = ( - "provider_environment_cursors" + "operations_environment_cursors" + if operations + else "provider_environment_cursors" if provider else "outbox_environment_cursors" if outbox diff --git a/src/firefly_weave/app.py b/src/firefly_weave/app.py index 4be74071..5da0dac7 100644 --- a/src/firefly_weave/app.py +++ b/src/firefly_weave/app.py @@ -102,6 +102,8 @@ from firefly_weave.operations.event_delivery import OutboxDispatcher from firefly_weave.operations.history import HistoryService from firefly_weave.operations.incidents import IncidentService +from firefly_weave.operations.loop import OperationsLoop +from firefly_weave.operations.loop_health import LoopHealth, register_background_loops from firefly_weave.operations.lumi import LumiService from firefly_weave.operations.lumi_gateway import LumiGatewayClient from firefly_weave.operations.outbox_loop import OutboxLoop @@ -263,8 +265,13 @@ async def lifespan(app: Starlette) -> AsyncIterator[None]: dispatcher = None kafka_loop = None provider_loop = None + operations_loop = None effects_open = False compatibility = None + # One registry for this process's background loops; every loop reports into it. + health = LoopHealth() + register_background_loops(health, settings) + app.state.loop_health = health try: if settings.broker.enabled: require_driver() @@ -376,6 +383,7 @@ async def open_control_loop(*, required: bool = False) -> None: pyfly.context.get_bean(RecoveryService), pyfly.context.get_bean(Scheduler), operational=lambda: compatibility.ready, + health=health, ) try: await candidate.open() @@ -389,15 +397,20 @@ async def open_control_loop(*, required: bool = False) -> None: app.state.recovery_loop = recovery_loop async def open_effects() -> None: - nonlocal outbox_loop, dispatcher, kafka_loop, provider_loop, effects_open + nonlocal outbox_loop, dispatcher, kafka_loop, provider_loop, operations_loop, effects_open if effects_open: return - if any(loop is not None for loop in (outbox_loop, dispatcher, kafka_loop, provider_loop)): + if any( + loop is not None for loop in (outbox_loop, dispatcher, kafka_loop, provider_loop, operations_loop) + ): await close_effects() await open_control_loop(required=True) try: outbox_loop = OutboxLoop( - settings, pyfly.context.get_bean(UnitOfWork), pyfly.context.get_bean(OutboxDispatcher) + settings, + pyfly.context.get_bean(UnitOfWork), + pyfly.context.get_bean(OutboxDispatcher), + health=health, ) await outbox_loop.open() app.state.outbox_loop = outbox_loop @@ -414,6 +427,7 @@ async def open_effects() -> None: pyfly.context.get_bean(UnitOfWork), pyfly.context.get_bean(KafkaTrigger), pyfly.context.get_bean(BrokerClients), + health=health, ) await kafka_loop.open() app.state.kafka_loop = kafka_loop @@ -422,9 +436,13 @@ async def open_effects() -> None: pyfly.context.get_bean(UnitOfWork), pyfly.context.get_bean(ProviderDispatcher), pyfly.context.get_bean(EmailSourceService), + health=health, ) await provider_loop.open() app.state.provider_loop = provider_loop + operations_loop = OperationsLoop(settings, pyfly.context.get_bean(UnitOfWork), health) + await operations_loop.open() + app.state.operations_loop = operations_loop except BaseException: try: await close_effects() @@ -434,11 +452,14 @@ async def open_effects() -> None: effects_open = True async def close_effects() -> None: - nonlocal outbox_loop, dispatcher, kafka_loop, provider_loop, effects_open + nonlocal outbox_loop, dispatcher, kafka_loop, provider_loop, operations_loop, effects_open effects_open = False - remaining = [provider_loop, kafka_loop, dispatcher, outbox_loop] + # The operations loop opened last, so it closes first. + remaining = [operations_loop, provider_loop, kafka_loop, dispatcher, outbox_loop] cleanup_error: BaseException | None = None - for index, name in enumerate(("provider_loop", "kafka_loop", "dispatcher", "outbox_loop")): + for index, name in enumerate( + ("operations_loop", "provider_loop", "kafka_loop", "dispatcher", "outbox_loop") + ): loop = remaining[index] if loop is not None: try: @@ -449,10 +470,11 @@ async def close_effects() -> None: remaining[index] = None setattr(app.state, name, remaining[index]) # Failed owners remain reachable for the next refresh or final cleanup. - provider_loop = None if remaining[0] is None else provider_loop - kafka_loop = None if remaining[1] is None else kafka_loop - dispatcher = None if remaining[2] is None else dispatcher - outbox_loop = None if remaining[3] is None else outbox_loop + operations_loop = None if remaining[0] is None else operations_loop + provider_loop = None if remaining[1] is None else provider_loop + kafka_loop = None if remaining[2] is None else kafka_loop + dispatcher = None if remaining[3] is None else dispatcher + outbox_loop = None if remaining[4] is None else outbox_loop if cleanup_error is not None: raise cleanup_error await open_control_loop() @@ -472,6 +494,11 @@ async def close_effects() -> None: await compatibility.close() except BaseException as error: cleanup_error = error + if operations_loop is not None: + try: + await operations_loop.close() + except BaseException as error: + cleanup_error = error if provider_loop is not None: try: await provider_loop.close() diff --git a/src/firefly_weave/operations/facts.py b/src/firefly_weave/operations/facts.py index a1441b60..9e6ce102 100644 --- a/src/firefly_weave/operations/facts.py +++ b/src/firefly_weave/operations/facts.py @@ -17,8 +17,10 @@ """Transaction-bound SQL for operational projections.""" import json +import re from dataclasses import dataclass from datetime import datetime +from functools import partial from typing import Any, Literal from uuid import UUID @@ -113,6 +115,26 @@ def listing_metadata(row: dict[str, Any]) -> dict[str, Any]: return result +REVALIDATION_BATCH = 100 + + +def revalidation_due(policy: str) -> str: + """SQL condition: the cached decision of fact row ``f`` is due for a refresh from its retained source. + + Due when it was recorded under another classification policy, or when it is a transient unsupported decision + whose pinned IR version and features this platform runs again. Bind ``runtime.admission.runnable_parameters()``. + The policy is written as a literal so the planner can prove that the partial index ``run_facts_revalidation``, + which holds the rows not classified under the current policy and the unsupported ones, applies. + """ + if re.fullmatch(r"[a-z0-9][a-z0-9.-]{0,63}", policy) is None: + raise ValueError("Classification policy must be a plain identifier") + return ( + f"(f.classification_policy<>'{policy}' OR (f.classification_state='unsupported' " + "AND f.pinned_ir_version=ANY(cast(:ir_versions AS text[])) " + "AND f.pinned_features <@ cast(:ir_features AS jsonb)))" + ) + + def task_fact_update(source: str) -> str: """Enlist task metadata in the existing authoritative source statement.""" return ( @@ -168,6 +190,92 @@ async def rows(self, sql: str, **values: Any) -> list[dict[str, Any]]: result = await self.tx.session.execute(text(sql), {**values, **self.params}) return [dict(row) for row in result.mappings()] + async def revalidation_cursor(self) -> UUID | None: + """The environment's keyset boundary: the last run whose refresh was started.""" + value: UUID | None = await self.tx.session.scalar( + text(f"SELECT after_run_id FROM operations_fact_cursors WHERE {SCOPE}"), self.params + ) + return value + + async def advance_revalidation_cursor(self, after: UUID | None) -> None: + """Move the boundary; bookkeeping only, with no fact or source change and no storage charge.""" + await self.execute( + "INSERT INTO operations_fact_cursors(tenant_id,project_id,environment_id,after_run_id) " + "VALUES(:tenant,:project,:environment,:after) ON CONFLICT(tenant_id,project_id,environment_id) " + "DO UPDATE SET after_run_id=excluded.after_run_id", + after=after, + ) + + async def revalidation_candidates(self, after: UUID | None) -> list[UUID]: + """Up to ``REVALIDATION_BATCH`` due runs after ``after`` in UUID order, from fact metadata only.""" + from firefly_weave.runtime import admission + + rows = await self.rows( + "SELECT f.run_id FROM run_facts f WHERE f.tenant_id=:tenant AND f.project_id=:project " + f"AND f.environment_id=:environment AND {revalidation_due(admission.POLICY)} " + f"AND (cast(:after AS uuid) IS NULL OR f.run_id>:after) ORDER BY f.run_id LIMIT {REVALIDATION_BATCH}", + after=after, + **admission.runnable_parameters(), + ) + return [row["run_id"] for row in rows] + + async def refresh_listing_metadata(self, run_id: UUID) -> None: + """Recompute one run's cached listing decision from its retained source. + + Only the five classification columns change: never the run's state, timing or status, and never a runtime + transition, capacity block or policy block. Runtime admission stays the authority for execution. A source + larger than the bounded fetch is recorded unavailable without being read. A run whose fact changed since it + was selected, so that it is no longer due, keeps that newer decision. + """ + from firefly_weave.operations.execution import execute_pure + from firefly_weave.runtime import admission + from firefly_weave.runtime.repository import RUN_FETCH_BYTES, RuntimeRepository + + values = {**self.params, "run": run_id} + # Lock the source first, in the order runtime writers lock a run and then its facts. + locked = await self.tx.session.scalar(text(f"SELECT id FROM runs WHERE {SCOPE} AND id=:run FOR UPDATE"), values) + if locked is None: + return + # Rechecked under the lock: an authoritative update since the selection keeps its newer decision. + due = await self.tx.session.scalar( + text( + "SELECT f.run_id FROM run_facts f WHERE f.tenant_id=:tenant AND f.project_id=:project " + f"AND f.environment_id=:environment AND f.run_id=:run AND {revalidation_due(admission.POLICY)} " + "FOR UPDATE" + ), + {**values, **admission.runnable_parameters()}, + ) + if due is None: + return + # Measured under the lock, so the size cannot change before the fetch. + size = await self.tx.session.scalar( + text(f"SELECT public.weave_runs_bytes(runs) FROM runs WHERE {SCOPE} AND id=:run"), values + ) + if size is None or size > RUN_FETCH_BYTES: + await self.execute( + "UPDATE run_facts SET classification_state='unavailable',classification_policy=:policy " + f"WHERE {SCOPE} AND run_id=:run", + policy=admission.POLICY, + run=run_id, + ) + return + row = await RuntimeRepository(self.tx).run(run_id, lock=True) + # A refresh is never a runtime candidate: a failed write must not leave a capacity block on the run. + self.tx.session.info.pop("weave_runtime_candidate", None) + metadata = await execute_pure(partial(listing_metadata, row)) + await self.execute( + "UPDATE run_facts SET classification_state=:classification_state," + "classification_policy=:classification_policy,pinned_ir_version=:pinned_ir_version," + "pinned_features=cast(:pinned_features AS jsonb),node_kinds=cast(:node_kinds AS jsonb) " + f"WHERE {SCOPE} AND run_id=:run", + run=run_id, + classification_state=metadata["classification_state"], + classification_policy=metadata["classification_policy"], + pinned_ir_version=metadata["pinned_ir_version"], + pinned_features=json.dumps(metadata["pinned_features"], ensure_ascii=False), + node_kinds=json.dumps(metadata["node_kinds"], ensure_ascii=False), + ) + async def insert_run( self, view: RunView, artifact: dict[str, Any], request: StartRunRequest, facts: RunStartFacts, at: datetime ) -> None: diff --git a/src/firefly_weave/operations/loop.py b/src/firefly_weave/operations/loop.py new file mode 100644 index 00000000..590d772f --- /dev/null +++ b/src/firefly_weave/operations/loop.py @@ -0,0 +1,229 @@ +# Copyright 2026 Firefly Software Foundation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# Author: Firefly Software Foundation +# SPDX-License-Identifier: Apache-2.0 + +"""The lifespan-owned operations loop: bounded per-environment quanta under execute-only traversal authority. + +Every minute it pages tenants with its own cursors and gives one environment per tenant a budget of one second of +database work. Today that work refreshes cached run listing decisions; it never grants execution authority. +""" + +import asyncio +import logging +import time +from collections.abc import AsyncIterator, Callable +from contextlib import asynccontextmanager, suppress +from contextvars import Context +from datetime import datetime + +from sqlalchemy import text +from sqlalchemy.ext.asyncio import AsyncEngine, create_async_engine + +from firefly_weave.access.authorization import AccessDenied +from firefly_weave.access.scheduler import _SchedulerScope, next_scope, tenant_page +from firefly_weave.operations.execution import ReservedSlots +from firefly_weave.operations.facts import FactRepository +from firefly_weave.operations.loop_health import OPERATIONS_SECONDS, LoopHealth, track, utc_now +from firefly_weave.persistence.migrations import check_schema +from firefly_weave.persistence.uow import Transaction, UnitOfWork +from firefly_weave.runtime.scheduler import failure_cause +from firefly_weave.settings import Settings + +# Database work one environment may use per cycle. A quantum that runs out resumes on a later cycle. +ENVIRONMENT_SECONDS = 1.0 +# The wall-clock guard waits this much longer, so the database's own timeout normally ends a slow statement. +CANCEL_GRACE_SECONDS = 0.25 +# A cycle runs one pure classification at a time; the spare slot absorbs one that outlived its cancelled quantum. +OPERATIONS_EXECUTION = ReservedSlots(2) + + +class QuantumIncomplete(Exception): + """An environment quantum that failed a row or ran out of budget; its committed progress stays.""" + + +class OperationsLoop: + def __init__( + self, + settings: Settings, + uow: UnitOfWork, + health: LoopHealth, + *, + clock: Callable[[], datetime] = utc_now, + monotonic: Callable[[], float] = time.monotonic, + ) -> None: + self.settings, self.uow, self.health = settings, uow, health + self.clock, self.monotonic = clock, monotonic + self.engine: AsyncEngine | None = None + self.task: asyncio.Task[None] | None = None + + async def open(self) -> None: + if self.task is not None or not self.settings.scheduler_enabled: + return + if self.settings.scheduler_database_url is None: + raise RuntimeError( + "The operations loop requires WEAVE_SCHEDULER_DATABASE_URL; API-only replicas explicitly disable " + "the scheduler" + ) + self.engine = create_async_engine(self.settings.scheduler_database_url.get_secret_value(), hide_parameters=True) + try: + async with asyncio.timeout(self.settings.database_timeout_seconds), self.engine.connect() as connection: + await check_schema(connection) + valid = await connection.scalar( + text( + "SELECT NOT rolsuper AND NOT rolbypassrls AND " + "pg_has_role(current_user,'weave_scheduler','MEMBER') AND NOT " + "pg_has_role(current_user,'weave_app','MEMBER') AND NOT " + "pg_has_role(current_user,'weave_catalog_reader','MEMBER') AND NOT " + "has_table_privilege(current_user,'run_facts','SELECT') AND NOT " + "has_table_privilege(current_user,'tenants','SELECT') AND " + "has_function_privilege(current_user,'weave_operations_tenants(integer)','EXECUTE') " + "FROM pg_roles WHERE rolname=current_user" + ) + ) + if not valid: + raise RuntimeError("Operations traversal requires execute-only scheduler authority") + # A request can open this loop. An empty context keeps the request's execution + # lease and identity, which end with its response, out of the long-lived task. + self.task = asyncio.create_task(self.poll(), name="weave-operations", context=Context()) + track(self.health, self.task, "operations_loop") + except BaseException: + await self.close() + raise + + async def poll(self) -> None: + while True: + self.health.started("operations_loop") + try: + healthy = await self.cycle() + except asyncio.CancelledError: + raise + except Exception as error: + logging.getLogger(__name__).error( + "Operations traversal failed (%s); retrying on the next cycle", failure_cause(error) + ) + healthy = False + # Coverage times are reported only once statistics exist, so a success carries none. + if healthy: + self.health.succeeded("operations_loop") + else: + self.health.failed("operations_loop") + await asyncio.sleep(OPERATIONS_SECONDS) + + async def cycle(self) -> bool: + """One bounded traversal; True only when every visited environment's quantum completed.""" + assert self.engine is not None + healthy = True + async with OPERATIONS_EXECUTION.execution(): + budget = min(self.settings.database_timeout_seconds, 5.0) + async with asyncio.timeout(budget): + tenants = await tenant_page(self.engine, operations=True) + for tenant in tenants: + try: + async with asyncio.timeout(budget): + authority = await next_scope(self.uow, tenant, operations=True) + if authority is None: + continue + await self._environment(authority) + except Exception as error: + logging.getLogger(__name__).error( + "Operations environment quantum failed (%s); traversal will continue", failure_cause(error) + ) + healthy = False + return healthy + + async def _environment(self, authority: _SchedulerScope) -> None: + deadline = self.monotonic() + ENVIRONMENT_SECONDS + async with asyncio.timeout(ENVIRONMENT_SECONDS + CANCEL_GRACE_SECONDS): + if not await self.revalidate(authority, deadline): + raise QuantumIncomplete() + + @asynccontextmanager + async def _unit( + self, authority: _SchedulerScope, deadline_monotonic: float, *, mutation: bool + ) -> AsyncIterator[Transaction]: + """An ordinary scoped transaction, verified as scheduler work and limited to the remaining budget.""" + if self.monotonic() >= deadline_monotonic: + raise QuantumIncomplete() + async with self.uow.open(authority.scope, mutation=mutation) as tx: + await authority.verify(tx) + remaining = deadline_monotonic - self.monotonic() + if remaining <= 0: + raise QuantumIncomplete() + limit = f"{max(1, min(1000, int(remaining * 1000)))}ms" + await tx.session.execute( + text("SELECT set_config('statement_timeout',:limit,true),set_config('lock_timeout',:limit,true)"), + {"limit": limit}, + ) + yield tx + + async def revalidate(self, authority: _SchedulerScope, deadline_monotonic: float) -> bool: + """Refresh due cached listing decisions in one environment; True when this quantum completed without failure. + + The keyset boundary is committed before each run's refresh, in its own transaction, so a run that fails or + outlives the budget never pins the environment: later runs advance, and the failed one is retried after + the boundary wraps. Each run's refresh commits on its own; a failure rolls back only that run. + """ + if not isinstance(authority, _SchedulerScope): + raise AccessDenied() + log = logging.getLogger(__name__) + try: + async with self._unit(authority, deadline_monotonic, mutation=False) as tx: + facts = FactRepository(tx) + after = await facts.revalidation_cursor() + candidates = await facts.revalidation_candidates(after) + except Exception as error: + log.error("Operations revalidation selection failed (%s)", failure_cause(error)) + return False + if not candidates: + if after is None: + return True + # Wrap once: the next cycle starts again from the first due run. + try: + async with self._unit(authority, deadline_monotonic, mutation=True) as tx: + await FactRepository(tx).advance_revalidation_cursor(None) + except Exception as error: + log.error("Operations revalidation cursor update failed (%s)", failure_cause(error)) + return False + return True + completed = True + for run_id in candidates: + if self.monotonic() >= deadline_monotonic: + return False + try: + async with self._unit(authority, deadline_monotonic, mutation=True) as tx: + await FactRepository(tx).advance_revalidation_cursor(run_id) + except Exception as error: + # Without a committed boundary the run is not attempted, so no progress is claimed. + log.error("Operations revalidation cursor update failed (%s)", failure_cause(error)) + return False + try: + async with self._unit(authority, deadline_monotonic, mutation=True) as tx: + await FactRepository(tx).refresh_listing_metadata(run_id) + except Exception as error: + log.error("Operations revalidation of a run failed (%s); later runs continue", failure_cause(error)) + completed = False + return completed + + async def close(self) -> None: + if self.task is not None: + self.task.cancel() + with suppress(asyncio.CancelledError): + await self.task + self.task = None + self.health.running("operations_loop", False) + # Cancellation completes before the engine it uses is disposed. + if self.engine is not None: + await self.engine.dispose() + self.engine = None diff --git a/src/firefly_weave/operations/loop_health.py b/src/firefly_weave/operations/loop_health.py new file mode 100644 index 00000000..12524fae --- /dev/null +++ b/src/firefly_weave/operations/loop_health.py @@ -0,0 +1,198 @@ +# Copyright 2026 Firefly Software Foundation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# Author: Firefly Software Foundation +# SPDX-License-Identifier: Apache-2.0 + +"""In-process health of the background loops; a loop reads as healthy only after it proved a recent cycle. + +The API process that runs the scheduler is a singleton, so this per-process registry is authoritative for it. +Each loop reports its own cycles. Reporting never raises into the loop, and nothing here is per tenant. +""" + +import asyncio +import logging +import math +from collections.abc import Callable +from dataclasses import dataclass +from datetime import UTC, datetime, timedelta +from typing import Literal + +from firefly_weave.settings import Settings + +type LoopState = Literal["ok", "degraded", "down", "disabled"] + +# The fixed set of components: health is per process and per loop, never per tenant or environment. +COMPONENTS = ( + "recovery_loop", + "schedules", + "outbox_loop", + "provider_loop", + "kafka_loop", + "operations_loop", + "alert_evaluator", +) +FAILURES_DEGRADED = 3 +INTERVALS_STALE = 3 +# The outbox, provider and broker traversals poll every second; the operations loop every minute. +TRAVERSAL_SECONDS = 1.0 +OPERATIONS_SECONDS = 60.0 +_FAILURE_CEILING = 1_000_000_000 + + +def utc_now() -> datetime: + return datetime.now(UTC) + + +@dataclass(frozen=True) +class LoopSnapshot: + component: str + state: LoopState + last_started_at: datetime | None + last_ok_at: datetime | None + consecutive_failures: int + complete_until: datetime | None + + +@dataclass +class _Record: + interval: timedelta + enabled: bool + running: bool = False + last_started_at: datetime | None = None + last_ok_at: datetime | None = None + consecutive_failures: int = 0 + complete_until: datetime | None = None + + +def _utc(value: datetime) -> datetime: + if not isinstance(value, datetime) or value.tzinfo is None or value.utcoffset() is None: + raise ValueError("Loop health times must be timezone-aware") + return value.astimezone(UTC) + + +class LoopHealth: + """Per-loop start, success and failure records, judged against the loop's poll interval. + + ``disabled`` when the loop is turned off, ``down`` while its task is not running, ``degraded`` after + ``FAILURES_DEGRADED`` consecutive failed cycles, before its first successful cycle, or when its last success is + older than ``INTERVALS_STALE`` intervals, and ``ok`` otherwise. Times come from ``clock``, are aware UTC and are + for display; loops measure their own budgets with a monotonic clock. + """ + + def __init__(self, clock: Callable[[], datetime] = utc_now) -> None: + self._clock = clock + self._records: dict[str, _Record] = {} + + def register(self, name: str, interval_seconds: float, *, enabled: bool) -> None: + """Declare a loop at startup; registering it again keeps what it already reported.""" + if name not in COMPONENTS: + raise ValueError("Unknown background loop") + if ( + isinstance(interval_seconds, bool) + or not isinstance(interval_seconds, (int, float)) + or not math.isfinite(interval_seconds) + or interval_seconds <= 0 + ): + raise ValueError("Loop interval must be a positive number of seconds") + if type(enabled) is not bool: + raise ValueError("Loop enablement must be a boolean") + interval = timedelta(seconds=interval_seconds) + record = self._records.get(name) + if record is None: + self._records[name] = _Record(interval=interval, enabled=enabled) + else: + record.interval, record.enabled = interval, enabled + + def running(self, name: str, value: bool) -> None: + self._update(name, lambda record: setattr(record, "running", value is True)) + + def started(self, name: str) -> None: + self._update(name, lambda record: setattr(record, "last_started_at", self._now())) + + def succeeded(self, name: str, *, complete_until: datetime | None = None) -> None: + def update(record: _Record) -> None: + now = self._now() + until = None if complete_until is None else _utc(complete_until) + record.last_ok_at, record.consecutive_failures, record.complete_until = now, 0, until + + self._update(name, update) + + def failed(self, name: str) -> None: + def update(record: _Record) -> None: + record.consecutive_failures = min(record.consecutive_failures + 1, _FAILURE_CEILING) + + self._update(name, update) + + def snapshot(self, name: str) -> LoopSnapshot: + record = self._records[name] + if not record.enabled: + state: LoopState = "disabled" + elif not record.running: + state = "down" + elif record.last_ok_at is None or record.consecutive_failures >= FAILURES_DEGRADED: + state = "degraded" + else: + state = "degraded" if self._now() - record.last_ok_at > record.interval * INTERVALS_STALE else "ok" + return LoopSnapshot( + component=name, + state=state, + last_started_at=record.last_started_at, + last_ok_at=record.last_ok_at, + consecutive_failures=record.consecutive_failures, + complete_until=record.complete_until, + ) + + def snapshots(self) -> tuple[LoopSnapshot, ...]: + return tuple(self.snapshot(name) for name in COMPONENTS if name in self._records) + + def _now(self) -> datetime: + return _utc(self._clock()) + + def _update(self, name: str, change: Callable[[_Record], None]) -> None: + # Unregistered names are ignored: an update never invents a component or fails the loop reporting it. + record = self._records.get(name) + if record is None: + return + try: + change(record) + except Exception as error: + logging.getLogger(__name__).warning("Loop health update skipped (%s)", type(error).__name__) + + +def register_background_loops(health: LoopHealth, settings: Settings) -> None: + """Every component, registered at startup so a loop that has not opened yet reads as down, not missing.""" + scheduled = settings.scheduler_enabled + for name in ("recovery_loop", "schedules"): + health.register(name, settings.scheduler_poll_seconds, enabled=scheduled) + for name in ("outbox_loop", "provider_loop"): + health.register(name, TRAVERSAL_SECONDS, enabled=scheduled) + # The broker consumer has its own switch; the scheduler setting does not stop it. + health.register( + "kafka_loop", TRAVERSAL_SECONDS, enabled=settings.broker.enabled and settings.kafka_consumer_enabled + ) + health.register("operations_loop", OPERATIONS_SECONDS, enabled=scheduled) + # Alert evaluation does not exist yet. + health.register("alert_evaluator", OPERATIONS_SECONDS, enabled=False) + + +def track(health: LoopHealth, task: "asyncio.Task[None]", *names: str) -> None: + """Report ``names`` running while ``task`` runs, and down as soon as it ends for any reason.""" + for name in names: + health.running(name, True) + + def ended(_: "asyncio.Task[None]") -> None: + for name in names: + health.running(name, False) + + task.add_done_callback(ended) diff --git a/src/firefly_weave/operations/outbox_loop.py b/src/firefly_weave/operations/outbox_loop.py index 4b3768e5..26152093 100644 --- a/src/firefly_weave/operations/outbox_loop.py +++ b/src/firefly_weave/operations/outbox_loop.py @@ -25,17 +25,23 @@ from firefly_weave.access.scheduler import next_scope from firefly_weave.operations.event_delivery import OutboxDispatcher, release, reserve +from firefly_weave.operations.loop_health import TRAVERSAL_SECONDS, LoopHealth, track from firefly_weave.persistence.migrations import check_schema from firefly_weave.persistence.uow import UnitOfWork from firefly_weave.settings import Settings class OutboxLoop: - def __init__(self, settings: Settings, uow: UnitOfWork, dispatcher: OutboxDispatcher) -> None: + def __init__( + self, settings: Settings, uow: UnitOfWork, dispatcher: OutboxDispatcher, *, health: LoopHealth | None = None + ) -> None: self.settings, self.uow, self.dispatcher = settings, uow, dispatcher + self.health = health if health is not None else LoopHealth() self.engine: AsyncEngine | None = None self.task: asyncio.Task[None] | None = None self.turns: set[asyncio.Task[None]] = set() + # A delivery turn runs in its own task; its failure fails the next traversal cycle. + self.turn_failed = False async def open(self) -> None: if not self.settings.scheduler_enabled: @@ -63,14 +69,17 @@ async def open(self) -> None: # A request can open this loop. An empty context keeps the request's execution # lease and identity, which end with its response, out of the long-lived task. self.task = asyncio.create_task(self.poll(), name="weave-outbox-traversal", context=Context()) + track(self.health, self.task, "outbox_loop") except BaseException: await self.close() raise - async def cycle(self) -> None: + async def cycle(self) -> bool: + """One bounded traversal turn; False when a delivery turn failed since the previous cycle.""" + healthy, self.turn_failed = not self.turn_failed, False count = reserve(1) if not count: - return + return healthy owned = True try: assert self.engine is not None @@ -80,15 +89,16 @@ async def cycle(self) -> None: await connection.execute(text("SET LOCAL lock_timeout='1000ms'")) tenant = await connection.scalar(text("SELECT weave_outbox_tenants(1)")) if tenant is None: - return + return healthy authority = await next_scope(self.uow, tenant, outbox=True) if authority is None: - return + return healthy async def turn() -> None: try: await self.dispatcher._dispatch(authority.scope, 1, None, AuditContext(), authority, reserved=count) except Exception: + self.turn_failed = True logging.getLogger(__name__).error("Outbox turn failed; durable lease recovery remains available") from firefly_weave.access.audit import AuditContext @@ -100,14 +110,21 @@ async def turn() -> None: finally: if owned: release(count) + return healthy async def poll(self) -> None: while True: + self.health.started("outbox_loop") try: - await self.cycle() + healthy = await self.cycle() except Exception: logging.getLogger(__name__).error("Outbox traversal failed; retrying bounded turn") - await asyncio.sleep(1) + healthy = False + if healthy: + self.health.succeeded("outbox_loop") + else: + self.health.failed("outbox_loop") + await asyncio.sleep(TRAVERSAL_SECONDS) async def close(self) -> None: if self.task is not None: @@ -115,6 +132,7 @@ async def close(self) -> None: with suppress(asyncio.CancelledError): await self.task self.task = None + self.health.running("outbox_loop", False) for task in tuple(self.turns): task.cancel() if self.turns: diff --git a/src/firefly_weave/providers/loop.py b/src/firefly_weave/providers/loop.py index 0a8e8f2c..4b9c4599 100644 --- a/src/firefly_weave/providers/loop.py +++ b/src/firefly_weave/providers/loop.py @@ -26,6 +26,7 @@ from firefly_weave.access.scheduler import next_scope from firefly_weave.email.source import EmailSourceService from firefly_weave.operations.execution import ReservedSlots +from firefly_weave.operations.loop_health import TRAVERSAL_SECONDS, LoopHealth, track from firefly_weave.persistence.migrations import check_schema from firefly_weave.persistence.uow import UnitOfWork from firefly_weave.providers.dispatcher import ProviderDispatcher @@ -43,9 +44,12 @@ def __init__( uow: UnitOfWork, dispatcher: ProviderDispatcher, email_sources: EmailSourceService | None = None, + *, + health: LoopHealth | None = None, ) -> None: self.settings, self.uow, self.dispatcher = settings, uow, dispatcher self.email_sources = email_sources + self.health = health if health is not None else LoopHealth() self.engine: AsyncEngine | None = None self.task: asyncio.Task[None] | None = None @@ -74,11 +78,13 @@ async def open(self) -> None: # A request can open this loop. An empty context keeps the request's execution # lease and identity, which end with its response, out of the long-lived task. self.task = asyncio.create_task(self.poll(), name="weave-provider-inbox", context=Context()) + track(self.health, self.task, "provider_loop") except BaseException: await self.close() raise - async def cycle(self) -> None: + async def cycle(self) -> bool: + """One bounded provider and email turn; a failure raises, so a returned cycle completed.""" assert self.engine is not None async with PROVIDER_EXECUTION.execution(): async with asyncio.timeout(5): @@ -87,20 +93,27 @@ async def cycle(self) -> None: await connection.execute(text("SET LOCAL lock_timeout='1000ms'")) tenant = await connection.scalar(text("SELECT weave_provider_tenants(1)")) if tenant is None: - return + return True authority = await next_scope(self.uow, tenant, provider=True) if authority is not None: await self.dispatcher.scan(authority, 10) if self.email_sources is not None: await self.email_sources.scan(authority, 1) + return True async def poll(self) -> None: while True: + self.health.started("provider_loop") try: - await self.cycle() + healthy = await self.cycle() except Exception: logging.getLogger(__name__).error("Provider traversal failed; retained intents remain retryable") - await asyncio.sleep(1) + healthy = False + if healthy: + self.health.succeeded("provider_loop") + else: + self.health.failed("provider_loop") + await asyncio.sleep(TRAVERSAL_SECONDS) async def close(self) -> None: if self.task is not None: @@ -108,6 +121,7 @@ async def close(self) -> None: with suppress(asyncio.CancelledError): await self.task self.task = None + self.health.running("provider_loop", False) if self.engine is not None: await self.engine.dispose() self.engine = None diff --git a/src/firefly_weave/runtime/scheduler.py b/src/firefly_weave/runtime/scheduler.py index 26b8eeb7..bd83088a 100644 --- a/src/firefly_weave/runtime/scheduler.py +++ b/src/firefly_weave/runtime/scheduler.py @@ -28,6 +28,7 @@ from firefly_weave.access.scheduler import next_scope, tenant_page from firefly_weave.definitions.models import CatalogError from firefly_weave.operations.execution import ReservedSlots +from firefly_weave.operations.loop_health import LoopHealth, track from firefly_weave.persistence.migrations import check_schema from firefly_weave.persistence.uow import UnitOfWork from firefly_weave.runtime.recovery import RecoveryService @@ -55,10 +56,12 @@ def __init__( schedules: Scheduler, *, operational: Callable[[], bool] = lambda: True, + health: LoopHealth | None = None, ) -> None: self.settings, self.uow, self.recovery = settings, uow, recovery self.schedules = schedules self.operational = operational + self.health = health if health is not None else LoopHealth() self.engine: AsyncEngine | None = None self.task: asyncio.Task[None] | None = None @@ -93,12 +96,37 @@ async def open(self) -> None: # A request can open this loop. An empty context keeps the request's execution # lease and identity, which end with its response, out of the long-lived task. self.task = asyncio.create_task(self.poll(), name="weave-recovery", context=Context()) + # Schedules fire inside the recovery cycle, so they run exactly while this task runs. + track(self.health, self.task, "recovery_loop", "schedules") except BaseException: await self.close() raise - async def cycle(self) -> None: + async def cycle(self) -> bool: + """One bounded traversal; True only when every tenant's recovery scan completed. + + Schedules are reported separately, and only while the platform is operational: a cycle counts as a + schedules success only when every visited tenant's schedule scan ran and completed. + """ + operational = self.operational() + if operational: + self.health.started("schedules") + try: + recovered, scheduled = await self._traverse() + except Exception: + if operational: + self.health.failed("schedules") + raise + if operational: + if scheduled: + self.health.succeeded("schedules") + else: + self.health.failed("schedules") + return recovered + + async def _traverse(self) -> tuple[bool, bool]: assert self.engine is not None + recovered = scheduled = True async with RECOVERY_EXECUTION.execution(): budget = min(self.settings.database_timeout_seconds, 5.0) async with asyncio.timeout(budget): @@ -120,8 +148,11 @@ async def cycle(self) -> None: logging.getLogger(__name__).error( "Tenant recovery scan failed (%s); traversal will continue", failure_cause(error) ) + # The tenant's schedules did not run in this cycle either. + recovered = scheduled = False continue if not self.operational(): + scheduled = False continue try: async with asyncio.timeout(budget): @@ -130,15 +161,24 @@ async def cycle(self) -> None: logging.getLogger(__name__).error( "Tenant schedule scan failed (%s); traversal will continue", failure_cause(error) ) + scheduled = False + return recovered, scheduled async def poll(self) -> None: while True: + self.health.started("recovery_loop") try: - await self.cycle() + recovered = await self.cycle() except Exception as error: logging.getLogger(__name__).error( "Recovery catalog scan failed (%s); retrying on next poll", failure_cause(error) ) + recovered = False + # A failure a cycle caught and logged is still a failed cycle, never a success. + if recovered: + self.health.succeeded("recovery_loop") + else: + self.health.failed("recovery_loop") await asyncio.sleep(self.settings.scheduler_poll_seconds) async def close(self) -> None: @@ -147,6 +187,8 @@ async def close(self) -> None: with suppress(asyncio.CancelledError): await self.task self.task = None + for name in ("recovery_loop", "schedules"): + self.health.running(name, False) if self.engine is not None: await self.engine.dispose() self.engine = None diff --git a/src/firefly_weave/triggers/kafka_loop.py b/src/firefly_weave/triggers/kafka_loop.py index 4e65cfc1..36a61c67 100644 --- a/src/firefly_weave/triggers/kafka_loop.py +++ b/src/firefly_weave/triggers/kafka_loop.py @@ -28,6 +28,7 @@ from firefly_weave.connectors.broker import BrokerConnectionConfig from firefly_weave.connectors.kafka_transport import BrokerClients, BrokerOwner from firefly_weave.operations.execution import ReservedSlots +from firefly_weave.operations.loop_health import TRAVERSAL_SECONDS, LoopHealth, track from firefly_weave.persistence.migrations import check_schema from firefly_weave.persistence.uow import UnitOfWork from firefly_weave.settings import Settings @@ -39,11 +40,22 @@ class KafkaLoop: - def __init__(self, settings: Settings, uow: UnitOfWork, triggers: KafkaTrigger, clients: BrokerClients) -> None: + def __init__( + self, + settings: Settings, + uow: UnitOfWork, + triggers: KafkaTrigger, + clients: BrokerClients, + *, + health: LoopHealth | None = None, + ) -> None: self.settings, self.uow, self.triggers, self.clients = settings, uow, triggers, clients + self.health = health if health is not None else LoopHealth() self.engine: AsyncEngine | None = None self.task: asyncio.Task[None] | None = None self.turns: set[asyncio.Task[None]] = set() + # A running consumer task does not prove a connected source: a failed turn fails the next cycle. + self.turn_failed = False async def open(self) -> None: if not self.settings.broker.enabled or not self.settings.kafka_consumer_enabled: @@ -73,16 +85,19 @@ async def open(self) -> None: # A request can open this loop. An empty context keeps the request's execution # lease and identity, which end with its response, out of the long-lived task. self.task = asyncio.create_task(self.poll(), name="weave-broker-traversal", context=Context()) + track(self.health, self.task, "kafka_loop") except BaseException: await self.close() raise - async def cycle(self) -> None: + async def cycle(self) -> bool: + """One bounded traversal turn; False when a consumer turn failed since the previous cycle.""" + healthy, self.turn_failed = not self.turn_failed, False if ( len(self.turns) >= self.settings.broker.max_clients - 1 or len(self.clients.owners) >= self.settings.broker.max_clients - 1 ): - return + return healthy assert self.engine is not None async with asyncio.timeout(5): async with self.engine.begin() as connection: @@ -90,15 +105,16 @@ async def cycle(self) -> None: await connection.execute(text("SET LOCAL lock_timeout='1000ms'")) tenant = await connection.scalar(text("SELECT weave_broker_tenants(1)")) if tenant is None: - return + return healthy authority = await next_scope(self.uow, tenant, broker=True) if authority is None: - return + return healthy source = await self.triggers.scheduled_claim(authority) if source is not None: task = asyncio.create_task(self.turn(source), name="weave-kafka-consumer") self.turns.add(task) task.add_done_callback(self.turns.discard) + return healthy async def turn(self, source: KafkaSource) -> None: owner: BrokerOwner | None = None @@ -149,6 +165,7 @@ async def monitor() -> None: except asyncio.CancelledError: raise except Exception: + self.turn_failed = True logging.getLogger(__name__).error( "Broker consumer turn failed; source authority and replay remain required" ) @@ -163,11 +180,17 @@ async def monitor() -> None: async def poll(self) -> None: while True: + self.health.started("kafka_loop") try: - await self.cycle() + healthy = await self.cycle() except Exception: logging.getLogger(__name__).error("Broker traversal failed; retrying bounded turn") - await asyncio.sleep(1) + healthy = False + if healthy: + self.health.succeeded("kafka_loop") + else: + self.health.failed("kafka_loop") + await asyncio.sleep(TRAVERSAL_SECONDS) async def close(self) -> None: if self.task: @@ -175,6 +198,7 @@ async def close(self) -> None: with suppress(asyncio.CancelledError): await self.task self.task = None + self.health.running("kafka_loop", False) turns = tuple(self.turns) for task in turns: task.cancel() diff --git a/tests/integration/test_operations_cache_refresh.py b/tests/integration/test_operations_cache_refresh.py new file mode 100644 index 00000000..b590e105 --- /dev/null +++ b/tests/integration/test_operations_cache_refresh.py @@ -0,0 +1,812 @@ +# Copyright 2026 Firefly Software Foundation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# Author: Firefly Software Foundation +# SPDX-License-Identifier: Apache-2.0 + +"""The operations loop revalidates cached run listing decisions in bounded, resumable quanta on real PostgreSQL. + +Runs pin ``text.concat``. A server rolled back below it is simulated by advertising no language features; +the real retained artifact and runtime admission decide every classification. +""" + +import json +import time +from collections import Counter +from contextlib import asynccontextmanager +from contextvars import ContextVar +from uuid import UUID + +import pytest +from pydantic import SecretStr +from sqlalchemy import event, text +from sqlalchemy.ext.asyncio import create_async_engine + +from firefly_weave.access.audit import AuditContext +from firefly_weave.access.authorization import AuthorizationService +from firefly_weave.contracts import language_features +from firefly_weave.contracts.access import Scope +from firefly_weave.contracts.run_views import RunStepQuery, RunSummaryQuery +from firefly_weave.definitions.models import CatalogError +from firefly_weave.operations.facts import FactRepository, listing_metadata +from firefly_weave.operations.loop import OperationsLoop +from firefly_weave.operations.loop_health import LoopHealth +from firefly_weave.operations.run_views import RunViewService +from firefly_weave.persistence.uow import UnitOfWork +from firefly_weave.runtime import admission +from firefly_weave.runtime.repository import RuntimeRepository +from firefly_weave.settings import Settings + +pytestmark = pytest.mark.integration + +GREETING = {"op": {"name": "concat", "args": [{"literal": "n="}, {"ref": "/input"}]}} +CLASSIFICATION = "classification_state,classification_policy,pinned_ir_version,pinned_features,node_kinds" +# Ordinary storage saturated: a write that grows a row fails, and one that shrinks it still succeeds. +SATURATE = text("UPDATE operation_policy SET limits=jsonb_set(limits,'{ordinary_bytes}','0')") +RESTORE = text("UPDATE operation_policy SET limits=jsonb_set(limits,'{ordinary_bytes}','1099511627776')") + + +def ids(prefix: int, count: int, start: int = 1) -> list[UUID]: + return [UUID(f"{prefix:08x}-0000-4000-8000-{index:012d}") for index in range(start, start + count)] + + +@pytest.fixture +def worker_runtime_fixture(worker_runtime_fixture): + """The shared worker flow whose task input is built by ``text.concat``.""" + document = json.loads(worker_runtime_fixture["source"]) + document["metadata"]["name"] = "greeting-flow" + document["spec"]["steps"][0]["with"] = GREETING + return {**worker_runtime_fixture, "source": json.dumps(document)} + + +@pytest.fixture +async def refresh(operations_case, scheduler_url): + """An operations loop on the execute-only scheduler login, cycled by the test instead of its poll task.""" + case = operations_case + health = LoopHealth() + health.register("operations_loop", 60, enabled=True) + loop = OperationsLoop( + Settings(scheduler_enabled=False, database_url=case.url.render_as_string(hide_password=False)), + UnitOfWork(case.sessions), + health, + ) + loop.engine = create_async_engine(scheduler_url) + try: + yield loop + finally: + await loop.engine.dispose() + + +class Reads(Counter): + """Retained sources fetched by listing-metadata refreshes, by run, and any fetch of a run already refreshed.""" + + after_success: list[UUID] + + +@pytest.fixture +def payload_reads(monkeypatch): + """Every retained source a listing-metadata refresh fetches, by run. + + A refresh cut short by the end of its quantum's budget rolls back and is retried after the keyset wraps, so a + run can be read again before its decision is stored. A read of a run whose stored decision is no longer due + would repeat completed work; those are listed in ``after_success``. + """ + reads = Reads() + reads.after_success = [] + refreshing: ContextVar[bool] = ContextVar("refreshing", default=False) + fetch, refresh = RuntimeRepository.run, FactRepository.refresh_listing_metadata + + async def run(self, identifier, *, lock=False): + if refreshing.get(): + reads[identifier] += 1 + stored = ( + ( + await self.tx.session.execute( + text( + "SELECT classification_state,classification_policy,pinned_ir_version,pinned_features " + "FROM run_facts WHERE run_id=:id" + ), + {"id": identifier}, + ) + ) + .mappings() + .one() + ) + supported = admission.runnable_parameters() + due = stored["classification_policy"] != admission.POLICY or ( + stored["classification_state"] == "unsupported" + and stored["pinned_ir_version"] in supported["ir_versions"] + and set(stored["pinned_features"]) <= set(json.loads(supported["ir_features"])) + ) + if not due: + reads.after_success.append(identifier) + return await fetch(self, identifier, lock=lock) + + async def refreshed(self, run_id): + token = refreshing.set(True) + try: + return await refresh(self, run_id) + finally: + refreshing.reset(token) + + monkeypatch.setattr(RuntimeRepository, "run", run) + monkeypatch.setattr(FactRepository, "refresh_listing_metadata", refreshed) + return reads + + +@asynccontextmanager +async def rolled_back(): + """A server whose capabilities list no language features.""" + with pytest.MonkeyPatch.context() as patch: + patch.setattr(language_features, "ADVERTISED_FEATURES", ()) + yield + + +async def source(case, run: UUID) -> dict: + async with case.owner() as session: + return dict((await session.execute(text("SELECT * FROM runs WHERE id=:id"), {"id": run})).mappings().one()) + + +async def cached(case, run: UUID) -> dict: + async with case.owner() as session: + return dict( + (await session.execute(text(f"SELECT {CLASSIFICATION} FROM run_facts WHERE run_id=:id"), {"id": run})) + .mappings() + .one() + ) + + +async def states(case, runs) -> dict[UUID, str]: + async with case.owner() as session: + rows = await session.execute( + text("SELECT run_id,classification_state FROM run_facts WHERE run_id=ANY(:ids)"), {"ids": list(runs)} + ) + return dict(rows.all()) + + +async def record_decision(case, run: UUID, metadata: dict) -> None: + """Store a listing decision the way the upgrade backfill records one.""" + async with case.owner.begin() as session: + await session.execute( + text( + "UPDATE run_facts SET classification_state=:classification_state," + "classification_policy=:classification_policy,pinned_ir_version=:pinned_ir_version," + "pinned_features=cast(:pinned_features AS jsonb),node_kinds=cast(:node_kinds AS jsonb) " + "WHERE run_id=:run" + ), + { + **metadata, + "pinned_features": json.dumps(metadata["pinned_features"]), + "node_kinds": json.dumps(metadata["node_kinds"]), + "run": run, + }, + ) + + +async def record_rolled_back_decision(case, run: UUID) -> dict: + """What a server rolled back below ``text.concat`` records for the run: a transient unsupported decision.""" + row = await source(case, run) + async with rolled_back(): + assert admission.admission(row) == "unsupported" + metadata = listing_metadata(row) + assert metadata["classification_state"] == "unsupported" and metadata["pinned_features"] == ["text.concat"] + await record_decision(case, run, metadata) + return metadata + + +async def cancelled_source(case) -> UUID: + run = await case.start("source") + async with case.tx() as tx: + await case.runtime.cancel(tx, run.id, "fixture", actor=case.actor, scope=case.scope, context=AuditContext()) + return run.id + + +async def clone(case, run: UUID, identifiers, *, environment=None, activation=None, future=False, node_kinds=None): + """Retained copies of a real run, each cached as unsupported like a backfill on a rolled-back server.""" + artifact = "jsonb_set(r.artifact,'{executable,irVersion}','\"weave/ir-future\"')" if future else "r.artifact" + async with case.owner.begin() as session: + await session.execute( + text( + "INSERT INTO runs SELECT (jsonb_populate_record(NULL::runs,to_jsonb(r) || jsonb_build_object(" + "'id',c.id,'environment_id',coalesce(cast(:environment AS uuid),r.environment_id)," + "'activation_id',coalesce(cast(:activation AS uuid),r.activation_id),'artifact'," + artifact + "))).* " + "FROM runs r CROSS JOIN unnest(cast(:ids AS uuid[])) AS c(id) WHERE r.id=:run" + ), + {"ids": list(identifiers), "run": run, "environment": environment, "activation": activation}, + ) + await session.execute( + text( + "INSERT INTO run_facts SELECT (jsonb_populate_record(NULL::run_facts,to_jsonb(f) || jsonb_build_object(" + "'run_id',c.id,'environment_id',coalesce(cast(:environment AS uuid),f.environment_id)," + "'activation_id',coalesce(cast(:activation AS uuid),f.activation_id)," + "'classification_state','unsupported','pinned_features','[\"text.concat\"]'::jsonb," + "'pinned_ir_version',CASE WHEN :future THEN 'weave/ir-future' ELSE f.pinned_ir_version END," + "'node_kinds',coalesce(cast(:node_kinds AS jsonb),'{}'::jsonb)))).* " + "FROM run_facts f CROSS JOIN unnest(cast(:ids AS uuid[])) AS c(id) WHERE f.run_id=:run" + ), + { + "ids": list(identifiers), + "run": run, + "environment": environment, + "activation": activation, + "future": future, + "node_kinds": None if node_kinds is None else json.dumps(node_kinds), + }, + ) + + +async def second_environment(case, access_db, provisioned) -> tuple[Scope, UUID]: + """Another environment of the same tenant and project, with a copy of the activation its runs pin.""" + admin = await access_db[2].load_principal(provisioned[0].id) + environment = await access_db[2].create_environment( + admin, Scope(tenant_id=case.scope.tenant_id, project_id=case.scope.project_id), "second" + ) + activation = UUID("a0000000-0000-4000-8000-000000000001") + async with case.owner.begin() as session: + await session.execute( + text( + "INSERT INTO activation_revisions SELECT (jsonb_populate_record(NULL::activation_revisions," + "to_jsonb(a) || jsonb_build_object('id',cast(:new AS uuid),'environment_id',cast(:environment AS uuid)" + "))).* FROM activation_revisions a WHERE a.id=:activation" + ), + {"new": activation, "environment": environment, "activation": case.activation.id}, + ) + return case.scope.model_copy(update={"environment_id": environment}), activation + + +async def cursors(case) -> dict: + async with case.owner() as session: + return { + "facts": sorted( + (await session.execute(text("SELECT environment_id,after_run_id FROM operations_fact_cursors"))).all() + ), + "environments": sorted( + ( + await session.execute(text("SELECT tenant_id,environment_id FROM operations_environment_cursors")) + ).all() + ), + "tenants": (await session.execute(text("SELECT tenant_id FROM operations_tenant_cursor"))).all(), + } + + +async def cache(case) -> list: + async with case.owner() as session: + return sorted( + (await session.execute(text(f"SELECT run_id,{CLASSIFICATION} FROM run_facts ORDER BY run_id"))).all() + ) + + +def service(case) -> RunViewService: + return RunViewService(UnitOfWork(case.sessions), AuthorizationService()) + + +async def summary(case, run: UUID): + page = await service(case).summaries( + case.actor, case.scope, RunSummaryQuery(limit=100), None, context=AuditContext() + ) + [item] = [item for item in page.items if item.id == run] + return item + + +async def http_reads_leave_the_cache(case, authenticated_client, run: UUID) -> None: + """Metadata reads over HTTP select no payload and change no cached decision or cursor.""" + client, tokens, _ = authenticated_client + app = client._transport.app + scope = case.scope + prefix = f"/api/v1/tenants/{scope.tenant_id}/projects/{scope.project_id}/environments/{scope.environment_id}" + statements = [] + + def capture(connection, cursor, statement, parameters, context, executemany): + statements.append(" ".join(statement.lower().split())) + + before = (await cache(case), await cursors(case)) + engine = app.state.resources.engine.sync_engine + event.listen(engine, "before_cursor_execute", capture) + try: + headers = {"Authorization": "Bearer " + tokens[0]} + assert (await client.get(prefix + "/run-summaries?limit=100", headers=headers)).status_code == 200 + assert (await client.get(prefix + f"/runs/{run}/steps", headers=headers)).status_code in {200, 409, 422} + finally: + event.remove(engine, "before_cursor_execute", capture) + assert (await cache(case), await cursors(case)) == before + assert any("from run_facts" in sql for sql in statements) + assert not any("from runs " in sql or "runs.*" in sql or "weave_runs_bytes" in sql for sql in statements) + assert not any( + "run_facts set" in sql or "operations_fact_cursors" in sql or "operations_environment_cursors" in sql + for sql in statements + ) + + +async def test_operations_traversal_is_execute_only_and_never_moves_recovery_cursors( + operations_case, refresh, scheduler_url, monkeypatch +): + case = operations_case + await case.start() + health = LoopHealth() + health.register("operations_loop", 60, enabled=True) + + async def idle(): + return None + + refused = OperationsLoop( + Settings( + database_url=case.url.render_as_string(hide_password=False), + scheduler_database_url=SecretStr(case.url.render_as_string(hide_password=False)), + ), + UnitOfWork(case.sessions), + health, + ) + with pytest.raises(RuntimeError, match="execute-only"): + await refused.open() + assert refused.engine is None and refused.task is None + accepted = OperationsLoop( + Settings( + database_url=case.url.render_as_string(hide_password=False), + scheduler_database_url=SecretStr(scheduler_url), + ), + UnitOfWork(case.sessions), + health, + ) + monkeypatch.setattr(accepted, "poll", idle) + await accepted.open() + await accepted.close() + async with case.owner.begin() as session: + recovery_before = ( + (await session.execute(text("SELECT tenant_id FROM scheduler_tenant_cursor"))).all(), + sorted((await session.execute(text("SELECT * FROM scheduler_environment_cursors"))).all()), + ) + await session.execute(text("SET LOCAL ROLE weave_scheduler")) + for value in (None, 0, 17): + with pytest.raises(Exception, match="Bounded catalog page"): + async with session.begin_nested(): + await session.execute(text("SELECT weave_operations_tenants(:size)"), {"size": value}) + async with case.owner() as session: + assert not await session.scalar( + text("SELECT has_function_privilege('weave_app','weave_operations_tenants(integer)','EXECUTE')") + ) + assert not await session.scalar( + text( + "SELECT EXISTS(SELECT 1 FROM pg_trigger WHERE tgname='operation_usage' AND tgrelid IN " + "('operations_tenant_cursor'::regclass,'operations_environment_cursors'::regclass," + "'operations_fact_cursors'::regclass))" + ) + ) + index = await session.scalar(text("SELECT indexdef FROM pg_indexes WHERE indexname='run_facts_revalidation'")) + assert "(tenant_id, project_id, environment_id, run_id)" in index + assert "classification_state = 'unsupported'" in index + assert f"classification_policy <> '{admission.POLICY}'" in index + assert await refresh.cycle() is True + async with case.owner() as session: + recovery_after = ( + (await session.execute(text("SELECT tenant_id FROM scheduler_tenant_cursor"))).all(), + sorted((await session.execute(text("SELECT * FROM scheduler_environment_cursors"))).all()), + ) + assert recovery_after == recovery_before + moved = await cursors(case) + # Both provisioned tenants were visited through the operations cursors only. + assert len(moved["environments"]) == 2 and moved["tenants"][0][0] is not None + + +async def test_unsupported_fact_becomes_available_after_support_returns(operations_case, refresh, payload_reads): + case = operations_case + run = (await case.start()).id + await record_rolled_back_decision(case, run) + row = await source(case, run) + async with rolled_back(): + assert (await summary(case, run)).unavailable + # Still rolled back: the pins stay unsupported, so the run is not due and its source is never read. + assert await refresh.cycle() is True + assert payload_reads == Counter() and (await cached(case, run))["classification_state"] == "unsupported" + # Support returns: runtime admission runs the run again, while the cached decision still withholds it. + assert admission.admission(row) == "available" + assert (await summary(case, run)).unavailable + with pytest.raises(CatalogError) as withheld: + await service(case).steps(case.actor, case.scope, run, RunStepQuery(), None, context=AuditContext()) + assert (withheld.value.status, withheld.value.code) == (422, "WV-IR-UNSUPPORTED") + assert await refresh.cycle() is True + assert payload_reads == Counter({run: 1}) + refreshed = await cached(case, run) + assert refreshed["classification_state"] == "available" and refreshed["classification_policy"] == admission.POLICY + assert refreshed["node_kinds"] and refreshed["pinned_features"] == ["text.concat"] + assert not getattr(await summary(case, run), "unavailable", False) + page = await service(case).steps(case.actor, case.scope, run, RunStepQuery(), None, context=AuditContext()) + assert page.items + assert await refresh.cycle() is True + assert payload_reads == Counter({run: 1}) + + +async def test_policy_change_withholds_until_cache_is_revalidated(operations_case, refresh, payload_reads, monkeypatch): + case = operations_case + first = (await case.start("first")).id + monkeypatch.setattr(admission, "POLICY", "classified-v2") + assert (await summary(case, first)).unavailable + assert await refresh.cycle() is True + assert payload_reads == Counter({first: 1}) + assert (await cached(case, first))["classification_policy"] == "classified-v2" + assert not getattr(await summary(case, first), "unavailable", False) + # A mismatch revalidated while rolled back becomes unsupported under the current policy and drops out. + second = (await case.start("second")).id + monkeypatch.setattr(admission, "POLICY", "classified-v3") + async with rolled_back(): + # Rows before the keyset boundary wait for its wrap, so a few cycles visit both runs once. + for _ in range(4): + assert await refresh.cycle() is True + assert payload_reads[second] == 1 and payload_reads[first] == 2 + assert (await cached(case, second))["classification_state"] == "unsupported" + for _ in range(2): + assert await refresh.cycle() is True + assert payload_reads[second] == 1 and payload_reads[first] == 2 + # Its pins are supported again, so it is due once more; a policy block still withholds it afterwards. + assert await refresh.cycle() is True + assert payload_reads[second] == 2 and (await cached(case, second))["classification_state"] == "available" + async with case.owner.begin() as session: + await session.execute( + text( + "INSERT INTO run_policy_blocks(tenant_id,project_id,environment_id,run_id,reason_code) " + "SELECT tenant_id,project_id,environment_id,id,'WV-LEGACY-UNAVAILABLE' FROM runs WHERE id=:run" + ), + {"run": second}, + ) + assert (await summary(case, second)).unavailable + + +async def test_cache_refresh_never_calls_runtime_start_or_persist(operations_case, refresh, monkeypatch): + from firefly_weave.runtime import kernel + from firefly_weave.runtime.service import RuntimeService + + case = operations_case + run = (await case.start()).id + await record_rolled_back_decision(case, run) + + async def forbidden(*args, **kwargs): + raise AssertionError("A cache refresh must not start, reduce or persist a run") + + for owner, name in ( + (RuntimeService, "start"), + (RuntimeRepository, "persist"), + (RuntimeRepository, "insert"), + (FactRepository, "project_run"), + (FactRepository, "project_terminal"), + (kernel, "transition"), + ): + monkeypatch.setattr(owner, name, forbidden) + + async def evidence(): + async with case.owner() as session: + return [ + await session.scalar(text("SELECT state FROM runs WHERE id=:id"), {"id": run}), + await session.scalar(text("SELECT count(*) FROM run_events WHERE run_id=:id"), {"id": run}), + await session.scalar(text("SELECT count(*) FROM task_intents WHERE run_id=:id"), {"id": run}), + await session.scalar(text("SELECT count(*) FROM runtime_capacity_blocks")), + await session.scalar(text("SELECT count(*) FROM run_policy_blocks")), + await session.scalar(text("SELECT count(*) FROM incidents")), + ] + + before = await evidence() + assert await refresh.cycle() is True + assert (await cached(case, run))["classification_state"] == "available" + assert await evidence() == before + + +async def test_a_refresh_refused_for_capacity_never_blocks_the_run(operations_case, refresh, monkeypatch): + from firefly_weave.operations import facts + from firefly_weave.runtime.capacity import RuntimeCapacityError + + case = operations_case + run = (await case.start()).id + await record_rolled_back_decision(case, run) + classify = facts.listing_metadata + + def refused(row): + # Classification meets a capacity limit; the unit of work then refuses the whole refresh. + RuntimeCapacityError() + return classify(row) + + async def blocks(): + async with case.owner() as session: + return (await session.execute(text("SELECT * FROM runtime_capacity_blocks ORDER BY run_id"))).all() + + before = await blocks() + monkeypatch.setattr(facts, "listing_metadata", refused) + assert await refresh.cycle() is False + assert (await cached(case, run))["classification_state"] == "unsupported" + # No capacity block is raised against the run, so its work goes on. + assert await blocks() == before + async with case.owner() as session: + assert not await session.scalar(text("SELECT count(*) FROM runtime_capacity_blocks WHERE active")) + assert await session.scalar( + text("SELECT count(*) FROM task_intents WHERE run_id=:id AND status='ready'"), {"id": run} + ) + + +async def test_cache_refresh_rolls_back_failed_row_and_keeps_prior_progress(operations_case, refresh, payload_reads): + case = operations_case + origin = await cancelled_source(case) + final = (await cached(case, origin))["node_kinds"] + first, failing, last = ids(0x20000000, 3) + # Only the middle row grows when refreshed; the others shrink and never need storage. + await clone(case, origin, [first, last], node_kinds=final) + await clone(case, origin, [failing]) + before = await cached(case, failing) + async with case.owner.begin() as session: + await session.execute(SATURATE) + assert await refresh.cycle() is False + assert await states(case, [first, failing, last]) == {first: "available", failing: "unsupported", last: "available"} + assert await cached(case, failing) == before + assert payload_reads == Counter({first: 1, failing: 1, last: 1}) + assert (await cursors(case))["facts"] == [(case.scope.environment_id, last)] + async with case.owner.begin() as session: + await session.execute(RESTORE) + # Nothing is due after the cursor: the environment wraps, and the next cycle retries the failed row. + assert await refresh.cycle() is True + assert (await cursors(case))["facts"] == [(case.scope.environment_id, None)] + assert await refresh.cycle() is True + assert (await states(case, [failing]))[failing] == "available" + assert payload_reads == Counter({first: 1, failing: 2, last: 1}) + + +async def test_an_oversized_source_is_recorded_unavailable_without_being_read(operations_case, refresh, payload_reads): + from firefly_weave.runtime import repository + + case = operations_case + run = (await case.start()).id + metadata = await record_rolled_back_decision(case, run) + async with case.owner() as session: + size = await session.scalar(text("SELECT public.weave_runs_bytes(r) FROM runs r WHERE id=:id"), {"id": run}) + with pytest.MonkeyPatch.context() as patch: + patch.setattr(repository, "RUN_FETCH_BYTES", size - 1) + assert await refresh.cycle() is True + assert payload_reads == Counter() + assert await cached(case, run) == { + **metadata, + "classification_state": "unavailable", + "classification_policy": admission.POLICY, + } + with pytest.raises(CatalogError) as withheld: + await service(case).steps(case.actor, case.scope, run, RunStepQuery(), None, context=AuditContext()) + assert withheld.value.code == "WV-LEGACY-UNAVAILABLE" + # A permanent decision is not due again, even once the source fits the fetch budget. + assert await refresh.cycle() is True + assert payload_reads == Counter() and (await cached(case, run))["classification_state"] == "unavailable" + + +async def test_cache_refresh_does_not_change_operational_updated_at(operations_case, refresh): + case = operations_case + run = (await case.start()).id + await record_rolled_back_decision(case, run) + columns = ( + "status,paused,test,origin,started_at,updated_at,ended_at,failed_node_id,failed_instance_key," + "failed_error_code,active_incidents,handled_errors,last_event_sequence,facts_started_at,business_key," + "correlation_key,workflow_name,workflow_version,activation_id,definition_version_id" + ) + + async def operational(): + async with case.owner() as session: + return ( + dict( + (await session.execute(text(f"SELECT {columns} FROM run_facts WHERE run_id=:id"), {"id": run})) + .mappings() + .one() + ), + await session.scalar(text("SELECT state FROM runs WHERE id=:id"), {"id": run}), + ) + + before = await operational() + assert await refresh.cycle() is True + assert (await cached(case, run))["classification_state"] == "available" + assert await operational() == before + + +async def due_progress(refresh, case, due, *, cycles, done) -> int: + for turn in range(1, cycles + 1): + await refresh.cycle() + if await done(): + return turn + remaining = Counter((await states(case, due)).values()) + raise AssertionError(f"Revalidation did not finish in {cycles} cycles: {remaining}") + + +async def test_cache_refresh_advances_across_250_due_rows_and_two_environments( + operations_case, refresh, payload_reads, access_db, provisioned, authenticated_client +): + case = operations_case + origin = await cancelled_source(case) + other, activation = await second_environment(case, access_db, provisioned) + still_unsupported = ids(0x10000000, 120) + due = ids(0x20000000, 250) + elsewhere = ids(0x30000000, 40) + # A leading batch whose pins stay unsupported precedes every due row in UUID order. + await clone(case, origin, still_unsupported, future=True) + await clone(case, origin, due) + await clone(case, origin, elsewhere, environment=other.environment_id, activation=activation) + await http_reads_leave_the_cache(case, authenticated_client, due[0]) + + async def finished() -> bool: + await http_reads_leave_the_cache(case, authenticated_client, due[0]) + return set((await states(case, due + elsewhere)).values()) == {"available"} + + turns = await due_progress(refresh, case, due + elsewhere, cycles=80, done=finished) + # Every due row was refreshed and none was read once its decision was stored; the still-unsupported rows never. + assert set(payload_reads) == set(due + elsewhere) and payload_reads.after_success == [] + assert set((await states(case, still_unsupported)).values()) == {"unsupported"} + environments = {environment for environment, _ in (await cursors(case))["facts"]} + assert environments == {case.scope.environment_id, other.environment_id} + settled = Counter(payload_reads) + for _ in range(4): + assert await refresh.cycle() is True + # Completed rows are never read again. + assert payload_reads == settled and payload_reads.after_success == [] + assert turns >= 4 + + +@pytest.mark.parametrize("failure", ["timeout", "quota"]) +async def test_failed_first_cache_candidate_does_not_starve_later_rows( + operations_case, refresh, payload_reads, access_db, provisioned, authenticated_client, monkeypatch, failure +): + case = operations_case + origin = await cancelled_source(case) + final = (await cached(case, origin))["node_kinds"] + other, activation = await second_environment(case, access_db, provisioned) + [failing] = ids(0x18000000, 1) + later = ids(0x20000000, 250) + elsewhere = ids(0x30000000, 20) + await clone(case, origin, ids(0x10000000, 10), future=True) + await clone(case, origin, [failing]) + await clone(case, origin, later, node_kinds=final) + await clone(case, origin, elsewhere, environment=other.environment_id, activation=activation, node_kinds=final) + attempts: list[UUID] = [] + original = FactRepository.refresh_listing_metadata + + async def attempt(self, run): + attempts.append(run) + return await original(self, run) + + monkeypatch.setattr(FactRepository, "refresh_listing_metadata", attempt) + wraps = [] + selection = FactRepository.revalidation_candidates + + async def candidates(self, after): + found = await selection(self, after) + if not found and after is not None: + wraps.append(self.tx.scope.environment_id) + return found + + monkeypatch.setattr(FactRepository, "revalidation_candidates", candidates) + lock = await case.owner.kw["bind"].connect() + transaction = await lock.begin() + try: + if failure == "timeout": + await lock.execute(text("SELECT 1 FROM runs WHERE id=:id FOR UPDATE"), {"id": failing}) + else: + async with case.owner.begin() as session: + await session.execute(SATURATE) + + async def finished() -> bool: + await http_reads_leave_the_cache(case, authenticated_client, later[0]) + return ( + set((await states(case, later + elsewhere)).values()) == {"available"} + and case.scope.environment_id in wraps + and attempts.count(failing) >= 2 + ) + + await due_progress(refresh, case, later + elsewhere, cycles=120, done=finished) + # The failed row stays withheld and was retried after the wrap; every later row advanced once. + assert (await states(case, [failing]))[failing] == "unsupported" + with pytest.raises(CatalogError) as withheld: + await service(case).steps(case.actor, case.scope, failing, RunStepQuery(), None, context=AuditContext()) + assert withheld.value.code == "WV-IR-UNSUPPORTED" + assert all(payload_reads[run] >= 1 for run in later + elsewhere) and payload_reads.after_success == [] + assert attempts.index(failing) < attempts.index(later[0]) + finally: + await transaction.rollback() + await lock.close() + if failure == "quota": + async with case.owner.begin() as session: + await session.execute(RESTORE) + retried = attempts.count(failing) + for _ in range(10): + await refresh.cycle() + if (await states(case, [failing]))[failing] == "available": + break + assert (await states(case, [failing]))[failing] == "available" and attempts.count(failing) > retried + + +async def test_environment_overrun_keeps_its_cursor_and_the_next_environment_progresses( + operations_case, refresh, payload_reads, access_db, provisioned, monkeypatch +): + case = operations_case + origin = await cancelled_source(case) + other, activation = await second_environment(case, access_db, provisioned) + due = ids(0x20000000, 5) + elsewhere = ids(0x30000000, 5) + await clone(case, origin, due) + await clone(case, origin, elsewhere, environment=other.environment_id, activation=activation) + boundary = UUID("15000000-0000-4000-8000-000000000000") + async with case.owner.begin() as session: + await session.execute( + text("INSERT INTO operations_fact_cursors VALUES(:tenant,:project,:environment,:after)"), + { + "tenant": case.scope.tenant_id, + "project": case.scope.project_id, + "environment": case.scope.environment_id, + "after": boundary, + }, + ) + offset = {"seconds": 0.0, "overrun": False} + refresh.monotonic = lambda: time.monotonic() + offset["seconds"] + selection = FactRepository.revalidation_candidates + + async def candidates(self, after): + found = await selection(self, after) + if self.tx.scope.environment_id == case.scope.environment_id and not offset["overrun"]: + # The first environment's database work takes longer than its whole budget. + offset["overrun"] = True + offset["seconds"] += 1.5 + return found + + monkeypatch.setattr(FactRepository, "revalidation_candidates", candidates) + results = [] + while not offset["overrun"]: + results.append(await refresh.cycle()) + assert len(results) <= 2 + # The overrun quantum fails its cycle, reads nothing and keeps its committed cursor. + assert results[-1] is False + assert not any(payload_reads[run] for run in due) + assert (case.scope.environment_id, boundary) in (await cursors(case))["facts"] + environment_turn = [ + environment for tenant, environment in (await cursors(case))["environments"] if tenant == case.scope.tenant_id + ] + assert environment_turn == [case.scope.environment_id] + # The traversal moved past the overrun environment, so the other one is next and progresses. + results.append(await refresh.cycle()) + assert all(payload_reads[run] >= 1 for run in elsewhere) + for _ in range(3): + results.append(await refresh.cycle()) + assert set((await states(case, due + elsewhere)).values()) == {"available"} + assert set(payload_reads) == set(due + elsewhere) and payload_reads.after_success == [] + + +async def test_each_quantum_lowers_its_database_timeouts_to_the_remaining_budget(operations_case, refresh, monkeypatch): + case = operations_case + run = (await case.start()).id + await record_rolled_back_decision(case, run) + observed = [] + + async def settings(tx): + observed.append( + tuple( + [ + await tx.session.scalar(text("SELECT current_setting(:name)"), {"name": name}) + for name in ("statement_timeout", "lock_timeout") + ] + ) + ) + + for name in ("revalidation_candidates", "advance_revalidation_cursor", "refresh_listing_metadata"): + original = getattr(FactRepository, name) + + def observing(original): + async def method(self, *args): + await settings(self.tx) + return await original(self, *args) + + return method + + monkeypatch.setattr(FactRepository, name, observing(original)) + assert await refresh.cycle() is True + assert len(observed) >= 3 + + def milliseconds(value: str) -> int: + return int(value[:-2]) if value.endswith("ms") else int(value[:-1]) * 1000 + + assert all(0 < milliseconds(value) <= 1000 for pair in observed for value in pair) diff --git a/tests/unit/operations/test_loop_context.py b/tests/unit/operations/test_loop_context.py index 2fd06ffb..4a492fe5 100644 --- a/tests/unit/operations/test_loop_context.py +++ b/tests/unit/operations/test_loop_context.py @@ -31,8 +31,11 @@ from firefly_weave.contracts.access import Scope from firefly_weave.definitions.models import CatalogError from firefly_weave.operations import compatibility, outbox_loop +from firefly_weave.operations import loop as operations_loop from firefly_weave.operations.compatibility import CompatibilityService from firefly_weave.operations.execution import debug_execution, execute_pure, request_execution +from firefly_weave.operations.loop import OperationsLoop +from firefly_weave.operations.loop_health import LoopHealth from firefly_weave.operations.outbox_loop import OutboxLoop from firefly_weave.providers import loop as provider_loop from firefly_weave.providers.loop import ProviderLoop @@ -133,6 +136,13 @@ def provider(monkeypatch, probe): return owner +def operations(monkeypatch, probe): + _without_database(monkeypatch, operations_loop) + owner = OperationsLoop(_settings(), None, LoopHealth()) # type: ignore[arg-type] + monkeypatch.setattr(owner, "poll", probe) + return owner + + def native(monkeypatch, probe): release = SimpleNamespace( id=uuid4(), image_digest=DIGEST, connector_bindings=[SimpleNamespace(task_reference=TASK, adapter="http")] @@ -199,7 +209,7 @@ def inventory_freshness(monkeypatch, probe): return owner -@pytest.mark.parametrize("owner", [recovery, outbox, kafka, provider, native, inventory_freshness]) +@pytest.mark.parametrize("owner", [recovery, outbox, kafka, provider, operations, native, inventory_freshness]) async def test_task_opened_during_a_request_runs_pure_work_after_the_request_ends(monkeypatch, owner): probe = Probe() opened = owner(monkeypatch, probe) diff --git a/tests/unit/operations/test_loop_health.py b/tests/unit/operations/test_loop_health.py new file mode 100644 index 00000000..f10de78d --- /dev/null +++ b/tests/unit/operations/test_loop_health.py @@ -0,0 +1,426 @@ +# Copyright 2026 Firefly Software Foundation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# Author: Firefly Software Foundation +# SPDX-License-Identifier: Apache-2.0 + +"""Background loop health on a fake clock: a loop is healthy only after it proved a recent cycle.""" + +import asyncio +from contextlib import asynccontextmanager +from datetime import UTC, datetime, timedelta, timezone +from types import SimpleNamespace +from uuid import uuid4 + +import pytest + +from firefly_weave.connectors.broker import BrokerPolicy +from firefly_weave.contracts.access import Scope +from firefly_weave.operations.loop_health import COMPONENTS, LoopHealth, register_background_loops, utc_now +from firefly_weave.settings import Settings + +DATABASE = "postgresql+asyncpg://weave_app:unused@127.0.0.1:65432/weave" +SCHEDULER = "postgresql+asyncpg://weave_scheduler:unused@127.0.0.1:65432/weave" + + +class Clock: + def __init__(self) -> None: + self.now = datetime(2026, 10, 9, tzinfo=UTC) + + def __call__(self) -> datetime: + return self.now + + +def running(clock: Clock, name: str = "outbox_loop", interval: float = 5) -> LoopHealth: + health = LoopHealth(clock) + health.register(name, interval, enabled=True) + health.running(name, True) + return health + + +def test_loop_health_never_confuses_disabled_and_healthy(): + now = datetime(2026, 10, 9, tzinfo=UTC) + health = LoopHealth(lambda: now) + health.register("operations_loop", 60, enabled=False) + assert health.snapshot("operations_loop").state == "disabled" + health.register("outbox_loop", 5, enabled=True) + assert health.snapshot("outbox_loop").state == "down" + health.running("outbox_loop", True) + health.started("outbox_loop") + health.succeeded("outbox_loop") + assert health.snapshot("outbox_loop").state == "ok" + now += timedelta(seconds=16) + assert health.snapshot("outbox_loop").state == "degraded" + + +def test_a_running_loop_without_a_success_is_degraded_never_ok(): + clock = Clock() + health = running(clock) + assert health.snapshot("outbox_loop").state == "degraded" + health.started("outbox_loop") + snapshot = health.snapshot("outbox_loop") + assert (snapshot.state, snapshot.last_started_at, snapshot.last_ok_at) == ("degraded", clock.now, None) + + +def test_exactly_three_consecutive_failures_degrade_and_a_successful_empty_scan_clears_them(): + clock = Clock() + health = running(clock) + health.succeeded("outbox_loop") + for expected in ("ok", "ok", "degraded"): + health.failed("outbox_loop") + assert health.snapshot("outbox_loop").state == expected + assert health.snapshot("outbox_loop").consecutive_failures == 3 + health.succeeded("outbox_loop") + snapshot = health.snapshot("outbox_loop") + assert (snapshot.state, snapshot.consecutive_failures) == ("ok", 0) + + +def test_last_success_counts_until_three_intervals_have_passed(): + clock = Clock() + health = running(clock) + health.succeeded("outbox_loop") + clock.now += timedelta(seconds=15) + assert health.snapshot("outbox_loop").state == "ok" + clock.now += timedelta(microseconds=1) + assert health.snapshot("outbox_loop").state == "degraded" + + +def test_a_stopped_loop_is_down_whatever_it_did_before(): + clock = Clock() + health = running(clock) + health.succeeded("outbox_loop") + health.running("outbox_loop", False) + snapshot = health.snapshot("outbox_loop") + assert (snapshot.state, snapshot.last_ok_at) == ("down", clock.now) + + +def test_times_are_aware_utc_and_complete_until_is_kept(): + offset = datetime(2026, 10, 9, 2, tzinfo=timezone(timedelta(hours=2))) + health = LoopHealth(lambda: offset) + health.register("operations_loop", 60, enabled=True) + health.running("operations_loop", True) + health.started("operations_loop") + health.succeeded("operations_loop", complete_until=offset - timedelta(minutes=5)) + snapshot = health.snapshot("operations_loop") + assert snapshot.last_ok_at == datetime(2026, 10, 9, tzinfo=UTC) and snapshot.last_ok_at.tzinfo is UTC + assert snapshot.complete_until == datetime(2026, 10, 8, 23, 55, tzinfo=UTC) + health.succeeded("operations_loop") + assert health.snapshot("operations_loop").complete_until is None + assert utc_now().tzinfo is UTC + + +def test_updates_never_raise_into_the_loop(): + def broken(): + raise RuntimeError("clock unavailable") + + health = LoopHealth(broken) + health.register("outbox_loop", 1, enabled=True) + health.running("outbox_loop", True) + health.started("outbox_loop") + health.succeeded("outbox_loop") + health.failed("outbox_loop") + naive = LoopHealth(lambda: datetime(2026, 10, 9)) + naive.register("outbox_loop", 1, enabled=True) + naive.succeeded("outbox_loop") + naive.running("outbox_loop", True) + assert naive.snapshot("outbox_loop").last_ok_at is None + # Unregistered components are not invented by an update. + health.started("alert_evaluator") + health.failed("recovery_loop") + assert [snapshot.component for snapshot in health.snapshots()] == ["outbox_loop"] + + +def test_only_fixed_components_with_a_positive_interval_register(): + health = LoopHealth() + for name, interval in (("tenant-1", 1), ("outbox_loop", 0), ("outbox_loop", float("inf")), ("outbox_loop", True)): + with pytest.raises(ValueError): + health.register(name, interval, enabled=True) + with pytest.raises(ValueError): + health.register("outbox_loop", 1, enabled="yes") # type: ignore[arg-type] + with pytest.raises(KeyError): + health.snapshot("outbox_loop") + + +def test_registering_again_keeps_what_the_loop_reported(): + clock = Clock() + health = running(clock) + health.succeeded("outbox_loop") + health.failed("outbox_loop") + health.register("outbox_loop", 1, enabled=True) + snapshot = health.snapshot("outbox_loop") + assert (snapshot.last_ok_at, snapshot.consecutive_failures, snapshot.state) == (clock.now, 1, "ok") + + +def test_scheduler_disabled_marks_every_scheduler_loop_disabled(): + health = LoopHealth() + register_background_loops(health, Settings(database_url=DATABASE, scheduler_enabled=False)) + assert {snapshot.component: snapshot.state for snapshot in health.snapshots()} == { + name: "disabled" for name in COMPONENTS + } + enabled = LoopHealth() + register_background_loops( + enabled, + Settings( + database_url=DATABASE, + scheduler_database_url=SCHEDULER, + broker=BrokerPolicy(enabled=True, max_clients=2), + kafka_consumer_enabled=True, + ), + ) + states = {snapshot.component: snapshot.state for snapshot in enabled.snapshots()} + # Nothing has run yet: enabled loops are down until their task starts, and alerts do not exist yet. + assert states == {**{name: "down" for name in COMPONENTS}, "alert_evaluator": "disabled"} + + +class Traversal: + """Tenant pages and environment turns for the loops, without a database.""" + + def __init__(self, monkeypatch, module: str, tenants: int) -> None: + self.tenants = [uuid4() for _ in range(tenants)] + + async def tenant_page(engine, *args, **kwargs): + return list(self.tenants) + + async def next_scope(uow, tenant, **kwargs): + return SimpleNamespace(scope=Scope(tenant_id=tenant, project_id=uuid4(), environment_id=uuid4())) + + monkeypatch.setattr(f"{module}.tenant_page", tenant_page, raising=False) + monkeypatch.setattr(f"{module}.next_scope", next_scope) + + +def recovery_loop(monkeypatch, *, tenants=2, recovery_errors=(), schedule_errors=(), operational=True): + from firefly_weave.runtime.scheduler import RecoveryLoop + + Traversal(monkeypatch, "firefly_weave.runtime.scheduler", tenants) + recovery_errors, schedule_errors = list(recovery_errors), list(schedule_errors) + + async def recover(authority, limit): + if recovery_errors: + raise recovery_errors.pop(0) + + async def fire(authority, limit): + if schedule_errors: + raise schedule_errors.pop(0) + + health = LoopHealth() + for name in ("recovery_loop", "schedules"): + health.register(name, 0.01, enabled=True) + health.running(name, True) + loop = RecoveryLoop( + SimpleNamespace(database_timeout_seconds=5.0, scheduler_poll_seconds=0.01), + None, # type: ignore[arg-type] + SimpleNamespace(_scan_scheduled=recover, _scan_terminal_scheduled=recover), # type: ignore[arg-type] + SimpleNamespace(_scan_scheduled=fire), # type: ignore[arg-type] + operational=lambda: operational, + health=health, + ) + loop.engine = object() # type: ignore[assignment] + return loop, health + + +async def one_poll(loop, name: str) -> None: + """Run the loop's poll until it has recorded the outcome of one cycle, then cancel it as close() does.""" + before = loop.health.snapshot(name) + + def recorded() -> bool: + after = loop.health.snapshot(name) + return after.last_started_at is not None and ( + after.consecutive_failures != before.consecutive_failures or after.last_ok_at != before.last_ok_at + ) + + task = asyncio.create_task(loop.poll()) + try: + async with asyncio.timeout(2): + while not recorded(): + await asyncio.sleep(0) + finally: + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + + +async def test_a_swallowed_tenant_failure_is_a_failed_cycle_and_never_erases_degraded(monkeypatch): + loop, health = recovery_loop(monkeypatch, tenants=1, recovery_errors=[RuntimeError("tenant")] * 10) + for _ in range(3): + assert await loop.cycle() is False + health.failed("recovery_loop") + assert health.snapshot("recovery_loop").state == "degraded" + # The poll body records the swallowed failure of its next cycle instead of a success. + await one_poll(loop, "recovery_loop") + snapshot = health.snapshot("recovery_loop") + assert (snapshot.state, snapshot.last_ok_at) == ("degraded", None) and snapshot.consecutive_failures >= 4 + # A clean cycle, including one with nothing to scan, clears the failures. + loop, health = recovery_loop(monkeypatch, tenants=0) + await one_poll(loop, "recovery_loop") + assert health.snapshot("recovery_loop").state == "ok" + + +async def test_schedules_report_separately_from_recovery(monkeypatch): + loop, health = recovery_loop(monkeypatch, schedule_errors=[RuntimeError("schedule")]) + assert await loop.cycle() is True + assert health.snapshot("schedules").consecutive_failures == 1 + assert health.snapshot("schedules").last_started_at is not None + assert await loop.cycle() is True + assert health.snapshot("schedules").consecutive_failures == 0 + # Schedules a failed recovery scan skipped did not run, so that cycle is not a schedules success. + loop, health = recovery_loop(monkeypatch, recovery_errors=[RuntimeError("tenant")]) + assert await loop.cycle() is False + assert health.snapshot("schedules").consecutive_failures == 1 + # A restricted platform fires no schedules and records no schedule cycle at all. + loop, health = recovery_loop(monkeypatch, operational=False) + assert await loop.cycle() is True + snapshot = health.snapshot("schedules") + assert (snapshot.last_started_at, snapshot.last_ok_at, snapshot.consecutive_failures) == (None, None, 0) + + +async def test_a_failed_catalog_page_fails_recovery_and_schedules(monkeypatch): + loop, health = recovery_loop(monkeypatch) + + async def tenant_page(engine): + raise RuntimeError("catalog") + + monkeypatch.setattr("firefly_weave.runtime.scheduler.tenant_page", tenant_page) + await one_poll(loop, "recovery_loop") + assert health.snapshot("recovery_loop").consecutive_failures == 1 + assert health.snapshot("schedules").consecutive_failures == 1 + + +async def test_outbox_and_broker_turn_failures_fail_the_next_cycle(monkeypatch): + from firefly_weave.operations import outbox_loop + from firefly_weave.operations.outbox_loop import OutboxLoop + from firefly_weave.triggers.kafka_loop import KafkaLoop + + tenant = uuid4() + + class Connection: + async def execute(self, statement): + return None + + async def scalar(self, statement): + return tenant + + class Engine: + @asynccontextmanager + async def begin(self): + yield Connection() + + async def dispose(self): + return None + + async def next_scope(uow, identifier, **kwargs): + return SimpleNamespace(scope=Scope(tenant_id=identifier, project_id=uuid4(), environment_id=uuid4())) + + async def failing_dispatch(*args, **kwargs): + raise RuntimeError("delivery") + + monkeypatch.setattr(outbox_loop, "next_scope", next_scope) + monkeypatch.setattr(outbox_loop, "reserve", lambda count: count) + monkeypatch.setattr(outbox_loop, "release", lambda count: None) + outbox = OutboxLoop(SimpleNamespace(), None, SimpleNamespace(_dispatch=failing_dispatch)) # type: ignore[arg-type] + outbox.engine = Engine() # type: ignore[assignment] + assert await outbox.cycle() is True + await asyncio.gather(*outbox.turns) + assert await outbox.cycle() is False + await asyncio.gather(*outbox.turns) + await outbox.close() + + async def credentials(authority): + raise RuntimeError("broker unreachable") + + async def release(authority): + return None + + broker = KafkaLoop( + SimpleNamespace(broker=SimpleNamespace(max_clients=4)), # type: ignore[arg-type] + None, # type: ignore[arg-type] + SimpleNamespace(credentials=credentials, release=release), # type: ignore[arg-type] + SimpleNamespace(owners=[]), # type: ignore[arg-type] + ) + await broker.turn(SimpleNamespace(authority=object())) # type: ignore[arg-type] + broker.engine = Engine() # type: ignore[assignment] + + async def nothing(authority): + return None + + monkeypatch.setattr("firefly_weave.triggers.kafka_loop.next_scope", next_scope) + broker.triggers.scheduled_claim = nothing + # A running consumer task is not a healthy source: the failed turn fails the next traversal. + assert await broker.cycle() is False + assert await broker.cycle() is True + + +async def test_provider_failures_fail_the_poll_cycle(monkeypatch): + from firefly_weave.providers.loop import ProviderLoop + + async def broken_cycle(): + raise RuntimeError("provider") + + health = LoopHealth() + health.register("provider_loop", 0.01, enabled=True) + health.running("provider_loop", True) + loop = ProviderLoop(SimpleNamespace(), None, None, health=health) # type: ignore[arg-type] + monkeypatch.setattr(loop, "cycle", broken_cycle) + monkeypatch.setattr("firefly_weave.providers.loop.TRAVERSAL_SECONDS", 0.01) + await one_poll(loop, "provider_loop") + assert health.snapshot("provider_loop").consecutive_failures == 1 + + async def clean_cycle(): + return True + + monkeypatch.setattr(loop, "cycle", clean_cycle) + await one_poll(loop, "provider_loop") + assert health.snapshot("provider_loop").state == "ok" + + +async def test_cancellation_marks_the_loop_down_and_propagates(monkeypatch): + from firefly_weave.runtime import scheduler + from firefly_weave.runtime.scheduler import RecoveryLoop + + class Engine: + @asynccontextmanager + async def connect(self): + yield SimpleNamespace(scalar=self.scalar) + + async def scalar(self, statement): + return True + + async def dispose(self): + return None + + async def check_schema(connection): + return None + + entered = asyncio.Event() + + async def cycle(): + entered.set() + await asyncio.Event().wait() + + monkeypatch.setattr(scheduler, "create_async_engine", lambda url, **options: Engine()) + monkeypatch.setattr(scheduler, "check_schema", check_schema) + health = LoopHealth() + settings = Settings(database_url=DATABASE, scheduler_database_url=SCHEDULER) + register_background_loops(health, settings) + loop = RecoveryLoop(settings, None, None, None, health=health) # type: ignore[arg-type] + monkeypatch.setattr(loop, "cycle", cycle) + await loop.open() + task = loop.task + assert task is not None + await entered.wait() + assert health.snapshot("recovery_loop").state == "degraded" + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + assert {health.snapshot(name).state for name in ("recovery_loop", "schedules")} == {"down"} + await loop.close() + assert health.snapshot("recovery_loop").state == "down" diff --git a/tests/unit/operations/test_operations_loop.py b/tests/unit/operations/test_operations_loop.py new file mode 100644 index 00000000..b9a28b78 --- /dev/null +++ b/tests/unit/operations/test_operations_loop.py @@ -0,0 +1,330 @@ +# Copyright 2026 Firefly Software Foundation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# Author: Firefly Software Foundation +# SPDX-License-Identifier: Apache-2.0 + +"""The operations loop owns one task and one engine, and reports only cycles it completed.""" + +import asyncio +from contextlib import asynccontextmanager +from types import SimpleNamespace +from uuid import uuid4 + +import pytest + +from firefly_weave.access.scheduler import _CATALOG_AUTHORITY, _SchedulerScope +from firefly_weave.contracts.access import Scope +from firefly_weave.operations import loop as operations +from firefly_weave.operations.execution import execute_pure +from firefly_weave.operations.loop import ENVIRONMENT_SECONDS, OperationsLoop +from firefly_weave.operations.loop_health import LoopHealth, register_background_loops +from firefly_weave.settings import Settings + +DATABASE = "postgresql+asyncpg://weave_app:unused@127.0.0.1:65432/weave" +SCHEDULER = "postgresql+asyncpg://weave_scheduler:unused@127.0.0.1:65432/weave" + + +class Engine: + created: list["Engine"] = [] + + def __init__(self, valid: bool = True) -> None: + self.valid, self.disposed, self.statements = valid, False, [] + Engine.created.append(self) + + @asynccontextmanager + async def connect(self): + yield SimpleNamespace(scalar=self.scalar) + + async def scalar(self, statement): + self.statements.append(str(statement)) + return self.valid + + async def dispose(self): + self.disposed = True + + +@pytest.fixture +def engines(monkeypatch): + Engine.created = [] + valid = {"value": True} + + async def check_schema(connection): + return None + + monkeypatch.setattr(operations, "create_async_engine", lambda url, **options: Engine(valid["value"])) + monkeypatch.setattr(operations, "check_schema", check_schema) + return valid + + +def settings(**overrides) -> Settings: + return Settings(database_url=DATABASE, scheduler_database_url=SCHEDULER, **overrides) + + +def health_for(value: Settings) -> LoopHealth: + health = LoopHealth() + register_background_loops(health, value) + return health + + +def tasks_named(name: str) -> list[asyncio.Task]: + return [task for task in asyncio.all_tasks() if task.get_name() == name and not task.done()] + + +async def test_open_requires_execute_only_operations_authority_and_cleans_up_a_partial_open(engines): + engines["value"] = False + value = settings() + loop = OperationsLoop(value, None, health_for(value)) # type: ignore[arg-type] + with pytest.raises(RuntimeError, match="execute-only"): + await loop.open() + [engine] = Engine.created + assert engine.disposed and loop.engine is None and loop.task is None + assert any("weave_operations_tenants(integer)" in sql for sql in engine.statements) + assert any("has_table_privilege(current_user,'run_facts','SELECT')" in sql for sql in engine.statements) + assert loop.health.snapshot("operations_loop").state == "down" + + +async def test_open_requires_the_scheduler_connection_and_honors_a_disabled_scheduler(engines): + with pytest.raises(RuntimeError, match="WEAVE_SCHEDULER_DATABASE_URL"): + await OperationsLoop(Settings(database_url=DATABASE), None, LoopHealth()).open() # type: ignore[arg-type] + value = Settings(database_url=DATABASE, scheduler_enabled=False) + loop = OperationsLoop(value, None, health_for(value)) # type: ignore[arg-type] + await loop.open() + assert Engine.created == [] and loop.task is None + assert loop.health.snapshot("operations_loop").state == "disabled" + + +async def test_one_open_owns_one_task_and_close_cancels_before_disposing(engines, monkeypatch): + value = settings() + loop = OperationsLoop(value, None, health_for(value)) # type: ignore[arg-type] + events = [] + started = asyncio.Event() + + async def poll(): + started.set() + try: + await asyncio.Event().wait() + except asyncio.CancelledError: + events.append("cancelled") + raise + + monkeypatch.setattr(loop, "poll", poll) + await loop.open() + first = loop.task + await loop.open() + assert loop.task is first and len(Engine.created) == 1 + assert len(tasks_named("weave-operations")) == 1 + await started.wait() + assert loop.health.snapshot("operations_loop").state == "degraded" + [engine] = Engine.created + original = engine.dispose + + async def dispose(): + events.append("disposed") + await original() + + engine.dispose = dispose # type: ignore[method-assign] + await loop.close() + assert events == ["cancelled", "disposed"] + assert loop.task is None and loop.engine is None and tasks_named("weave-operations") == [] + assert loop.health.snapshot("operations_loop").state == "down" + await loop.close() + + +async def test_poll_reports_success_failure_and_cancellation(engines, monkeypatch): + value = settings() + health = health_for(value) + loop = OperationsLoop(value, None, health) # type: ignore[arg-type] + outcomes = iter([True, False, RuntimeError("catalog"), True]) + seen = [] + finished = asyncio.Event() + + async def cycle(): + outcome = next(outcomes, None) + if outcome is None: + finished.set() + await asyncio.Event().wait() + snapshot = health.snapshot("operations_loop") + seen.append((snapshot.state, snapshot.consecutive_failures)) + if isinstance(outcome, Exception): + raise outcome + return outcome + + monkeypatch.setattr(loop, "cycle", cycle) + monkeypatch.setattr(operations, "OPERATIONS_SECONDS", 0) + await loop.open() + await finished.wait() + # No success before the first cycle; a failed or raised cycle is a failure, never a success. + assert seen == [("degraded", 0), ("ok", 0), ("ok", 1), ("ok", 2)] + snapshot = health.snapshot("operations_loop") + assert (snapshot.state, snapshot.consecutive_failures, snapshot.complete_until) == ("ok", 0, None) + task = loop.task + assert task is not None + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + assert health.snapshot("operations_loop").state == "down" + await loop.close() + + +def authority() -> _SchedulerScope: + return _SchedulerScope(Scope(tenant_id=uuid4(), project_id=uuid4(), environment_id=uuid4()), _CATALOG_AUTHORITY) + + +async def test_cycle_traverses_its_own_cursors_one_environment_per_tenant(monkeypatch): + tenants = [uuid4(), uuid4()] + pages, turns, quanta = [], [], [] + + async def tenant_page(engine, limit=16, *, operations=False): + pages.append((limit, operations)) + return tenants + + async def next_scope(uow, tenant, **kinds): + turns.append((tenant, kinds)) + return None if tenant == tenants[1] else authority() + + async def revalidate(scope, deadline): + quanta.append(scope) + return True + + monkeypatch.setattr(operations, "tenant_page", tenant_page) + monkeypatch.setattr(operations, "next_scope", next_scope) + loop = OperationsLoop(settings(), None, LoopHealth()) # type: ignore[arg-type] + loop.engine = object() # type: ignore[assignment] + monkeypatch.setattr(loop, "revalidate", revalidate) + assert await loop.cycle() is True + # The operations traversal never moves the recovery cursors. + assert pages == [(16, True)] + assert turns == [(tenants[0], {"operations": True}), (tenants[1], {"operations": True})] + assert len(quanta) == 1 + + +async def test_an_environment_overrun_fails_the_cycle_but_later_tenants_still_progress(monkeypatch): + tenants = [uuid4(), uuid4(), uuid4()] + clock = {"now": 100.0} + progressed = [] + + async def tenant_page(engine, limit=16, *, operations=False): + return tenants + + async def next_scope(uow, tenant, **kinds): + value = authority() + progressed.append((tenant, value)) + return value + + async def revalidate(scope, deadline): + assert deadline == clock["now"] + ENVIRONMENT_SECONDS + if len(progressed) == 1: + # The first environment's database work outlives its budget. + clock["now"] += ENVIRONMENT_SECONDS + 0.5 + return False + if len(progressed) == 2: + raise RuntimeError("environment") + return True + + monkeypatch.setattr(operations, "tenant_page", tenant_page) + monkeypatch.setattr(operations, "next_scope", next_scope) + loop = OperationsLoop(settings(), None, LoopHealth(), monotonic=lambda: clock["now"]) # type: ignore[arg-type] + loop.engine = object() # type: ignore[assignment] + monkeypatch.setattr(loop, "revalidate", revalidate) + assert await loop.cycle() is False + assert [tenant for tenant, _ in progressed] == tenants + + +async def test_a_quantum_that_outlives_its_wall_clock_guard_is_cancelled(monkeypatch): + async def tenant_page(engine, limit=16, *, operations=False): + return [uuid4()] + + async def next_scope(uow, tenant, **kinds): + return authority() + + async def revalidate(scope, deadline): + await asyncio.sleep(10) + return True + + monkeypatch.setattr(operations, "tenant_page", tenant_page) + monkeypatch.setattr(operations, "next_scope", next_scope) + monkeypatch.setattr(operations, "ENVIRONMENT_SECONDS", 0.05) + monkeypatch.setattr(operations, "CANCEL_GRACE_SECONDS", 0.05) + loop = OperationsLoop(settings(), None, LoopHealth()) # type: ignore[arg-type] + loop.engine = object() # type: ignore[assignment] + monkeypatch.setattr(loop, "revalidate", revalidate) + async with asyncio.timeout(2): + assert await loop.cycle() is False + + +async def test_cycle_runs_pure_work_from_its_reservation_while_requests_hold_every_slot(monkeypatch): + from test_background_admission import saturated_requests + + async def tenant_page(engine, limit=16, *, operations=False): + return [uuid4()] + + async def next_scope(uow, tenant, **kinds): + return authority() + + ran = [] + + async def revalidate(scope, deadline): + ran.append(await execute_pure(lambda: "classified")) + return True + + monkeypatch.setattr(operations, "tenant_page", tenant_page) + monkeypatch.setattr(operations, "next_scope", next_scope) + loop = OperationsLoop(settings(), None, LoopHealth()) # type: ignore[arg-type] + loop.engine = object() # type: ignore[assignment] + monkeypatch.setattr(loop, "revalidate", revalidate) + async with saturated_requests(): + assert await loop.cycle() is True + assert ran == ["classified"] + + +async def test_revalidation_refuses_an_authority_the_catalog_did_not_mint(): + loop = OperationsLoop(settings(), None, LoopHealth()) # type: ignore[arg-type] + from firefly_weave.access.authorization import AccessDenied + + forged = _SchedulerScope(Scope(tenant_id=uuid4(), project_id=uuid4(), environment_id=uuid4()), object()) + with pytest.raises(AccessDenied): + await loop.revalidate(SimpleNamespace(scope=forged.scope), 1.0) # type: ignore[arg-type] + loop.uow = SimpleNamespace(open=None) # type: ignore[assignment] + + class Session: + def get_transaction(self): + return SimpleNamespace(is_active=True) + + class UnitOfWork: + @asynccontextmanager + async def open(self, scope, *, mutation=True): + yield SimpleNamespace(session=Session(), scope=scope) + + loop.uow = UnitOfWork() # type: ignore[assignment] + # A forged issuer is refused inside the transaction, before any fact is read. + assert await loop.revalidate(forged, loop.monotonic() + 1) is False + + +def test_the_revalidation_scan_matches_an_index_built_for_the_current_policy(): + from pathlib import Path + + from firefly_weave.operations.facts import revalidation_due + from firefly_weave.runtime import admission + + literal = f"classification_policy<>'{admission.POLICY}'" + migrations = Path(__file__).resolve().parents[3] / "migrations/versions" + # A new classification policy needs a migration that rebuilds run_facts_revalidation for it. + assert any( + "revalidation" in source and literal in source + for source in (path.read_text() for path in sorted(migrations.glob("*.py"))) + ) + assert literal in revalidation_due(admission.POLICY) + for unsafe in ("x' OR true --", "", "Classified"): + with pytest.raises(ValueError): + revalidation_due(unsafe) From e338f6bf3be42d2f0ce5c7fc75a7ec793c914bb7 Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Mon, 12 Oct 2026 07:07:07 -0700 Subject: [PATCH 20/63] Keep unavailable run listing decisions across classification policy changes --- CHANGELOG.md | 19 +- docs/operations/upgrades.md | 18 +- docs/reference/api.md | 6 +- src/firefly_weave/operations/facts.py | 14 +- .../test_operations_cache_refresh.py | 209 ++++++++++++++++-- tests/unit/cli/test_platform_ai_setup.py | 7 + tests/unit/operations/test_loop_health.py | 32 ++- 7 files changed, 263 insertions(+), 42 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1af6b6c1..c1fc5b6b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,14 +35,17 @@ SPDX-License-Identifier: Apache-2.0 Outputs of agent, tool or yield instance keys stay withheld until they have their own schemas. A run waiting for a platform that runs it answers `422 WV-IR-UNSUPPORTED` as `runs.read` does. -- The scheduler re-checks stored run listing decisions in the background. Each - minute it visits up to 16 tenants and gives each one's next environment up to - one second of database work and up to 100 runs: runs recorded while this - platform could not run them, which it now runs, and runs recorded under an - earlier classification policy. Until a run is re-checked, run summaries, step timelines and incident - lists keep withholding it even when `runs.read` serves it again, and its - timeline answers `422 WV-IR-UNSUPPORTED`. A retained run larger than 128 MiB - is recorded as unavailable without being read. +- The scheduler re-checks stored run listing decisions in the background: runs + recorded while this platform could not run them, which it now runs, and + available or unsupported decisions recorded under an earlier classification + policy. Unavailable decisions stay unavailable. Each minute it visits up to 16 + tenants and gives one environment of each up to one second of database work + and at most 100 runs. Until a run is re-checked, run summaries, step timelines + and incident lists keep withholding it, even when `runs.read` serves it again: + its timeline answers `422 WV-IR-UNSUPPORTED` when this platform runs it again, + and `409 WV-LEGACY-UNAVAILABLE` when it was recorded under an earlier policy. + A retained run larger than 128 MiB is recorded as unavailable without being + read. - The Python SDK's `list_runs` rejects a `query` combined with keyword filters. - Request telemetry records authorized `501 WV-UNAVAILABLE` answers with the status `unavailable` instead of `failed`. diff --git a/docs/operations/upgrades.md b/docs/operations/upgrades.md index 1ac506fb..ccdc9768 100644 --- a/docs/operations/upgrades.md +++ b/docs/operations/upgrades.md @@ -363,8 +363,9 @@ provenance. Backfill inventories retained data without granting extra admission capacity: a platform already above a quota can still migrate, but new work must satisfy the unchanged quotas. Purging an authorized archived run also removes its run, step, task and incident facts; deleting a worker removes its facts. -The cursor tables hold one row in all, per tenant or per environment and have -no business payload or retained history. +The cursor tables hold a single row for tenants, one row per tenant for +environments, and one row per environment for re-checks. They have no business +payload or retained history. ## Start and inspect compatibility @@ -488,9 +489,16 @@ timelines and incident lists keep withholding it until the scheduler re-checks its stored listing decision; its timeline answers `422 WV-IR-UNSUPPORTED` meanwhile. The scheduler does this in the background: each minute it visits up to 16 tenants, gives each one's next environment up to one second of database -work, and re-checks up to 100 runs there, so a large backlog takes several -minutes or more. Runs recorded -under an earlier classification policy are re-checked the same way. API +work, and re-checks at most 100 runs there, fewer when the second runs out +first. A tenant's environments take turns, one per minute. So in an installation +with one tenant and one environment, 10,000 waiting runs take at least 100 +minutes, and a million about a week. More environments in a tenant, or more +than 16 tenants, make each environment wait longer. + +Runs recorded under an earlier classification policy are re-checked the same +way, and their timelines answer `409 WV-LEGACY-UNAVAILABLE` meanwhile. A policy +change re-checks available and unsupported decisions only: unavailable ones, +such as runs whose history the upgrade could not trust, stay unavailable. API replicas that disable the scheduler rely on the scheduler replica for this. A run whose re-check fails, for example because its record cannot be locked in time or ordinary storage is full, stays withheld, is logged by error class, and diff --git a/docs/reference/api.md b/docs/reference/api.md index 82387a8f..f82b6bd7 100644 --- a/docs/reference/api.md +++ b/docs/reference/api.md @@ -644,8 +644,10 @@ evidence answers `409 WV-LEGACY-UNAVAILABLE`. Timelines, summaries and incident lists follow the run's stored listing decision. After an upgrade lets the platform run such a run again, `runs.read` serves it at once, but its timeline keeps answering `422 WV-IR-UNSUPPORTED` until the scheduler re-checks the -decision in the background, up to 100 runs per environment each minute. In -`runs.list`, only a recorded policy block withholds a run before its own read +decision in the background. Each minute the scheduler visits up to 16 tenants +and re-checks at most 100 runs in one environment of each, so a large backlog +takes a long time; see [Upgrades](../operations/upgrades.md#runs-that-need-a-newer-server). +In `runs.list`, only a recorded policy block withholds a run before its own read classifies it. **Incidents.** `incidents.list` and the paged `incidents.run_list` read diff --git a/src/firefly_weave/operations/facts.py b/src/firefly_weave/operations/facts.py index 9e6ce102..47de7639 100644 --- a/src/firefly_weave/operations/facts.py +++ b/src/firefly_weave/operations/facts.py @@ -121,15 +121,19 @@ def listing_metadata(row: dict[str, Any]) -> dict[str, Any]: def revalidation_due(policy: str) -> str: """SQL condition: the cached decision of fact row ``f`` is due for a refresh from its retained source. - Due when it was recorded under another classification policy, or when it is a transient unsupported decision - whose pinned IR version and features this platform runs again. Bind ``runtime.admission.runnable_parameters()``. - The policy is written as a literal so the planner can prove that the partial index ``run_facts_revalidation``, - which holds the rows not classified under the current policy and the unsupported ones, applies. + Due when an available or unsupported decision was recorded under another classification policy, or when it is a + transient unsupported decision whose pinned IR version and features this platform runs again. A policy change + never re-checks an unavailable decision: it can rest on evidence the source row does not show, such as an + incoherent event history found by the upgrade, so it stays unavailable. Bind + ``runtime.admission.runnable_parameters()``. The policy is written as a literal so the planner can prove that the + partial index ``run_facts_revalidation``, which holds the rows not classified under the current policy and the + unsupported ones, applies. """ if re.fullmatch(r"[a-z0-9][a-z0-9.-]{0,63}", policy) is None: raise ValueError("Classification policy must be a plain identifier") return ( - f"(f.classification_policy<>'{policy}' OR (f.classification_state='unsupported' " + f"((f.classification_policy<>'{policy}' AND f.classification_state<>'unavailable') " + "OR (f.classification_state='unsupported' " "AND f.pinned_ir_version=ANY(cast(:ir_versions AS text[])) " "AND f.pinned_features <@ cast(:ir_features AS jsonb)))" ) diff --git a/tests/integration/test_operations_cache_refresh.py b/tests/integration/test_operations_cache_refresh.py index b590e105..dea79e42 100644 --- a/tests/integration/test_operations_cache_refresh.py +++ b/tests/integration/test_operations_cache_refresh.py @@ -39,7 +39,7 @@ from firefly_weave.contracts.run_views import RunStepQuery, RunSummaryQuery from firefly_weave.definitions.models import CatalogError from firefly_weave.operations.facts import FactRepository, listing_metadata -from firefly_weave.operations.loop import OperationsLoop +from firefly_weave.operations.loop import ENVIRONMENT_SECONDS, OperationsLoop from firefly_weave.operations.loop_health import LoopHealth from firefly_weave.operations.run_views import RunViewService from firefly_weave.persistence.uow import UnitOfWork @@ -81,12 +81,31 @@ async def refresh(operations_case, scheduler_url): health, ) loop.engine = create_async_engine(scheduler_url) + # Quanta whose budget ran out: each can cut short at most the one refresh it was running. + loop.cut_short = 0 # type: ignore[attr-defined] + environment = loop._environment + + async def quantum(authority): + started = loop.monotonic() + try: + await environment(authority) + except Exception: + if loop.monotonic() >= started + ENVIRONMENT_SECONDS - 0.01: + loop.cut_short += 1 # type: ignore[attr-defined] + raise + + loop._environment = quantum # type: ignore[method-assign] try: yield loop finally: await loop.engine.dispose() +def extra_reads(reads: Counter, runs) -> int: + """Reads beyond one per run: refreshes cut short at a budget edge and retried after the keyset wrapped.""" + return sum(reads[run] for run in runs) - len(runs) + + class Reads(Counter): """Retained sources fetched by listing-metadata refreshes, by run, and any fetch of a run already refreshed.""" @@ -211,8 +230,18 @@ async def cancelled_source(case) -> UUID: return run.id -async def clone(case, run: UUID, identifiers, *, environment=None, activation=None, future=False, node_kinds=None): - """Retained copies of a real run, each cached as unsupported like a backfill on a rolled-back server.""" +async def clone( + case, + run: UUID, + identifiers, + *, + environment=None, + activation=None, + future=False, + node_kinds=None, + state="unsupported", +): + """Retained copies of a real run, each cached (unsupported by default) like a backfill on a rolled-back server.""" artifact = "jsonb_set(r.artifact,'{executable,irVersion}','\"weave/ir-future\"')" if future else "r.artifact" async with case.owner.begin() as session: await session.execute( @@ -229,7 +258,7 @@ async def clone(case, run: UUID, identifiers, *, environment=None, activation=No "INSERT INTO run_facts SELECT (jsonb_populate_record(NULL::run_facts,to_jsonb(f) || jsonb_build_object(" "'run_id',c.id,'environment_id',coalesce(cast(:environment AS uuid),f.environment_id)," "'activation_id',coalesce(cast(:activation AS uuid),f.activation_id)," - "'classification_state','unsupported','pinned_features','[\"text.concat\"]'::jsonb," + "'classification_state',cast(:state AS text),'pinned_features','[\"text.concat\"]'::jsonb," "'pinned_ir_version',CASE WHEN :future THEN 'weave/ir-future' ELSE f.pinned_ir_version END," "'node_kinds',coalesce(cast(:node_kinds AS jsonb),'{}'::jsonb)))).* " "FROM run_facts f CROSS JOIN unnest(cast(:ids AS uuid[])) AS c(id) WHERE f.run_id=:run" @@ -240,6 +269,7 @@ async def clone(case, run: UUID, identifiers, *, environment=None, activation=No "environment": environment, "activation": activation, "future": future, + "state": state, "node_kinds": None if node_kinds is None else json.dumps(node_kinds), }, ) @@ -430,6 +460,9 @@ async def test_policy_change_withholds_until_cache_is_revalidated(operations_cas first = (await case.start("first")).id monkeypatch.setattr(admission, "POLICY", "classified-v2") assert (await summary(case, first)).unavailable + with pytest.raises(CatalogError) as withheld: + await service(case).steps(case.actor, case.scope, first, RunStepQuery(), None, context=AuditContext()) + assert (withheld.value.status, withheld.value.code) == (409, "WV-LEGACY-UNAVAILABLE") assert await refresh.cycle() is True assert payload_reads == Counter({first: 1}) assert (await cached(case, first))["classification_policy"] == "classified-v2" @@ -638,6 +671,8 @@ async def finished() -> bool: turns = await due_progress(refresh, case, due + elsewhere, cycles=80, done=finished) # Every due row was refreshed and none was read once its decision was stored; the still-unsupported rows never. assert set(payload_reads) == set(due + elsewhere) and payload_reads.after_success == [] + # A row is read again only after a quantum ran out of budget while refreshing it. + assert extra_reads(payload_reads, due + elsewhere) <= refresh.cut_short assert set((await states(case, still_unsupported)).values()) == {"unsupported"} environments = {environment for environment, _ in (await cursors(case))["facts"]} assert environments == {case.scope.environment_id, other.environment_id} @@ -706,6 +741,7 @@ async def finished() -> bool: await service(case).steps(case.actor, case.scope, failing, RunStepQuery(), None, context=AuditContext()) assert withheld.value.code == "WV-IR-UNSUPPORTED" assert all(payload_reads[run] >= 1 for run in later + elsewhere) and payload_reads.after_success == [] + assert extra_reads(payload_reads, later + elsewhere) <= refresh.cut_short assert attempts.index(failing) < attempts.index(later[0]) finally: await transaction.rollback() @@ -774,39 +810,170 @@ async def candidates(self, after): results.append(await refresh.cycle()) assert set((await states(case, due + elsewhere)).values()) == {"available"} assert set(payload_reads) == set(due + elsewhere) and payload_reads.after_success == [] + assert extra_reads(payload_reads, due + elsewhere) <= refresh.cut_short async def test_each_quantum_lowers_its_database_timeouts_to_the_remaining_budget(operations_case, refresh, monkeypatch): case = operations_case run = (await case.start()).id await record_rolled_back_decision(case, run) - observed = [] + clock = {"now": 1000.0} - async def settings(tx): - observed.append( - tuple( - [ - await tx.session.scalar(text("SELECT current_setting(:name)"), {"name": name}) - for name in ("statement_timeout", "lock_timeout") - ] - ) - ) + def tick() -> float: + # Every reading advances the budget clock by an eighth of a second, exactly representable. + clock["now"] += 0.125 + return clock["now"] + refresh.monotonic = tick + observed = [] for name in ("revalidation_candidates", "advance_revalidation_cursor", "refresh_listing_metadata"): original = getattr(FactRepository, name) - def observing(original): + def observing(original, name): async def method(self, *args): - await settings(self.tx) + values = [ + await self.tx.session.scalar(text("SELECT current_setting(:name)"), {"name": setting}) + for setting in ("statement_timeout", "lock_timeout") + ] + if self.tx.scope.environment_id == case.scope.environment_id: + observed.append((name, *values)) return await original(self, *args) return method - monkeypatch.setattr(FactRepository, name, observing(original)) + monkeypatch.setattr(FactRepository, name, observing(original, name)) assert await refresh.cycle() is True - assert len(observed) >= 3 + # The quantum starts with one second; each transaction gets what is left after it verified its authority. + assert observed == [ + ("revalidation_candidates", "750ms", "750ms"), + ("advance_revalidation_cursor", "375ms", "375ms"), + ("refresh_listing_metadata", "125ms", "125ms"), + ] + assert (await cached(case, run))["classification_state"] == "available" + - def milliseconds(value: str) -> int: - return int(value[:-2]) if value.endswith("ms") else int(value[:-1]) * 1000 +async def test_a_decision_changed_after_selection_keeps_its_newer_value( + operations_case, refresh, payload_reads, monkeypatch +): + case = operations_case + run = (await case.start()).id + await record_rolled_back_decision(case, run) + selection = FactRepository.revalidation_candidates + + async def candidates(self, after): + found = await selection(self, after) + if run in found: + # An authoritative writer withholds the run between the selection and its refresh. + async with case.owner.begin() as session: + await session.execute( + text("UPDATE run_facts SET classification_state='unavailable' WHERE run_id=:run"), {"run": run} + ) + return found - assert all(0 < milliseconds(value) <= 1000 for pair in observed for value in pair) + monkeypatch.setattr(FactRepository, "revalidation_candidates", candidates) + assert await refresh.cycle() is True + # The recheck under the lock sees the newer decision: nothing is read and nothing is overwritten. + assert payload_reads == Counter() + assert (await cached(case, run))["classification_state"] == "unavailable" + + +async def test_the_due_scan_uses_the_revalidation_index(operations_case, refresh, monkeypatch): + case = operations_case + origin = await cancelled_source(case) + final = (await cached(case, origin))["node_kinds"] + await clone(case, origin, ids(0x20000000, 3000), node_kinds=final, state="available") + due = ids(0x30000000, 20) + await clone(case, origin, due) + async with case.owner.begin() as session: + await session.execute(text("ANALYZE run_facts")) + plans = [] + rows = FactRepository.rows + + async def explained(self, sql, **values): + if "ORDER BY f.run_id LIMIT" in sql: + plan = await self.tx.session.scalar(text("EXPLAIN (FORMAT JSON) " + sql), {**values, **self.params}) + plans.append(json.dumps(plan)) + return await rows(self, sql, **values) + + monkeypatch.setattr(FactRepository, "rows", explained) + assert await refresh.cycle() is True + assert plans and all('"Index Name": "run_facts_revalidation"' in plan for plan in plans) + assert set((await states(case, due)).values()) == {"available"} + + +async def test_a_policy_change_never_upgrades_a_decision_the_upgrade_withheld( + empty_settings, payload_reads, monkeypatch +): + """Runs whose history the upgrade could not trust stay unavailable, whatever later policy re-checks the rest.""" + import os + import secrets + from datetime import UTC, datetime + from pathlib import Path + from uuid import uuid4 + + from operations_support import seed_prior_head + from sqlalchemy import make_url + from sqlalchemy.ext.asyncio import async_sessionmaker + from test_incident_migration import upgrade + + owner_url = make_url(empty_settings.database_url.get_secret_value()) + owner = create_async_engine(owner_url, hide_parameters=True) + logins = {} + async with owner.begin() as connection: + await upgrade(connection, "0030_worker_presence") + identifiers = await seed_prior_head(connection, count=20) + await upgrade(connection, "0031_operations_facts") + for role in ("weave_app", "weave_scheduler"): + name, password = f"{role}_{uuid4().hex}", secrets.token_urlsafe(32) + await connection.execute(text(f"CREATE ROLE {name} LOGIN PASSWORD '{password}' IN ROLE {role}")) + logins[role] = owner_url.set(username=name, password=password) + manifest = os.environ.get("WEAVE_TEST_RESOURCE_MANIFEST") + if manifest: + with Path(manifest).open("a") as handle: + entry = {"kind": "role", "name": name, "observed_at": datetime.now(UTC).isoformat()} + handle.write(json.dumps(entry) + "\n") + application = create_async_engine(logins["weave_app"], hide_parameters=True) + withheld = [identifiers[name] for name in ("incoherent", "no_events")] + + async def facts(runs): + async with owner.connect() as connection: + rows = await connection.execute( + text(f"SELECT run_id,{CLASSIFICATION} FROM run_facts WHERE run_id=ANY(:ids) ORDER BY run_id"), + {"ids": list(runs)}, + ) + return [dict(row) for row in rows.mappings()] + + try: + before = await facts(withheld) + # The upgrade withheld them for their history, although each run row alone would classify as available. + assert {row["classification_state"] for row in before} == {"unavailable"} + async with owner.connect() as connection: + row = dict( + (await connection.execute(text("SELECT * FROM runs WHERE id=:id"), {"id": identifiers["incoherent"]})) + .mappings() + .one() + ) + assert listing_metadata(row)["classification_state"] == "available" + monkeypatch.setattr(admission, "POLICY", "classified-v2") + health = LoopHealth() + health.register("operations_loop", 60, enabled=True) + loop = OperationsLoop( + Settings(scheduler_enabled=False, database_url=logins["weave_app"].render_as_string(hide_password=False)), + UnitOfWork(async_sessionmaker(application, expire_on_commit=False)), + health, + ) + loop.engine = create_async_engine(logins["weave_scheduler"], hide_parameters=True) + try: + # The seeded tenant has two environments and each cycle visits one of them. + for _ in range(4): + assert await loop.cycle() is True + finally: + await loop.engine.dispose() + assert not any(payload_reads[run] for run in withheld) and await facts(withheld) == before + # Decisions that are available under the earlier policy are re-checked under the new one. + [first] = await facts([identifiers["first_run"]]) + assert payload_reads[identifiers["first_run"]] == 1 + assert (first["classification_state"], first["classification_policy"]) == ("available", "classified-v2") + finally: + await application.dispose() + await owner.dispose() diff --git a/tests/unit/cli/test_platform_ai_setup.py b/tests/unit/cli/test_platform_ai_setup.py index ceba6091..69ed25ba 100644 --- a/tests/unit/cli/test_platform_ai_setup.py +++ b/tests/unit/cli/test_platform_ai_setup.py @@ -295,6 +295,13 @@ async def test_presence_ignores_workers_of_another_release(): assert await setup.presence(Client(workers=[other]), str(RELEASE)) == {"presence": "unknown", "last_seen_at": None} +async def test_presence_ignores_revoked_workers_of_the_release(): + revoked = SimpleNamespace(id=uuid4(), release_id=RELEASE, presence="recent", revoked=True, last_seen_at=None) + own = SimpleNamespace(id=uuid4(), release_id=RELEASE, presence="stale", revoked=False, last_seen_at=None) + found = await setup.presence(Client(workers=[revoked, own]), str(RELEASE)) + assert found == {"presence": "stale", "last_seen_at": None, "worker_id": str(own.id)} + + async def test_the_connection_test_returns_its_result_with_a_timestamp(): client = Client() result = await setup.test_connection(client, str(REVISION), "qwen2.5:1.5b") diff --git a/tests/unit/operations/test_loop_health.py b/tests/unit/operations/test_loop_health.py index f10de78d..4c0c7ebe 100644 --- a/tests/unit/operations/test_loop_health.py +++ b/tests/unit/operations/test_loop_health.py @@ -180,6 +180,21 @@ def test_scheduler_disabled_marks_every_scheduler_loop_disabled(): states = {snapshot.component: snapshot.state for snapshot in enabled.snapshots()} # Nothing has run yet: enabled loops are down until their task starts, and alerts do not exist yet. assert states == {**{name: "down" for name in COMPONENTS}, "alert_evaluator": "disabled"} + # The broker consumer follows its own switch, whatever the scheduler setting. + for scheduler, consumer, expected in ((False, True, "down"), (True, False, "disabled")): + health = LoopHealth() + register_background_loops( + health, + Settings( + database_url=DATABASE, + scheduler_enabled=scheduler, + scheduler_database_url=SCHEDULER, + broker=BrokerPolicy(enabled=True, max_clients=2), + kafka_consumer_enabled=consumer, + ), + ) + assert health.snapshot("kafka_loop").state == expected + assert health.snapshot("outbox_loop").state == ("down" if scheduler else "disabled") class Traversal: @@ -199,6 +214,7 @@ async def next_scope(uow, tenant, **kwargs): def recovery_loop(monkeypatch, *, tenants=2, recovery_errors=(), schedule_errors=(), operational=True): + """A recovery loop over fake tenants; ``operational`` is a fixed answer or a sequence of answers.""" from firefly_weave.runtime.scheduler import RecoveryLoop Traversal(monkeypatch, "firefly_weave.runtime.scheduler", tenants) @@ -221,7 +237,7 @@ async def fire(authority, limit): None, # type: ignore[arg-type] SimpleNamespace(_scan_scheduled=recover, _scan_terminal_scheduled=recover), # type: ignore[arg-type] SimpleNamespace(_scan_scheduled=fire), # type: ignore[arg-type] - operational=lambda: operational, + operational=(lambda: operational) if isinstance(operational, bool) else iter(operational).__next__, health=health, ) loop.engine = object() # type: ignore[assignment] @@ -281,6 +297,11 @@ async def test_schedules_report_separately_from_recovery(monkeypatch): assert await loop.cycle() is True snapshot = health.snapshot("schedules") assert (snapshot.last_started_at, snapshot.last_ok_at, snapshot.consecutive_failures) == (None, None, 0) + # Restricted after the cycle began: the schedules it skipped make it a failed schedules cycle. + loop, health = recovery_loop(monkeypatch, tenants=1, operational=[True, True, False]) + assert await loop.cycle() is True + snapshot = health.snapshot("schedules") + assert snapshot.last_started_at is not None and snapshot.consecutive_failures == 1 async def test_a_failed_catalog_page_fails_recovery_and_schedules(monkeypatch): @@ -323,6 +344,9 @@ async def next_scope(uow, identifier, **kwargs): async def failing_dispatch(*args, **kwargs): raise RuntimeError("delivery") + async def delivered(*args, **kwargs): + return None + monkeypatch.setattr(outbox_loop, "next_scope", next_scope) monkeypatch.setattr(outbox_loop, "reserve", lambda count: count) monkeypatch.setattr(outbox_loop, "release", lambda count: None) @@ -332,6 +356,12 @@ async def failing_dispatch(*args, **kwargs): await asyncio.gather(*outbox.turns) assert await outbox.cycle() is False await asyncio.gather(*outbox.turns) + # The failure was reported once; the next cycle starts clean. + outbox.dispatcher = SimpleNamespace(_dispatch=delivered) + assert await outbox.cycle() is False + await asyncio.gather(*outbox.turns) + assert await outbox.cycle() is True + await asyncio.gather(*outbox.turns) await outbox.close() async def credentials(authority): From d6f32427852058d6dc4c378d414e58cdbc38a219 Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Mon, 12 Oct 2026 08:30:41 -0700 Subject: [PATCH 21/63] Accept the loop health option in the composition test's background effect stand-ins --- tests/integration/test_composition.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_composition.py b/tests/integration/test_composition.py index 2aa0c6bf..5c9e6a3e 100644 --- a/tests/integration/test_composition.py +++ b/tests/integration/test_composition.py @@ -459,7 +459,7 @@ async def close(self): kinds = ["OutboxLoop", "NativeDispatcher", "KafkaLoop", "ProviderLoop"] for kind in kinds: - monkeypatch.setattr(application, kind, lambda *args, kind=kind: Loop(kind, *args)) + monkeypatch.setattr(application, kind, lambda *args, kind=kind, **options: Loop(kind, *args)) class Engine: @asynccontextmanager From c798a3a452b4a443d646cfb697b69cb38fd372b9 Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Mon, 12 Oct 2026 08:48:12 -0700 Subject: [PATCH 22/63] Test the budget each operations transaction gets after its authority check --- .../test_operations_cache_refresh.py | 30 +++++++++---------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/tests/integration/test_operations_cache_refresh.py b/tests/integration/test_operations_cache_refresh.py index dea79e42..61911096 100644 --- a/tests/integration/test_operations_cache_refresh.py +++ b/tests/integration/test_operations_cache_refresh.py @@ -34,6 +34,7 @@ from firefly_weave.access.audit import AuditContext from firefly_weave.access.authorization import AuthorizationService +from firefly_weave.access.scheduler import _SchedulerScope from firefly_weave.contracts import language_features from firefly_weave.contracts.access import Scope from firefly_weave.contracts.run_views import RunStepQuery, RunSummaryQuery @@ -818,13 +819,15 @@ async def test_each_quantum_lowers_its_database_timeouts_to_the_remaining_budget run = (await case.start()).id await record_rolled_back_decision(case, run) clock = {"now": 1000.0} + verify = _SchedulerScope.verify - def tick() -> float: - # Every reading advances the budget clock by an eighth of a second, exactly representable. - clock["now"] += 0.125 - return clock["now"] + async def slow_verify(self, tx): + # Each authority check takes a quarter of the budget on the fake clock, which moves only here. + await verify(self, tx) + clock["now"] += 0.25 - refresh.monotonic = tick + monkeypatch.setattr(_SchedulerScope, "verify", slow_verify) + refresh.monotonic = lambda: clock["now"] observed = [] for name in ("revalidation_candidates", "advance_revalidation_cursor", "refresh_listing_metadata"): original = getattr(FactRepository, name) @@ -843,11 +846,12 @@ async def method(self, *args): monkeypatch.setattr(FactRepository, name, observing(original, name)) assert await refresh.cycle() is True - # The quantum starts with one second; each transaction gets what is left after it verified its authority. + # The quantum starts with one second; each transaction gets what is left after it verified its authority, so + # a timeout taken before the check would read 1000, 750 and 500 ms. assert observed == [ ("revalidation_candidates", "750ms", "750ms"), - ("advance_revalidation_cursor", "375ms", "375ms"), - ("refresh_listing_metadata", "125ms", "125ms"), + ("advance_revalidation_cursor", "500ms", "500ms"), + ("refresh_listing_metadata", "250ms", "250ms"), ] assert (await cached(case, run))["classification_state"] == "available" @@ -905,12 +909,10 @@ async def test_a_policy_change_never_upgrades_a_decision_the_upgrade_withheld( empty_settings, payload_reads, monkeypatch ): """Runs whose history the upgrade could not trust stay unavailable, whatever later policy re-checks the rest.""" - import os import secrets - from datetime import UTC, datetime - from pathlib import Path from uuid import uuid4 + from conftest import retain_resource from operations_support import seed_prior_head from sqlalchemy import make_url from sqlalchemy.ext.asyncio import async_sessionmaker @@ -927,11 +929,7 @@ async def test_a_policy_change_never_upgrades_a_decision_the_upgrade_withheld( name, password = f"{role}_{uuid4().hex}", secrets.token_urlsafe(32) await connection.execute(text(f"CREATE ROLE {name} LOGIN PASSWORD '{password}' IN ROLE {role}")) logins[role] = owner_url.set(username=name, password=password) - manifest = os.environ.get("WEAVE_TEST_RESOURCE_MANIFEST") - if manifest: - with Path(manifest).open("a") as handle: - entry = {"kind": "role", "name": name, "observed_at": datetime.now(UTC).isoformat()} - handle.write(json.dumps(entry) + "\n") + retain_resource("role", name) application = create_async_engine(logins["weave_app"], hide_parameters=True) withheld = [identifiers[name] for name in ("incoherent", "no_events")] From 1dbb2b3f81e05bbdeaf8f8b155eae7d705027623 Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Mon, 12 Oct 2026 08:53:19 -0700 Subject: [PATCH 23/63] Judge background loop health by a monotonic clock, cycle budgets and recent failures Loop health now ages successes with a monotonic clock and keeps the wall clock only for the times it shows, so a clock step neither hides a stalled loop nor degrades a healthy one. A cycle still running within its loop's cycle budget is on time: 5 seconds for the outbox and broker traversals, 30 seconds for recovery and schedules, and 60 seconds for provider and mailbox polling. A loop also reads degraded when 3 of its last 20 cycles failed, so a failure that alternates with successes, such as one run that keeps failing in a rotating traversal, is no longer hidden. An outbox cycle that finds no free delivery slot no longer counts as a success: like a cycle that waits for readiness, it is neither a success nor a failure, and the outbox reads degraded once no turn has started for more than three intervals. --- src/firefly_weave/operations/loop_health.py | 117 +++++++++--- src/firefly_weave/operations/outbox_loop.py | 16 +- tests/unit/operations/test_loop_health.py | 188 ++++++++++++++++++-- 3 files changed, 275 insertions(+), 46 deletions(-) diff --git a/src/firefly_weave/operations/loop_health.py b/src/firefly_weave/operations/loop_health.py index 12524fae..07aa1a41 100644 --- a/src/firefly_weave/operations/loop_health.py +++ b/src/firefly_weave/operations/loop_health.py @@ -23,9 +23,10 @@ import asyncio import logging import math +import time from collections.abc import Callable from dataclasses import dataclass -from datetime import UTC, datetime, timedelta +from datetime import UTC, datetime from typing import Literal from firefly_weave.settings import Settings @@ -44,10 +45,20 @@ ) FAILURES_DEGRADED = 3 INTERVALS_STALE = 3 +# Failures are also counted over this many recent cycles, so failures that alternate with successes, such as one +# permanently failing run or tenant in a rotating traversal, still degrade the loop. +RECENT_CYCLES = 20 # The outbox, provider and broker traversals poll every second; the operations loop every minute. TRAVERSAL_SECONDS = 1.0 OPERATIONS_SECONDS = 60.0 +# How long one healthy cycle may take. A cycle still running within its budget is on time, however many poll +# intervals that is. Outbox and broker traversals time out after 5 s, a recovery cycle scans up to 16 tenants, and a +# provider cycle can wait on one mailbox poll. +TRAVERSAL_BUDGET_SECONDS = 5.0 +RECOVERY_BUDGET_SECONDS = 30.0 +PROVIDER_BUDGET_SECONDS = 60.0 _FAILURE_CEILING = 1_000_000_000 +_RECENT_MASK = (1 << RECENT_CYCLES) - 1 def utc_now() -> datetime: @@ -62,17 +73,26 @@ class LoopSnapshot: last_ok_at: datetime | None consecutive_failures: int complete_until: datetime | None + # Failed cycles among the last RECENT_CYCLES cycles. + recent_failures: int = 0 @dataclass class _Record: - interval: timedelta + interval: float + budget: float enabled: bool running: bool = False + # Wall-clock times, for display only. last_started_at: datetime | None = None last_ok_at: datetime | None = None - consecutive_failures: int = 0 complete_until: datetime | None = None + # Monotonic times, for aging: the start of the cycle in flight and the last success. + started: float | None = None + ok: float | None = None + consecutive_failures: int = 0 + # The outcomes of the most recent cycles, newest in the lowest bit; a set bit is a failed cycle. + outcomes: int = 0 def _utc(value: datetime) -> datetime: @@ -81,69 +101,96 @@ def _utc(value: datetime) -> datetime: return value.astimezone(UTC) +def _seconds(value: object, *, positive: bool) -> float: + if ( + isinstance(value, bool) + or not isinstance(value, (int, float)) + or not math.isfinite(value) + or value < 0 + or (positive and value == 0) + ): + raise ValueError("Loop intervals and budgets must be finite numbers of seconds") + return float(value) + + class LoopHealth: """Per-loop start, success and failure records, judged against the loop's poll interval. - ``disabled`` when the loop is turned off, ``down`` while its task is not running, ``degraded`` after - ``FAILURES_DEGRADED`` consecutive failed cycles, before its first successful cycle, or when its last success is - older than ``INTERVALS_STALE`` intervals, and ``ok`` otherwise. Times come from ``clock``, are aware UTC and are - for display; loops measure their own budgets with a monotonic clock. + ``disabled`` when the loop is turned off, ``down`` while its task is not running, ``degraded`` before its first + successful cycle, after ``FAILURES_DEGRADED`` consecutive failed cycles or as many among its last + ``RECENT_CYCLES``, or when its last success is older than ``INTERVALS_STALE`` intervals, and ``ok`` otherwise. A + cycle still running within the loop's cycle budget does not age the loop: it is judged by when that cycle + started. Aging uses ``monotonic``, so a wall-clock step neither hides a stalled loop nor degrades a healthy one; + ``clock`` gives the aware UTC times that are shown. """ - def __init__(self, clock: Callable[[], datetime] = utc_now) -> None: + def __init__(self, clock: Callable[[], datetime] = utc_now, *, monotonic: Callable[[], float] = time.monotonic): self._clock = clock + self._monotonic = monotonic self._records: dict[str, _Record] = {} - def register(self, name: str, interval_seconds: float, *, enabled: bool) -> None: + def register(self, name: str, interval_seconds: float, *, enabled: bool, budget_seconds: float = 0.0) -> None: """Declare a loop at startup; registering it again keeps what it already reported.""" if name not in COMPONENTS: raise ValueError("Unknown background loop") - if ( - isinstance(interval_seconds, bool) - or not isinstance(interval_seconds, (int, float)) - or not math.isfinite(interval_seconds) - or interval_seconds <= 0 - ): - raise ValueError("Loop interval must be a positive number of seconds") + interval = _seconds(interval_seconds, positive=True) + budget = _seconds(budget_seconds, positive=False) if type(enabled) is not bool: raise ValueError("Loop enablement must be a boolean") - interval = timedelta(seconds=interval_seconds) record = self._records.get(name) if record is None: - self._records[name] = _Record(interval=interval, enabled=enabled) + self._records[name] = _Record(interval=interval, budget=budget, enabled=enabled) else: - record.interval, record.enabled = interval, enabled + record.interval, record.budget, record.enabled = interval, budget, enabled def running(self, name: str, value: bool) -> None: self._update(name, lambda record: setattr(record, "running", value is True)) def started(self, name: str) -> None: - self._update(name, lambda record: setattr(record, "last_started_at", self._now())) + def update(record: _Record) -> None: + shown, at = self._now(), self._monotonic() + record.last_started_at, record.started = shown, at + + self._update(name, update) def succeeded(self, name: str, *, complete_until: datetime | None = None) -> None: def update(record: _Record) -> None: - now = self._now() + shown, at = self._now(), self._monotonic() until = None if complete_until is None else _utc(complete_until) - record.last_ok_at, record.consecutive_failures, record.complete_until = now, 0, until + record.last_ok_at, record.ok, record.complete_until = shown, at, until + record.started, record.consecutive_failures = None, 0 + record.outcomes = (record.outcomes << 1) & _RECENT_MASK self._update(name, update) def failed(self, name: str) -> None: def update(record: _Record) -> None: record.consecutive_failures = min(record.consecutive_failures + 1, _FAILURE_CEILING) + record.started = None + record.outcomes = ((record.outcomes << 1) | 1) & _RECENT_MASK self._update(name, update) + def skipped(self, name: str) -> None: + """A cycle that did no work, such as one waiting for readiness: neither a success nor a failure.""" + self._update(name, lambda record: setattr(record, "started", None)) + def snapshot(self, name: str) -> LoopSnapshot: record = self._records[name] + recent = record.outcomes.bit_count() if not record.enabled: state: LoopState = "disabled" elif not record.running: state = "down" - elif record.last_ok_at is None or record.consecutive_failures >= FAILURES_DEGRADED: + elif ( + record.ok is None + or record.consecutive_failures >= FAILURES_DEGRADED + or recent >= FAILURES_DEGRADED + or self._behind(record, record.ok) + ): state = "degraded" else: - state = "degraded" if self._now() - record.last_ok_at > record.interval * INTERVALS_STALE else "ok" + state = "ok" return LoopSnapshot( component=name, state=state, @@ -151,11 +198,19 @@ def snapshot(self, name: str) -> LoopSnapshot: last_ok_at=record.last_ok_at, consecutive_failures=record.consecutive_failures, complete_until=record.complete_until, + recent_failures=recent, ) def snapshots(self) -> tuple[LoopSnapshot, ...]: return tuple(self.snapshot(name) for name in COMPONENTS if name in self._records) + def _behind(self, record: _Record, ok: float) -> bool: + now, limit = self._monotonic(), record.interval * INTERVALS_STALE + if record.started is not None and now - record.started <= record.budget: + # The cycle in flight is within its budget: the loop is on time if that cycle started on time. + return record.started - ok > limit + return now - ok > limit + def _now(self) -> datetime: return _utc(self._clock()) @@ -174,13 +229,19 @@ def register_background_loops(health: LoopHealth, settings: Settings) -> None: """Every component, registered at startup so a loop that has not opened yet reads as down, not missing.""" scheduled = settings.scheduler_enabled for name in ("recovery_loop", "schedules"): - health.register(name, settings.scheduler_poll_seconds, enabled=scheduled) - for name in ("outbox_loop", "provider_loop"): - health.register(name, TRAVERSAL_SECONDS, enabled=scheduled) + health.register( + name, settings.scheduler_poll_seconds, enabled=scheduled, budget_seconds=RECOVERY_BUDGET_SECONDS + ) + health.register("outbox_loop", TRAVERSAL_SECONDS, enabled=scheduled, budget_seconds=TRAVERSAL_BUDGET_SECONDS) + health.register("provider_loop", TRAVERSAL_SECONDS, enabled=scheduled, budget_seconds=PROVIDER_BUDGET_SECONDS) # The broker consumer has its own switch; the scheduler setting does not stop it. health.register( - "kafka_loop", TRAVERSAL_SECONDS, enabled=settings.broker.enabled and settings.kafka_consumer_enabled + "kafka_loop", + TRAVERSAL_SECONDS, + enabled=settings.broker.enabled and settings.kafka_consumer_enabled, + budget_seconds=TRAVERSAL_BUDGET_SECONDS, ) + # Its cycles finish well within three of its one-minute intervals. health.register("operations_loop", OPERATIONS_SECONDS, enabled=scheduled) # Alert evaluation does not exist yet. health.register("alert_evaluator", OPERATIONS_SECONDS, enabled=False) diff --git a/src/firefly_weave/operations/outbox_loop.py b/src/firefly_weave/operations/outbox_loop.py index 30c76258..fb312c25 100644 --- a/src/firefly_weave/operations/outbox_loop.py +++ b/src/firefly_weave/operations/outbox_loop.py @@ -77,8 +77,9 @@ async def open(self) -> None: async def cycle(self) -> bool | None: """One bounded traversal turn. - False when a delivery turn failed since the previous cycle. None while the platform is not ready: no turn - starts then, so the cycle is neither a success nor a failure, as for schedules. + False when a delivery turn failed since the previous cycle. None when no turn could start, because the + platform is not ready or no delivery slot is free: the cycle is neither a success nor a failure, so the + outbox reads degraded once that lasts longer than three intervals. """ healthy, self.turn_failed = not self.turn_failed, False # A turn started now would be refused, so wait for readiness without moving a cursor. @@ -86,7 +87,8 @@ async def cycle(self) -> bool | None: return None if healthy else False count = reserve(1) if not count: - return healthy + # Every slot is held by a delivery in flight, or was never given back. + return None if healthy else False owned = True try: assert self.engine is not None @@ -138,10 +140,12 @@ async def poll(self) -> None: except Exception: logging.getLogger(__name__).error("Outbox traversal failed; retrying bounded turn") healthy = False - # While the platform is not ready the cycle proves nothing, so its last success keeps aging. - if healthy is True: + if healthy is None: + # No turn could start, so the cycle proves nothing and the last success keeps aging. + self.health.skipped("outbox_loop") + elif healthy: self.health.succeeded("outbox_loop") - elif healthy is False: + else: self.health.failed("outbox_loop") await asyncio.sleep(TRAVERSAL_SECONDS) diff --git a/tests/unit/operations/test_loop_health.py b/tests/unit/operations/test_loop_health.py index a757a934..262f91c0 100644 --- a/tests/unit/operations/test_loop_health.py +++ b/tests/unit/operations/test_loop_health.py @@ -26,7 +26,13 @@ from firefly_weave.connectors.broker import BrokerPolicy from firefly_weave.contracts.access import Scope -from firefly_weave.operations.loop_health import COMPONENTS, LoopHealth, register_background_loops, utc_now +from firefly_weave.operations.loop_health import ( + COMPONENTS, + RECENT_CYCLES, + LoopHealth, + register_background_loops, + utc_now, +) from firefly_weave.settings import Settings DATABASE = "postgresql+asyncpg://weave_app:unused@127.0.0.1:65432/weave" @@ -34,23 +40,33 @@ class Clock: + """The wall clock shown in times and the monotonic clock that ages them; `advance` moves both.""" + def __init__(self) -> None: self.now = datetime(2026, 10, 9, tzinfo=UTC) + self.elapsed = 1000.0 def __call__(self) -> datetime: return self.now + def monotonic(self) -> float: + return self.elapsed + + def advance(self, seconds: float) -> None: + self.now += timedelta(seconds=seconds) + self.elapsed += seconds + -def running(clock: Clock, name: str = "outbox_loop", interval: float = 5) -> LoopHealth: - health = LoopHealth(clock) - health.register(name, interval, enabled=True) +def running(clock: Clock, name: str = "outbox_loop", interval: float = 5, budget: float = 0) -> LoopHealth: + health = LoopHealth(clock, monotonic=clock.monotonic) + health.register(name, interval, enabled=True, budget_seconds=budget) health.running(name, True) return health def test_loop_health_never_confuses_disabled_and_healthy(): - now = datetime(2026, 10, 9, tzinfo=UTC) - health = LoopHealth(lambda: now) + clock = Clock() + health = LoopHealth(clock, monotonic=clock.monotonic) health.register("operations_loop", 60, enabled=False) assert health.snapshot("operations_loop").state == "disabled" health.register("outbox_loop", 5, enabled=True) @@ -59,7 +75,7 @@ def test_loop_health_never_confuses_disabled_and_healthy(): health.started("outbox_loop") health.succeeded("outbox_loop") assert health.snapshot("outbox_loop").state == "ok" - now += timedelta(seconds=16) + clock.advance(16) assert health.snapshot("outbox_loop").state == "degraded" @@ -72,7 +88,7 @@ def test_a_running_loop_without_a_success_is_degraded_never_ok(): assert (snapshot.state, snapshot.last_started_at, snapshot.last_ok_at) == ("degraded", clock.now, None) -def test_exactly_three_consecutive_failures_degrade_and_a_successful_empty_scan_clears_them(): +def test_exactly_three_consecutive_failures_degrade_and_a_success_clears_the_count(): clock = Clock() health = running(clock) health.succeeded("outbox_loop") @@ -82,16 +98,88 @@ def test_exactly_three_consecutive_failures_degrade_and_a_successful_empty_scan_ assert health.snapshot("outbox_loop").consecutive_failures == 3 health.succeeded("outbox_loop") snapshot = health.snapshot("outbox_loop") - assert (snapshot.state, snapshot.consecutive_failures) == ("ok", 0) + # The three failures are still among the recent cycles, so the loop stays degraded until they age out. + assert (snapshot.state, snapshot.consecutive_failures, snapshot.recent_failures) == ("degraded", 0, 3) + for _ in range(RECENT_CYCLES - 4): + health.succeeded("outbox_loop") + assert health.snapshot("outbox_loop").state == "degraded" + health.succeeded("outbox_loop") + snapshot = health.snapshot("outbox_loop") + assert (snapshot.state, snapshot.recent_failures) == ("ok", 2) + + +def test_failures_that_alternate_with_successes_degrade_the_loop(): + clock = Clock() + health = running(clock, "operations_loop", 60) + # One permanently failing run in a rotating traversal: a failed pass, then a clean wrap, again and again. + for _ in range(2): + health.failed("operations_loop") + health.succeeded("operations_loop") + assert health.snapshot("operations_loop").state == "ok" + health.failed("operations_loop") + health.succeeded("operations_loop") + snapshot = health.snapshot("operations_loop") + assert (snapshot.state, snapshot.consecutive_failures, snapshot.recent_failures) == ("degraded", 0, 3) + + +def test_aging_follows_the_monotonic_clock_and_wall_time_is_only_shown(): + clock = Clock() + health = running(clock) + health.succeeded("outbox_loop") + shown = clock.now + # A wall clock stepped back an hour does not hide a loop that stopped cycling. + clock.now -= timedelta(hours=1) + clock.elapsed += 16 + snapshot = health.snapshot("outbox_loop") + assert (snapshot.state, snapshot.last_ok_at) == ("degraded", shown) + health.succeeded("outbox_loop") + # A wall clock stepped forward does not age a loop that just succeeded. + clock.now += timedelta(hours=2) + assert health.snapshot("outbox_loop").state == "ok" + + +def test_a_long_cycle_is_on_time_until_its_budget_runs_out(): + clock = Clock() + health = running(clock, "provider_loop", 1, budget=60) + health.succeeded("provider_loop") + clock.advance(1) + health.started("provider_loop") + # A slow mailbox poll: far more than three intervals, still within the loop's own cycle budget. + clock.advance(59) + assert health.snapshot("provider_loop").state == "ok" + clock.advance(1.5) + assert health.snapshot("provider_loop").state == "degraded" + health.succeeded("provider_loop") + assert health.snapshot("provider_loop").state == "ok" + # Without a budget, time spent in the cycle counts like any other. + health = running(clock, "outbox_loop", 1) + health.succeeded("outbox_loop") + health.started("outbox_loop") + clock.advance(3.5) + assert health.snapshot("outbox_loop").state == "degraded" + + +def test_a_cycle_that_did_no_work_neither_succeeds_nor_fails(): + clock = Clock() + health = running(clock, "outbox_loop", 1, budget=5) + health.succeeded("outbox_loop") + clock.advance(1) + health.started("outbox_loop") + health.skipped("outbox_loop") + # The skipped cycle is no longer in flight, so its budget does not hold the loop on time. + clock.advance(2.5) + snapshot = health.snapshot("outbox_loop") + assert (snapshot.state, snapshot.consecutive_failures, snapshot.recent_failures) == ("degraded", 0, 0) + assert snapshot.last_started_at == clock.now - timedelta(seconds=2.5) def test_last_success_counts_until_three_intervals_have_passed(): clock = Clock() health = running(clock) health.succeeded("outbox_loop") - clock.now += timedelta(seconds=15) + clock.advance(15) assert health.snapshot("outbox_loop").state == "ok" - clock.now += timedelta(microseconds=1) + clock.advance(0.000001) assert health.snapshot("outbox_loop").state == "degraded" @@ -145,6 +233,9 @@ def test_only_fixed_components_with_a_positive_interval_register(): for name, interval in (("tenant-1", 1), ("outbox_loop", 0), ("outbox_loop", float("inf")), ("outbox_loop", True)): with pytest.raises(ValueError): health.register(name, interval, enabled=True) + for budget in (-1, float("nan"), float("inf"), True): + with pytest.raises(ValueError): + health.register("outbox_loop", 1, enabled=True, budget_seconds=budget) with pytest.raises(ValueError): health.register("outbox_loop", 1, enabled="yes") # type: ignore[arg-type] with pytest.raises(KeyError): @@ -444,11 +535,84 @@ async def counted(): assert (snapshot.state, snapshot.consecutive_failures, snapshot.last_ok_at) == ("ok", 0, ok_at) assert reserved == [] # Its last success keeps aging, so a platform that stays not ready leaves the outbox behind, not failing. - clock.now += timedelta(seconds=4) + clock.advance(4) assert health.snapshot("outbox_loop").state == "degraded" assert health.snapshot("outbox_loop").consecutive_failures == 0 +async def test_an_outbox_without_a_free_delivery_slot_is_behind_not_healthy(monkeypatch): + from firefly_weave.operations import outbox_loop + from firefly_weave.operations.outbox_loop import OutboxLoop + + clock = Clock() + health = running(clock, interval=1, budget=5) + monkeypatch.setattr(outbox_loop, "reserve", lambda count: 0) + monkeypatch.setattr(outbox_loop, "TRAVERSAL_SECONDS", 0) + outbox = OutboxLoop( + SimpleNamespace(), # type: ignore[arg-type] + None, # type: ignore[arg-type] + SimpleNamespace(operational=lambda: True), # type: ignore[arg-type] + health=health, + ) + health.succeeded("outbox_loop") + # Every slot is taken, by slow deliveries or by slots that were never given back: no turn starts. + assert await outbox.cycle() is None + outbox.turn_failed = True + assert await outbox.cycle() is False + cycles = 0 + cycle = outbox.cycle + + async def counted(): + nonlocal cycles + try: + return await cycle() + finally: + cycles += 1 + + monkeypatch.setattr(outbox, "cycle", counted) + health.succeeded("outbox_loop") + task = asyncio.create_task(outbox.poll()) + try: + async with asyncio.timeout(2): + while cycles < 5: + await asyncio.sleep(0) + finally: + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + assert health.snapshot("outbox_loop").state == "ok" + clock.advance(3.5) + snapshot = health.snapshot("outbox_loop") + # Without a free slot for more than three intervals the outbox reads degraded, with no failure invented. + assert (snapshot.state, snapshot.consecutive_failures) == ("degraded", 0) + + +def test_each_background_loop_judges_a_cycle_in_flight_by_its_own_budget(): + clock = Clock() + health = LoopHealth(clock, monotonic=clock.monotonic) + register_background_loops(health, Settings(database_url=DATABASE, scheduler_database_url=SCHEDULER)) + budgets = {} + for name in ("recovery_loop", "schedules", "outbox_loop", "provider_loop", "operations_loop"): + health.running(name, True) + health.succeeded(name) + start = clock.elapsed + for name in ("recovery_loop", "schedules", "outbox_loop", "provider_loop", "operations_loop"): + health.started(name) + for seconds in (4.9, 5.1, 29.9, 30.1, 59.9, 60.1, 179.9, 180.1): + clock.elapsed = start + seconds + for name in ("recovery_loop", "schedules", "outbox_loop", "provider_loop", "operations_loop"): + if health.snapshot(name).state == "ok": + budgets[name] = seconds + # The traversal timeout bounds outbox cycles; tenant scans and mailbox polls take longer. + assert budgets == { + "outbox_loop": 4.9, + "recovery_loop": 29.9, + "schedules": 29.9, + "provider_loop": 59.9, + "operations_loop": 179.9, + } + + async def test_provider_failures_fail_the_poll_cycle(monkeypatch): from firefly_weave.providers.loop import ProviderLoop From 62785581ca566361599ea8a5ae980b2b36312fd1 Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Mon, 12 Oct 2026 09:16:35 -0700 Subject: [PATCH 24/63] Serve a capability-filtered environment summary and safe telemetry settings operations_summary.read answers an environment's health, load, workers, targets, incidents and up to 20 items that need attention. It checks status.read on the token before opening one read transaction, reloads the caller's grants there, and gives each section a savepoint with two seconds of database time. A section the caller may not read is forbidden before any of its statements run; one that times out or meets a transient database failure is unavailable while the others are still served, and a lost connection, cancellation or any other error fails the request. Targets, jobs, runners and approvals count only the targets both principals may read, approvals also need deployment.approve on the same target, and run figures leave out test runs while task and worker figures measure shared capacity. Ready tasks count only once they are due. operations_telemetry.read answers whether telemetry export is on, the collector host names, the metrics interval and the dropped count, never endpoint paths, credentials or headers. Both reads are in the CLI as weave operations summary and weave operations telemetry, and Studio's host bridge forwards them together with releases.usage. --- CHANGELOG.md | 18 + docs/operations/upgrades.md | 26 + docs/reference/api.md | 76 ++ src/firefly_weave/api/operations_summary.py | 53 ++ src/firefly_weave/app.py | 19 +- src/firefly_weave/cli/cluster_operations.py | 6 +- .../contracts/operations_summary.py | 153 ++++ src/firefly_weave/contracts/schema_export.py | 3 + src/firefly_weave/contracts/surface.py | 7 + src/firefly_weave/operations/summary.py | 612 ++++++++++++++ src/firefly_weave/studio/service.py | 5 +- .../test_operations_summary_surface.py | 172 ++++ tests/integration/test_operations_summary.py | 759 ++++++++++++++++++ .../test_operations_summary_rules.py | 343 ++++++++ tests/unit/test_studio_bridge_families.py | 85 +- 15 files changed, 2330 insertions(+), 7 deletions(-) create mode 100644 src/firefly_weave/api/operations_summary.py create mode 100644 src/firefly_weave/contracts/operations_summary.py create mode 100644 src/firefly_weave/operations/summary.py create mode 100644 tests/contracts/test_operations_summary_surface.py create mode 100644 tests/integration/test_operations_summary.py create mode 100644 tests/unit/operations/test_operations_summary_rules.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 54e02bff..bb8a6d3c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -132,6 +132,24 @@ SPDX-License-Identifier: Apache-2.0 combined with keyword arguments. - Kubernetes runners no longer read a pod's free-text status reason, and a pod whose readiness is unknown is listed as a member that is not ready. +- New operation `operations_summary.read` (`weave operations summary`) answers + an environment's health, load, workers, targets, incidents and up to 20 items + that need attention, in one read that reloads the caller's grants. Each + section and item needs its own capability; one the caller may not read is + `{"state": "forbidden"}`, and one that runs out of its two seconds of + database time is `{"state": "unavailable"}` while the rest are still served. + Run figures leave out test runs. +- New operation `operations_telemetry.read` (`weave operations telemetry`) + answers whether telemetry export is on, the collector host names, the + metrics interval and how many items were dropped, never endpoint paths, + credentials or headers. It needs `status.read` for the project. +- Background loop health, which the summary reports, ages successes with a + monotonic clock, counts a cycle still within its loop's budget as on time, + and reads degraded after 3 failed cycles in a row or among the last 20. An + outbox cycle that cannot start a delivery, because the platform is not ready + or no delivery slot is free, is neither a success nor a failure. +- Studio's host bridge forwards the environment summary, the telemetry + settings and `releases.usage`. - Keep typing in the selected form field when deferred row autofocus completes. Rapid row additions focus the newest item without overriding a later choice. diff --git a/docs/operations/upgrades.md b/docs/operations/upgrades.md index ccdc9768..73d034b9 100644 --- a/docs/operations/upgrades.md +++ b/docs/operations/upgrades.md @@ -328,6 +328,32 @@ leases per command, and `releases.usage` (`weave workers releases usage`, SDK `release_usage`) reports where a release runs. Neither changes an existing answer. Earlier servers do not serve them and answer `404` or `405`. +## Environment summary and telemetry settings + +Two reads are new, and neither changes an existing answer: + +- `operations_summary.read` (`weave operations summary`) answers an + environment's health, load, workers, targets, incidents and the items that + need attention. It needs `status.read`; each section beyond health, load and + workers needs its own capability, and a section you may not read answers + `{"state": "forbidden"}` instead of zeros. Run figures leave out test runs. +- `operations_telemetry.read` (`weave operations telemetry`) answers whether + telemetry export is on and which collector hosts it uses. It needs + `status.read` granted for the project, not only for an environment, and + never returns endpoint paths, credentials or headers. + +Earlier servers do not serve them and answer `404` or `405`. Loop health is +measured in each API process and starts again when the process restarts, so a +freshly started process reads `degraded` until each loop completes a cycle. An +answer from an API-only replica reports every scheduler loop as `disabled`; the +summary's `instance` ID tells you which process answered. The reference explains +how each state is decided; see +[Discovery, debugging, and history](../reference/api.md#discovery-debugging-and-history). + +The Studio host bridge now forwards both reads and `releases.usage`. An earlier +Studio host or desktop app refuses them with `404`, so upgrade Studio and the +desktop app with the server before relying on the pages that use them. + ## Operational fact storage The `0031_operations_facts` migration adds five scoped operational projections diff --git a/docs/reference/api.md b/docs/reference/api.md index f82b6bd7..e452d061 100644 --- a/docs/reference/api.md +++ b/docs/reference/api.md @@ -779,6 +779,80 @@ answers `{workers, deployments, activations, next_cursor}`: readable targets. - A release of another environment answers `404 WV-NOT-FOUND`. +**Environment summary.** `operations_summary.read` +(`GET …/environments/{environment}/operations/summary`, `weave operations +summary`) answers one environment's platform health in a single read. It needs +`status.read` on both the token and the current principal, and reads every +section in the same transaction that reloads those grants. + +- `observed_at` is the database time of the answer. `api` names the server + `version`, its image `build` digest when the server knows it, the `instance` + ID of the process that answered (new each time it starts), its + `uptime_seconds`, and `compatibility` (`mode` and the number of this + project's `findings`). +- `health` lists `database` (with `latency_ms` of a bounded probe), each + background loop (`recovery_loop`, `schedules`, `outbox_loop`, + `provider_loop`, `kafka_loop`, `operations_loop`, `alert_evaluator`), + `native_executors`, and `telemetry`, each with a `state` of `ok`, `degraded`, + `down`, or `disabled`. Loops add `last_started_at` and `last_ok_at` when + known, `consecutive_failures`, and `recent_failures` (failed cycles among + the last 20). Health is measured by the process that answered; no exception + text is returned. +- `load`: `runs_active` (test runs left out), `tasks_ready` and + `oldest_ready_seconds` (tasks due now; a retry joins when its next attempt is + due), `tasks_leased`, `waits_active`, `waits_overdue` and + `oldest_overdue_seconds`, `outbox_pending` and `outbox_oldest_seconds`, + `provider_pending`, and `sources_enabled`. Task and worker figures count test + runs' work too, because they measure shared capacity. An age is `null` when + nothing waits. +- `workers` counts current workers as `workers.list` lists them by default: + `online` (seen in the last 60 seconds), `offline`, `draining`, the `capacity` + of online workers that are not draining, and the `active_leases` workers hold. +- `targets` needs `deployment.read` and counts only the targets both the token + and the current principal may read: `total`, `runner_online`, `busy` (an + active job), `reconciliation_required`, and `awaiting_approval` (plans that + can still be approved: unexpired, on a fresh observation of the current + target and deployment revisions, and neither approved nor applied). +- `incidents` needs `incident.read`: `active` and `opened_24h`, leaving out + incidents of test runs. +- `attention` lists at most 20 items, `{kind, id, title}` plus `expires_at` for + approvals, in this order: jobs that need reconciliation, plans you may + approve (soonest expiry first), offline workers that still hold active + leases, active incidents, targets whose runners are all offline, a restricted + compatibility verdict (`id` is `compatibility`), and runs that failed in the + last hour. Within a kind, items come oldest first, then by ID. Each kind + needs the capability that reads its records: `deployment.read` on the target + for jobs and runners, `deployment.read` and `deployment.approve` on the same + target for approvals, `incident.read` for incidents, and `run.read` for + failed runs. Items you may not read are left out; test runs never appear. +- A section you may not read is `{"state": "forbidden"}`. Each section has two + seconds of database time; one that runs out, or meets a transient database + failure, is `{"state": "unavailable"}` while the others are still served with + `200`. Neither state is ever reported as zero. The answer has no 24-hour run + figures yet. + +Loop states follow these rules. A loop is `disabled` when its setting turns it +off (API-only replicas disable every scheduler loop) and `down` while its task +is not running. It is `degraded` before its first successful cycle, after 3 +failed cycles in a row or 3 failed cycles among its last 20, and when its last +success is older than three poll intervals. A cycle still running within its +loop's budget is on time: 5 seconds for the outbox and broker traversals, 30 +seconds for recovery and schedules, and 60 seconds for provider and mailbox +polling. A cycle that starts no work, such as an outbox cycle while the platform +is not ready or while no delivery slot is free, is neither a success nor a +failure, so a loop that keeps waiting turns `degraded` without counting +failures. Ages use the process's monotonic clock; the times shown come from +its wall clock. + +**Telemetry settings.** `operations_telemetry.read` +(`GET …/projects/{project}/operations/telemetry`, `weave operations +telemetry`) needs `status.read` for the project, on both the token and the +current principal; an environment grant is not enough. It answers whether +telemetry export is `enabled`, whether `traces` and `metrics` have a collector +`configured` and its `host` name, the metrics `interval_seconds`, and how many +telemetry items this process has `dropped`. It never returns endpoint paths, +ports, queries, credentials, or headers. + **Answers that say "unavailable".** Cancellation and task completion can return an unavailable acknowledgment when the platform cannot classify the historical data or compare payloads; task completion then includes @@ -827,6 +901,8 @@ operation in the [full API reference](api-explorer.md); to generate a client, | `lumi.status` | `GET /api/v1/tenants/{tenant}/projects/{project}/environments/{environment}/lumi/status` | lumi.use | | `lumi.ask` | `POST /api/v1/tenants/{tenant}/projects/{project}/environments/{environment}/lumi/ask` | lumi.use | | `compatibility.read` | `GET /api/v1/tenants/{tenant}/projects/{project}/operations/compatibility` | status.read | +| `operations_summary.read` | `GET /api/v1/tenants/{tenant}/projects/{project}/environments/{environment}/operations/summary` | status.read | +| `operations_telemetry.read` | `GET /api/v1/tenants/{tenant}/projects/{project}/operations/telemetry` | status.read | | `compatibility.check` | `POST /api/v1/tenants/{tenant}/projects/{project}/operations/compatibility/check` | compatibility.check | | `retention.plan` | `POST /api/v1/tenants/{tenant}/projects/{project}/operations/retention/plans` | retention.plan | | `retention.read` | `GET /api/v1/tenants/{tenant}/projects/{project}/operations/retention/plans/{identifier}` | retention.plan | diff --git a/src/firefly_weave/api/operations_summary.py b/src/firefly_weave/api/operations_summary.py new file mode 100644 index 00000000..adcdde07 --- /dev/null +++ b/src/firefly_weave/api/operations_summary.py @@ -0,0 +1,53 @@ +# Copyright 2026 Firefly Software Foundation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# Author: Firefly Software Foundation +# SPDX-License-Identifier: Apache-2.0 + +"""Native routes for the environment summary and the project's telemetry settings.""" + +from pyfly.container.stereotypes import rest_controller +from pyfly.web import request_mapping +from starlette.requests import Request +from starlette.responses import JSONResponse + +from firefly_weave.api.access import request_scope +from firefly_weave.api.surface import operation +from firefly_weave.definitions.models import CatalogError +from firefly_weave.operations.summary import OperationsSummaryService + + +def _service(request: Request) -> OperationsSummaryService: + # The service belongs to the process lifespan: it carries this process's instance ID and loop health. + service = getattr(request.app.state, "operations_summary", None) + if not isinstance(service, OperationsSummaryService): + raise CatalogError(503, "WV-UNAVAILABLE", "The platform summary is not available yet") + return service + + +@rest_controller +@request_mapping("") +class OperationsSummaryController: + @operation("operations_summary.read") + async def summary(self, request: Request) -> JSONResponse: + result = await _service(request).read( + request.state.principal, request_scope(request, environment=True), context=request.state.audit_context + ) + return JSONResponse(result.model_dump(mode="json")) + + @operation("operations_telemetry.read") + async def telemetry(self, request: Request) -> JSONResponse: + result = await _service(request).telemetry( + request.state.principal, request_scope(request), context=request.state.audit_context + ) + return JSONResponse(result.model_dump(mode="json")) diff --git a/src/firefly_weave/app.py b/src/firefly_weave/app.py index 5da0dac7..fd40b411 100644 --- a/src/firefly_weave/app.py +++ b/src/firefly_weave/app.py @@ -19,10 +19,11 @@ import asyncio import logging import os +import time from collections.abc import AsyncIterator from contextlib import asynccontextmanager from pathlib import Path -from uuid import UUID +from uuid import UUID, uuid4 from pyfly.client.ports.outbound import BoundedHttpClientPort from pyfly.container.bean import bean @@ -56,6 +57,7 @@ from firefly_weave.api.human_files import HumanFileController from firefly_weave.api.lumi import LumiController from firefly_weave.api.operations import OperationsController +from firefly_weave.api.operations_summary import OperationsSummaryController from firefly_weave.api.providers import ProviderController from firefly_weave.api.run_views import RunViewController from firefly_weave.api.runs import RunController @@ -108,6 +110,7 @@ from firefly_weave.operations.lumi_gateway import LumiGatewayClient from firefly_weave.operations.outbox_loop import OutboxLoop from firefly_weave.operations.run_views import RunViewService +from firefly_weave.operations.summary import OperationsSummaryService from firefly_weave.operations.telemetry import TelemetryService from firefly_weave.persistence.resources import DatabaseResources from firefly_weave.persistence.uow import UnitOfWork @@ -272,6 +275,8 @@ async def lifespan(app: Starlette) -> AsyncIterator[None]: health = LoopHealth() register_background_loops(health, settings) app.state.loop_health = health + # The summary names the process that answered and how long it has been up. + instance, started = uuid4(), time.monotonic() try: if settings.broker.enabled: require_driver() @@ -364,12 +369,24 @@ async def refresh_models( RunViewService, RunViewController, WorkerController, + OperationsSummaryController, ): pyfly.context.get_bean(required) app.state.authentication = pyfly.context.get_bean(AuthenticationService) await resources.check_startup() compatibility = pyfly.context.get_bean(CompatibilityService) app.state.compatibility = compatibility + app.state.operations_summary = OperationsSummaryService( + pyfly.context.get_bean(UnitOfWork), + pyfly.context.get_bean(AuthorizationService), + settings, + health, + compatibility, + drops=telemetry_drops, + instance=instance, + started_monotonic=started, + native=lambda: getattr(app.state, "dispatcher", None), + ) app.state.telemetry_service = pyfly.context.get_bean(TelemetryService) registry.set_operational_guard(lambda: compatibility.ready, retry_after=compatibility.retry_after_seconds) diff --git a/src/firefly_weave/cli/cluster_operations.py b/src/firefly_weave/cli/cluster_operations.py index 785178ad..5e508b16 100644 --- a/src/firefly_weave/cli/cluster_operations.py +++ b/src/firefly_weave/cli/cluster_operations.py @@ -26,7 +26,7 @@ import click -from firefly_weave.cli.remote import family, machine_result +from firefly_weave.cli.remote import command, family, machine_result @click.group("operations") @@ -35,6 +35,8 @@ def operations() -> None: Observe a target, set desired components, create a plan, review and approve its digest, then apply it. The runner keeps provider credentials locally. + `summary` and `telemetry` read the environment's platform health and the + project's telemetry settings. """ @@ -47,6 +49,8 @@ def operations() -> None: ("runners", "deployment_runners"), ): operations.add_command(family(name, prefix)) +operations.add_command(command("operations_summary.read", "summary")) +operations.add_command(command("operations_telemetry.read", "telemetry")) @operations.group("runner") diff --git a/src/firefly_weave/contracts/operations_summary.py b/src/firefly_weave/contracts/operations_summary.py new file mode 100644 index 00000000..9a18e0d1 --- /dev/null +++ b/src/firefly_weave/contracts/operations_summary.py @@ -0,0 +1,153 @@ +# Copyright 2026 Firefly Software Foundation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# Author: Firefly Software Foundation +# SPDX-License-Identifier: Apache-2.0 + +"""Bounded environment summary and telemetry configuration answers. + +A section the caller may not read is ``{"state": "forbidden"}`` and a section the server could not compute in time +is ``{"state": "unavailable"}``; neither is ever reported as zero. Run figures leave out test runs. +""" + +from typing import Annotated, Literal +from uuid import UUID + +from pydantic import AwareDatetime, Field + +from firefly_weave.contracts.definitions import ContractModel + +type Count = Annotated[int, Field(ge=0)] +type Seconds = Annotated[float, Field(ge=0)] +type HealthState = Literal["ok", "degraded", "down", "disabled"] +type HealthComponentName = Literal[ + "database", + "recovery_loop", + "schedules", + "outbox_loop", + "provider_loop", + "kafka_loop", + "operations_loop", + "alert_evaluator", + "native_executors", + "telemetry", +] +type AttentionKind = Literal["job", "approval", "worker", "incident", "runner", "compatibility", "run"] +type AttentionTitle = Annotated[str, Field(min_length=1, max_length=200)] +type Host = Annotated[str, Field(min_length=1, max_length=253)] + +ATTENTION_LIMIT = 20 +HEALTH_LIMIT = 16 +# The one attention item that is not a record: the platform's compatibility verdict. +COMPATIBILITY_ITEM: Literal["compatibility"] = "compatibility" + + +class ForbiddenSection(ContractModel): + state: Literal["forbidden"] + + +class UnavailableSection(ContractModel): + state: Literal["unavailable"] + + +class CompatibilitySummary(ContractModel): + mode: Literal["ready", "restricted"] + findings: Count + + +class ApiSection(ContractModel): + version: str = Field(min_length=1, max_length=64) + build: str | None = Field(pattern=r"^sha256:[a-f0-9]{64}$") + instance: UUID + uptime_seconds: Seconds + compatibility: CompatibilitySummary + + +class HealthComponent(ContractModel): + component: HealthComponentName + state: HealthState + last_started_at: AwareDatetime | None = Field(default=None, exclude_if=lambda value: value is None) + last_ok_at: AwareDatetime | None = Field(default=None, exclude_if=lambda value: value is None) + consecutive_failures: Count | None = Field(default=None, exclude_if=lambda value: value is None) + recent_failures: Count | None = Field(default=None, exclude_if=lambda value: value is None) + complete_until: AwareDatetime | None = Field(default=None, exclude_if=lambda value: value is None) + latency_ms: Seconds | None = Field(default=None, exclude_if=lambda value: value is None) + + +class LoadSection(ContractModel): + runs_active: Count + tasks_ready: Count + tasks_leased: Count + oldest_ready_seconds: Seconds | None + waits_active: Count + waits_overdue: Count + oldest_overdue_seconds: Seconds | None + outbox_pending: Count + outbox_oldest_seconds: Seconds | None + provider_pending: Count + sources_enabled: Count + + +class WorkersSection(ContractModel): + online: Count + offline: Count + draining: Count + capacity: Count + active_leases: Count + + +class TargetsSection(ContractModel): + total: Count + runner_online: Count + busy: Count + reconciliation_required: Count + awaiting_approval: Count + + +class IncidentsSection(ContractModel): + active: Count + opened_24h: Count + + +class AttentionItem(ContractModel): + kind: AttentionKind + id: UUID | Literal["compatibility"] + title: AttentionTitle + expires_at: AwareDatetime | None = Field(default=None, exclude_if=lambda value: value is None) + + +class OperationsSummary(ContractModel): + observed_at: AwareDatetime + api: ApiSection + health: list[HealthComponent] = Field(max_length=HEALTH_LIMIT) + load: LoadSection | UnavailableSection + workers: WorkersSection | UnavailableSection + targets: TargetsSection | ForbiddenSection | UnavailableSection + incidents: IncidentsSection | ForbiddenSection | UnavailableSection + attention: Annotated[list[AttentionItem], Field(max_length=ATTENTION_LIMIT)] | UnavailableSection + + +class TelemetryEndpoint(ContractModel): + configured: bool + host: Host | None + + +class TelemetryMetricEndpoint(TelemetryEndpoint): + interval_seconds: Seconds | None + + +class OperationsTelemetry(ContractModel): + enabled: bool + traces: TelemetryEndpoint + metrics: TelemetryMetricEndpoint + dropped: Count diff --git a/src/firefly_weave/contracts/schema_export.py b/src/firefly_weave/contracts/schema_export.py index dbc6916a..1d5118d0 100644 --- a/src/firefly_weave/contracts/schema_export.py +++ b/src/firefly_weave/contracts/schema_export.py @@ -178,6 +178,7 @@ def contract_models() -> dict[str, SchemaModel]: RecordedEvidence, ReplayReport, ) + from firefly_weave.contracts.operations_summary import OperationsSummary, OperationsTelemetry from firefly_weave.contracts.providers import ( ProviderEvent, ProviderIngressResponse, @@ -413,6 +414,8 @@ def contract_models() -> dict[str, SchemaModel]: "retire-stale-result": RetireStaleResult, "release-usage-query": ReleaseUsageQuery, "release-usage": ReleaseUsage, + "operations-summary": OperationsSummary, + "operations-telemetry": OperationsTelemetry, "lease-proof": LeaseProof, "task-lease": TaskLease, "completion-receipt": CompletionReceipt, diff --git a/src/firefly_weave/contracts/surface.py b/src/firefly_weave/contracts/surface.py index ce8a21e9..8cc234f3 100644 --- a/src/firefly_weave/contracts/surface.py +++ b/src/firefly_weave/contracts/surface.py @@ -134,6 +134,7 @@ IncidentView, ReplayReport, ) +from firefly_weave.contracts.operations_summary import OperationsSummary, OperationsTelemetry from firefly_weave.contracts.providers import ( ProviderIngressResponse, ProviderReceipt, @@ -794,6 +795,12 @@ def native(self) -> OpenAPIOperation: CompatibilityReport, "compatibility.check", ), + Operation( + "operations_summary.read", ENVIRONMENT + "/operations/summary", "GET", OperationsSummary, "status.read" + ), + Operation( + "operations_telemetry.read", PROJECT + "/operations/telemetry", "GET", OperationsTelemetry, "status.read" + ), Operation( "retention.plan", PROJECT + "/operations/retention/plans", diff --git a/src/firefly_weave/operations/summary.py b/src/firefly_weave/operations/summary.py new file mode 100644 index 00000000..02c81bbd --- /dev/null +++ b/src/firefly_weave/operations/summary.py @@ -0,0 +1,612 @@ +# Copyright 2026 Firefly Software Foundation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# Author: Firefly Software Foundation +# SPDX-License-Identifier: Apache-2.0 + +"""The environment summary and the telemetry settings an operator may see. + +The summary is read in one transaction that also reloads the caller's grants. Beyond `status.read`, each section +and attention item needs its own capability on both the token and the principal stored now; a section the caller +may not read is forbidden before any of its statements run. Each section runs in a savepoint under a two-second +statement timeout, and one that times out is unavailable while the others are still served. Run figures leave out +test runs; queue and worker figures measure shared capacity and count every task. +""" + +import time +from collections.abc import Awaitable, Callable, Iterable +from dataclasses import dataclass +from datetime import UTC, datetime +from decimal import Decimal +from typing import Any, Literal, Protocol, cast +from urllib.parse import urlsplit +from uuid import UUID + +from sqlalchemy import text +from sqlalchemy.exc import DBAPIError + +from firefly_weave import __version__ +from firefly_weave.access.audit import AuditContext +from firefly_weave.access.authorization import AccessDenied, AuthorizationService +from firefly_weave.access.models import Principal +from firefly_weave.access.repository import load_principal +from firefly_weave.contracts.access import Scope +from firefly_weave.contracts.operations_summary import ( + ATTENTION_LIMIT, + COMPATIBILITY_ITEM, + ApiSection, + AttentionItem, + AttentionKind, + CompatibilitySummary, + ForbiddenSection, + HealthComponent, + HealthComponentName, + HealthState, + IncidentsSection, + LoadSection, + OperationsSummary, + OperationsTelemetry, + TargetsSection, + TelemetryEndpoint, + TelemetryMetricEndpoint, + UnavailableSection, + WorkersSection, +) +from firefly_weave.observability import TelemetryDrops +from firefly_weave.operations.compatibility import CompatibilityService +from firefly_weave.operations.fact_reads import available +from firefly_weave.operations.loop_health import LoopHealth, LoopSnapshot +from firefly_weave.operations.permissions import target_permissions +from firefly_weave.persistence.uow import Transaction, UnitOfWork +from firefly_weave.settings import Settings + +# Each section's statements share this budget; a stricter timeout already in force is kept. +SECTION_TIMEOUT_MS = 2000 +# Query failures a section reports as unavailable: a cancelled or timed-out statement, a lock timeout, a +# serialization conflict, a deadlock and exhausted database resources. Every other error propagates. +_TRANSIENT = ("57014", "55P03", "40001", "40P01", "53") +# Priority of each attention kind; critical alerts (1) arrive with alert storage. +PRIORITY: dict[str, int] = { + "job": 2, + "approval": 3, + "worker": 4, + "incident": 5, + "runner": 6, + "compatibility": 7, + "run": 8, +} +_TITLE = 200 +_SCOPE = "{alias}tenant_id=:tenant AND {alias}project_id=:project AND {alias}environment_id=:environment" +_TARGETS = "(:all_targets OR {column}=ANY(cast(:target_ids AS uuid[])))" +_SAME_SCOPE = "{a}.tenant_id={b}.tenant_id AND {a}.project_id={b}.project_id AND {a}.environment_id={b}.environment_id" +NOW = "cast(:now AS timestamptz)" +# Plans the approve operation would accept now: unexpired, on a fresh observation of the target's current revision +# and the deployment's current revision, for an intent the enabled target allows, and neither approved nor applied. +_AWAITING_APPROVAL = ( + "SELECT p.id,(p.payload->>'expires_at')::timestamptz AS expires_at,p.payload->>'intent' AS intent," + "t.payload->>'name' AS target_name FROM deployment_plans p " + f"JOIN deployment_targets t ON {_SAME_SCOPE.format(a='t', b='p')} AND t.id=p.target_id " + f"JOIN deployment_deployments d ON {_SAME_SCOPE.format(a='d', b='p')} AND d.id=p.deployment_id " + f"JOIN deployment_observations o ON {_SAME_SCOPE.format(a='o', b='p')} " + "AND o.id=(p.payload->>'observation_id')::uuid " + f"WHERE {_SCOPE.format(alias='p.')} AND {_TARGETS.format(column='p.target_id')} " + f"AND (p.payload->>'expires_at')::timestamptz>{NOW} AND (o.payload->>'expires_at')::timestamptz>{NOW} " + "AND (t.payload->>'revision')::integer=(p.payload->>'target_revision')::integer " + "AND NOT coalesce((t.payload->>'disabled')::boolean,false) " + "AND t.payload->'capabilities' @> jsonb_build_array(p.payload->>'intent') " + "AND (d.payload->>'revision')::integer=(p.payload->>'deployment_revision')::integer " + "AND o.payload->>'digest'=p.payload->>'observation_digest' " + f"AND NOT EXISTS(SELECT 1 FROM deployment_approvals a WHERE {_SAME_SCOPE.format(a='a', b='p')} AND a.id=p.id) " + f"AND NOT EXISTS(SELECT 1 FROM deployment_jobs j WHERE {_SAME_SCOPE.format(a='j', b='p')} " + "AND j.payload->>'plan_id'=p.id::text)" +) +_LIVE_RUNNER = ( + f"SELECT 1 FROM deployment_runners r WHERE {_SAME_SCOPE.format(a='r', b='t')} AND r.target_id=t.id " + f"AND NOT coalesce((r.payload->>'revoked')::boolean,false) AND (r.payload->>'expires_at')::timestamptz>{NOW}" +) +_LIVE_LEASES = ( + "SELECT owner,count(*) AS leases FROM task_leases WHERE " + + _SCOPE.format(alias="") + + f" AND status='active' AND expires_at>{NOW} AND deadline>{NOW} GROUP BY owner" +) +_RECENT = f"w.last_seen_at>{NOW}-interval '60 seconds'" +_CURRENT = f"NOT w.revoked AND coalesce(w.last_seen_at,f.registered_at)>{NOW}-interval '1 hour'" +_WORKER_FACTS = f"LEFT JOIN worker_facts f ON {_SAME_SCOPE.format(a='f', b='w')} AND f.worker_id=w.id" +_RUN_CLASSIFICATION = ( + "r.classification_state,r.classification_policy,r.pinned_ir_version,r.pinned_features," + f"EXISTS(SELECT 1 FROM run_policy_blocks b WHERE {_SAME_SCOPE.format(a='b', b='r')} AND b.run_id=r.run_id) " + "AS policy_blocked" +) + + +class NativeExecutors(Protocol): + def healthy(self) -> bool: ... + + +def telemetry_host(endpoint: str | None) -> str | None: + """Only the host name of a collector endpoint: never its credentials, port, path or query.""" + return urlsplit(endpoint).hostname if endpoint else None + + +def _transient(error: DBAPIError) -> bool: + state = getattr(error.orig, "sqlstate", None) + return isinstance(state, str) and state.startswith(_TRANSIENT) + + +async def read_section[T](tx: Transaction, read: Callable[[], Awaitable[T]]) -> T | UnavailableSection: + """Run one section's statements in a savepoint under the section timeout. + + A timed-out or otherwise transient query failure rolls the savepoint back, which restores the earlier local + timeout, and the section is unavailable. A lost connection, cancellation, an authorization failure and any other + error propagate, so a broken transaction never answers later sections. + """ + session = tx.session + row = ( + await session.execute( + text( + "SELECT current_setting('statement_timeout') AS prior,setting::bigint AS prior_ms FROM pg_settings " + "WHERE name='statement_timeout'" + ) + ) + ).one() + bound = min(row.prior_ms, SECTION_TIMEOUT_MS) if row.prior_ms else SECTION_TIMEOUT_MS + try: + async with session.begin_nested(): + await session.execute(text("SELECT set_config('statement_timeout',:value,true)"), {"value": f"{bound}ms"}) + value = await read() + await session.execute(text("SELECT set_config('statement_timeout',:value,true)"), {"value": row.prior}) + return value + except DBAPIError as error: + if error.connection_invalidated or not _transient(error): + raise + return UnavailableSection(state="unavailable") + + +async def read_ready_counts(tx: Transaction, observed_at: datetime) -> tuple[int, float | None]: + """Ready tasks that are due at `observed_at`, and how long the oldest of them has waited. + + A retry whose next attempt is still in the future is stored as ready from that instant, so it joins the count + exactly when it becomes due; nothing is written to make it so. Claims stay the authority on what runs. + """ + row = ( + ( + await tx.session.execute( + text( + "SELECT count(*) AS ready,extract(epoch FROM (cast(:now AS timestamptz)-min(ready_since))) " + f"AS oldest FROM task_facts WHERE {_SCOPE.format(alias='')} AND status='ready' " + "AND ready_since<=cast(:now AS timestamptz)" + ), + {**_bound(tx), "now": observed_at}, + ) + ) + .mappings() + .one() + ) + return int(row["ready"]), _seconds(row["oldest"]) + + +def _bound(tx: Transaction) -> dict[str, Any]: + return { + "tenant": tx.scope.tenant_id, + "project": tx.scope.project_id, + "environment": tx.scope.environment_id, + } + + +def _seconds(value: Decimal | float | None) -> float | None: + return None if value is None else max(0.0, float(value)) + + +def _title(value: str) -> str: + return value if len(value) <= _TITLE else value[: _TITLE - 1] + "…" + + +def _plural(count: int, noun: str) -> str: + return f"{count} {noun}" + ("" if count == 1 else "s") + + +def native_state(settings: Settings, dispatcher: NativeExecutors | None) -> HealthState: + """Native executors as readiness reports them: off when none are configured, down while not open.""" + if not settings.native_executors: + return "disabled" + if dispatcher is None: + return "down" + return "ok" if dispatcher.healthy() else "degraded" + + +def health_components( + latency: float | UnavailableSection, + snapshots: Iterable[LoopSnapshot], + *, + native: HealthState, + telemetry_enabled: bool, +) -> list[HealthComponent]: + """The database, each registered loop, native executors and telemetry export; states and counts, never messages.""" + database = ( + HealthComponent(component="database", state="ok", latency_ms=latency) + if isinstance(latency, float) + else HealthComponent(component="database", state="degraded") + ) + loops = [ + HealthComponent( + component=cast(HealthComponentName, snapshot.component), + state=snapshot.state, + last_started_at=snapshot.last_started_at, + last_ok_at=snapshot.last_ok_at, + consecutive_failures=snapshot.consecutive_failures, + recent_failures=snapshot.recent_failures, + complete_until=snapshot.complete_until, + ) + for snapshot in snapshots + ] + return [ + database, + *loops, + HealthComponent(component="native_executors", state=native), + HealthComponent(component="telemetry", state="ok" if telemetry_enabled else "disabled"), + ] + + +@dataclass(frozen=True) +class Candidate: + """An attention item before ranking: `at` orders items of one kind, and is the expiry for approvals.""" + + kind: AttentionKind + id: UUID | Literal["compatibility"] + title: str + at: datetime | None = None + expires_at: datetime | None = None + + +def attention_order(candidates: Iterable[Candidate]) -> list[AttentionItem]: + """At most 20 items: by priority, then soonest time (unknown last), then ID, deterministically.""" + + def key(item: Candidate) -> tuple[int, bool, datetime, int]: + identifier = item.id.int if isinstance(item.id, UUID) else 0 + return PRIORITY[item.kind], item.at is None, item.at or datetime.min.replace(tzinfo=UTC), identifier + + return [ + AttentionItem(kind=item.kind, id=item.id, title=_title(item.title), expires_at=item.expires_at) + for item in sorted(candidates, key=key)[:ATTENTION_LIMIT] + ] + + +@dataclass(frozen=True) +class _Reach: + """What the caller may read beyond `status.read`, from both the token and the principal stored now. + + Target sets are None for every target and empty for none. + """ + + incidents: bool + runs: bool + targets: tuple[UUID, ...] | None + approvals: tuple[UUID, ...] | None + + +def _intersect(left: tuple[UUID, ...] | None, right: tuple[UUID, ...] | None) -> tuple[UUID, ...] | None: + if left is None: + return right + if right is None: + return left + return tuple(sorted(set(left) & set(right))) + + +class _Reader: + def __init__(self, tx: Transaction, observed_at: datetime) -> None: + self.tx = tx + self.bound = {**_bound(tx), "now": observed_at} + + async def rows(self, sql: str, **values: Any) -> list[dict[str, Any]]: + result = await self.tx.session.execute(text(sql), {**self.bound, **values}) + return [dict(row) for row in result.mappings()] + + async def row(self, sql: str, **values: Any) -> dict[str, Any]: + return (await self.rows(sql, **values))[0] + + +def _targets(targets: tuple[UUID, ...] | None) -> dict[str, Any]: + return {"all_targets": targets is None, "target_ids": list(targets or ())} + + +class OperationsSummaryService: + """Lifespan-owned: one instance ID, one health registry and the exporters' own drop counter per process.""" + + def __init__( + self, + uow: UnitOfWork, + authorization: AuthorizationService, + settings: Settings, + health: LoopHealth, + compatibility: CompatibilityService, + *, + drops: TelemetryDrops, + instance: UUID, + started_monotonic: float, + native: Callable[[], NativeExecutors | None] = lambda: None, + monotonic: Callable[[], float] = time.monotonic, + ) -> None: + self.uow, self.authorization, self.settings = uow, authorization, settings + self.health, self.compatibility = health, compatibility + self.drops, self.instance, self.started_monotonic = drops, instance, started_monotonic + self.native, self.monotonic = native, monotonic + + async def read(self, actor: Principal, scope: Scope, *, context: AuditContext) -> OperationsSummary: + if scope.environment_id is None: + raise AccessDenied() + self.authorization.require(actor, scope, "status.read", context=context) + async with self.uow.open(scope, mutation=False) as tx: + current = await load_principal(tx.session, actor.id) + self.authorization.require(current, scope, "status.read", context=context) + reach = self._reach(actor, current, scope, context) + observed_at = await tx.session.scalar(text("SELECT clock_timestamp()")) + assert isinstance(observed_at, datetime) + reader = _Reader(tx, observed_at) + latency = await read_section(tx, lambda: self._latency(tx)) + load = await read_section(tx, lambda: self._load(reader)) + workers = await read_section(tx, lambda: self._workers(reader)) + targets: TargetsSection | ForbiddenSection | UnavailableSection = ( + ForbiddenSection(state="forbidden") + if reach.targets == () + else await read_section(tx, lambda: self._targets(reader, reach.targets)) + ) + incidents: IncidentsSection | ForbiddenSection | UnavailableSection = ( + await read_section(tx, lambda: self._incidents(reader)) + if reach.incidents + else ForbiddenSection(state="forbidden") + ) + attention = await read_section(tx, lambda: self._attention(reader, reach)) + return OperationsSummary( + observed_at=observed_at, + api=self._api(scope), + health=self._health(latency), + load=load, + workers=workers, + targets=targets, + incidents=incidents, + attention=attention, + ) + + async def telemetry(self, actor: Principal, scope: Scope, *, context: AuditContext) -> OperationsTelemetry: + """Whether telemetry is exported and to which collector hosts; never headers, credentials or full endpoints.""" + if scope.project_id is None: + raise AccessDenied() + self.authorization.require(actor, scope, "status.read", context=context) + async with self.uow.open(scope, mutation=False) as tx: + current = await load_principal(tx.session, actor.id) + self.authorization.require(current, scope, "status.read", context=context) + options = self.settings.telemetry + return OperationsTelemetry( + enabled=options.enabled, + traces=TelemetryEndpoint( + configured=options.traces_endpoint is not None, host=telemetry_host(options.traces_endpoint) + ), + metrics=TelemetryMetricEndpoint( + configured=options.metrics_endpoint is not None, + host=telemetry_host(options.metrics_endpoint), + interval_seconds=options.metrics_interval_seconds if options.metrics_endpoint else None, + ), + dropped=self.drops.count, + ) + + def _reach(self, actor: Principal, current: Principal, scope: Scope, context: AuditContext) -> _Reach: + def permitted(capability: str) -> bool: + try: + for principal in (actor, current): + self.authorization.require(principal, scope, capability, context=context) + except AccessDenied: + return False + return True + + readable = target_permissions(self.authorization, actor, current, scope, "deployment.read", context) + approvable = target_permissions(self.authorization, actor, current, scope, "deployment.approve", context) + return _Reach( + incidents=permitted("incident.read"), + runs=permitted("run.read"), + targets=readable, + approvals=_intersect(readable, approvable), + ) + + def _api(self, scope: Scope) -> ApiSection: + report = self.compatibility.report + # Only this project's findings are counted, as the compatibility report shows them. + findings = sum( + 1 + for finding in report.findings + if finding.scope is None + or (finding.scope.tenant_id == scope.tenant_id and finding.scope.project_id == scope.project_id) + ) + return ApiSection( + version=__version__, + build=self.settings.native_image_digest, + instance=self.instance, + uptime_seconds=max(0.0, self.monotonic() - self.started_monotonic), + compatibility=CompatibilitySummary( + mode="ready" if self.compatibility.ready else "restricted", findings=findings + ), + ) + + def _health(self, latency: float | UnavailableSection) -> list[HealthComponent]: + return health_components( + latency, + self.health.snapshots(), + native=native_state(self.settings, self.native()), + telemetry_enabled=self.settings.telemetry.enabled, + ) + + async def _latency(self, tx: Transaction) -> float: + started = self.monotonic() + await tx.session.execute(text("SELECT 1")) + return round(max(0.0, self.monotonic() - started) * 1000, 3) + + async def _load(self, reader: _Reader) -> LoadSection: + usage = { + row["metric"]: int(row["value"]) + for row in await reader.rows( + "SELECT metric,value FROM operation_usage WHERE " + + _SCOPE.format(alias="") + + " AND metric IN ('runs_active','waits_active','outbox_pending','provider_pending','sources_enabled')" + ) + } + tests = await reader.row( + f"SELECT count(*) AS active FROM run_facts WHERE {_SCOPE.format(alias='')} AND test AND ended_at IS NULL" + ) + ready, oldest_ready = await read_ready_counts(reader.tx, reader.bound["now"]) + leased = await reader.row( + "SELECT count(DISTINCT l.task_id) AS leased FROM task_leases l JOIN task_intents i " + f"ON {_SAME_SCOPE.format(a='i', b='l')} AND i.id=l.task_id WHERE {_SCOPE.format(alias='l.')} " + f"AND l.status='active' AND l.expires_at>{NOW} AND l.deadline>{NOW} AND i.status='leased'" + ) + overdue = await reader.row( + f"SELECT count(*) AS overdue,extract(epoch FROM ({NOW}-min(deadline))) AS oldest FROM run_deadlines " + f"WHERE {_SCOPE.format(alias='')} AND NOT consumed AND deadline<={NOW}" + ) + outbox = await reader.row( + f"SELECT extract(epoch FROM ({NOW}-min(next_at))) AS oldest FROM event_deliveries " + f"WHERE {_SCOPE.format(alias='')} AND status IN ('pending','retry') AND next_at<={NOW}" + ) + return LoadSection( + runs_active=max(0, usage.get("runs_active", 0) - int(tests["active"])), + tasks_ready=ready, + tasks_leased=int(leased["leased"]), + oldest_ready_seconds=oldest_ready, + waits_active=usage.get("waits_active", 0), + waits_overdue=int(overdue["overdue"]), + oldest_overdue_seconds=_seconds(overdue["oldest"]), + outbox_pending=usage.get("outbox_pending", 0), + outbox_oldest_seconds=_seconds(outbox["oldest"]), + provider_pending=usage.get("provider_pending", 0), + sources_enabled=usage.get("sources_enabled", 0), + ) + + async def _workers(self, reader: _Reader) -> WorkersSection: + """One grouped statement over the environment's workers, counted as the Workers page lists current ones.""" + row = await reader.row( + f"WITH live AS ({_LIVE_LEASES}) SELECT " + f"count(*) FILTER (WHERE {_CURRENT} AND NOT w.draining AND {_RECENT}) AS online," + f"count(*) FILTER (WHERE {_CURRENT} AND NOT w.draining AND NOT coalesce({_RECENT},false)) AS offline," + f"count(*) FILTER (WHERE {_CURRENT} AND w.draining) AS draining," + f"coalesce(sum((w.payload->>'capacity')::integer) FILTER (WHERE {_CURRENT} AND NOT w.draining " + f"AND {_RECENT}),0) AS capacity,coalesce(sum(a.leases),0) AS active_leases " + f"FROM worker_instances w {_WORKER_FACTS} LEFT JOIN live a ON a.owner=w.id " + f"WHERE {_SCOPE.format(alias='w.')}" + ) + return WorkersSection(**{key: int(value) for key, value in row.items()}) + + async def _targets(self, reader: _Reader, targets: tuple[UUID, ...] | None) -> TargetsSection: + """Each readable target counted once, however many runners, jobs or plans it has.""" + row = await reader.row( + "SELECT count(*) AS total," + f"count(*) FILTER (WHERE EXISTS({_LIVE_RUNNER})) AS runner_online," + "count(*) FILTER (WHERE EXISTS(SELECT 1 FROM deployment_jobs j WHERE " + f"{_SAME_SCOPE.format(a='j', b='t')} AND j.target_id=t.id " + "AND j.state IN ('queued','claimed','running','verifying'))) AS busy," + "count(*) FILTER (WHERE EXISTS(SELECT 1 FROM deployment_jobs j WHERE " + f"{_SAME_SCOPE.format(a='j', b='t')} AND j.target_id=t.id " + "AND j.state='reconciliation_required')) AS reconciliation_required " + f"FROM deployment_targets t WHERE {_SCOPE.format(alias='t.')} AND {_TARGETS.format(column='t.id')}", + **_targets(targets), + ) + plans = await reader.row(f"SELECT count(*) AS awaiting FROM ({_AWAITING_APPROVAL}) p", **_targets(targets)) + return TargetsSection( + **{key: int(value) for key, value in row.items()}, awaiting_approval=int(plans["awaiting"]) + ) + + async def _incidents(self, reader: _Reader) -> IncidentsSection: + """Incidents of test runs are left out; an incident without facts predates test runs and counts.""" + active = await reader.row( + "SELECT count(*) AS active FROM incidents i LEFT JOIN incident_facts f ON " + f"{_SAME_SCOPE.format(a='f', b='i')} AND f.incident_id=i.id " + f"WHERE {_SCOPE.format(alias='i.')} AND i.status='active' AND NOT coalesce(f.test,false)" + ) + opened = await reader.row( + f"SELECT count(*) AS opened FROM incident_facts WHERE {_SCOPE.format(alias='')} AND NOT test " + f"AND opened_at>{NOW}-interval '24 hours'" + ) + return IncidentsSection(active=int(active["active"]), opened_24h=int(opened["opened"])) + + async def _attention(self, reader: _Reader, reach: _Reach) -> list[AttentionItem]: + """Up to 20 items per kind the caller may read, ranked together; alerts join once they are stored.""" + candidates: list[Candidate] = [] + limit = ATTENTION_LIMIT + if reach.targets != (): + for row in await reader.rows( + "SELECT j.id,t.payload->>'name' AS target_name," + "coalesce((j.payload->>'reconciliation_started_at')::timestamptz,j.created_at) AS at " + f"FROM deployment_jobs j JOIN deployment_targets t ON {_SAME_SCOPE.format(a='t', b='j')} " + f"AND t.id=j.target_id WHERE {_SCOPE.format(alias='j.')} AND {_TARGETS.format(column='j.target_id')} " + "AND j.state='reconciliation_required' ORDER BY at,j.id LIMIT :limit", + limit=limit, + **_targets(reach.targets), + ): + title = f"Deployment job on {row['target_name']} needs reconciliation" + candidates.append(Candidate("job", row["id"], title, row["at"])) + if reach.approvals != (): + for row in await reader.rows( + f"SELECT p.* FROM ({_AWAITING_APPROVAL}) p ORDER BY p.expires_at,p.id LIMIT :limit", + limit=limit, + **_targets(reach.approvals), + ): + intent = str(row["intent"]).replace("_", " ") + title = f"Approve the plan to {intent} on {row['target_name']}" + candidates.append(Candidate("approval", row["id"], title, row["expires_at"], row["expires_at"])) + for row in await reader.rows( + f"WITH live AS ({_LIVE_LEASES}) SELECT w.id,a.leases,coalesce(w.last_seen_at,f.registered_at) AS at " + f"FROM worker_instances w JOIN live a ON a.owner=w.id {_WORKER_FACTS} " + f"WHERE {_SCOPE.format(alias='w.')} AND NOT w.revoked AND NOT coalesce({_RECENT},false) " + "ORDER BY at ASC NULLS LAST,w.id LIMIT :limit", + limit=limit, + ): + leases = int(row["leases"]) + title = f"Offline worker still holds {_plural(leases, 'active lease')}" + candidates.append(Candidate("worker", row["id"], title, row["at"])) + if reach.incidents: + for row in await reader.rows( + f"SELECT i.id,f.opened_at,f.workflow_name,{_RUN_CLASSIFICATION} FROM incidents i " + f"LEFT JOIN incident_facts f ON {_SAME_SCOPE.format(a='f', b='i')} AND f.incident_id=i.id " + f"LEFT JOIN run_facts r ON {_SAME_SCOPE.format(a='r', b='i')} AND r.run_id=i.run_id " + f"WHERE {_SCOPE.format(alias='i.')} AND i.status='active' AND NOT coalesce(f.test,false) " + "ORDER BY f.opened_at ASC NULLS LAST,i.id LIMIT :limit", + limit=limit, + ): + named = row["workflow_name"] is not None and available(row) + title = f"Active incident in {row['workflow_name']}" if named else "Active incident" + candidates.append(Candidate("incident", row["id"], title, row["opened_at"])) + if reach.targets != (): + for row in await reader.rows( + "SELECT * FROM (SELECT DISTINCT ON (r.target_id) r.id,t.payload->>'name' AS target_name," + "(r.payload->>'expires_at')::timestamptz AS at FROM deployment_runners r " + f"JOIN deployment_targets t ON {_SAME_SCOPE.format(a='t', b='r')} AND t.id=r.target_id " + f"WHERE {_SCOPE.format(alias='r.')} AND {_TARGETS.format(column='r.target_id')} " + "AND NOT coalesce((t.payload->>'disabled')::boolean,false) " + "AND NOT coalesce((r.payload->>'revoked')::boolean,false) " + f"AND NOT EXISTS({_LIVE_RUNNER}) " + "ORDER BY r.target_id,(r.payload->>'expires_at')::timestamptz DESC,r.id DESC) offline " + "ORDER BY at,id LIMIT :limit", + limit=limit, + **_targets(reach.targets), + ): + title = f"No runner is online for {row['target_name']}" + candidates.append(Candidate("runner", row["id"], title, row["at"])) + if not self.compatibility.ready: + candidates.append(Candidate("compatibility", COMPATIBILITY_ITEM, "Execution compatibility is restricted")) + if reach.runs: + for row in await reader.rows( + f"SELECT r.run_id,r.ended_at,r.workflow_name,{_RUN_CLASSIFICATION} FROM run_facts r " + f"WHERE {_SCOPE.format(alias='r.')} AND r.status='failed' AND NOT r.test " + f"AND r.ended_at>{NOW}-interval '1 hour' ORDER BY r.ended_at,r.run_id LIMIT :limit", + limit=limit, + ): + title = f"A run of {row['workflow_name']} failed" if available(row) else "A run failed" + candidates.append(Candidate("run", row["run_id"], title, row["ended_at"])) + return attention_order(candidates) diff --git a/src/firefly_weave/studio/service.py b/src/firefly_weave/studio/service.py index 0116a265..b79de9db 100644 --- a/src/firefly_weave/studio/service.py +++ b/src/firefly_weave/studio/service.py @@ -92,6 +92,9 @@ "deployment_runners", "environments", "incidents", + # The environment summary and telemetry settings re-check each section's capability server-side. + "operations_summary", + "operations_telemetry", # Draft save/retire and Simulate are authorized server-side like every other family. "drafts", "debug", @@ -99,7 +102,7 @@ ) # Single read operations from families that otherwise stay operator-only: releases.create # registers image identities and remains a CLI/operator step. -STUDIO_OPERATIONS = frozenset({"releases.list", "releases.read"}) +STUDIO_OPERATIONS = frozenset({"releases.list", "releases.read", "releases.usage"}) CATALOG_COLLECTIONS = frozenset({"drafts", "workflows", "actions", "connectors", "decision-tables"}) diff --git a/tests/contracts/test_operations_summary_surface.py b/tests/contracts/test_operations_summary_surface.py new file mode 100644 index 00000000..0b88bd6e --- /dev/null +++ b/tests/contracts/test_operations_summary_surface.py @@ -0,0 +1,172 @@ +# Copyright 2026 Firefly Software Foundation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# Author: Firefly Software Foundation +# SPDX-License-Identifier: Apache-2.0 + +"""The environment summary and telemetry reads are registered, documented, exported and reachable from the CLI.""" + +import json +from datetime import UTC, datetime +from functools import cache +from uuid import UUID + +import pytest +from click.testing import CliRunner +from pydantic import ValidationError + +from firefly_weave.contracts.openapi import export_openapi +from firefly_weave.contracts.operations_summary import ( + ApiSection, + AttentionItem, + CompatibilitySummary, + ForbiddenSection, + HealthComponent, + OperationsSummary, + OperationsTelemetry, + TelemetryEndpoint, + TelemetryMetricEndpoint, + UnavailableSection, +) +from firefly_weave.contracts.schema_export import export_schemas +from firefly_weave.contracts.surface import OPERATIONS + +PROJECT = "/api/v1/tenants/{tenant}/projects/{project}" +ENVIRONMENT = PROJECT + "/environments/{environment}" +EXPECTED = { + "operations_summary.read": (ENVIRONMENT + "/operations/summary", OperationsSummary), + "operations_telemetry.read": (PROJECT + "/operations/telemetry", OperationsTelemetry), +} +AT = datetime(2026, 10, 12, 12, tzinfo=UTC) +INSTANCE = UUID(int=7) + +spec = cache(export_openapi) + + +def summary(**sections) -> OperationsSummary: + unavailable = UnavailableSection(state="unavailable") + values = { + "observed_at": AT, + "api": ApiSection( + version="0.1.0a15", + build=None, + instance=INSTANCE, + uptime_seconds=12.5, + compatibility=CompatibilitySummary(mode="ready", findings=0), + ), + "health": [HealthComponent(component="database", state="ok", latency_ms=1.25)], + "load": unavailable, + "workers": unavailable, + "targets": ForbiddenSection(state="forbidden"), + "incidents": ForbiddenSection(state="forbidden"), + "attention": [], + } + return OperationsSummary(**{**values, **sections}) + + +@pytest.mark.parametrize("identifier", sorted(EXPECTED)) +def test_operations_are_registered_reads_that_need_status_read(identifier): + operation = OPERATIONS[identifier] + path, response = EXPECTED[identifier] + assert (operation.method, operation.canonical_path, operation.response) == ("GET", path, response) + assert operation.capability == "status.read" + assert operation.served and not operation.page and operation.request is None and operation.query is None + + +@pytest.mark.parametrize("identifier", sorted(EXPECTED)) +def test_openapi_documents_bearer_security_and_the_response_schema(identifier): + operation = OPERATIONS[identifier] + document = spec()["paths"][operation.canonical_path]["get"] + assert document["summary"] == identifier.replace(".", " ") + assert document["security"] == [{"bearer": []}] + assert "Required capability: status.read" in document["description"] + reference = document["responses"]["200"]["content"]["application/json"]["schema"]["$ref"] + assert reference.endswith("/" + EXPECTED[identifier][1].__name__) + assert {"401", "403"} <= set(document["responses"]) + + +def test_sections_are_successful_objects_or_explicit_states_and_never_runs_24h(): + body = summary().model_dump(mode="json") + assert set(body) == {"observed_at", "api", "health", "load", "workers", "targets", "incidents", "attention"} + assert body["targets"] == body["incidents"] == {"state": "forbidden"} + assert body["load"] == body["workers"] == {"state": "unavailable"} + # Only what a component reports is sent: a database probe has a latency and no failure counts. + assert body["health"] == [{"component": "database", "state": "ok", "latency_ms": 1.25}] + assert summary(attention=UnavailableSection(state="unavailable")).model_dump(mode="json")["attention"] == { + "state": "unavailable" + } + + +def test_attention_is_bounded_to_twenty_items_with_bounded_titles_and_known_kinds(): + item = AttentionItem(kind="approval", id=INSTANCE, title="Approve", expires_at=AT) + assert item.model_dump(mode="json") == { + "kind": "approval", + "id": str(INSTANCE), + "title": "Approve", + "expires_at": "2026-10-12T12:00:00Z", + } + assert AttentionItem(kind="compatibility", id="compatibility", title="Restricted").model_dump(mode="json") == { + "kind": "compatibility", + "id": "compatibility", + "title": "Restricted", + } + with pytest.raises(ValidationError): + summary(attention=[item] * 21) + for invalid in ( + {"kind": "alert", "id": INSTANCE, "title": "x"}, + {"kind": "run", "id": "other", "title": "x"}, + {"kind": "run", "id": INSTANCE, "title": "x" * 201}, + {"kind": "run", "id": INSTANCE, "title": ""}, + ): + with pytest.raises(ValidationError): + AttentionItem(**invalid) + with pytest.raises(ValidationError): + summary(health=[HealthComponent(component="database", state="ok")] * 17) + with pytest.raises(ValidationError): + HealthComponent(component="tenant-1", state="ok") # type: ignore[arg-type] + + +def test_telemetry_answers_hosts_only(): + answer = OperationsTelemetry( + enabled=True, + traces=TelemetryEndpoint(configured=True, host="collector.example"), + metrics=TelemetryMetricEndpoint(configured=False, host=None, interval_seconds=None), + dropped=3, + ) + assert answer.model_dump(mode="json") == { + "enabled": True, + "traces": {"configured": True, "host": "collector.example"}, + "metrics": {"configured": False, "host": None, "interval_seconds": None}, + "dropped": 3, + } + assert set(OperationsTelemetry.model_fields) == {"enabled", "traces", "metrics", "dropped"} + assert set(TelemetryMetricEndpoint.model_fields) == {"configured", "host", "interval_seconds"} + + +def test_exported_schemas_include_both_answers_without_runs_24h(): + schemas = export_schemas() + assert "runs_24h" not in schemas["operations-summary"]["properties"] + assert schemas["operations-summary"]["properties"]["attention"] + assert set(schemas["operations-telemetry"]["properties"]) == {"enabled", "traces", "metrics", "dropped"} + assert "authorization" not in json.dumps(schemas["operations-telemetry"]) + + +@pytest.mark.parametrize( + ("command", "identifier"), [("summary", "operations_summary.read"), ("telemetry", "operations_telemetry.read")] +) +def test_the_cli_reads_both_under_operations(command, identifier): + from firefly_weave.cli.main import cli + + result = CliRunner().invoke(cli, ["operations", command, "--help"]) + assert result.exit_code == 0, result.output + assert identifier + ": status.read" in result.output diff --git a/tests/integration/test_operations_summary.py b/tests/integration/test_operations_summary.py new file mode 100644 index 00000000..81d29c7c --- /dev/null +++ b/tests/integration/test_operations_summary.py @@ -0,0 +1,759 @@ +# Copyright 2026 Firefly Software Foundation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# Author: Firefly Software Foundation +# SPDX-License-Identifier: Apache-2.0 + +"""Real PostgreSQL environment summary: per-capability sections, one read transaction and safe partial answers.""" + +import asyncio +import json +import time +from datetime import datetime, timedelta +from types import SimpleNamespace +from uuid import UUID, uuid4 + +import pytest +from sqlalchemy import event, text +from sqlalchemy.exc import DBAPIError +from test_worker_fleet_facts import add_deployment, add_observation, add_target, add_workers, clock, principal_with + +from firefly_weave.access.audit import AuditContext +from firefly_weave.access.authorization import AccessDenied, AuthorizationService +from firefly_weave.access.models import Grant, Principal +from firefly_weave.access.oidc import AuthenticationFailed +from firefly_weave.access.roles import ROLE_CAPABILITIES +from firefly_weave.contracts.compatibility import CompatibilityReport +from firefly_weave.contracts.deployments import ( + ComponentSpec, + DeploymentJob, + DeploymentPlan, + DeploymentRunner, + PlanApproval, + PlanStep, +) +from firefly_weave.observability import TelemetryDrops +from firefly_weave.operations.loop_health import LoopHealth, register_background_loops +from firefly_weave.operations.summary import OperationsSummaryService, read_ready_counts +from firefly_weave.persistence.uow import UnitOfWork +from firefly_weave.settings import Settings + +pytestmark = pytest.mark.integration + + +@pytest.fixture +def worker_runtime_fixture(worker_runtime_fixture): + """The echo Action retries, so an expired lease is requeued for a later attempt instead of opening an incident.""" + from firefly_weave.compiler.catalog import CatalogSnapshot + from firefly_weave.contracts.definitions import load_definition + + action = worker_runtime_fixture["action"].model_dump(by_alias=True) + action["spec"]["retry"] = {"maxAttempts": 3, "initialDelaySeconds": 1, "maxDelaySeconds": 2} + action = load_definition(action) + contract = worker_runtime_fixture["catalog"].resolve("TaskCapability", "echo@1.2.0").definition.value + return { + **worker_runtime_fixture, + "action": action, + "catalog": CatalogSnapshot.from_definitions([action], tasks=[contract]), + } + + +READY = SimpleNamespace(report=CompatibilityReport(mode="ready", complete=True), ready=True) +RESTRICTED = SimpleNamespace(report=CompatibilityReport(mode="restricted", complete=True), ready=False) +DIGEST = "sha256:" + "a" * 64 +ZERO = UUID(int=0) + + +def summary_service(case, *, compatibility=READY) -> OperationsSummaryService: + settings = Settings(database_url=case.url.render_as_string(hide_password=False)) + health = LoopHealth() + register_background_loops(health, settings) + return OperationsSummaryService( + UnitOfWork(case.sessions), + AuthorizationService(), + settings, + health, + compatibility, # type: ignore[arg-type] + drops=TelemetryDrops(), + instance=uuid4(), + started_monotonic=time.monotonic(), + ) + + +async def summarize(case, actor: Principal, service: OperationsSummaryService | None = None) -> dict: + result = await (service or summary_service(case)).read(actor, case.scope, context=AuditContext()) + return result.model_dump(mode="json") + + +def kinds(body: dict) -> set[str]: + return {item["kind"] for item in body["attention"]} + + +async def owner_execute(case, sql: str, **values) -> None: + async with case.owner.begin() as session: + await session.execute( + text(sql), + { + "tenant": case.scope.tenant_id, + "project": case.scope.project_id, + "environment": case.scope.environment_id, + **values, + }, + ) + + +async def add_incident(case, run: UUID, key: str, *, opened=None, test=False, status="active", facts=True) -> UUID: + identifier = uuid4() + await owner_execute( + case, + "INSERT INTO incidents(id,tenant_id,project_id,environment_id,run_id,incident_key,node_id,generation," + "origin_code,code,status,revision) VALUES(:id,:tenant,:project,:environment,:run,:key,'work',1," + "'WV-TASK-FAILED','WV-TASK-FAILED',:status,1)", + id=identifier, + run=run, + key=key, + status=status, + ) + if facts: + await owner_execute( + case, + "INSERT INTO incident_facts(tenant_id,project_id,environment_id,incident_id,node_id,instance_key," + "opened_at,updated_at,workflow_name,workflow_version,test,opened_at_estimated) SELECT :tenant,:project," + ":environment,:id,'work','',cast(:opened AS timestamptz),cast(:opened AS timestamptz),workflow_name," + "workflow_version,:test,false FROM run_facts WHERE run_id=:run", + id=identifier, + run=run, + opened=opened, + test=test, + ) + return identifier + + +async def insert_deployment_row(case, table: str, identifier: UUID, target: UUID, payload, **columns) -> None: + names = "".join("," + name for name in columns) + values = "".join(",:" + name for name in columns) + await owner_execute( + case, + f"INSERT INTO {table}(id,tenant_id,project_id,environment_id,target_id,payload{names}) " + f"VALUES(:id,:tenant,:project,:environment,:target,cast(:payload AS jsonb){values})", + id=identifier, + target=target, + payload=payload.model_dump_json(), + **columns, + ) + + +async def add_plan(case, target: UUID, deployment: UUID, observation: UUID, *, expires: datetime, intent="update"): + now = await clock(case) + identifier = uuid4() + plan = DeploymentPlan( + id=identifier, + scope=case.scope, + target_id=target, + target_revision=1, + deployment_id=deployment, + deployment_revision=1, + adapter="docker-compose", + intent=intent, + observation_id=observation, + observation_digest="0" * 64, + steps=[ + PlanStep( + action=intent, + component=ComponentSpec( + name="engine", + kind="worker", + image=f"registry.example/engine@{DIGEST}", + configuration="current", + replicas=2, + worker_release_id=case.instances[0].release_id, + ), + ) + ], + risks=[], + created_at=now, + expires_at=expires, + digest="1" * 64, + ) + await insert_deployment_row(case, "deployment_plans", identifier, target, plan, deployment_id=deployment) + return identifier + + +async def add_job(case, target: UUID, state: str, *, plan: UUID | None = None, at: datetime | None = None) -> UUID: + now = at or await clock(case) + identifier = uuid4() + job = DeploymentJob( + id=identifier, + scope=case.scope, + target_id=target, + target_revision=1, + kind="apply" if plan else "observe", + plan_id=plan, + plan_digest="1" * 64 if plan else None, + state=state, + revision=1, + operation_key=uuid4(), + created_at=now, + deadline=now + timedelta(hours=1), + reconciliation_started_at=now if state == "reconciliation_required" else None, + ) + await insert_deployment_row(case, "deployment_jobs", identifier, target, job, state=state) + return identifier + + +async def add_runner(case, target: UUID, *, expires: datetime, revoked=False) -> UUID: + identifier = uuid4() + runner = DeploymentRunner( + target_id=target, + adapter="docker-compose", + capabilities=["observe"], + id=identifier, + principal_id=uuid4(), + last_seen=expires - timedelta(seconds=90), + expires_at=expires, + revoked=revoked, + ) + await insert_deployment_row(case, "deployment_runners", identifier, target, runner) + return identifier + + +async def capable_target(case, name: str) -> tuple[UUID, UUID, UUID]: + """A target that allows updates, with a deployment and a fresh complete observation of its revision.""" + target = await add_target(case, name) + await owner_execute( + case, + "UPDATE deployment_targets SET payload=jsonb_set(payload,'{capabilities}','[\"observe\",\"update\"]') " + "WHERE id=:id", + id=target, + ) + deployment = await add_deployment(case, target) + observation = await add_observation(case, target, {"engine": ["host-" + name]}) + return target, deployment, observation + + +async def fail_run(case, run: UUID, *, minutes_ago: float, test=False) -> None: + await owner_execute( + case, + "UPDATE run_facts SET status='failed',test=:test,origin=CASE WHEN :test THEN 'test' ELSE origin END," + "ended_at=clock_timestamp()-make_interval(secs=>:seconds) WHERE run_id=:run", + run=run, + test=test, + seconds=minutes_ago * 60, + ) + + +async def test_summary_denied_sections_are_not_healthy_zero(operations_case): + case = operations_case + run = await case.start() + await add_incident(case, run.id, "denied", opened=await clock(case)) + target, deployment, observation = await capable_target(case, "alpha") + await add_job(case, target, "reconciliation_required") + await add_plan(case, target, deployment, observation, expires=await clock(case) + timedelta(minutes=5)) + await fail_run(case, (await case.start("failed")).id, minutes_ago=5) + actor_id = await case.access.create_principal(case.admin, "human") + await case.access.grant(case.admin, actor_id, Grant(role="worker_operator", scope=case.scope)) + actor = await case.access.load_principal(actor_id) + body = await summarize(case, actor) + assert body["incidents"] == {"state": "forbidden"} + assert body["targets"] == {"state": "forbidden"} + assert all(item["kind"] not in {"incident", "approval", "job", "runner", "run"} for item in body["attention"]) + assert "runs_24h" not in body + # The same environment, read by its operator with every capability, lists what the worker operator may not see. + everything = await principal_with( + case, + Grant(role="viewer", scope=case.scope), + Grant(role="operator", scope=case.scope), + Grant(role="deployment_approver", scope=case.scope), + ) + full = await summarize(case, everything) + assert {"incident", "approval", "job", "run"} <= kinds(full) + assert full["incidents"] == {"active": 1, "opened_24h": 1} + assert full["targets"]["reconciliation_required"] == 1 and full["targets"]["awaiting_approval"] == 1 + + +SECTION_ROLES = sorted(role for role in ROLE_CAPABILITIES if role != "platform_admin") + + +@pytest.mark.parametrize("role", SECTION_ROLES) +async def test_every_role_needs_status_read_and_each_section_its_own_capability(operations_case, role): + case = operations_case + actor = await principal_with(case, Grant(role=role, scope=case.scope)) + capabilities = ROLE_CAPABILITIES[role] + if "status.read" not in capabilities: + with pytest.raises(AccessDenied): + await summarize(case, actor) + return + body = await summarize(case, actor) + assert (body["incidents"] == {"state": "forbidden"}) is ("incident.read" not in capabilities) + assert (body["targets"] == {"state": "forbidden"}) is ("deployment.read" not in capabilities) + for section in ("load", "workers"): + assert "state" not in body[section] + assert isinstance(body["attention"], list) + + +async def test_token_and_stored_grants_are_each_reduced_in_the_read_transaction(operations_case): + case = operations_case + viewer = Grant(role="viewer", scope=case.scope) + incidents = Grant(role="operator", scope=case.scope) + stored = await principal_with(case, viewer) + # A token that claims more than the stored grants: the stored principal decides. + token = stored.model_copy(update={"grants": (viewer, incidents)}) + assert (await summarize(case, token))["incidents"] == {"state": "forbidden"} + # Stored grants that exceed the token: the token decides. + stored = await principal_with(case, viewer, incidents) + assert (await summarize(case, stored.model_copy(update={"grants": (viewer,)})))["incidents"] == { + "state": "forbidden" + } + assert "state" not in (await summarize(case, stored))["incidents"] + # A principal deactivated after its token was issued reads nothing: it no longer authenticates. + await case.access.set_active(case.admin, stored.id, False) + with pytest.raises(AuthenticationFailed): + await summarize(case, stored) + + +async def test_targets_and_attention_follow_target_restricted_grants(operations_case): + case = operations_case + alpha, alpha_deployment, alpha_observation = await capable_target(case, "alpha") + bravo, bravo_deployment, bravo_observation = await capable_target(case, "bravo") + now = await clock(case) + for target, deployment, observation in ( + (alpha, alpha_deployment, alpha_observation), + (bravo, bravo_deployment, bravo_observation), + ): + await add_plan(case, target, deployment, observation, expires=now + timedelta(minutes=10)) + await add_job(case, target, "reconciliation_required") + viewer = Grant(role="viewer", scope=case.scope) + only_alpha = {"total": 1, "runner_online": 0, "busy": 0, "reconciliation_required": 1, "awaiting_approval": 1} + reader = await principal_with( + case, viewer, Grant(role="deployment_reader", scope=case.scope, resources=(str(alpha),)) + ) + body = await summarize(case, reader) + # Counts cover only the readable target, never a global total, and reading is not approving. + assert body["targets"] == only_alpha + assert [(item["kind"], item["title"]) for item in body["attention"]] == [ + ("job", "Deployment job on alpha needs reconciliation") + ] + # A token for every target still counts only what the stored grant allows now. + token = reader.model_copy(update={"grants": (viewer, Grant(role="deployment_reader", scope=case.scope))}) + assert (await summarize(case, token))["targets"] == only_alpha + # Approval items need the approval capability on that same target. + approver = await principal_with( + case, + viewer, + Grant(role="deployment_reader", scope=case.scope), + Grant(role="deployment_approver", scope=case.scope, resources=(str(bravo),)), + ) + body = await summarize(case, approver) + assert body["targets"]["total"] == 2 and body["targets"]["awaiting_approval"] == 2 + approvals = [item for item in body["attention"] if item["kind"] == "approval"] + assert [item["title"] for item in approvals] == ["Approve the plan to update on bravo"] + assert {item["title"] for item in body["attention"] if item["kind"] == "job"} == { + "Deployment job on alpha needs reconciliation", + "Deployment job on bravo needs reconciliation", + } + # Readable targets limited to ones that do not exist leave a readable, empty section. + nowhere = await principal_with( + case, viewer, Grant(role="deployment_reader", scope=case.scope, resources=(str(uuid4()),)) + ) + assert (await summarize(case, nowhere))["targets"] == { + "total": 0, + "runner_online": 0, + "busy": 0, + "reconciliation_required": 0, + "awaiting_approval": 0, + } + + +async def test_targets_count_each_target_once_and_only_plans_that_can_still_be_approved(operations_case): + case = operations_case + alpha, alpha_deployment, alpha_observation = await capable_target(case, "alpha") + bravo, _, _ = await capable_target(case, "bravo") + now = await clock(case) + await add_runner(case, alpha, expires=now + timedelta(seconds=60)) + await add_runner(case, alpha, expires=now + timedelta(seconds=30)) + await add_runner(case, bravo, expires=now - timedelta(seconds=1)) + await add_runner(case, bravo, expires=now + timedelta(seconds=60), revoked=True) + await add_job(case, alpha, "running") + await add_job(case, alpha, "reconciliation_required") + await add_job(case, alpha, "reconciliation_required") + await add_job(case, bravo, "succeeded") + awaiting = await add_plan(case, alpha, alpha_deployment, alpha_observation, expires=now + timedelta(minutes=5)) + await add_plan(case, alpha, alpha_deployment, alpha_observation, expires=now - timedelta(seconds=1)) + approved = await add_plan(case, alpha, alpha_deployment, alpha_observation, expires=now + timedelta(minutes=5)) + await insert_deployment_row( + case, + "deployment_approvals", + approved, + alpha, + PlanApproval(plan_id=approved, digest="1" * 64, principal_id=case.actor.id, approved_at=now), + ) + applied = await add_plan(case, alpha, alpha_deployment, alpha_observation, expires=now + timedelta(minutes=5)) + await add_job(case, alpha, "succeeded", plan=applied) + await add_plan( + case, alpha, alpha_deployment, alpha_observation, expires=now + timedelta(minutes=5), intent="deploy" + ) + reader = await principal_with( + case, + Grant(role="viewer", scope=case.scope), + Grant(role="deployment_approver", scope=case.scope), + ) + body = await summarize(case, reader) + assert body["targets"] == { + "total": 2, + "runner_online": 1, + "busy": 1, + "reconciliation_required": 1, + "awaiting_approval": 1, + } + assert [item["id"] for item in body["attention"] if item["kind"] == "approval"] == [str(awaiting)] + runners = [item for item in body["attention"] if item["kind"] == "runner"] + assert [item["title"] for item in runners] == ["No runner is online for bravo"] + + +async def test_attention_ranks_at_most_twenty_items_by_priority_then_expiry(operations_case, task_service, worker_ids): + case = operations_case + run = await case.start() + async with case.tx() as tx: + await task_service.claim(tx, worker_ids[0], 1) + # The worker holding the lease stops reporting. + await owner_execute( + case, + "UPDATE worker_instances SET last_seen_at=clock_timestamp()-interval '5 minutes' WHERE id=:id", + id=worker_ids[0], + ) + target, deployment, observation = await capable_target(case, "alpha") + now = await clock(case) + jobs = [ + await add_job(case, target, "reconciliation_required", at=now - timedelta(minutes=minutes)) + for minutes in (1, 2) + ] + later = await add_plan(case, target, deployment, observation, expires=now + timedelta(minutes=9)) + sooner = await add_plan(case, target, deployment, observation, expires=now + timedelta(minutes=3)) + opened = [ + await add_incident(case, run.id, f"incident-{index}", opened=now - timedelta(minutes=30 - index)) + for index in range(20) + ] + await fail_run(case, (await case.start("failed")).id, minutes_ago=10) + reader = await principal_with( + case, + Grant(role="viewer", scope=case.scope), + Grant(role="operator", scope=case.scope), + Grant(role="deployment_approver", scope=case.scope), + ) + body = await summarize(case, reader, summary_service(case, compatibility=RESTRICTED)) + ranked = [(item["kind"], item["id"]) for item in body["attention"]] + assert len(ranked) == 20 + assert ranked[:5] == [ + ("job", str(jobs[1])), + ("job", str(jobs[0])), + ("approval", str(sooner)), + ("approval", str(later)), + ("worker", str(worker_ids[0])), + ] + # Oldest incidents first fill the rest; lower priorities are left out. + assert ranked[5:] == [("incident", str(identifier)) for identifier in opened[:15]] + assert body["attention"][2]["expires_at"] is not None and "expires_at" not in body["attention"][0] + assert body["attention"][4]["title"] == "Offline worker still holds 1 active lease" + # With fewer incidents the lower priorities follow in order. + await owner_execute(case, "UPDATE incidents SET status='resolved' WHERE run_id=:run", run=run.id) + body = await summarize(case, reader, summary_service(case, compatibility=RESTRICTED)) + assert [item["kind"] for item in body["attention"]] == [ + "job", + "job", + "approval", + "approval", + "worker", + "compatibility", + "run", + ] + assert body["attention"][-2]["title"] == "Execution compatibility is restricted" + assert body["attention"][-1]["title"].startswith("A run of ") and body["attention"][-1]["title"].endswith(" failed") + + +async def test_test_runs_leave_run_figures_but_count_in_shared_capacity(operations_case): + case = operations_case + real, test = await case.start("real"), await case.start("test") + await owner_execute(case, "UPDATE run_facts SET test=true,origin='test' WHERE run_id=:run", run=test.id) + now = await clock(case) + counted = {await add_incident(case, real.id, "real", opened=now)} + await add_incident(case, test.id, "test", opened=now, test=True) + # An incident recorded before incident facts existed still counts: it predates test runs. + counted.add(await add_incident(case, real.id, "legacy", facts=False)) + reader = await principal_with( + case, Grant(role="viewer", scope=case.scope), Grant(role="operator", scope=case.scope) + ) + body = await summarize(case, reader) + [usage] = await case.rows( + "SELECT value FROM operation_usage WHERE environment_id=:environment AND metric='runs_active'" + ) + # The usage counter counts every active run; the summary leaves out the active test run. + assert usage["value"] == 2 and body["load"]["runs_active"] == 1 + # Each started run waits on one ready task, the test run's too: queues measure shared capacity. + assert body["load"]["tasks_ready"] == 2 + assert body["incidents"] == {"active": 2, "opened_24h": 1} + assert {UUID(item["id"]) for item in body["attention"] if item["kind"] == "incident"} == counted + # A failed test run is not a failed run to look at; a failed real run is. + await fail_run(case, test.id, minutes_ago=1, test=True) + await fail_run(case, real.id, minutes_ago=1) + body = await summarize(case, reader) + assert [item["id"] for item in body["attention"] if item["kind"] == "run"] == [str(real.id)] + + +async def test_counts_never_include_another_environment_or_project_wide_usage(operations_case): + case = operations_case + await case.start() + sibling = uuid4() + for environment in (sibling, ZERO): + for metric in ("runs_active", "waits_active", "outbox_pending", "provider_pending", "sources_enabled"): + await owner_execute( + case, + "INSERT INTO operation_usage(tenant_id,project_id,environment_id,metric,value) " + "VALUES(:tenant,:project,:other,:metric,1000) ON CONFLICT(tenant_id,project_id,environment_id,metric) " + "DO UPDATE SET value=1000", + other=environment, + metric=metric, + ) + viewer = await principal_with(case, Grant(role="viewer", scope=case.scope)) + load = (await summarize(case, viewer))["load"] + assert load["runs_active"] == 1 + assert {key: load[key] for key in ("waits_active", "outbox_pending", "provider_pending", "sources_enabled")} == { + "waits_active": 0, + "outbox_pending": 0, + "provider_pending": 0, + "sources_enabled": 0, + } + + +async def test_workers_and_waits_come_from_bounded_aggregates(operations_case, task_service, worker_ids): + case = operations_case + run = await case.start() + now = await clock(case) + await owner_execute(case, "UPDATE worker_instances SET revoked=true WHERE id<>:id", id=worker_ids[0]) + await owner_execute( + case, "UPDATE worker_instances SET last_seen_at=clock_timestamp() WHERE id=:id", id=worker_ids[0] + ) + async with case.tx() as tx: + await task_service.claim(tx, worker_ids[0], 1) + await add_workers(case, 2, seen=now) + await add_workers(case, 1, seen=now - timedelta(minutes=5)) + await add_workers(case, 1, seen=now, draining=True) + await add_workers(case, 1, seen=now - timedelta(hours=2)) + await add_workers(case, 1, seen=None, registered=now - timedelta(minutes=10)) + await add_workers(case, 1, seen=now, revoked=True) + await owner_execute( + case, + "INSERT INTO run_deadlines(tenant_id,project_id,environment_id,run_id,node_id,deadline,consumed) VALUES " + "(:tenant,:project,:environment,:run,'late',clock_timestamp()-interval '90 seconds',false)," + "(:tenant,:project,:environment,:run,'later',clock_timestamp()+interval '1 hour',false)," + "(:tenant,:project,:environment,:run,'done',clock_timestamp()-interval '1 hour',true)", + run=run.id, + ) + viewer = await principal_with(case, Grant(role="viewer", scope=case.scope)) + body = await summarize(case, viewer) + # Current workers as the Workers page lists them; only online, undrained ones add capacity. + assert body["workers"] == {"online": 3, "offline": 2, "draining": 1, "capacity": 3, "active_leases": 1} + load = body["load"] + assert (load["tasks_ready"], load["tasks_leased"], load["oldest_ready_seconds"]) == (0, 1, None) + assert (load["waits_active"], load["waits_overdue"]) == (2, 1) + assert 89 <= load["oldest_overdue_seconds"] < 120 + assert load["outbox_oldest_seconds"] is None + + +async def test_delayed_retry_enters_ready_gauges_at_due_time_without_mutation( + operations_case, task_service, worker_ids, services +): + from firefly_weave.runtime.recovery import RecoveryService + + case = operations_case + await case.start() + async with case.tx() as tx: + old = (await task_service.claim(tx, worker_ids[0], 1))[0] + async with case.owner.begin() as session: + await session.execute(text("UPDATE task_leases SET expires_at=clock_timestamp()-interval '1 second'")) + report = ( + await services(case.sessions) + .resolve(RecoveryService) + .scan(case.scope, 100, actor=case.actor, context=AuditContext()) + ) + assert report.resumed == 1 + state = ( + "SELECT t.status,t.next_attempt_at,f.status AS fact,f.ready_since FROM task_intents t " + "JOIN task_facts f ON f.task_id=t.id" + ) + [before] = await case.rows(state) + due = before["next_attempt_at"] + assert before["status"] == before["fact"] == "ready" and before["ready_since"] == due and due > await clock(case) + observed = [] + for at in (due - timedelta(microseconds=1), due, due + timedelta(seconds=5)): + async with UnitOfWork(case.sessions).open(case.scope, mutation=False) as tx: + observed.append(await read_ready_counts(tx, at)) + assert observed == [(0, None), (1, 0.0), (1, 5.0)] + # Reading eligibility wrote nothing. + assert await case.rows(state) == [before] + await owner_execute( + case, + "UPDATE task_intents SET next_attempt_at=clock_timestamp()-interval '1 second' WHERE id=:id", + id=old.proof.task_id, + ) + async with case.tx() as tx: + assert len(await task_service.claim(tx, worker_ids[1], 1)) == 1 + async with UnitOfWork(case.sessions).open(case.scope, mutation=False) as tx: + assert await read_ready_counts(tx, await clock(case) + timedelta(hours=1)) == (0, None) + + +async def test_a_started_task_is_ready_at_once_and_cancelled_or_handled_ones_never(operations_case): + case = operations_case + first, second = await case.start("first"), await case.start("second") + now = await clock(case) + async with UnitOfWork(case.sessions).open(case.scope, mutation=False) as tx: + count, oldest = await read_ready_counts(tx, now) + assert count == 2 and oldest is not None and oldest >= 0 + async with case.tx() as tx: + await case.runtime.cancel( + tx, first.id, "operator_cancelled", actor=case.actor, scope=case.scope, context=AuditContext() + ) + await owner_execute( + case, "UPDATE task_facts SET status='handled',ready_since=NULL WHERE run_id=:run", run=second.id + ) + async with UnitOfWork(case.sessions).open(case.scope, mutation=False) as tx: + assert await read_ready_counts(tx, await clock(case)) == (0, None) + + +async def test_a_slow_section_is_unavailable_while_the_others_are_served(operations_case, monkeypatch): + case = operations_case + await case.start() + workers = OperationsSummaryService._workers + + async def slow(self, reader): + await reader.tx.session.execute(text("SELECT pg_sleep(2.1)")) + return await workers(self, reader) + + timeouts = [] + incidents = OperationsSummaryService._incidents + + async def observed(self, reader): + timeouts.append(await reader.tx.session.scalar(text("SELECT current_setting('statement_timeout')"))) + return await incidents(self, reader) + + monkeypatch.setattr(OperationsSummaryService, "_workers", slow) + monkeypatch.setattr(OperationsSummaryService, "_incidents", observed) + reader = await principal_with( + case, Grant(role="viewer", scope=case.scope), Grant(role="operator", scope=case.scope) + ) + started = time.monotonic() + body = await summarize(case, reader) + assert time.monotonic() - started < 10 + assert body["workers"] == {"state": "unavailable"} + assert body["load"]["runs_active"] == 1 and body["incidents"] == {"active": 0, "opened_24h": 0} + assert isinstance(body["attention"], list) and body["health"][0]["component"] == "database" + # A later section still runs, under its own two-second bound, in the same transaction. + assert timeouts == ["2s"] + + +async def test_a_lost_connection_or_a_cancelled_read_is_never_an_unavailable_section(operations_case, monkeypatch): + case = operations_case + reader = await principal_with(case, Grant(role="viewer", scope=case.scope)) + + async def lost(self, reader): + await reader.tx.session.execute(text("SELECT pg_terminate_backend(pg_backend_pid())")) + + monkeypatch.setattr(OperationsSummaryService, "_workers", lost) + with pytest.raises(DBAPIError): + await summarize(case, reader) + entered = asyncio.Event() + + async def held(self, reader): + entered.set() + await reader.tx.session.execute(text("SELECT pg_sleep(1.5)")) + + monkeypatch.setattr(OperationsSummaryService, "_workers", held) + task = asyncio.create_task(summarize(case, reader)) + await asyncio.wait_for(entered.wait(), 5) + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + # The pool still serves a complete answer afterwards. + monkeypatch.undo() + assert "state" not in (await summarize(case, reader))["workers"] + + +async def test_the_principal_and_every_section_share_one_read_transaction(operations_case): + case = operations_case + await case.start() + reader = await principal_with( + case, + Grant(role="viewer", scope=case.scope), + Grant(role="operator", scope=case.scope), + Grant(role="deployment_reader", scope=case.scope), + ) + statements: list[str] = [] + engine = case.sessions.kw["bind"].sync_engine + + def record(connection, cursor, statement, parameters, context, executemany): + statements.append(" ".join(statement.lower().split())) + + begins = [] + + def begin(connection): + begins.append(connection) + + event.listen(engine, "before_cursor_execute", record) + event.listen(engine, "begin", begin) + try: + await summarize(case, reader) + finally: + event.remove(engine, "before_cursor_execute", record) + event.remove(engine, "begin", begin) + assert len(begins) == 1 + assert sum("repeatable read, read only" in statement for statement in statements) == 1 + # No section reads run payloads, run state or step outputs. + assert not any("from runs " in statement or "state->" in statement for statement in statements) + + +async def test_native_routes_serve_the_summary_and_telemetry_hosts(operations_case, authenticated_client): + case = operations_case + client, tokens, scopes = authenticated_client + assert scopes[0] == case.scope + project = f"/api/v1/tenants/{case.scope.tenant_id}/projects/{case.scope.project_id}" + environment = f"{project}/environments/{case.scope.environment_id}" + headers = {"Authorization": "Bearer " + tokens[0]} + first = await client.get(environment + "/operations/summary", headers=headers) + assert first.status_code == 200, first.text + body = first.json() + assert set(body) == {"observed_at", "api", "health", "load", "workers", "targets", "incidents", "attention"} + assert body["incidents"] == body["targets"] == {"state": "forbidden"} + components = [item["component"] for item in body["health"]] + assert components[0] == "database" and components[-2:] == ["native_executors", "telemetry"] + # Scheduler loops are disabled on this API-only process; the instance is fixed for its lifetime. + assert {item["state"] for item in body["health"] if item["component"].endswith("_loop")} == {"disabled"} + second = (await client.get(environment + "/operations/summary", headers=headers)).json() + assert second["api"]["instance"] == body["api"]["instance"] + assert second["api"]["uptime_seconds"] >= body["api"]["uptime_seconds"] + # Another tenant's token is refused. + other = {"Authorization": "Bearer " + tokens[1]} + assert (await client.get(environment + "/operations/summary", headers=other)).status_code == 403 + # Telemetry settings are per project: an environment grant is not enough. + assert (await client.get(project + "/operations/telemetry", headers=headers)).status_code == 403 + async with case.owner.begin() as session: + caller = await session.scalar(text("SELECT principal_id FROM identity_links WHERE subject='0'")) + project_scope = case.scope.model_copy(update={"environment_id": None}) + await case.access.grant(case.admin, caller, Grant(role="viewer", scope=project_scope)) + telemetry = await client.get(project + "/operations/telemetry", headers=headers) + assert telemetry.status_code == 200 + assert telemetry.json() == { + "enabled": False, + "traces": {"configured": False, "host": None}, + "metrics": {"configured": False, "host": None, "interval_seconds": None}, + "dropped": 0, + } + assert json.loads(telemetry.text) == telemetry.json() diff --git a/tests/unit/operations/test_operations_summary_rules.py b/tests/unit/operations/test_operations_summary_rules.py new file mode 100644 index 00000000..620d54f2 --- /dev/null +++ b/tests/unit/operations/test_operations_summary_rules.py @@ -0,0 +1,343 @@ +# Copyright 2026 Firefly Software Foundation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# Author: Firefly Software Foundation +# SPDX-License-Identifier: Apache-2.0 + +"""Service-free summary rules: section isolation, attention order, health components and safe telemetry answers.""" + +import asyncio +import json +from contextlib import asynccontextmanager +from datetime import UTC, datetime, timedelta +from types import SimpleNamespace +from uuid import UUID, uuid4 + +import pytest +from pydantic import SecretStr +from sqlalchemy.exc import DBAPIError + +from firefly_weave.access.audit import AuditContext +from firefly_weave.access.authorization import AccessDenied, AuthorizationService +from firefly_weave.access.models import Grant, Principal +from firefly_weave.contracts.access import Scope +from firefly_weave.contracts.operations_summary import UnavailableSection +from firefly_weave.contracts.telemetry import TelemetryOptions +from firefly_weave.observability import TelemetryDrops +from firefly_weave.operations import summary +from firefly_weave.operations.loop_health import LoopSnapshot +from firefly_weave.operations.summary import ( + Candidate, + OperationsSummaryService, + attention_order, + health_components, + native_state, + read_section, + telemetry_host, +) +from firefly_weave.settings import Settings + +DATABASE = "postgresql+asyncpg://weave_app:unused@127.0.0.1:65432/weave" +AT = datetime(2026, 10, 12, 12, tzinfo=UTC) +SCOPE = Scope(tenant_id=uuid4(), project_id=uuid4(), environment_id=uuid4()) +PROJECT = Scope(tenant_id=SCOPE.tenant_id, project_id=SCOPE.project_id) +CANARY = "canary-credential-7f3a" + + +class Failure(Exception): + def __init__(self, sqlstate: str) -> None: + super().__init__(sqlstate) + self.sqlstate = sqlstate + + +def failure(sqlstate: str, *, invalidated: bool = False) -> DBAPIError: + return DBAPIError("SELECT", {}, Failure(sqlstate), connection_invalidated=invalidated) + + +class Session: + """Records statements and savepoints; the section timeout in force is what the last set_config chose.""" + + def __init__(self, prior: str = "0", prior_ms: int = 0) -> None: + self.prior, self.prior_ms = prior, prior_ms + self.timeout = prior + self.savepoints: list[str] = [] + + async def execute(self, statement, values=None): + sql = str(statement) + if "pg_settings" in sql: + return SimpleNamespace(one=lambda: SimpleNamespace(prior=self.prior, prior_ms=self.prior_ms)) + if "set_config" in sql: + self.timeout = values["value"] + return None + + @asynccontextmanager + async def begin_nested(self): + saved = self.timeout + try: + yield + except BaseException: + # Rolling back to the savepoint restores the setting that was in force before it. + self.timeout = saved + self.savepoints.append("rolled back") + raise + self.savepoints.append("released") + + +def transaction(session: Session) -> SimpleNamespace: + return SimpleNamespace(session=session, scope=SCOPE) + + +def test_only_the_collector_host_is_kept(): + assert telemetry_host(None) is None + assert telemetry_host("https://collector.example:4318/v1/traces") == "collector.example" + assert ( + telemetry_host(f"https://user:{CANARY}@Collector.Example:4318/{CANARY}?token={CANARY}") == "collector.example" + ) + assert telemetry_host("http://[::1]:4318/v1/metrics") == "::1" + + +async def test_a_section_runs_under_two_seconds_and_restores_the_timeout_in_force(): + session = Session(prior="30s", prior_ms=30000) + seen = [] + + async def read(): + seen.append(session.timeout) + return 7 + + assert await read_section(transaction(session), read) == 7 + assert (seen, session.timeout, session.savepoints) == (["2000ms"], "30s", ["released"]) + # A stricter timeout already in force is kept, and no timeout at all becomes the section budget. + for prior, prior_ms, expected in (("500ms", 500, "500ms"), ("0", 0, "2000ms")): + session = Session(prior=prior, prior_ms=prior_ms) + seen.clear() + assert await read_section(transaction(session), read) == 7 + assert seen == [expected] and session.timeout == prior + + +@pytest.mark.parametrize("sqlstate", ["57014", "55P03", "40001", "40P01", "53200"]) +async def test_a_timed_out_or_transient_section_is_unavailable_and_rolled_back(sqlstate): + session = Session(prior="30s", prior_ms=30000) + + async def read(): + raise failure(sqlstate) + + assert await read_section(transaction(session), read) == UnavailableSection(state="unavailable") + assert session.savepoints == ["rolled back"] and session.timeout == "30s" + + +@pytest.mark.parametrize( + "error", + [ + failure("57014", invalidated=True), + failure("08006"), + failure("42601"), + failure("42501"), + AccessDenied(), + asyncio.CancelledError(), + RuntimeError("programmer error"), + ], +) +async def test_a_lost_connection_cancellation_denial_or_programming_error_is_never_unavailable(error): + session = Session() + + async def read(): + raise error + + with pytest.raises(type(error)): + await read_section(transaction(session), read) + assert session.savepoints == ["rolled back"] + + +def item(kind, identifier, at=None, **values) -> Candidate: + return Candidate(kind, identifier, f"{kind} {identifier}", at, **values) + + +def test_attention_is_ranked_by_priority_then_time_then_id_and_capped_at_twenty(): + first, second, third = UUID(int=1), UUID(int=2), UUID(int=3) + candidates = [ + item("run", first, AT), + item("compatibility", "compatibility"), + item("runner", second, AT), + item("incident", third, None), + item("incident", second, AT + timedelta(minutes=1)), + item("incident", first, AT + timedelta(minutes=1)), + item("worker", first, AT), + item("approval", second, AT + timedelta(minutes=2), expires_at=AT + timedelta(minutes=2)), + item("approval", third, AT + timedelta(minutes=1), expires_at=AT + timedelta(minutes=1)), + item("job", third, AT), + ] + ranked = [(entry.kind, entry.id) for entry in attention_order(reversed(candidates))] + assert ranked == [ + ("job", third), + # Approvals that expire first come first. + ("approval", third), + ("approval", second), + ("worker", first), + # Equal times are ordered by ID; unknown times come last. + ("incident", first), + ("incident", second), + ("incident", third), + ("runner", second), + ("compatibility", "compatibility"), + ("run", first), + ] + assert attention_order(candidates)[1].expires_at == AT + timedelta(minutes=1) + many = [item("run", UUID(int=index), AT) for index in range(30, 0, -1)] + [item("job", UUID(int=99), AT)] + ranked_many = attention_order(many) + assert len(ranked_many) == 20 and ranked_many[0].kind == "job" + assert [entry.id for entry in ranked_many[1:]] == [UUID(int=index) for index in range(1, 20)] + + +def test_long_titles_are_cut_to_the_contract_bound(): + [entry] = attention_order([Candidate("run", UUID(int=1), "A run of " + "w" * 300 + " failed", AT)]) + assert len(entry.title) == 200 and entry.title.endswith("…") + + +def snapshot(name, state="ok", **values) -> LoopSnapshot: + return LoopSnapshot( + component=name, + state=state, + last_started_at=values.get("started"), + last_ok_at=values.get("ok"), + consecutive_failures=values.get("failures", 0), + complete_until=None, + recent_failures=values.get("recent", 0), + ) + + +def test_health_lists_the_database_each_loop_native_executors_and_telemetry_without_messages(): + components = health_components( + 4.25, + ( + snapshot("outbox_loop", "degraded", started=AT, ok=AT, failures=1, recent=3), + snapshot("alert_evaluator", "disabled"), + ), + native="down", + telemetry_enabled=False, + ) + assert [component.model_dump(mode="json") for component in components] == [ + {"component": "database", "state": "ok", "latency_ms": 4.25}, + { + "component": "outbox_loop", + "state": "degraded", + "last_started_at": "2026-10-12T12:00:00Z", + "last_ok_at": "2026-10-12T12:00:00Z", + "consecutive_failures": 1, + "recent_failures": 3, + }, + {"component": "alert_evaluator", "state": "disabled", "consecutive_failures": 0, "recent_failures": 0}, + {"component": "native_executors", "state": "down"}, + {"component": "telemetry", "state": "disabled"}, + ] + # A database probe that timed out is degraded, never a zero latency. + [database, *_] = health_components(UnavailableSection(state="unavailable"), (), native="ok", telemetry_enabled=True) + assert database.model_dump(mode="json") == {"component": "database", "state": "degraded"} + + +def test_native_executors_follow_their_existing_readiness(): + configured = SimpleNamespace(native_executors=(object(),)) + assert native_state(SimpleNamespace(native_executors=()), None) == "disabled" + assert native_state(configured, None) == "down" + assert native_state(configured, SimpleNamespace(healthy=lambda: True)) == "ok" + assert native_state(configured, SimpleNamespace(healthy=lambda: False)) == "degraded" + + +class Units: + """A unit of work that only counts how often it opened; the stored principal comes from `stored`.""" + + def __init__(self, stored: Principal) -> None: + self.stored, self.opened = stored, 0 + + @asynccontextmanager + async def open(self, scope, *, mutation=True): + assert mutation is False + self.opened += 1 + yield SimpleNamespace(session=self, scope=scope) + + +def principal(*grants: Grant, identifier: UUID | None = None) -> Principal: + return Principal(id=identifier or uuid4(), kind="human", grants=grants) + + +def service(monkeypatch, units: Units, *, telemetry: TelemetryOptions | None = None) -> OperationsSummaryService: + async def load_principal(session, identifier): + assert session is units + return units.stored + + monkeypatch.setattr(summary, "load_principal", load_principal) + settings = Settings(database_url=DATABASE).model_copy(update={"telemetry": telemetry or TelemetryOptions()}) + drops = TelemetryDrops() + drops.add(5) + return OperationsSummaryService( + units, # type: ignore[arg-type] + AuthorizationService(), + settings, + SimpleNamespace(snapshots=lambda: ()), # type: ignore[arg-type] + SimpleNamespace(), # type: ignore[arg-type] + drops=drops, + instance=uuid4(), + started_monotonic=0.0, + ) + + +async def test_telemetry_never_answers_credentials_headers_ports_paths_or_queries(monkeypatch): + reader = principal(Grant(role="viewer", scope=PROJECT)) + options = TelemetryOptions.model_construct( + enabled=True, + # A value planted around validation still yields only the host. + traces_endpoint=f"https://exporter:{CANARY}@traces.example:4318/v1/{CANARY}?key={CANARY}", + metrics_endpoint="https://metrics.example/v1/metrics", + authorization=SecretStr(f"Bearer {CANARY}"), + timeout_seconds=5.0, + metrics_interval_seconds=15.0, + ) + units = Units(reader) + answer = await service(monkeypatch, units, telemetry=options).telemetry(reader, PROJECT, context=AuditContext()) + body = json.dumps(answer.model_dump(mode="json")) + assert json.loads(body) == { + "enabled": True, + "traces": {"configured": True, "host": "traces.example"}, + "metrics": {"configured": True, "host": "metrics.example", "interval_seconds": 15.0}, + "dropped": 5, + } + assert CANARY not in body and "4318" not in body and "/v1" not in body and units.opened == 1 + off = await service(monkeypatch, Units(reader)).telemetry(reader, PROJECT, context=AuditContext()) + assert off.model_dump(mode="json") == { + "enabled": False, + "traces": {"configured": False, "host": None}, + "metrics": {"configured": False, "host": None, "interval_seconds": None}, + "dropped": 5, + } + + +async def test_both_reads_check_the_token_before_opening_and_the_stored_principal_inside(monkeypatch): + viewer = Grant(role="viewer", scope=PROJECT) + outsider = principal(Grant(role="developer", scope=PROJECT)) + for read, scope in ((OperationsSummaryService.telemetry, PROJECT), (OperationsSummaryService.read, SCOPE)): + units = Units(principal(viewer)) + with pytest.raises(AccessDenied): + await read(service(monkeypatch, units), outsider, scope, context=AuditContext()) + assert units.opened == 0 + token = principal(viewer) + units = Units(principal(Grant(role="developer", scope=PROJECT), identifier=token.id)) + with pytest.raises(AccessDenied): + await read(service(monkeypatch, units), token, scope, context=AuditContext()) + assert units.opened == 1 + # The summary is per environment and the telemetry settings per project. + reader = principal(viewer) + with pytest.raises(AccessDenied): + await service(monkeypatch, Units(reader)).read(reader, PROJECT, context=AuditContext()) + with pytest.raises(AccessDenied): + await service(monkeypatch, Units(reader)).telemetry( + reader, Scope(tenant_id=PROJECT.tenant_id), context=AuditContext() + ) diff --git a/tests/unit/test_studio_bridge_families.py b/tests/unit/test_studio_bridge_families.py index 07817b4d..2248bb5e 100644 --- a/tests/unit/test_studio_bridge_families.py +++ b/tests/unit/test_studio_bridge_families.py @@ -91,6 +91,9 @@ def pair(browser): ("GET", f"{ENVIRONMENT}/ai/readiness"), ("POST", f"{ENVIRONMENT}/ai/setup/publish"), ("POST", f"{ENVIRONMENT}/ai/setup/grant"), + ("GET", f"{ENVIRONMENT}/operations/summary"), + ("GET", f"{PROJECT}/operations/telemetry"), + ("GET", f"{ENVIRONMENT}/worker-releases/{RESOURCE}/usage"), ], ) def test_studio_journeys_are_bridged(tmp_path, method, path): @@ -147,6 +150,15 @@ def test_studio_journeys_are_bridged(tmp_path, method, path): ("POST", f"{ENVIRONMENT}/ai/endpoints", 404), ("GET", ENVIRONMENT.replace("000000000003", "000000000009") + "/ai/models", 403), ("GET", ENVIRONMENT.replace("000000000003", "000000000009") + "/ai/endpoints", 403), + # The summary, the telemetry settings and release usage are reads of the paired scope only. + ("GET", ENVIRONMENT.replace("000000000003", "000000000009") + "/operations/summary", 403), + ("GET", f"{OTHER_PROJECT}/operations/telemetry", 403), + ("GET", ENVIRONMENT.replace("000000000003", "000000000009") + f"/worker-releases/{RESOURCE}/usage", 403), + ("POST", f"{ENVIRONMENT}/operations/summary", 404), + ("PUT", f"{PROJECT}/operations/telemetry", 404), + ("POST", f"{ENVIRONMENT}/worker-releases/{RESOURCE}/usage", 404), + ("GET", f"{ENVIRONMENT}/worker-releases/not-a-uuid/usage", 404), + ("GET", f"{PROJECT}/operations/compatibility/summary", 404), ], ) def test_bridge_boundaries_hold(tmp_path, method, path, status): @@ -216,8 +228,8 @@ def test_every_operation_crosses_the_bridge_exactly_when_its_family_or_id_is_adm } assert {"releases.create", "admin.grant", "admin.tenant"}.isdisjoint(crossing) # The AI setup operations add exactly these six, direct incident reads join the incidents - # family and stale-worker retirement joins the workers family; every earlier crossing stays - # as it was. Release usage stays outside until the releases allowlist names it. + # family, stale-worker retirement joins the workers family, and the environment summary, + # telemetry settings and release usage join as reads; every earlier crossing stays as it was. approved_additions = { "ai_endpoints.list", "ai_models.list", @@ -227,6 +239,9 @@ def test_every_operation_crosses_the_bridge_exactly_when_its_family_or_id_is_adm "bindable_connections.list", "incidents.read", "workers.retire_stale", + "operations_summary.read", + "operations_telemetry.read", + "releases.usage", } assert approved_additions <= crossing preserved = crossing - approved_additions @@ -235,9 +250,71 @@ def test_every_operation_crosses_the_bridge_exactly_when_its_family_or_id_is_adm "3ba280e1522358905e2cdb2e4bf66976fdc988018cdc08742c4956b87d1ab159" ) assert {"workers.grant", "ai_connections.test"} <= preserved - assert {"releases.create", "tasks.credentials", "admin.grant", "admin.tenant", "releases.usage"}.isdisjoint( - crossing + assert {"releases.create", "tasks.credentials", "admin.grant", "admin.tenant"}.isdisjoint(crossing) + + +def test_releases_and_summary_reads_cross_by_this_literal_set(tmp_path): + from firefly_weave.contracts.surface import OPERATIONS + from firefly_weave.studio.service import STUDIO_FAMILIES, STUDIO_OPERATIONS + + # Releases cross as three named reads, never as a family: registering an image stays an operator step. + assert {"releases.list", "releases.read", "releases.usage"} == STUDIO_OPERATIONS + assert "releases" not in STUDIO_FAMILIES and "tasks" not in STUDIO_FAMILIES + assert {"operations_summary", "operations_telemetry"} <= STUDIO_FAMILIES + with client(tmp_path) as browser: + studio = browser.app.state.studio + crossing = { + identifier + for identifier, operation in OPERATIONS.items() + if identifier.startswith(("releases.", "operations_")) + and studio.check_scope( + operation.canonical_path.format( + tenant=TENANT, project=PROJECT_ID, environment=ENVIRONMENT_ID, identifier=RESOURCE + ), + operation.method, + ) + == 200 + } + assert crossing == { + "releases.list", + "releases.read", + "releases.usage", + "operations_summary.read", + "operations_telemetry.read", + } + + +@pytest.mark.parametrize("upstream_status", [200, 403]) +def test_release_usage_and_summary_reads_keep_scope_method_and_upstream_authority(tmp_path, upstream_status): + seen = [] + + async def upstream(request): + seen.append((request.method, request.url.path, request.url.query.decode(), request.headers["authorization"])) + return httpx.Response(upstream_status, json={"code": "WV-FORBIDDEN"} if upstream_status == 403 else {}) + + paths = ( + (f"{ENVIRONMENT}/worker-releases/{RESOURCE}/usage", "limit=10"), + (f"{ENVIRONMENT}/operations/summary", ""), + (f"{PROJECT}/operations/telemetry", ""), ) + with client(tmp_path, transport=httpx.MockTransport(upstream)) as browser: + targets = [f"/studio/api{path}" + (f"?{query}" if query else "") for path, query in paths] + # Nothing crosses before pairing. + assert {browser.get(target, headers={"Origin": ORIGIN}).status_code for target in targets} == {401} + headers = pair(browser) + for (path, _), target in zip(paths, targets, strict=True): + assert browser.get(target, headers=headers).status_code == upstream_status + assert browser.post(f"/studio/api{path}", json={}, headers=headers).status_code == 404 + assert browser.delete(f"/studio/api{path}", headers=headers).status_code == 404 + assert ( + browser.get(f"/studio/api{OTHER_ENVIRONMENT}/worker-releases/{RESOURCE}/usage", headers=headers).status_code + == 403 + ) + assert browser.get(f"/studio/api{OTHER_ENVIRONMENT}/operations/summary", headers=headers).status_code == 403 + assert browser.get(f"/studio/api{OTHER_PROJECT}/operations/telemetry", headers=headers).status_code == 403 + # Registering a release image still never crosses. + assert browser.post(f"/studio/api{ENVIRONMENT}/worker-releases", json={}, headers=headers).status_code == 404 + assert seen == [("GET", path, query, "Bearer t") for path, query in paths] @pytest.mark.parametrize( From 65eaabfe65b8f92b3cdfde5b4df63c4e8279724c Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Mon, 12 Oct 2026 11:54:53 -0700 Subject: [PATCH 25/63] Share each summary section's time budget, judge telemetry export by recent drops and pin the summary's access and figure rules --- CHANGELOG.md | 10 +- docs/operations/observability.md | 4 +- docs/operations/upgrades.md | 6 +- docs/reference/api.md | 24 ++- migrations/versions/0031_operations_facts.py | 7 +- src/firefly_weave/access/authorization.py | 25 ++- src/firefly_weave/contracts/telemetry.py | 2 + src/firefly_weave/observability.py | 18 +- src/firefly_weave/operations/summary.py | 160 ++++++++++++------ .../test_operations_facts_migration.py | 5 +- tests/integration/test_operations_summary.py | 108 +++++++++++- tests/unit/operations/test_loop_health.py | 72 +++++++- .../test_operations_summary_rules.py | 140 +++++++++++++-- tests/unit/operations/test_telemetry.py | 38 ++++- 14 files changed, 520 insertions(+), 99 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bb8a6d3c..88b9fa94 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -136,13 +136,17 @@ SPDX-License-Identifier: Apache-2.0 an environment's health, load, workers, targets, incidents and up to 20 items that need attention, in one read that reloads the caller's grants. Each section and item needs its own capability; one the caller may not read is - `{"state": "forbidden"}`, and one that runs out of its two seconds of - database time is `{"state": "unavailable"}` while the rest are still served. - Run figures leave out test runs. + `{"state": "forbidden"}` without running its queries or recording a denial, + and one whose queries use up its two seconds of database time between them + is `{"state": "unavailable"}` while the rest are still served. Run figures + leave out test runs, and opening times the upgrade estimated are not counted. + Telemetry export health is degraded while exports are being dropped. - New operation `operations_telemetry.read` (`weave operations telemetry`) answers whether telemetry export is on, the collector host names, the metrics interval and how many items were dropped, never endpoint paths, credentials or headers. It needs `status.read` for the project. + `WEAVE_TELEMETRY` now refuses a collector host name longer than 253 + characters. - Background loop health, which the summary reports, ages successes with a monotonic clock, counts a cycle still within its loop's budget as on time, and reads degraded after 3 failed cycles in a row or among the last 20. An diff --git a/docs/operations/observability.md b/docs/operations/observability.md index b8d9c2a1..7fe17d30 100644 --- a/docs/operations/observability.md +++ b/docs/operations/observability.md @@ -142,7 +142,7 @@ Export failure never changes workflow state or grants execution authority. | Field | Default | Rule | | --- | --- | --- | | `enabled` | `false` | JSON `true` or `false`; at least one endpoint is required when `true` | -| `traces_endpoint`, `metrics_endpoint` | Absent | Exact `http` or `https` URL, at most 2048 characters; `http` only for `localhost`, `127.0.0.1`, or `::1`; no user name, password, query, or fragment. Either may be omitted | +| `traces_endpoint`, `metrics_endpoint` | Absent | Exact `http` or `https` URL, at most 2048 characters, whose host name has at most 253 characters; `http` only for `localhost`, `127.0.0.1`, or `::1`; no user name, password, query, or fragment. Either may be omitted | | `authorization` | Absent | The complete `Authorization` header value the collector expects, 1 to 4096 printable ASCII characters | | `timeout_seconds` | `5` | Greater than 0 and at most 5 | | `metrics_interval_seconds` | `30` | From 5 to 300 | @@ -219,7 +219,7 @@ volume needs to be deleted. | What you see | Why | What to do | | --- | --- | --- | | No export traffic at all | The API process never received `WEAVE_TELEMETRY` with `"enabled": true`, or it runs under `weave platform start` | Restart or redeploy the API with the explicit value; `OTEL_*` variables are not enough | -| The API stops with `Invalid WEAVE_TELEMETRY configuration` | The JSON is malformed, has an unknown field, an `http` URL to a non-loopback host, a URL with credentials or a query, or a value out of range | Compare each field with the table above | +| The API stops with `Invalid WEAVE_TELEMETRY configuration` | The JSON is malformed, has an unknown field, an `http` URL to a non-loopback host, a URL with credentials or a query, a host name longer than 253 characters, or a value out of range | Compare each field with the table above | | Connection refused | The collector is not listening where the API looks | A loopback URL works only for an API on the same host; use a reachable trusted URL for a container API | | The collector rejects requests | Receiver type, exact `/v1/traces` or `/v1/metrics` path, TLS, or authorization | Correct the receiver or the configuration; never put credentials in the URL | | Metrics look doubled | Several replicas report the same global inventory | Use max for duplicate inventory observations and check freshness | diff --git a/docs/operations/upgrades.md b/docs/operations/upgrades.md index 73d034b9..0c28b3b5 100644 --- a/docs/operations/upgrades.md +++ b/docs/operations/upgrades.md @@ -342,7 +342,11 @@ Two reads are new, and neither changes an existing answer: `status.read` granted for the project, not only for an environment, and never returns endpoint paths, credentials or headers. -Earlier servers do not serve them and answer `404` or `405`. Loop health is +Earlier servers do not serve them and answer `404` or `405`. For the first 24 +hours after the upgrade, `opened_24h` leaves out incidents whose opening time +the upgrade could only estimate. A `WEAVE_TELEMETRY` collector URL whose host +name is longer than 253 characters, which no DNS name is, now stops startup with +`Invalid WEAVE_TELEMETRY configuration`. Loop health is measured in each API process and starts again when the process restarts, so a freshly started process reads `degraded` until each loop completes a cycle. An answer from an API-only replica reports every scheduler loop as `disabled`; the diff --git a/docs/reference/api.md b/docs/reference/api.md index e452d061..cd2c2aef 100644 --- a/docs/reference/api.md +++ b/docs/reference/api.md @@ -796,9 +796,12 @@ section in the same transaction that reloads those grants. `native_executors`, and `telemetry`, each with a `state` of `ok`, `degraded`, `down`, or `disabled`. Loops add `last_started_at` and `last_ok_at` when known, `consecutive_failures`, and `recent_failures` (failed cycles among - the last 20). Health is measured by the process that answered; no exception - text is returned. -- `load`: `runs_active` (test runs left out), `tasks_ready` and + the last 20). `telemetry` is `disabled` while export is off, `degraded` when + the process dropped telemetry within the last three metrics export intervals + (90 seconds by default), and `ok` otherwise. Health is measured by the + process that answered; no exception text is returned. +- `load`: `runs_active` (active by status, as the run usage counter counts + them, with test runs left out), `tasks_ready` and `oldest_ready_seconds` (tasks due now; a retry joins when its next attempt is due), `tasks_leased`, `waits_active`, `waits_overdue` and `oldest_overdue_seconds`, `outbox_pending` and `outbox_oldest_seconds`, @@ -814,7 +817,8 @@ section in the same transaction that reloads those grants. can still be approved: unexpired, on a fresh observation of the current target and deployment revisions, and neither approved nor applied). - `incidents` needs `incident.read`: `active` and `opened_24h`, leaving out - incidents of test runs. + incidents of test runs. `opened_24h` counts only known opening times: an + incident whose opening time the upgrade estimated is never counted. - `attention` lists at most 20 items, `{kind, id, title}` plus `expires_at` for approvals, in this order: jobs that need reconciliation, plans you may approve (soonest expiry first), offline workers that still hold active @@ -825,11 +829,13 @@ section in the same transaction that reloads those grants. for jobs and runners, `deployment.read` and `deployment.approve` on the same target for approvals, `incident.read` for incidents, and `run.read` for failed runs. Items you may not read are left out; test runs never appear. -- A section you may not read is `{"state": "forbidden"}`. Each section has two - seconds of database time; one that runs out, or meets a transient database - failure, is `{"state": "unavailable"}` while the others are still served with - `200`. Neither state is ever reported as zero. The answer has no 24-hour run - figures yet. +- A section you may not read is `{"state": "forbidden"}`; none of its queries + run, and leaving it out records no authorization denial. Each section has two + seconds of database time in all (or a stricter statement timeout already in + force), shared by its queries: each runs under what is left. One that runs + out, or meets a transient database failure, is `{"state": "unavailable"}` + while the others are still served with `200`. Neither state is ever reported + as zero. The answer has no 24-hour run figures yet. Loop states follow these rules. A loop is `disabled` when its setting turns it off (API-only replicas disable every scheduler loop) and `down` while its task diff --git a/migrations/versions/0031_operations_facts.py b/migrations/versions/0031_operations_facts.py index 7b17b2ba..466eb44f 100644 --- a/migrations/versions/0031_operations_facts.py +++ b/migrations/versions/0031_operations_facts.py @@ -179,7 +179,12 @@ def indexes() -> None: ("caller", "caller_run_id", "WHERE caller_run_id IS NOT NULL"), ("retry", "retried_from_run_id", "WHERE retried_from_run_id IS NOT NULL"), ("ended", "ended_at", "WHERE ended_at IS NOT NULL"), - ("active_test", "", "WHERE test AND ended_at IS NULL"), + # Active test runs as the run usage counter counts runs: by status, never by a missing end time. + ( + "active_test", + "", + "WHERE test AND (status IS NULL OR status NOT IN ('succeeded','failed','cancelled','timed_out'))", + ), # Rows a listing-metadata refresh can need: another policy's decisions and transient unsupported ones. ( "revalidation", diff --git a/src/firefly_weave/access/authorization.py b/src/firefly_weave/access/authorization.py index 14ca3fac..3c36fb56 100644 --- a/src/firefly_weave/access/authorization.py +++ b/src/firefly_weave/access/authorization.py @@ -40,6 +40,22 @@ def contains(parent: Scope | None, child: Scope | None) -> bool: @service class AuthorizationService: + def allows( + self, principal: Principal, scope: Scope | None, capability: str, *, resource: str | None = None + ) -> bool: + """Whether `require` would allow this, without recording anything. + + For reads that only shape what they return, such as which sections of a summary to fill; the access itself + is still checked with `require`. + """ + allowed = principal.active and any( + capability in ROLE_CAPABILITIES[grant.role] + and contains(grant.scope, scope) + and (not grant.resources or resource in grant.resources) + for grant in principal.grants + ) + return allowed and not (principal.kind == "worker" and capability not in ROLE_CAPABILITIES["worker"]) + def require( self, principal: Principal, @@ -49,14 +65,7 @@ def require( resource: str | None = None, context: AuditContext | None = None, ) -> None: - allowed = principal.active and any( - capability in ROLE_CAPABILITIES[grant.role] - and contains(grant.scope, scope) - and (not grant.resources or resource in grant.resources) - for grant in principal.grants - ) - if principal.kind == "worker" and capability not in ROLE_CAPABILITIES["worker"]: - allowed = False + allowed = self.allows(principal, scope, capability, resource=resource) emit(event(principal, scope, capability, "allow" if allowed else "deny", context or AuditContext())) if not allowed: raise AccessDenied() diff --git a/src/firefly_weave/contracts/telemetry.py b/src/firefly_weave/contracts/telemetry.py index f0b0fac1..b86d8124 100644 --- a/src/firefly_weave/contracts/telemetry.py +++ b/src/firefly_weave/contracts/telemetry.py @@ -42,6 +42,8 @@ def endpoint(cls, value: str | None) -> str | None: valid = ( url.scheme in {"http", "https"} and url.hostname + # A DNS name has at most 253 characters, and the host is all that is ever shown. + and len(url.hostname) <= 253 and url.port != 0 and not url.username and not url.password diff --git a/src/firefly_weave/observability.py b/src/firefly_weave/observability.py index dee4a9d8..168bb843 100644 --- a/src/firefly_weave/observability.py +++ b/src/firefly_weave/observability.py @@ -21,6 +21,8 @@ """ import threading +import time +from collections.abc import Callable from opentelemetry.metrics import MeterProvider as MeterProviderAPI from opentelemetry.sdk.metrics import AlwaysOffExemplarFilter, MeterProvider @@ -47,11 +49,16 @@ class TelemetryDrops: - """A saturating diagnostic counter remains bounded even when no exporter is available.""" + """A saturating diagnostic counter remains bounded even when no exporter is available. - def __init__(self) -> None: + It also keeps the monotonic time of the last drop, so export health can be judged by recent drops. + """ + + def __init__(self, *, monotonic: Callable[[], float] = time.monotonic) -> None: self.count = 0 self._lock = threading.Lock() + self._monotonic = monotonic + self._last: float | None = None self.provider: MeterProviderAPI | None = None def add(self, count: int) -> None: @@ -59,12 +66,19 @@ def add(self, count: int) -> None: return with self._lock: self.count = min((1 << 63) - 1, self.count + count) + self._last = self._monotonic() provider = self.provider if provider is not None: provider.get_meter(SCOPE).create_counter("weave.telemetry_dropped").add( count, attributes("other", "rejected") ) + def dropped_within(self, seconds: float) -> bool: + """Whether anything was dropped in the last `seconds`.""" + with self._lock: + last = self._last + return last is not None and self._monotonic() - last <= seconds + @configuration class OwnedMeterConfiguration: diff --git a/src/firefly_weave/operations/summary.py b/src/firefly_weave/operations/summary.py index 02c81bbd..90f77911 100644 --- a/src/firefly_weave/operations/summary.py +++ b/src/firefly_weave/operations/summary.py @@ -18,9 +18,11 @@ The summary is read in one transaction that also reloads the caller's grants. Beyond `status.read`, each section and attention item needs its own capability on both the token and the principal stored now; a section the caller -may not read is forbidden before any of its statements run. Each section runs in a savepoint under a two-second -statement timeout, and one that times out is unavailable while the others are still served. Run figures leave out -test runs; queue and worker figures measure shared capacity and count every task. +may not read is forbidden before any of its statements run, and finding that out records nothing. Each section runs +in a savepoint with two seconds of database time in all: every statement runs under what is left of it, and a +section that runs out is unavailable while the others are still served. Run figures leave out test runs, and times +the platform only estimated are never counted as known; queue and worker figures measure shared capacity and count +every task. """ import time @@ -32,8 +34,10 @@ from urllib.parse import urlsplit from uuid import UUID -from sqlalchemy import text +from sqlalchemy import Result, text from sqlalchemy.exc import DBAPIError +from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy.sql import Executable from firefly_weave import __version__ from firefly_weave.access.audit import AuditContext @@ -62,6 +66,7 @@ UnavailableSection, WorkersSection, ) +from firefly_weave.contracts.telemetry import TelemetryOptions from firefly_weave.observability import TelemetryDrops from firefly_weave.operations.compatibility import CompatibilityService from firefly_weave.operations.fact_reads import available @@ -72,6 +77,8 @@ # Each section's statements share this budget; a stricter timeout already in force is kept. SECTION_TIMEOUT_MS = 2000 +# Telemetry export is degraded while anything was dropped within this many metric export intervals. +TELEMETRY_INTERVALS_RECENT = 3 # Query failures a section reports as unavailable: a cancelled or timed-out statement, a lock timeout, a # serialization conflict, a deadlock and exhausted database resources. Every other error propagates. _TRANSIENT = ("57014", "55P03", "40001", "40P01", "53") @@ -122,6 +129,13 @@ _RECENT = f"w.last_seen_at>{NOW}-interval '60 seconds'" _CURRENT = f"NOT w.revoked AND coalesce(w.last_seen_at,f.registered_at)>{NOW}-interval '1 hour'" _WORKER_FACTS = f"LEFT JOIN worker_facts f ON {_SAME_SCOPE.format(a='f', b='w')} AND f.worker_id=w.id" +# Active test runs, exactly as the run usage counter counts runs: by status, so a finished run without a recorded end +# is not active. The partial index run_facts_active_test has this predicate. +_ACTIVE_TEST = "test AND (status IS NULL OR status NOT IN ('succeeded','failed','cancelled','timed_out'))" +_READY_COUNTS = ( + "SELECT count(*) AS ready,extract(epoch FROM (cast(:now AS timestamptz)-min(ready_since))) AS oldest " + f"FROM task_facts WHERE {_SCOPE.format(alias='')} AND status='ready' AND ready_since<=cast(:now AS timestamptz)" +) _RUN_CLASSIFICATION = ( "r.classification_state,r.classification_policy,r.pinned_ir_version,r.pinned_features," f"EXISTS(SELECT 1 FROM run_policy_blocks b WHERE {_SAME_SCOPE.format(a='b', b='r')} AND b.run_id=r.run_id) " @@ -143,12 +157,40 @@ def _transient(error: DBAPIError) -> bool: return isinstance(state, str) and state.startswith(_TRANSIENT) -async def read_section[T](tx: Transaction, read: Callable[[], Awaitable[T]]) -> T | UnavailableSection: - """Run one section's statements in a savepoint under the section timeout. +class SectionTimedOut(Exception): + """A section used up its database time before its next statement.""" + + +class Budget: + """One section's database time, shared by its statements: each runs under what is left of it.""" + + def __init__(self, session: AsyncSession, limit_ms: int, monotonic: Callable[[], float]) -> None: + self.session, self.monotonic = session, monotonic + self.deadline = monotonic() + limit_ms / 1000 + + async def limit(self) -> None: + """Bound the next statement by what is left; nothing left means the section ran out.""" + remaining = int((self.deadline - self.monotonic()) * 1000) + if remaining <= 0: + raise SectionTimedOut() + await self.session.execute( + text("SELECT set_config('statement_timeout',:value,true)"), {"value": f"{remaining}ms"} + ) + + async def execute(self, statement: Executable, values: dict[str, Any] | None = None) -> Result[Any]: + await self.limit() + return await self.session.execute(statement, values) + + +async def read_section[T]( + tx: Transaction, read: Callable[[Budget], Awaitable[T]], *, monotonic: Callable[[], float] = time.monotonic +) -> T | UnavailableSection: + """Run one section's statements in a savepoint, sharing one section budget. - A timed-out or otherwise transient query failure rolls the savepoint back, which restores the earlier local - timeout, and the section is unavailable. A lost connection, cancellation, an authorization failure and any other - error propagate, so a broken transaction never answers later sections. + The budget is two seconds, or a stricter timeout already in force. A section that runs out of it, or meets a + transient query failure, rolls the savepoint back, which restores the earlier local timeout, and is unavailable. + A lost connection, cancellation, an authorization failure and any other error propagate, so a broken transaction + never answers later sections. """ session = tx.session row = ( @@ -162,37 +204,31 @@ async def read_section[T](tx: Transaction, read: Callable[[], Awaitable[T]]) -> bound = min(row.prior_ms, SECTION_TIMEOUT_MS) if row.prior_ms else SECTION_TIMEOUT_MS try: async with session.begin_nested(): + # Every statement of the section is bounded by the whole budget, and lowered to what is left of it. await session.execute(text("SELECT set_config('statement_timeout',:value,true)"), {"value": f"{bound}ms"}) - value = await read() + value = await read(Budget(session, bound, monotonic)) await session.execute(text("SELECT set_config('statement_timeout',:value,true)"), {"value": row.prior}) return value + except SectionTimedOut: + return UnavailableSection(state="unavailable") except DBAPIError as error: if error.connection_invalidated or not _transient(error): raise return UnavailableSection(state="unavailable") +def _ready(row: Any) -> tuple[int, float | None]: + return int(row["ready"]), _seconds(row["oldest"]) + + async def read_ready_counts(tx: Transaction, observed_at: datetime) -> tuple[int, float | None]: """Ready tasks that are due at `observed_at`, and how long the oldest of them has waited. A retry whose next attempt is still in the future is stored as ready from that instant, so it joins the count exactly when it becomes due; nothing is written to make it so. Claims stay the authority on what runs. """ - row = ( - ( - await tx.session.execute( - text( - "SELECT count(*) AS ready,extract(epoch FROM (cast(:now AS timestamptz)-min(ready_since))) " - f"AS oldest FROM task_facts WHERE {_SCOPE.format(alias='')} AND status='ready' " - "AND ready_since<=cast(:now AS timestamptz)" - ), - {**_bound(tx), "now": observed_at}, - ) - ) - .mappings() - .one() - ) - return int(row["ready"]), _seconds(row["oldest"]) + result = await tx.session.execute(text(_READY_COUNTS), {**_bound(tx), "now": observed_at}) + return _ready(result.mappings().one()) def _bound(tx: Transaction) -> dict[str, Any]: @@ -224,12 +260,23 @@ def native_state(settings: Settings, dispatcher: NativeExecutors | None) -> Heal return "ok" if dispatcher.healthy() else "degraded" +def telemetry_state(options: TelemetryOptions, drops: TelemetryDrops) -> HealthState: + """Off unless export is on; degraded while this process dropped telemetry within the last three metric intervals. + + Configuration alone proves nothing: export is healthy only while nothing is being dropped. + """ + if not options.enabled: + return "disabled" + recent = TELEMETRY_INTERVALS_RECENT * options.metrics_interval_seconds + return "degraded" if drops.dropped_within(recent) else "ok" + + def health_components( latency: float | UnavailableSection, snapshots: Iterable[LoopSnapshot], *, native: HealthState, - telemetry_enabled: bool, + telemetry: HealthState, ) -> list[HealthComponent]: """The database, each registered loop, native executors and telemetry export; states and counts, never messages.""" database = ( @@ -253,7 +300,7 @@ def health_components( database, *loops, HealthComponent(component="native_executors", state=native), - HealthComponent(component="telemetry", state="ok" if telemetry_enabled else "disabled"), + HealthComponent(component="telemetry", state=telemetry), ] @@ -303,12 +350,14 @@ def _intersect(left: tuple[UUID, ...] | None, right: tuple[UUID, ...] | None) -> class _Reader: - def __init__(self, tx: Transaction, observed_at: datetime) -> None: - self.tx = tx + """One section's statements, each under what is left of the section's budget.""" + + def __init__(self, tx: Transaction, observed_at: datetime, budget: Budget) -> None: + self.tx, self.budget = tx, budget self.bound = {**_bound(tx), "now": observed_at} async def rows(self, sql: str, **values: Any) -> list[dict[str, Any]]: - result = await self.tx.session.execute(text(sql), {**self.bound, **values}) + result = await self.budget.execute(text(sql), {**self.bound, **values}) return [dict(row) for row in result.mappings()] async def row(self, sql: str, **values: Any) -> dict[str, Any]: @@ -351,21 +400,24 @@ async def read(self, actor: Principal, scope: Scope, *, context: AuditContext) - reach = self._reach(actor, current, scope, context) observed_at = await tx.session.scalar(text("SELECT clock_timestamp()")) assert isinstance(observed_at, datetime) - reader = _Reader(tx, observed_at) - latency = await read_section(tx, lambda: self._latency(tx)) - load = await read_section(tx, lambda: self._load(reader)) - workers = await read_section(tx, lambda: self._workers(reader)) + + async def section[T](read: Callable[[_Reader], Awaitable[T]]) -> T | UnavailableSection: + return await read_section( + tx, lambda budget: read(_Reader(tx, observed_at, budget)), monotonic=self.monotonic + ) + + latency = await read_section(tx, self._latency, monotonic=self.monotonic) + load = await section(self._load) + workers = await section(self._workers) targets: TargetsSection | ForbiddenSection | UnavailableSection = ( ForbiddenSection(state="forbidden") if reach.targets == () - else await read_section(tx, lambda: self._targets(reader, reach.targets)) + else await section(lambda reader: self._targets(reader, reach.targets)) ) incidents: IncidentsSection | ForbiddenSection | UnavailableSection = ( - await read_section(tx, lambda: self._incidents(reader)) - if reach.incidents - else ForbiddenSection(state="forbidden") + await section(self._incidents) if reach.incidents else ForbiddenSection(state="forbidden") ) - attention = await read_section(tx, lambda: self._attention(reader, reach)) + attention = await section(lambda reader: self._attention(reader, reach)) return OperationsSummary( observed_at=observed_at, api=self._api(scope), @@ -400,12 +452,16 @@ async def telemetry(self, actor: Principal, scope: Scope, *, context: AuditConte ) def _reach(self, actor: Principal, current: Principal, scope: Scope, context: AuditContext) -> _Reach: + """Which sections to fill. A capability the caller lacks is not an attempt to use it, so it records nothing; + reading a section the caller may read is recorded as allowed, as target-restricted reads are. + """ + def permitted(capability: str) -> bool: - try: - for principal in (actor, current): - self.authorization.require(principal, scope, capability, context=context) - except AccessDenied: + principals = (actor, current) + if not all(self.authorization.allows(principal, scope, capability) for principal in principals): return False + for principal in principals: + self.authorization.require(principal, scope, capability, context=context) return True readable = target_permissions(self.authorization, actor, current, scope, "deployment.read", context) @@ -441,12 +497,13 @@ def _health(self, latency: float | UnavailableSection) -> list[HealthComponent]: latency, self.health.snapshots(), native=native_state(self.settings, self.native()), - telemetry_enabled=self.settings.telemetry.enabled, + telemetry=telemetry_state(self.settings.telemetry, self.drops), ) - async def _latency(self, tx: Transaction) -> float: + async def _latency(self, budget: Budget) -> float: + await budget.limit() started = self.monotonic() - await tx.session.execute(text("SELECT 1")) + await budget.session.execute(text("SELECT 1")) return round(max(0.0, self.monotonic() - started) * 1000, 3) async def _load(self, reader: _Reader) -> LoadSection: @@ -459,9 +516,9 @@ async def _load(self, reader: _Reader) -> LoadSection: ) } tests = await reader.row( - f"SELECT count(*) AS active FROM run_facts WHERE {_SCOPE.format(alias='')} AND test AND ended_at IS NULL" + f"SELECT count(*) AS active FROM run_facts WHERE {_SCOPE.format(alias='')} AND {_ACTIVE_TEST}" ) - ready, oldest_ready = await read_ready_counts(reader.tx, reader.bound["now"]) + ready, oldest_ready = _ready(await reader.row(_READY_COUNTS)) leased = await reader.row( "SELECT count(DISTINCT l.task_id) AS leased FROM task_leases l JOIN task_intents i " f"ON {_SAME_SCOPE.format(a='i', b='l')} AND i.id=l.task_id WHERE {_SCOPE.format(alias='l.')} " @@ -523,7 +580,10 @@ async def _targets(self, reader: _Reader, targets: tuple[UUID, ...] | None) -> T ) async def _incidents(self, reader: _Reader) -> IncidentsSection: - """Incidents of test runs are left out; an incident without facts predates test runs and counts.""" + """Incidents of test runs are left out; an incident without facts predates test runs and counts. + + Only known opening times count as opened in the last 24 hours: one the upgrade estimated is never counted. + """ active = await reader.row( "SELECT count(*) AS active FROM incidents i LEFT JOIN incident_facts f ON " f"{_SAME_SCOPE.format(a='f', b='i')} AND f.incident_id=i.id " @@ -531,7 +591,7 @@ async def _incidents(self, reader: _Reader) -> IncidentsSection: ) opened = await reader.row( f"SELECT count(*) AS opened FROM incident_facts WHERE {_SCOPE.format(alias='')} AND NOT test " - f"AND opened_at>{NOW}-interval '24 hours'" + f"AND NOT opened_at_estimated AND opened_at>{NOW}-interval '24 hours'" ) return IncidentsSection(active=int(active["active"]), opened_24h=int(opened["opened"])) diff --git a/tests/integration/test_operations_facts_migration.py b/tests/integration/test_operations_facts_migration.py index 9a654fd2..e217f80b 100644 --- a/tests/integration/test_operations_facts_migration.py +++ b/tests/integration/test_operations_facts_migration.py @@ -839,7 +839,10 @@ async def test_fact_query_indexes_cover_scope_and_ordering(migration_db): assert "(" + scope + ", " + suffix + ")" in definitions["run_facts_" + name] for name in ("business_key", "correlation_key", "caller", "retry", "ended", "active_test"): assert scope in definitions["run_facts_" + name] - assert "WHERE (test AND (ended_at IS NULL))" in definitions["run_facts_active_test"] + assert definitions["run_facts_active_test"].endswith( + "WHERE (test AND ((status IS NULL) OR (status <> ALL (ARRAY['succeeded'::text, 'failed'::text, " + "'cancelled'::text, 'timed_out'::text]))))" + ) for name in ( "step_facts_ended", "task_facts_ready", diff --git a/tests/integration/test_operations_summary.py b/tests/integration/test_operations_summary.py index 81d29c7c..39032de3 100644 --- a/tests/integration/test_operations_summary.py +++ b/tests/integration/test_operations_summary.py @@ -18,6 +18,7 @@ import asyncio import json +import logging import time from datetime import datetime, timedelta from types import SimpleNamespace @@ -112,7 +113,9 @@ async def owner_execute(case, sql: str, **values) -> None: ) -async def add_incident(case, run: UUID, key: str, *, opened=None, test=False, status="active", facts=True) -> UUID: +async def add_incident( + case, run: UUID, key: str, *, opened=None, test=False, status="active", facts=True, estimated=False +) -> UUID: identifier = uuid4() await owner_execute( case, @@ -130,11 +133,12 @@ async def add_incident(case, run: UUID, key: str, *, opened=None, test=False, st "INSERT INTO incident_facts(tenant_id,project_id,environment_id,incident_id,node_id,instance_key," "opened_at,updated_at,workflow_name,workflow_version,test,opened_at_estimated) SELECT :tenant,:project," ":environment,:id,'work','',cast(:opened AS timestamptz),cast(:opened AS timestamptz),workflow_name," - "workflow_version,:test,false FROM run_facts WHERE run_id=:run", + "workflow_version,:test,:estimated FROM run_facts WHERE run_id=:run", id=identifier, run=run, opened=opened, test=test, + estimated=estimated, ) return identifier @@ -687,6 +691,106 @@ async def held(self, reader): assert "state" not in (await summarize(case, reader))["workers"] +async def test_the_statements_of_a_section_share_its_two_seconds(operations_case, monkeypatch): + case = operations_case + await case.start() + workers = OperationsSummaryService._workers + slept = [] + + async def two_slow_statements(self, reader): + # Each statement alone is well within two seconds; together they are not. + for _ in range(2): + slept.append(await reader.row("SELECT pg_sleep(1.2) IS NULL AS slept")) + return await workers(self, reader) + + monkeypatch.setattr(OperationsSummaryService, "_workers", two_slow_statements) + reader = await principal_with(case, Grant(role="viewer", scope=case.scope)) + started = time.monotonic() + body = await summarize(case, reader) + elapsed = time.monotonic() - started + # The second statement got what was left of the section's budget and was cancelled when it ran out. + assert body["workers"] == {"state": "unavailable"} and len(slept) == 1 + assert elapsed < 6 + assert body["load"]["runs_active"] == 1 and isinstance(body["attention"], list) + + +async def test_a_forbidden_section_runs_no_statement_and_records_no_denial(operations_case, caplog): + case = operations_case + await add_incident(case, (await case.start()).id, "hidden", opened=await clock(case)) + target, deployment, observation = await capable_target(case, "alpha") + await add_job(case, target, "reconciliation_required") + await add_plan(case, target, deployment, observation, expires=await clock(case) + timedelta(minutes=5)) + await add_runner(case, target, expires=await clock(case) - timedelta(seconds=1)) + await fail_run(case, (await case.start("failed")).id, minutes_ago=5) + worker_operator = await principal_with(case, Grant(role="worker_operator", scope=case.scope)) + statements: list[str] = [] + engine = case.sessions.kw["bind"].sync_engine + + def record(connection, cursor, statement, parameters, context, executemany): + statements.append(" ".join(statement.lower().split())) + + caplog.set_level(logging.INFO, logger="weave.authorization") + event.listen(engine, "before_cursor_execute", record) + try: + body = await summarize(case, worker_operator) + finally: + event.remove(engine, "before_cursor_execute", record) + assert body["targets"] == body["incidents"] == {"state": "forbidden"} + assert kinds(body) <= {"worker", "compatibility"} + # Sections and attention kinds the caller may not read are never queried, not queried and discarded. + assert statements and not [ + statement + for statement in statements + if "deployment_" in statement or "incident" in statement or "status='failed'" in statement + ] + # Finding out which sections to leave out is not an access attempt: the summary records no denial. + records = [json.loads(item.getMessage()) for item in caplog.records if item.name == "weave.authorization"] + assert records and {record["outcome"] for record in records} == {"allow"} + assert {record["capability"] for record in records} == {"status.read"} + + +async def test_failed_run_items_need_run_read_not_incident_read(operations_case): + case = operations_case + failed = await case.start("failed") + await fail_run(case, failed.id, minutes_ago=5) + incidents_only = await principal_with( + case, Grant(role="worker_operator", scope=case.scope), Grant(role="operator", scope=case.scope) + ) + runs_only = await principal_with(case, Grant(role="viewer", scope=case.scope)) + incident_body = await summarize(case, incidents_only) + run_body = await summarize(case, runs_only) + assert "state" not in incident_body["incidents"] and "run" not in kinds(incident_body) + assert run_body["incidents"] == {"state": "forbidden"} + assert [item["id"] for item in run_body["attention"] if item["kind"] == "run"] == [str(failed.id)] + + +async def test_estimated_opening_times_and_finished_test_runs_are_not_counted(operations_case): + case = operations_case + real, finished = await case.start("real"), await case.start("finished-test") + async with case.tx() as tx: + await case.runtime.cancel( + tx, finished.id, "operator_cancelled", actor=case.actor, scope=case.scope, context=AuditContext() + ) + # An upgraded test run whose history lacked its end: finished by status, with no end time recorded. + await owner_execute( + case, "UPDATE run_facts SET test=true,origin='test',ended_at=NULL WHERE run_id=:run", run=finished.id + ) + now = await clock(case) + await add_incident(case, real.id, "known", opened=now) + # The upgrade estimated this opening time from the run's last event: it is at or after the real one. + await add_incident(case, real.id, "estimated", opened=now, estimated=True) + reader = await principal_with( + case, Grant(role="viewer", scope=case.scope), Grant(role="operator", scope=case.scope) + ) + body = await summarize(case, reader) + [usage] = await case.rows( + "SELECT value FROM operation_usage WHERE environment_id=:environment AND metric='runs_active'" + ) + # The usage counter counts runs by status, and so does the summary: the finished test run was never counted. + assert usage["value"] == 1 and body["load"]["runs_active"] == 1 + assert body["incidents"] == {"active": 2, "opened_24h": 1} + + async def test_the_principal_and_every_section_share_one_read_transaction(operations_case): case = operations_case await case.start() diff --git a/tests/unit/operations/test_loop_health.py b/tests/unit/operations/test_loop_health.py index 262f91c0..5106e726 100644 --- a/tests/unit/operations/test_loop_health.py +++ b/tests/unit/operations/test_loop_health.py @@ -108,6 +108,23 @@ def test_exactly_three_consecutive_failures_degrade_and_a_success_clears_the_cou assert (snapshot.state, snapshot.recent_failures) == ("ok", 2) +def test_failures_are_counted_over_the_last_twenty_cycles(): + clock = Clock() + health = running(clock) + health.succeeded("outbox_loop") + for _ in range(3): + health.failed("outbox_loop") + # The documented window is twenty cycles: the third failure ages out after seventeen successes, no sooner. + for _ in range(17): + assert health.snapshot("outbox_loop").state == "degraded" + health.succeeded("outbox_loop") + snapshot = health.snapshot("outbox_loop") + assert (snapshot.state, snapshot.recent_failures) == ("degraded", 3) + health.succeeded("outbox_loop") + snapshot = health.snapshot("outbox_loop") + assert (snapshot.state, snapshot.recent_failures) == ("ok", 2) + + def test_failures_that_alternate_with_successes_degrade_the_loop(): clock = Clock() health = running(clock, "operations_loop", 60) @@ -484,6 +501,32 @@ async def nothing(authority): assert await broker.cycle() is True +async def poll_cycles(monkeypatch, outbox, clock: Clock, count: int) -> None: + """Drive the real `poll` for `count` cycles that each take one second, then stop it.""" + cycles = 0 + cycle = outbox.cycle + + async def timed(): + nonlocal cycles + try: + clock.advance(1) + return await cycle() + finally: + cycles += 1 + + monkeypatch.setattr(outbox, "cycle", timed) + task = asyncio.create_task(outbox.poll()) + try: + async with asyncio.timeout(2): + while cycles < count: + await asyncio.sleep(0) + finally: + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + monkeypatch.setattr(outbox, "cycle", cycle) + + async def test_an_outbox_waiting_for_readiness_neither_succeeds_nor_fails(monkeypatch): from firefly_weave.operations import outbox_loop from firefly_weave.operations.outbox_loop import OutboxLoop @@ -538,6 +581,10 @@ async def counted(): clock.advance(4) assert health.snapshot("outbox_loop").state == "degraded" assert health.snapshot("outbox_loop").consecutive_failures == 0 + # Waiting cycles that take time never count as successes: the poll records none of them. + await poll_cycles(monkeypatch, outbox, clock, 5) + snapshot = health.snapshot("outbox_loop") + assert (snapshot.state, snapshot.consecutive_failures, snapshot.last_ok_at) == ("degraded", 0, ok_at) async def test_an_outbox_without_a_free_delivery_slot_is_behind_not_healthy(monkeypatch): @@ -585,27 +632,42 @@ async def counted(): snapshot = health.snapshot("outbox_loop") # Without a free slot for more than three intervals the outbox reads degraded, with no failure invented. assert (snapshot.state, snapshot.consecutive_failures) == ("degraded", 0) + # Cycles that take time without a free slot never count as successes: the poll records none of them. + ok_at = snapshot.last_ok_at + await poll_cycles(monkeypatch, outbox, clock, 5) + snapshot = health.snapshot("outbox_loop") + assert (snapshot.state, snapshot.consecutive_failures, snapshot.last_ok_at) == ("degraded", 0, ok_at) def test_each_background_loop_judges_a_cycle_in_flight_by_its_own_budget(): clock = Clock() health = LoopHealth(clock, monotonic=clock.monotonic) - register_background_loops(health, Settings(database_url=DATABASE, scheduler_database_url=SCHEDULER)) + register_background_loops( + health, + Settings( + database_url=DATABASE, + scheduler_database_url=SCHEDULER, + broker=BrokerPolicy(enabled=True, max_clients=2), + kafka_consumer_enabled=True, + ), + ) + loops = ("recovery_loop", "schedules", "outbox_loop", "provider_loop", "kafka_loop", "operations_loop") budgets = {} - for name in ("recovery_loop", "schedules", "outbox_loop", "provider_loop", "operations_loop"): + for name in loops: health.running(name, True) health.succeeded(name) start = clock.elapsed - for name in ("recovery_loop", "schedules", "outbox_loop", "provider_loop", "operations_loop"): + for name in loops: health.started(name) for seconds in (4.9, 5.1, 29.9, 30.1, 59.9, 60.1, 179.9, 180.1): clock.elapsed = start + seconds - for name in ("recovery_loop", "schedules", "outbox_loop", "provider_loop", "operations_loop"): + for name in loops: if health.snapshot(name).state == "ok": budgets[name] = seconds - # The traversal timeout bounds outbox cycles; tenant scans and mailbox polls take longer. + # The traversal timeout bounds outbox and broker cycles; tenant scans and mailbox polls take longer. assert budgets == { "outbox_loop": 4.9, + "kafka_loop": 4.9, "recovery_loop": 29.9, "schedules": 29.9, "provider_loop": 59.9, diff --git a/tests/unit/operations/test_operations_summary_rules.py b/tests/unit/operations/test_operations_summary_rules.py index 620d54f2..6bf48490 100644 --- a/tests/unit/operations/test_operations_summary_rules.py +++ b/tests/unit/operations/test_operations_summary_rules.py @@ -18,6 +18,7 @@ import asyncio import json +import logging from contextlib import asynccontextmanager from datetime import UTC, datetime, timedelta from types import SimpleNamespace @@ -25,12 +26,14 @@ import pytest from pydantic import SecretStr +from sqlalchemy import text from sqlalchemy.exc import DBAPIError from firefly_weave.access.audit import AuditContext from firefly_weave.access.authorization import AccessDenied, AuthorizationService from firefly_weave.access.models import Grant, Principal from firefly_weave.contracts.access import Scope +from firefly_weave.contracts.compatibility import CompatibilityFinding, CompatibilityReport from firefly_weave.contracts.operations_summary import UnavailableSection from firefly_weave.contracts.telemetry import TelemetryOptions from firefly_weave.observability import TelemetryDrops @@ -44,6 +47,7 @@ native_state, read_section, telemetry_host, + telemetry_state, ) from firefly_weave.settings import Settings @@ -65,12 +69,20 @@ def failure(sqlstate: str, *, invalidated: bool = False) -> DBAPIError: class Session: - """Records statements and savepoints; the section timeout in force is what the last set_config chose.""" + """Records statements and savepoints; the section timeout in force is what the last set_config chose. - def __init__(self, prior: str = "0", prior_ms: int = 0) -> None: - self.prior, self.prior_ms = prior, prior_ms + Each other statement takes `cost` seconds of the fake monotonic clock and records the timeout it ran under. + """ + + def __init__(self, prior: str = "0", prior_ms: int = 0, cost: float = 0.0) -> None: + self.prior, self.prior_ms, self.cost = prior, prior_ms, cost self.timeout = prior self.savepoints: list[str] = [] + self.elapsed = 100.0 + self.ran_under: list[str] = [] + + def monotonic(self) -> float: + return self.elapsed async def execute(self, statement, values=None): sql = str(statement) @@ -78,6 +90,9 @@ async def execute(self, statement, values=None): return SimpleNamespace(one=lambda: SimpleNamespace(prior=self.prior, prior_ms=self.prior_ms)) if "set_config" in sql: self.timeout = values["value"] + return None + self.ran_under.append(self.timeout) + self.elapsed += self.cost return None @asynccontextmanager @@ -106,32 +121,71 @@ def test_only_the_collector_host_is_kept(): assert telemetry_host("http://[::1]:4318/v1/metrics") == "::1" +async def section(session: Session, read): + return await read_section(transaction(session), read, monotonic=session.monotonic) + + async def test_a_section_runs_under_two_seconds_and_restores_the_timeout_in_force(): session = Session(prior="30s", prior_ms=30000) seen = [] - async def read(): + async def read(budget): seen.append(session.timeout) + await budget.execute(text("SELECT 1")) return 7 - assert await read_section(transaction(session), read) == 7 - assert (seen, session.timeout, session.savepoints) == (["2000ms"], "30s", ["released"]) + assert await section(session, read) == 7 + assert (seen, session.ran_under, session.timeout, session.savepoints) == ( + ["2000ms"], + ["2000ms"], + "30s", + ["released"], + ) # A stricter timeout already in force is kept, and no timeout at all becomes the section budget. for prior, prior_ms, expected in (("500ms", 500, "500ms"), ("0", 0, "2000ms")): session = Session(prior=prior, prior_ms=prior_ms) seen.clear() - assert await read_section(transaction(session), read) == 7 - assert seen == [expected] and session.timeout == prior + assert await section(session, read) == 7 + assert seen == [expected] == session.ran_under and session.timeout == prior + + +async def test_the_statements_of_a_section_share_one_budget(): + # Each statement takes 0.75 s: the next one gets what is left, and the fourth finds nothing left. + session = Session(prior="30s", prior_ms=30000, cost=0.75) + + async def read(budget): + for _ in range(4): + await budget.execute(text("SELECT 1")) + return 7 + + assert await section(session, read) == UnavailableSection(state="unavailable") + assert session.ran_under == ["2000ms", "1250ms", "500ms"] + assert (session.timeout, session.savepoints) == ("30s", ["rolled back"]) + # Three statements fit: the section is served and the earlier timeout comes back. + session = Session(prior="30s", prior_ms=30000, cost=0.5) + + async def three(budget): + for _ in range(3): + await budget.execute(text("SELECT 1")) + return 3 + + assert await section(session, three) == 3 + assert session.ran_under == ["2000ms", "1500ms", "1000ms"] + assert (session.timeout, session.savepoints) == ("30s", ["released"]) + # A stricter timeout in force is the whole section's budget. + session = Session(prior="500ms", prior_ms=500, cost=0.25) + assert await section(session, three) == UnavailableSection(state="unavailable") + assert session.ran_under == ["500ms", "250ms"] @pytest.mark.parametrize("sqlstate", ["57014", "55P03", "40001", "40P01", "53200"]) async def test_a_timed_out_or_transient_section_is_unavailable_and_rolled_back(sqlstate): session = Session(prior="30s", prior_ms=30000) - async def read(): + async def read(budget): raise failure(sqlstate) - assert await read_section(transaction(session), read) == UnavailableSection(state="unavailable") + assert await section(session, read) == UnavailableSection(state="unavailable") assert session.savepoints == ["rolled back"] and session.timeout == "30s" @@ -150,11 +204,11 @@ async def read(): async def test_a_lost_connection_cancellation_denial_or_programming_error_is_never_unavailable(error): session = Session() - async def read(): + async def read(budget): raise error with pytest.raises(type(error)): - await read_section(transaction(session), read) + await section(session, read) assert session.savepoints == ["rolled back"] @@ -223,7 +277,7 @@ def test_health_lists_the_database_each_loop_native_executors_and_telemetry_with snapshot("alert_evaluator", "disabled"), ), native="down", - telemetry_enabled=False, + telemetry="disabled", ) assert [component.model_dump(mode="json") for component in components] == [ {"component": "database", "state": "ok", "latency_ms": 4.25}, @@ -240,10 +294,34 @@ def test_health_lists_the_database_each_loop_native_executors_and_telemetry_with {"component": "telemetry", "state": "disabled"}, ] # A database probe that timed out is degraded, never a zero latency. - [database, *_] = health_components(UnavailableSection(state="unavailable"), (), native="ok", telemetry_enabled=True) + [database, *_] = health_components(UnavailableSection(state="unavailable"), (), native="ok", telemetry="ok") assert database.model_dump(mode="json") == {"component": "database", "state": "degraded"} +def test_telemetry_health_follows_recent_drops_never_the_configuration_alone(): + elapsed = [1000.0] + drops = TelemetryDrops(monotonic=lambda: elapsed[0]) + exporting = TelemetryOptions(enabled=True, metrics_endpoint="https://collector.example/v1/metrics") + assert telemetry_state(TelemetryOptions(), drops) == "disabled" + assert telemetry_state(exporting, drops) == "ok" + # A failed export makes it degraded for three metric intervals (90 s by default), then it is ok again. + drops.add(2) + assert telemetry_state(exporting, drops) == "degraded" + elapsed[0] += 90 + assert telemetry_state(exporting, drops) == "degraded" + elapsed[0] += 0.5 + assert telemetry_state(exporting, drops) == "ok" + slower = exporting.model_copy(update={"metrics_interval_seconds": 60.0}) + drops.add(1) + elapsed[0] += 179 + assert telemetry_state(slower, drops) == "degraded" + elapsed[0] += 2 + assert telemetry_state(slower, drops) == "ok" + # Drops while export is off do not make it a health claim. + drops.add(1) + assert telemetry_state(TelemetryOptions(), drops) == "disabled" + + def test_native_executors_follow_their_existing_readiness(): configured = SimpleNamespace(native_executors=(object(),)) assert native_state(SimpleNamespace(native_executors=()), None) == "disabled" @@ -341,3 +419,37 @@ async def test_both_reads_check_the_token_before_opening_and_the_stored_principa await service(monkeypatch, Units(reader)).telemetry( reader, Scope(tenant_id=PROJECT.tenant_id), context=AuditContext() ) + + +def test_compatibility_findings_are_counted_for_this_project_only(monkeypatch): + other_project = Scope(tenant_id=SCOPE.tenant_id, project_id=uuid4()) + other_tenant = Scope(tenant_id=uuid4(), project_id=uuid4()) + sibling = Scope(tenant_id=SCOPE.tenant_id, project_id=SCOPE.project_id, environment_id=uuid4()) + report = CompatibilityReport( + mode="restricted", + findings=[ + CompatibilityFinding(kind="inventory", code="inventory_incomplete"), + CompatibilityFinding(kind="run", code="ir_unsupported", scope=SCOPE), + CompatibilityFinding(kind="activation", code="policy_mismatch", scope=sibling), + CompatibilityFinding(kind="run", code="ir_unsupported", scope=other_project), + CompatibilityFinding(kind="worker", code="worker_protocol_unsupported", scope=other_tenant), + ], + ) + reader = service(monkeypatch, Units(principal())) + reader.compatibility = SimpleNamespace(report=report, ready=False) # type: ignore[assignment] + # The platform-wide finding and this project's own two, never another project's or tenant's. + assert reader._api(SCOPE).compatibility.model_dump() == {"mode": "restricted", "findings": 3} + + +def test_section_probes_never_write_deny_records(monkeypatch, caplog): + viewer = principal(Grant(role="viewer", scope=PROJECT)) + reader = service(monkeypatch, Units(viewer)) + caplog.set_level(logging.INFO, logger="weave.authorization") + reach = reader._reach(viewer, viewer, SCOPE, AuditContext()) + assert (reach.incidents, reach.runs, reach.targets, reach.approvals) == (False, True, (), ()) + records = [json.loads(record.getMessage()) for record in caplog.records if record.name == "weave.authorization"] + # Reading runs is recorded as allowed for both principals; the capabilities the viewer lacks are not attempts. + assert [(record["capability"], record["outcome"]) for record in records] == [ + ("run.read", "allow"), + ("run.read", "allow"), + ] diff --git a/tests/unit/operations/test_telemetry.py b/tests/unit/operations/test_telemetry.py index 2715113c..23e45f91 100644 --- a/tests/unit/operations/test_telemetry.py +++ b/tests/unit/operations/test_telemetry.py @@ -29,8 +29,9 @@ from pyfly.core.config import Config from firefly_weave.contracts.telemetry import TelemetryOptions -from firefly_weave.observability import OwnedMeterConfiguration, OwnedTracingConfiguration +from firefly_weave.observability import OwnedMeterConfiguration, OwnedTracingConfiguration, TelemetryDrops from firefly_weave.operations.telemetry import TelemetryService, WeaveMetricsRecorder +from firefly_weave.settings import Settings def test_options_reject_secret_urls_and_invalid_bounds(): @@ -48,6 +49,41 @@ def test_options_reject_secret_urls_and_invalid_bounds(): TelemetryOptions(enabled=True, traces_endpoint="http://127.0.0.1:4318/v1/traces", timeout_seconds=6) +def test_options_refuse_a_collector_host_name_longer_than_a_dns_name(monkeypatch): + label = "a" * 63 + longest = ".".join([label, label, label, "b" * 61]) + assert len(longest) == 253 + options = TelemetryOptions(enabled=True, traces_endpoint=f"https://{longest}:4318/v1/traces") + assert options.traces_endpoint is not None + for host in (longest + "c", ".".join([label] * 4) + ".example"): + with pytest.raises(ValidationError) as error: + TelemetryOptions(enabled=True, metrics_endpoint=f"https://{host}/v1/metrics") + # The same refusal as every other unusable endpoint, without echoing the value. + assert "An explicit TLS collector endpoint or loopback development endpoint is required" in str(error.value) + assert host not in str(error.value) + monkeypatch.setenv("WEAVE_DATABASE_URL", "postgresql+asyncpg://weave_app:unused@127.0.0.1:65432/weave") + monkeypatch.setenv( + "WEAVE_TELEMETRY", json.dumps({"enabled": True, "metrics_endpoint": f"https://{host}/v1/metrics"}) + ) + with pytest.raises(ValueError, match="Invalid WEAVE_TELEMETRY configuration"): + Settings.from_env() + + +def test_drops_remember_when_the_last_one_happened(): + elapsed = [50.0] + drops = TelemetryDrops(monotonic=lambda: elapsed[0]) + assert (drops.count, drops.dropped_within(90)) == (0, False) + drops.add(0) + assert (drops.count, drops.dropped_within(90)) == (0, False) + drops.add(3) + elapsed[0] += 90 + assert (drops.count, drops.dropped_within(90)) == (3, True) + elapsed[0] += 1 + assert drops.dropped_within(90) is False + drops.add(1) + assert (drops.count, drops.dropped_within(0.5)) == (4, True) + + def test_disabled_never_constructs_exporter_from_environment(monkeypatch): monkeypatch.setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "https://ambient-CANARY.test") monkeypatch.setenv("OTEL_EXPORTER_OTLP_HEADERS", "Authorization=CANARY") From 35647cc7ce0c9bf02cfd85f5eefe042f7322dbb3 Mon Sep 17 00:00:00 2001 From: Andres Contreras Date: Mon, 12 Oct 2026 13:54:15 -0700 Subject: [PATCH 26/63] Show platform health and chronological run timelines in Operate --- CHANGELOG.md | 14 + docs/guides/execution-management.md | 20 +- docs/guides/studio.md | 89 +- studio/src/app/app.html | 12 + studio/src/app/app.ts | 83 +- studio/src/app/operate/operate-routes.ts | 8 +- .../app/operate/overview/overview-model.ts | 508 +++++ .../src/app/operate/overview/overview-page.ts | 644 ++++++ studio/src/app/operate/overview/overview.css | 531 +++++ studio/src/app/operate/runs/runs-model.ts | 335 ++++ studio/src/app/operate/runs/runs-page.ts | 1757 +++++++++++++++++ studio/src/app/operate/runs/runs.css | 693 +++++++ studio/src/app/operate/runs/step-timeline.ts | 420 ++++ studio/src/app/operations/records-view.ts | 305 +-- studio/src/app/operations/run-detail.ts | 15 +- studio/src/app/status-labels.ts | 10 +- studio/tests/browser/connected.spec.ts | 20 +- studio/tests/browser/connection.spec.ts | 19 +- studio/tests/browser/design-review.spec.ts | 2 + studio/tests/browser/layout-fixes.spec.ts | 2 + .../tests/browser/operate-navigation.spec.ts | 92 +- studio/tests/browser/operate-overview.spec.ts | 1044 ++++++++++ studio/tests/browser/operate-runs.spec.ts | 1312 ++++++++++++ studio/tests/browser/operations.spec.ts | 86 +- studio/tests/browser/overlay-sheets.spec.ts | 14 +- studio/tests/browser/responsive.spec.ts | 9 +- studio/tests/browser/run-actions.spec.ts | 3 +- studio/tests/browser/shell-navigation.spec.ts | 1 + studio/tests/browser/start-run.spec.ts | 3 +- studio/tests/browser/support.ts | 72 + studio/tests/browser/visual-tour.spec.ts | 206 ++ studio/tests/operate-routes.test.ts | 27 +- studio/tests/overview-model.test.ts | 514 +++++ studio/tests/runs-model.test.ts | 326 +++ studio/tests/step-timeline.test.ts | 103 + 35 files changed, 8874 insertions(+), 425 deletions(-) create mode 100644 studio/src/app/operate/overview/overview-model.ts create mode 100644 studio/src/app/operate/overview/overview-page.ts create mode 100644 studio/src/app/operate/overview/overview.css create mode 100644 studio/src/app/operate/runs/runs-model.ts create mode 100644 studio/src/app/operate/runs/runs-page.ts create mode 100644 studio/src/app/operate/runs/runs.css create mode 100644 studio/src/app/operate/runs/step-timeline.ts create mode 100644 studio/tests/browser/operate-overview.spec.ts create mode 100644 studio/tests/browser/operate-runs.spec.ts create mode 100644 studio/tests/overview-model.test.ts create mode 100644 studio/tests/runs-model.test.ts create mode 100644 studio/tests/step-timeline.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 88b9fa94..c5b32ae8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -154,6 +154,20 @@ SPDX-License-Identifier: Apache-2.0 or no delivery slot is free, is neither a success nor a failure. - Studio's host bridge forwards the environment summary, the telemetry settings and `releases.usage`. +- Studio's Operate area opens on a new Overview page. It shows the health of + the API, the database, the scheduler, the outbox, compatibility and + telemetry export, each with its details; the current load; workers, + clusters and incidents; and up to 20 items that need attention, each opening + its record. It reads the summary every 15 seconds while visible. A section + you may not read names the permission it needs instead of showing zeros. +- Operate › Runs reads run summaries, newest first, with time range, workflow, + version, origin, status, test, handled-error, incident, top-level and + archive filters. The address keeps every filter except the business and + correlation keys. Load more continues the same list, Auto-refresh reads the + first page every 10 seconds, and up to 50 selected runs can be cancelled or + retried together. A run's detail draws its steps with queue wait and run + time, groups loop iterations, links to called runs, shows a step's output + on request and follows a running run every 2 seconds. - Keep typing in the selected form field when deferred row autofocus completes. Rapid row additions focus the newest item without overriding a later choice. diff --git a/docs/guides/execution-management.md b/docs/guides/execution-management.md index 6995287a..08b2d68c 100644 --- a/docs/guides/execution-management.md +++ b/docs/guides/execution-management.md @@ -100,12 +100,17 @@ start, the dialog stays open and says "The run didn't start." with the reason. **Find runs later** with the filters above the list in **Runs**. The list updates as soon as you change a filter; there is no button to apply them. -- Type an exact business key in **Search by business key**. -- Choose a status: **All**, **Waiting**, **Failed**, or **Succeeded**. +- Type an exact business key in **Business key**. +- Choose one or more statuses, such as **Waiting** and **Failed**; choose a + status again to remove it. +- Choose a **Time range**: the last 24 hours by default, a longer or shorter + one, or a custom range. - Open **More filters** to enter an exact **Correlation key** or to check **Include archived runs**. -To remove a filter, empty its field, choose **All**, or clear the check box. +To remove a filter, empty its field, choose its status again, or clear the +check box; **Clear filters** removes them all. Business and correlation keys +never appear in the page's address. ### With the CLI @@ -179,10 +184,11 @@ explicitly which run should receive the event. Open a run in Studio's **Runs** view. Its detail shows, from top to bottom: the status, business key, and run ID (**Copy** copies the full ID); a **Now** line that says what the run is doing; its commands; the graph of the workflow -version it runs, with **Now** on the current step; a **Timeline** of its -recorded events; **Technical details**; and **Archive**. The detail does not -update by itself: select **Refresh** at the bottom of the detail to read the -run again. +version it runs, with **Now** on the current step; its **Steps**, each with the +time it waited and the time it ran; its recorded **Events**; **Technical +details**; and **Archive**. While the run is still going, Studio reads it again +every 2 seconds; select **Refresh** at the bottom of the detail to read it at +once. The **Now** line, or the card that replaces it, tells you what the run waits for. STEP stands for the step's ID: diff --git a/docs/guides/studio.md b/docs/guides/studio.md index a029a891..38e207f4 100644 --- a/docs/guides/studio.md +++ b/docs/guides/studio.md @@ -274,7 +274,8 @@ Open the assistant in any of these ways: - While you work locally, select **Connect to a platform** on **Home**, in the platform menu at the top right, in the designer next to "Saving, publishing and runs need a platform.", or on the **Connections**, **My tasks**, - **Email**, **Runs**, **Incidents**, **Workers**, and **Clusters** pages. + **Email**, **Overview**, **Runs**, **Incidents**, **Workers**, and + **Clusters** pages. - In **Settings → Platforms**, select **Add platform** while you use a platform, or **Connect to a platform** while you work locally. @@ -1247,27 +1248,81 @@ Every start creates a new run, even with the same business key; a business key groups related runs. Studio remembers the last input for each version in this browser, leaving out fields the schema marks as secret. -**Follow and manage runs in Runs.** - -- **Find runs** with the filters above the list: type an exact business key in - **Search by business key**, choose **All**, **Waiting**, **Failed**, or - **Succeeded**, or open **More filters** for an exact **Correlation key** and - **Include archived runs**. The list updates as you change a filter. In a - narrow window, the filters fold behind a button such as "Filters (2)". +**See the platform at a glance in Overview.** **Operate** opens on +**Overview**, which needs `status.read`. It reads the environment's summary +every 15 seconds while the page is visible, and after a failed read it waits +longer, up to a minute. + +- **Health** has one tile each for the API, the database, the scheduler, the + outbox, compatibility and telemetry export, with a state (**OK**, + **Degraded**, **Down** or **Off**) and one figure such as "Latency 4 ms" or + "Last success 2 min ago". Each tile's details button lists its facts or its + background loops, with each loop's last success and failures in a row. +- **Load** shows active runs (test runs left out), ready tasks with the oldest + wait, waits with how many are overdue, the outbox backlog and the provider + backlog. +- **Workers and clusters** counts online, offline and draining workers, free + task slots against ready tasks, runners, busy targets, targets that need + reconciliation and plans waiting for approval, and active incidents. +- **Needs attention** lists up to 20 items, most urgent first: jobs that need + reconciliation, plans you may approve, offline workers that still hold work, + active incidents, targets without an online runner, a restricted + compatibility verdict and runs that failed in the last hour. Each one opens + its record. + +A section you may not read says which permission it needs, such as "You need +`incident.read` in this environment.", instead of showing zeros; a section the +platform could not compute in time says "This section could not be loaded." +with **Try again**, while the rest of the page stays usable. + +**Follow and manage runs in Runs.** The list shows the newest runs first: +each run's status (a run that succeeded after handling errors reads "Succeeded +with errors"), its workflow and version, how it started (**Retry of** and +**Called by** open the run it came from), when it started, how long it ran +(counting up while it runs), its active incidents, its business key and its +short run ID with **Copy**. + +- **Find runs** with the filters above the list: **Time range** (the last 15 + minutes, hour, 24 hours, 7 days or 30 days, or a custom range; the last 24 + hours by default), **Workflow** and **Version**, **Origin**, an exact + **Business key**, one or more statuses, and **Include test runs**. **More + filters** adds an exact **Correlation key**, **Handled errors**, **Order**, + **Has active incident**, **Top-level runs only** and **Include archived + runs**. The list updates as you change a filter, and **Clear filters** resets + them all. The address keeps every filter except the business and correlation + keys, so you can share a view without sharing business data. In a narrow + window, the filters fold behind a button such as "Filters (2)". +- **Keep the list current.** With **Auto-refresh** on (the default), the list + reads its first page again every 10 seconds while the page is visible; a + relative time range then ends at that moment. **Refresh** does the same at + once. **Load more runs** continues exactly the list you are reading, so pages + never shift under you; while more than the first page is open, or runs are + selected, the list waits for **Refresh**. +- **Act on several runs.** Select runs, up to 50, and choose **Cancel runs** + (one reason for all) or **Retry runs** (each on its own version with its own + input). Studio sends them one at a time and reports the result for each run. - **Open a run** to watch it. The **Now** line says what it is doing, such as "Waiting for a person at review, assigned to reviewers", and offers **Open task** or **Open My tasks** when a person must act ([what each reason means](execution-management.md#3-understand-why-a-run-is-waiting)). The graph of the version it runs marks the current step **Now** and finished - steps with a check. **Timeline** lists the recorded events in plain sentences, - each with **View raw event**, and **Technical details** holds the run ID, the - correlation key, the artifact, and the run data. The view does not update by - itself: select **Refresh** at the bottom of the run's detail. -- **Pause run** and **Resume run** need their own permissions, ask for a - **Reason**, and appear only while the run hasn't finished; only one of them - shows at a time. Studio has no controls to cancel or retry a run or to resolve - an incident; a blocked run shows "This run stopped at" and the step. Use the - CLI or the API, as in [incident operations](../reference/incident-operations.md). + steps with a check. **Steps** draws one bar per step: the time it waited in + the queue and the time it ran. Loop iterations sit under their step, and a + step that called a workflow opens the run it called. **Input/Output** shows a + step's recorded output only when you select **Show output**, and says why + when an output can't be shown. **Logs** aren't available yet. **Events** + lists the recorded events in plain sentences, each with **View raw event**, + and **Technical details** holds the run ID, the correlation key, the + artifact, and the run data. While the run is still going, Studio reads it and + its steps every 2 seconds; **Refresh** at the bottom of the detail reads it at + once. +- **Cancel run**, **Retry run**, **Send signal** and **Export history** work as + [Manage runs and business cases](execution-management.md#3-understand-why-a-run-is-waiting) + describes. **Pause run** and **Resume run** need their own permissions, ask + for a **Reason**, and appear only while the run hasn't finished; only one of + them shows at a time. A blocked run shows "This run stopped at" and the step; + resolve its incident in **Incidents**, or as in + [incident operations](../reference/incident-operations.md). - **Archive run**, under **Archive**, hides a finished run from the list and keeps its history. Open **More filters** and check **Include archived runs** to show it again; **Restore run** returns it to the list. diff --git a/studio/src/app/app.html b/studio/src/app/app.html index 88ffb274..38160409 100644 --- a/studio/src/app/app.html +++ b/studio/src/app/app.html @@ -423,6 +423,18 @@

Your workflows, woven together.

} + } @else if (view === "overview") { +
+ @defer (on immediate) { + + } +
+ } @else if (view === "runs") { +
+ @defer (on immediate) { + + } +
} @else if (view === "incidents") {
@defer (on immediate) { diff --git a/studio/src/app/app.ts b/studio/src/app/app.ts index 699cfd59..c9389021 100644 --- a/studio/src/app/app.ts +++ b/studio/src/app/app.ts @@ -164,6 +164,8 @@ import { import { ClustersPage } from "./operate/clusters/clusters-page"; import { WorkersPage } from "./operate/workers/workers-page"; import { IncidentsPage } from "./operate/incidents/incidents-page"; +import { OverviewPage } from "./operate/overview/overview-page"; +import { RunsPage } from "./operate/runs/runs-page"; import { navEntryVisible, viewFromPath, @@ -189,6 +191,7 @@ export type View = | "home" | "workflows" | "designer" + | "overview" | "runs" | "incidents" | "tasks" @@ -351,6 +354,8 @@ const sideEffects: Record = { ClustersPage, WorkersPage, IncidentsPage, + OverviewPage, + RunsPage, SettingsPage, LumiPanel, ], @@ -408,7 +413,6 @@ export class App implements CanvasHost { runHistory: Record | null = null; runCanvas: StructuredCanvasAdapter | null = null; runLifecycle: Record | null = null; - runIncludeArchived = false; debugSession: Record | null = null; pendingMutation: { path: string; @@ -614,6 +618,7 @@ export class App implements CanvasHost { { id: "connections", label: "Connections" }, { id: "tasks", label: "My tasks" }, { id: "email", label: "Email" }, + { id: "overview", label: "Overview" }, { id: "runs", label: "Runs" }, { id: "incidents", label: "Incidents" }, { id: "workers", label: "Workers" }, @@ -624,7 +629,7 @@ export class App implements CanvasHost { ["", ["home"]], ["Build", ["workflows", "connections"]], ["Work", ["tasks", "email"]], - ["Operate", ["runs", "incidents", "workers", "clusters"]], + ["Operate", ["overview", "runs", "incidents", "workers", "clusters"]], ]; private navCache: { key: string; @@ -883,7 +888,6 @@ export class App implements CanvasHost { commandKey: string | null = null; unknownCommand = false; taskFilter = ""; - runFilters = { business_key: "", correlation_key: "", status: "" }; showStartRun = false; /** The activation the start-run dialog preselects. */ runStartActivation = ""; @@ -5375,11 +5379,18 @@ export class App implements CanvasHost { } if ( this.view === "connect" || + this.view === "overview" || this.view === "clusters" || this.view === "workers" || this.view === "incidents" ) return; + if (this.view === "runs") { + // Operate › Runs reads its own list; run titles and Retry run still + // read the published versions. + void this.loadWorkflowVersions(); + return; + } if (this.view === "home") { await this.refreshHome(); return; @@ -5402,7 +5413,6 @@ export class App implements CanvasHost { const requestedView = this.view; const collections: Partial> = { workflows: [this.libraryCollection, false], - runs: ["runs", true], connections: ["connections", true], tasks: ["human-tasks", true], email: ["email/conversations", true], @@ -5412,7 +5422,6 @@ export class App implements CanvasHost { this.loading = false; return; } - if (this.view === "runs") void this.loadWorkflowVersions(); if (this.view === "workflows" && this.libraryCollection === "workflows") void this.loadActivations(sequence); try { @@ -5420,32 +5429,18 @@ export class App implements CanvasHost { c[0], c[1], append ? (this.nextCursor ?? undefined) : undefined, - this.view === "runs" - ? { - ...this.runFilters, - include_archived: this.runIncludeArchived ? "true" : "", - } - : this.view === "tasks" && this.taskFilter - ? { status: this.taskFilter } - : {}, + this.view === "tasks" && this.taskFilter + ? { status: this.taskFilter } + : {}, ); if (sequence !== this.listSequence || requestedView !== this.view) return; this.records = append ? [...this.records, ...result.items] : result.items; this.nextCursor = result.next_cursor; - if (this.view === "runs") { - this.rememberRuns(result.items); - this.runsRefused = false; - } if (this.view === "workflows" && this.libraryCollection === "workflows") for (const item of result.items) this.rememberVersion(item); this.error = ""; } catch (e) { - // A refused Runs list names the access it needs on the page, instead - // of a banner over an empty list that claims there are no runs. - if (requestedView === "runs" && describeError(e).status === 403) { - if (sequence === this.listSequence && requestedView === this.view) - this.runsRefused = true; - } else this.fail(e); + this.fail(e); } finally { this.cdr.markForCheck(); if (sequence === this.listSequence) this.loading = false; @@ -6057,48 +6052,6 @@ export class App implements CanvasHost { if (!listed && this.runRefused === run["id"]) this.records = this.records.filter((r) => r !== run); } - applyRunFilters() { - if (this.runFilterTimer) clearTimeout(this.runFilterTimer); - this.runFilterTimer = null; - this.nextCursor = null; - this.records = []; - this.closeRecord(); - void this.refresh(); - } - private runFilterTimer: ReturnType | null = null; - /** - * Runs filters apply as they change: the typed keys after a 400 ms pause, - * the status and "Include archived runs" at once. - */ - setRunFilter( - key: "business_key" | "correlation_key" | "status", - value: string, - ) { - if (this.runFilters[key] === value) return; - this.runFilters = { ...this.runFilters, [key]: value }; - if (key === "status") return this.applyRunFilters(); - if (this.runFilterTimer) clearTimeout(this.runFilterTimer); - this.runFilterTimer = setTimeout(() => this.applyRunFilters(), 400); - } - toggleArchivedRuns() { - this.runIncludeArchived = !this.runIncludeArchived; - this.applyRunFilters(); - } - /** How many Runs filters are set ("Filters (2)" on narrow windows). */ - get runFilterCount() { - return ( - [ - this.runFilters.business_key, - this.runFilters.correlation_key, - this.runFilters.status, - ].filter(Boolean).length + (this.runIncludeArchived ? 1 : 0) - ); - } - clearRunFilters() { - this.runIncludeArchived = false; - this.runFilters = { business_key: "", correlation_key: "", status: "" }; - this.applyRunFilters(); - } applyTaskFilter(event: Event) { this.taskFilter = this.value(event); this.taskRunFilter = ""; diff --git a/studio/src/app/operate/operate-routes.ts b/studio/src/app/operate/operate-routes.ts index 7941376c..b6d6fd89 100644 --- a/studio/src/app/operate/operate-routes.ts +++ b/studio/src/app/operate/operate-routes.ts @@ -22,8 +22,9 @@ SPDX-License-Identifier: Apache-2.0 import type { Routes } from "@angular/router"; import { SETTINGS_TABS } from "../settings/settings-routes"; -/** The Operate pages, in navigation order. */ +/** The Operate pages, in navigation order; Overview is the default. */ export const operatePages = [ + "overview", "runs", "incidents", "workers", @@ -44,6 +45,7 @@ export type ClusterTab = (typeof clusterTabs)[number]; /** The capability each Operate page needs somewhere in the workspace. */ export const operateCapabilities: Record = { + overview: "status.read", runs: "run.read", incidents: "incident.read", workers: "status.read", @@ -76,7 +78,7 @@ export function viewFromPath(pathname: string): ViewRoute | null { const parts = pathname.split("/").filter(Boolean); if (!parts.length) return null; if (parts[0] === "operate") { - const page = parts[1] ?? "runs"; + const page = parts[1] ?? "overview"; if (!isOperatePage(page)) return null; const detail = (page === "runs" || page === "workers") && @@ -156,7 +158,7 @@ export const operateRoutes: Routes = [ /** Paths from before Operate, kept working as redirects. */ export const legacyRedirects: Routes = [ - { path: "operate", redirectTo: "operate/runs", pathMatch: "full" }, + { path: "operate", redirectTo: "operate/overview", pathMatch: "full" }, { path: "runs", redirectTo: "operate/runs", pathMatch: "full" }, { path: "workers", redirectTo: "operate/workers", pathMatch: "full" }, { path: "operations", redirectTo: "operate/clusters", pathMatch: "full" }, diff --git a/studio/src/app/operate/overview/overview-model.ts b/studio/src/app/operate/overview/overview-model.ts new file mode 100644 index 00000000..cebc8e40 --- /dev/null +++ b/studio/src/app/operate/overview/overview-model.ts @@ -0,0 +1,508 @@ +/* +Copyright 2026 Firefly Software Foundation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +Author: Firefly Software Foundation +SPDX-License-Identifier: Apache-2.0 +*/ +// Overview: the environment summary (operations_summary.read) and what the +// page says about it. Mirrors src/firefly_weave/contracts/operations_summary.py; +// wire fields are snake_case. A section the person may not read is +// "forbidden" and one the platform could not compute in time "unavailable": +// neither is ever shown as zero. Pure: the page renders it, tests check it. +import type { StudioApi } from "../../api"; +import { durationText, relativeTime, shortId } from "../../format"; +import { statusLabel } from "../../status-labels"; +import { + clustersRecordPath, + clustersTabPath, + viewPath, +} from "../operate-routes"; + +export const healthStates = ["ok", "degraded", "down", "disabled"] as const; +export type HealthState = (typeof healthStates)[number]; +export const healthComponents = [ + "database", + "recovery_loop", + "schedules", + "outbox_loop", + "provider_loop", + "kafka_loop", + "operations_loop", + "alert_evaluator", + "native_executors", + "telemetry", +] as const; +export type HealthComponentName = (typeof healthComponents)[number]; +export const attentionKinds = [ + "job", + "approval", + "worker", + "incident", + "runner", + "compatibility", + "run", +] as const; +export type AttentionKind = (typeof attentionKinds)[number]; + +export interface ForbiddenSection { + state: "forbidden"; +} +export interface UnavailableSection { + state: "unavailable"; +} +export interface ApiSection { + version: string; + /** The server image's digest, when the server knows it. */ + build: string | null; + /** The process that answered; new each time it starts. */ + instance: string; + uptime_seconds: number; + compatibility: { mode: "ready" | "restricted"; findings: number }; +} +/** Times and counts are present only when the component reports them. */ +export interface HealthComponent { + component: HealthComponentName; + state: HealthState; + last_started_at?: string; + last_ok_at?: string; + consecutive_failures?: number; + recent_failures?: number; + complete_until?: string; + latency_ms?: number; +} +export interface LoadSection { + runs_active: number; + tasks_ready: number; + tasks_leased: number; + oldest_ready_seconds: number | null; + waits_active: number; + waits_overdue: number; + oldest_overdue_seconds: number | null; + outbox_pending: number; + outbox_oldest_seconds: number | null; + provider_pending: number; + sources_enabled: number; +} +export interface WorkersSection { + online: number; + offline: number; + draining: number; + capacity: number; + active_leases: number; +} +export interface TargetsSection { + total: number; + runner_online: number; + busy: number; + reconciliation_required: number; + awaiting_approval: number; +} +export interface IncidentsSection { + active: number; + opened_24h: number; +} +export interface AttentionItem { + kind: AttentionKind; + /** A record ID, or "compatibility" for the platform's verdict. */ + id: string; + title: string; + /** Approvals only: when the plan can no longer be approved. */ + expires_at?: string; +} +export interface OperationsSummary { + observed_at: string; + api: ApiSection; + health: HealthComponent[]; + load: LoadSection | UnavailableSection; + workers: WorkersSection | UnavailableSection; + targets: TargetsSection | ForbiddenSection | UnavailableSection; + incidents: IncidentsSection | ForbiddenSection | UnavailableSection; + attention: AttentionItem[] | UnavailableSection; +} + +/** The environment summary for the environment Studio is in. */ +export function readSummary(api: Pick) { + return api.request( + `${api.environment}/operations/summary`, + ); +} + +/** A section as the page shows it: its figures, or why there are none. */ +export type Section = + | { kind: "ready"; value: T } + | { kind: "forbidden" } + | { kind: "unavailable" }; +export function sectionOf( + value: T | ForbiddenSection | UnavailableSection, +): Section { + if (Array.isArray(value)) return { kind: "ready", value }; + if ("state" in value) { + const state = (value as { state: unknown }).state; + if (state === "forbidden") return { kind: "forbidden" }; + if (state === "unavailable") return { kind: "unavailable" }; + } + return { kind: "ready", value: value as T }; +} + +/** + * How far the server's clock is from this computer's, in milliseconds to add + * to local time: ages and countdowns tick locally on the server's clock. + */ +export function clockOffset(observedAt: string, receivedAt: number): number { + const observed = Date.parse(observedAt); + return Number.isFinite(observed) ? observed - receivedAt : 0; +} + +/** How long the answering process has run by `now` (the server's clock). */ +export function uptimeSeconds(summary: OperationsSummary, now: Date): number { + const since = (now.getTime() - Date.parse(summary.observed_at)) / 1000; + return Math.max( + 0, + Math.floor(summary.api.uptime_seconds + Math.max(0, since)), + ); +} + +const componentLabels: Record = { + database: "Database", + recovery_loop: "Recovery", + schedules: "Schedules", + outbox_loop: "Outbox", + provider_loop: "Providers", + kafka_loop: "Kafka", + operations_loop: "Operations", + alert_evaluator: "Alerts", + native_executors: "Native executors", + telemetry: "Telemetry export", +}; +/** The loops the Scheduler tile stands for. */ +const schedulerLoops: HealthComponentName[] = [ + "recovery_loop", + "schedules", + "provider_loop", + "kafka_loop", + "operations_loop", + "alert_evaluator", +]; +const severity: Record = { + disabled: 0, + ok: 1, + degraded: 2, + down: 3, +}; + +export type TileId = + | "api" + | "database" + | "scheduler" + | "outbox" + | "compatibility" + | "telemetry"; +/** One loop in a tile's Details: its last success and failures in a row. */ +export interface LoopRow { + label: string; + state: HealthState; + /** "2 min ago", "Never" before a first success, "" when switched off. */ + lastSuccess: string; + lastSuccessAt: string | null; + failures: number; +} +export interface HealthTile { + id: TileId; + label: string; + state: HealthState; + /** The tile's one figure. */ + figure: string; + /** Details: facts, as label and value. */ + facts: [string, string][]; + /** Details: the loops behind the tile. */ + rows: LoopRow[]; +} + +function loopRow(component: HealthComponent, now: Date): LoopRow { + const at = component.last_ok_at ?? null; + return { + label: componentLabels[component.component], + state: component.state, + lastSuccess: + component.state === "disabled" && !at + ? "" + : at + ? relativeTime(at, now) || "Unknown" + : "Never", + lastSuccessAt: at, + failures: component.consecutive_failures ?? 0, + }; +} + +/** "3 of 3 loops running", "1 of 3 loops down", or off on an API-only server. */ +function loopsTile( + components: HealthComponent[], +): Pick { + const enabled = components.filter((item) => item.state !== "disabled"); + if (!enabled.length) + return { state: "disabled", figure: "Off on this server" }; + const state = enabled.reduce( + (worst, item) => + severity[item.state] > severity[worst] ? item.state : worst, + "ok", + ); + const count = enabled.filter((item) => item.state === state).length; + const word = + state === "ok" ? "running" : statusLabel("health", state).toLowerCase(); + return { + state, + figure: `${count} of ${enabled.length} ${enabled.length === 1 ? "loop" : "loops"} ${word}`, + }; +} + +/** + * The health strip: API, Database, Scheduler, Outbox, Compatibility and + * Telemetry export. `now` is the server's clock. + */ +export function healthTiles( + summary: OperationsSummary, + now: Date, +): HealthTile[] { + const byName = new Map(summary.health.map((item) => [item.component, item])); + const component = (name: HealthComponentName): HealthComponent => + byName.get(name) ?? { component: name, state: "down" }; + const native = component("native_executors"); + const database = component("database"); + const outbox = component("outbox_loop"); + const telemetry = component("telemetry"); + const loops = schedulerLoops.map(component); + const { mode, findings } = summary.api.compatibility; + const uptime = durationText(uptimeSeconds(summary, now) * 1000); + const outboxLast = outbox.last_ok_at + ? relativeTime(outbox.last_ok_at, now) + : ""; + return [ + { + id: "api", + label: "API", + // The API answered; executors it cannot reach make it degraded. + state: + native.state === "degraded" || native.state === "down" + ? "degraded" + : "ok", + figure: `Up ${uptime}`, + facts: [ + ["Version", summary.api.version], + [ + "Build", + summary.api.build + ? shortId(summary.api.build.replace(/^sha256:/, "")) + : "Not reported", + ], + ["Instance", shortId(summary.api.instance)], + ["Native executors", statusLabel("health", native.state)], + ], + rows: [], + }, + { + id: "database", + label: "Database", + state: database.state, + figure: + database.latency_ms === undefined + ? "Probe timed out" + : `Latency ${Math.round(database.latency_ms)} ms`, + facts: + database.latency_ms === undefined + ? [["Latency", "Probe timed out"]] + : [["Latency", `${database.latency_ms} ms`]], + rows: [], + }, + { + id: "scheduler", + label: "Scheduler", + ...loopsTile(loops), + facts: [], + rows: loops.map((loop) => loopRow(loop, now)), + }, + { + id: "outbox", + label: "Outbox", + state: outbox.state, + figure: + outbox.state === "disabled" + ? "Off on this server" + : outboxLast + ? `Last success ${outboxLast}` + : "No success yet", + facts: [], + rows: [loopRow(outbox, now)], + }, + { + id: "compatibility", + label: "Compatibility", + state: mode === "ready" ? "ok" : "degraded", + figure: findings + ? `${countText(findings)} ${findings === 1 ? "finding" : "findings"}` + : "No findings", + facts: [ + ["Mode", mode === "ready" ? "Ready" : "Restricted"], + ["Findings in this project", countText(findings)], + ], + rows: [], + }, + { + id: "telemetry", + label: "Telemetry export", + state: telemetry.state, + figure: + telemetry.state === "disabled" + ? "Export is off" + : telemetry.state === "ok" + ? "Exporting" + : "Dropping telemetry", + facts: [["State", statusLabel("health", telemetry.state)]], + rows: [], + }, + ]; +} + +/** A count with digit grouping, for tabular figures: "1,204". */ +export function countText(value: number): string { + return value.toLocaleString("en-US"); +} +const seconds = (value: number) => durationText(value * 1000); + +export interface LoadFigure { + id: keyof LoadSection; + label: string; + value: number; + detail: string; +} +/** The Load band: five figures and what each one waits on. */ +export function loadFigures(load: LoadSection): LoadFigure[] { + const sources = load.sources_enabled; + return [ + { + id: "runs_active", + label: "Active runs", + value: load.runs_active, + detail: "Test runs left out", + }, + { + id: "tasks_ready", + label: "Ready tasks", + value: load.tasks_ready, + detail: + load.oldest_ready_seconds === null + ? "Nothing waiting" + : `Oldest waits ${seconds(load.oldest_ready_seconds)}`, + }, + { + id: "waits_active", + label: "Waits", + value: load.waits_active, + detail: load.waits_overdue + ? `${countText(load.waits_overdue)} overdue` + + (load.oldest_overdue_seconds === null + ? "" + : `, oldest by ${seconds(load.oldest_overdue_seconds)}`) + : "None overdue", + }, + { + id: "outbox_pending", + label: "Outbox backlog", + value: load.outbox_pending, + detail: + load.outbox_oldest_seconds === null + ? "Nothing due" + : `Oldest waits ${seconds(load.outbox_oldest_seconds)}`, + }, + { + id: "provider_pending", + label: "Provider backlog", + value: load.provider_pending, + detail: `${countText(sources)} ${sources === 1 ? "source" : "sources"} enabled`, + }, + ]; +} + +/** Free task slots against ready tasks: the fill is ready ÷ free, at most full. */ +export interface SlotMeter { + free: number; + ready: number | null; + fill: number; + over: boolean; + text: string; +} +export function slotMeter( + workers: WorkersSection, + load: LoadSection | null, +): SlotMeter { + const free = Math.max(0, workers.capacity - workers.active_leases); + const slots = `${countText(free)} free ${free === 1 ? "slot" : "slots"}`; + if (!load) return { free, ready: null, fill: 0, over: false, text: slots }; + const ready = load.tasks_ready; + const over = ready > free; + return { + free, + ready, + fill: over ? 1 : free ? ready / free : 0, + over, + text: `${countText(ready)} ready, ${slots}`, + }; +} + +/** Where an attention item's record opens; null when it has no record. */ +export function attentionLink(item: AttentionItem): string | null { + switch (item.kind) { + case "job": + return clustersRecordPath("jobs", item.id); + case "approval": + return clustersRecordPath("plans", item.id); + case "worker": + return viewPath("workers", item.id); + case "incident": + return `${viewPath("incidents")}?incident=${encodeURIComponent(item.id)}`; + case "runner": + return clustersTabPath("runners"); + case "run": + return viewPath("runs", item.id); + default: + return null; + } +} + +/** The icon and the short word that lead an attention item. */ +export const attentionLabels: Record = { + job: ["Job", "warning"], + approval: ["Approval", "approvals"], + worker: ["Worker", "workers"], + incident: ["Incident", "incidents"], + runner: ["Runner", "clusters"], + compatibility: ["Compatibility", "degraded"], + run: ["Failed run", "failCircle"], +}; + +/** Nothing running, waiting or needing attention in what the person may read. */ +export function isQuiet(summary: OperationsSummary): boolean { + const load = sectionOf(summary.load); + const attention = sectionOf(summary.attention); + if (load.kind !== "ready" || attention.kind !== "ready") return false; + const figures = load.value; + return ( + attention.value.length === 0 && + figures.runs_active === 0 && + figures.tasks_ready === 0 && + figures.tasks_leased === 0 && + figures.waits_active === 0 && + figures.outbox_pending === 0 && + figures.provider_pending === 0 + ); +} diff --git a/studio/src/app/operate/overview/overview-page.ts b/studio/src/app/operate/overview/overview-page.ts new file mode 100644 index 00000000..a75ef58f --- /dev/null +++ b/studio/src/app/operate/overview/overview-page.ts @@ -0,0 +1,644 @@ +/* +Copyright 2026 Firefly Software Foundation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +Author: Firefly Software Foundation +SPDX-License-Identifier: Apache-2.0 +*/ +// Operate › Overview: platform health, load, workers and clusters, and what +// needs attention, from one environment summary every 15 s while the page is +// visible. Each section shows its own access or loading problem; the rest of +// the page stays usable. Ages tick on the server's clock between loads, and +// health is only ever what the last answer said. +import { + ChangeDetectionStrategy, + ChangeDetectorRef, + Component, + DoCheck, + Input, + OnDestroy, + OnInit, + inject, +} from "@angular/core"; +import { NgTemplateOutlet } from "@angular/common"; +import { Router } from "@angular/router"; +import type { App, View } from "../../app"; +import { describeError, type PlainError } from "../../errors"; +import { relativeTime } from "../../format"; +import { AnchoredPopover } from "../../forms/ui/anchored-popover"; +import { Icon } from "../../icon"; +import { statusLabel, statusTone, toneAttribute } from "../../status-labels"; +import { environmentOf, listFailure, scopeKeyOf } from "../operate-list"; +import { viewFromPath } from "../operate-routes"; +import { OperateState } from "../operate-state"; +import { Poller } from "../operate-store"; +import { timeAbsolute, timeIso } from "../operate-time"; +import { RefreshStatus } from "../refresh-status"; +import { + attentionLabels, + attentionLink, + clockOffset, + countText, + healthTiles, + isQuiet, + loadFigures, + readSummary, + sectionOf, + slotMeter, + type AttentionItem, + type HealthState, + type HealthTile, + type OperationsSummary, + type TileId, +} from "./overview-model"; + +@Component({ + selector: "weave-overview-page", + standalone: true, + changeDetection: ChangeDetectionStrategy.Eager, + imports: [ + AnchoredPopover, + Icon, + NgTemplateOutlet, + OperateState, + RefreshStatus, + ], + styleUrls: ["../operate.css", "./overview.css"], + template: ` +
+
+

Overview

+

+ Platform health and the work in this environment. +

+
+ @if (readable && summary) { +
+ +
+ } +
+ @if (!host.profile) { +
+

Platform health appears here once you connect

+

Overview reads the health and load of a platform environment.

+ +
+ } @else if (host.signInEnded || host.platformSignedOut) { +
+

Sign in to see platform health

+

+ {{ + host.signInEnded + ? "Studio shows it as soon as you're signed in again." + : "You're signed out of " + host.profile.name + "." + }} +

+ +
+ } @else if (!host.identity) { + + } @else if (!readable || forbidden) { + + } @else if (!summary && error) { + + } @else if (!summary) { + + } @else { + @let s = summary; + @if (error) { + + } +
+
+

Health

+
    + @for (tile of tiles(); track tile.id) { +
  • +
    + {{ tile.label }} +
    + {{ stateLabel(tile.state) }} + {{ tile.figure }} + @if (open === tile.id) { + + } +
    • + } +
+
+ +
+

Load

+ @let load = sectionOf(s.load); + @if (load.kind === "ready") { + @if (quiet()) { +
+

+ No activity yet. Run a workflow to see throughput, failures + and queues here. +

+ +
+ } @else { +
    + @for (figure of figures(); track figure.id) { +
  • + {{ figure.label }}{{ + count(figure.value) + }}{{ figure.detail }} +
    • + } +
+ } + } @else { + + } +
+ +
+

+ Workers and clusters +

+
+
+

Workers

+ @let workers = sectionOf(s.workers); + @if (workers.kind === "ready") { + @let w = workers.value; +
+
+
Online
+
{{ count(w.online) }}
+
+
+
Offline
+
{{ count(w.offline) }}
+
+
+
Draining
+
{{ count(w.draining) }}
+
+
+ @let meter = meterOf(s); + @if (meter) { +
+ Free slots and ready tasks + +
+ } + } @else { + + } +
+
+

Clusters

+ @let targets = sectionOf(s.targets); + @if (targets.kind === "ready") { + @let t = targets.value; +
+
+
Runners online
+
{{ count(t.runner_online) }}
+
+
+
Runners offline
+
+ {{ count(t.total - t.runner_online) }} +
+
+
+
Busy targets
+
{{ count(t.busy) }}
+
+
+
Need reconciliation
+
+ {{ count(t.reconciliation_required) }} +
+
+
+
Plans awaiting approval
+
{{ count(t.awaiting_approval) }}
+
+
+ } @else if (targets.kind === "forbidden") { + + } @else { + + } +
+
+

Incidents

+ @let incidents = sectionOf(s.incidents); + @if (incidents.kind === "ready") { +
+
+
Active
+
+ {{ count(incidents.value.active) }} +
+
+
+
Opened in the last 24 hours
+
+ {{ count(incidents.value.opened_24h) }} +
+
+
+ } @else if (incidents.kind === "forbidden") { + + } @else { + + } +
+
+
+ +
+

Needs attention

+ @let attention = sectionOf(s.attention); + @if (attention.kind === "ready") { + @if (attention.value.length) { +
    + @for (item of attention.value; track item.kind + item.id) { + @let link = linkOf(item); +
  • + {{ labelOf(item)[0] }} + @if (link) { + {{ item.title }} + } @else { + {{ item.title }} + } + @if (item.expires_at) { + + } +
    • + } +
+ } @else { +

Nothing needs attention.

+ } + } @else { + + } +
+
+ } + +

+ You need {{ capability }} in this environment. Ask an + administrator in Settings › People and access. +

+ + +
+ Loading platform health… + + +
+ + `, +}) +export class OverviewPage implements DoCheck, OnInit, OnDestroy { + @Input({ required: true }) host!: App; + private cdr = inject(ChangeDetectorRef); + private router = inject(Router); + readonly poller = new Poller(() => this.load(), 15_000); + readonly sectionOf = sectionOf; + readonly count = countText; + readonly iso = timeIso; + readonly absolute = timeAbsolute; + summary: OperationsSummary | null = null; + error: PlainError | null = null; + forbidden = false; + /** The tile whose Details are open. */ + open: TileId | null = null; + /** Milliseconds from this computer's clock to the server's. */ + private offset = 0; + private scopeKey = ""; + private generation = 0; + // Ages and uptime count on between loads; health itself never does. + private ticker = setInterval(() => this.cdr.markForCheck(), 1000); + + get readable() { + return this.host.canAnywhere("status.read"); + } + ngOnInit() { + this.poller.start(false); + } + ngOnDestroy() { + clearInterval(this.ticker); + this.poller.stop(); + this.generation++; + } + ngDoCheck() { + const key = scopeKeyOf(this.host); + if (key === this.scopeKey) return; + // Another platform, environment or person: nothing from before stays. + this.scopeKey = key; + this.generation++; + this.summary = null; + this.error = null; + this.forbidden = false; + this.open = null; + this.offset = 0; + if (environmentOf(this.host) && this.host.identity && this.readable) + void this.poller.refresh(); + } + private async load(): Promise { + if (!environmentOf(this.host) || !this.host.identity || !this.readable) + return false; + const key = scopeKeyOf(this.host), + generation = this.generation; + try { + const value = await readSummary(this.host.api); + if (generation !== this.generation || key !== scopeKeyOf(this.host)) + return false; + this.summary = value; + this.offset = clockOffset(value.observed_at, Date.now()); + this.error = null; + this.forbidden = false; + return true; + } catch (error) { + if (generation !== this.generation || key !== scopeKeyOf(this.host)) + return false; + ({ error: this.error, forbidden: this.forbidden } = listFailure(error)); + // An ended sign-in is the shell's to explain, with its own Sign in again. + const plain = describeError(error); + if (plain.status === 401 && !plain.code.startsWith("WV-STUDIO-")) + this.host.fail(error); + throw error; + } finally { + this.cdr.markForCheck(); + } + } + /** Now, by the server's clock. */ + private now() { + return new Date(Date.now() + this.offset); + } + tiles(): HealthTile[] { + return this.summary ? healthTiles(this.summary, this.now()) : []; + } + toggle(id: TileId) { + this.open = this.open === id ? null : id; + } + /** Focus moved on past the tile: its Details close, so they never cover it. */ + leave(event: FocusEvent, id: TileId) { + const tile = event.currentTarget as HTMLElement; + if (this.open === id && !tile.contains(event.relatedTarget as Node | null)) + this.open = null; + } + /** Closes Details and gives focus back to the button that opened them. */ + close(anchor: HTMLElement) { + this.open = null; + anchor.focus(); + } + figures() { + const load = this.summary ? sectionOf(this.summary.load) : null; + return load?.kind === "ready" ? loadFigures(load.value) : []; + } + quiet() { + return !!this.summary && isQuiet(this.summary); + } + meterOf(summary: OperationsSummary) { + const workers = sectionOf(summary.workers); + const load = sectionOf(summary.load); + return workers.kind === "ready" + ? slotMeter(workers.value, load.kind === "ready" ? load.value : null) + : null; + } + stateLabel(state: HealthState) { + return statusLabel("health", state); + } + tone(state: HealthState) { + return toneAttribute(statusTone("health", state)); + } + labelOf(item: AttentionItem) { + return attentionLabels[item.kind]; + } + linkOf(item: AttentionItem) { + return attentionLink(item); + } + /** "2 min ago" or "in 4 min", by the server's clock. */ + ago(value: string) { + return relativeTime(value, this.now()) || "at an unknown time"; + } + go(view: View) { + void this.host.navigate(view); + } + /** Opens an attention item's record in its page; a modified click opens a new tab. */ + async follow(event: MouseEvent, link: string) { + if (event.button !== 0 || event.metaKey || event.ctrlKey || event.shiftKey) + return; + event.preventDefault(); + const [path] = link.split("?"); + const route = viewFromPath(path); + if (!route) return; + if (route.view === "runs" && route.id) { + await this.host.viewRun({ id: route.id }); + return; + } + await this.host.navigate(route.view as View); + if (location.pathname + location.search !== link) + await this.router.navigateByUrl(link, { replaceUrl: true }); + } +} diff --git a/studio/src/app/operate/overview/overview.css b/studio/src/app/operate/overview/overview.css new file mode 100644 index 00000000..f08a6d42 --- /dev/null +++ b/studio/src/app/operate/overview/overview.css @@ -0,0 +1,531 @@ +/* +Copyright 2026 Firefly Software Foundation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +Author: Firefly Software Foundation +SPDX-License-Identifier: Apache-2.0 +*/ +/* Operate › Overview, in design tokens only. A fine pointer on a wide screen + gets 36 px controls, 32 px rows and 24/12/4 spacing; a narrow screen or a + touch pointer gets 44 px targets and 16/8 spacing. Every band shares the + page's left and right edges, and stacks into one column when narrow. */ +:host { + --section-gap: var(--space-6); + --group-gap: var(--space-3); + --inline-gap: var(--space-1); + --row: var(--control-sm); + --target: var(--control-md); + display: block; + min-width: 0; + container: overview / inline-size; +} +@media (max-width: 767px), (pointer: coarse) { + :host { + --section-gap: var(--space-4); + --group-gap: var(--space-2); + --row: var(--control-touch); + --target: var(--control-touch); + } +} +.overview { + display: grid; + gap: var(--section-gap); + min-width: 0; +} +weave-operate-state { + margin-bottom: var(--group-gap); +} +.band { + display: grid; + gap: var(--group-gap); + min-width: 0; + margin: 0; +} +.band > weave-operate-state { + margin: 0; +} +.band-title { + margin: 0; + font: var(--type-title-sm); +} + +/* Health: one strip of six tiles. Their lines sit on shared rows, so labels, + states, figures and Details line up across the strip. */ +.health-strip { + display: grid; + grid-template-columns: repeat(6, minmax(0, 1fr)); + gap: var(--group-gap); + margin: 0; + padding: 0; + list-style: none; +} +.health-tile { + display: grid; + grid-row: span 3; + grid-template-rows: subgrid; + align-items: center; + justify-items: start; + row-gap: var(--space-2); + min-width: 0; + padding: var(--group-gap); + border: thin solid var(--border); + border-radius: var(--radius-md); + background: var(--surface); +} +.tile-head { + display: flex; + align-items: center; + justify-content: space-between; + gap: var(--space-2); + justify-self: stretch; + min-width: 0; +} +.tile-label { + font: var(--type-label); + color: var(--text); +} +.tile-figure { + font: var(--type-small); + color: var(--muted); + overflow-wrap: anywhere; +} +.details-toggle { + flex: none; + width: var(--row); + min-height: var(--row); + margin-block: calc(var(--inline-gap) * -1); + color: var(--muted); +} +.details-toggle[aria-expanded="true"] { + background: var(--hover); + color: var(--text); +} +/* Details: a popover under the tile's button, above the page. */ +.health-details { + display: grid; + align-content: start; + gap: var(--space-2); + min-width: 0; + padding: var(--space-3); + border: thin solid var(--border); + border-radius: var(--radius-lg); + background: var(--raised); + box-shadow: var(--shadow-2); + color: var(--text); + overflow: auto; +} +.details-head { + display: flex; + align-items: center; + justify-content: space-between; + gap: var(--space-2); + min-height: var(--target); +} +.details-head h3 { + margin: 0; + font: var(--type-label); +} +.details-head .icon-button { + min-height: var(--target); + width: var(--target); +} +.detail-facts { + display: grid; + grid-template-columns: repeat( + auto-fit, + minmax(min(100%, calc(var(--space-12) * 4)), 1fr) + ); + gap: var(--inline-gap) var(--group-gap); + margin: 0; +} +.detail-fact { + display: flex; + align-items: center; + justify-content: space-between; + gap: var(--space-2); + min-height: var(--row); + min-width: 0; + border-bottom: thin solid var(--line); +} +.detail-fact dt { + font: var(--type-small); + color: var(--muted); +} +.detail-fact dd { + margin: 0; + font: var(--type-small); + color: var(--text); + text-align: right; + overflow-wrap: anywhere; +} +.loop-table { + display: grid; + min-width: 0; + border: thin solid var(--line); + border-radius: var(--radius-sm); +} +.loop-row { + display: grid; + grid-template-columns: minmax(0, 1fr) auto auto auto; + align-items: center; + gap: var(--space-3); + min-height: var(--row); + padding: 0 var(--space-2); + border-bottom: thin solid var(--line); + font: var(--type-small); + color: var(--muted); +} +.loop-row:last-child { + border-bottom: 0; +} +.loop-head { + font: var(--type-caption); + background: var(--sunken); +} +.loop-name { + color: var(--text); +} +.numeric-cell { + min-width: var(--space-10); + text-align: right; +} +.details-note { + margin: 0; + font: var(--type-caption); + font-weight: 400; + color: var(--muted); +} + +/* Load: five figures side by side, or labelled rows when narrow. */ +.figures { + display: grid; + grid-template-columns: repeat(5, minmax(0, 1fr)); + gap: var(--group-gap); + margin: 0; + padding: 0; + list-style: none; +} +.figure { + display: grid; + grid-row: span 3; + grid-template-rows: subgrid; + row-gap: var(--inline-gap); + min-width: 0; + padding: var(--group-gap); + border: thin solid var(--border); + border-radius: var(--radius-md); + background: var(--surface); +} +.figure-label { + font: var(--type-caption); + color: var(--muted); +} +.figure-value { + font: var(--type-title-md); + color: var(--text); +} +.figure-detail { + font: var(--type-small); + color: var(--muted); + overflow-wrap: anywhere; +} +.band-empty { + display: grid; + justify-items: start; + gap: var(--group-gap); + padding: var(--group-gap); + border: thin solid var(--border); + border-radius: var(--radius-md); + background: var(--surface); +} +.band-empty p, +.band-note { + margin: 0; + color: var(--muted); +} +.band-empty button { + min-height: var(--target); +} + +/* Workers and clusters: three panels of counts. */ +.panels { + display: grid; + grid-template-columns: repeat(3, minmax(0, 1fr)); + gap: var(--group-gap); + min-width: 0; +} +.panel { + display: grid; + align-content: start; + gap: var(--space-2); + min-width: 0; + padding: var(--group-gap); + border: thin solid var(--border); + border-radius: var(--radius-md); + background: var(--surface); +} +.panel h3 { + margin: 0; + font: var(--type-label); + line-height: var(--row); +} +.panel weave-operate-state { + margin: 0; +} +.counts { + display: grid; + grid-template-columns: minmax(0, 1fr); + gap: 0; + margin: 0; +} +.count-row { + display: flex; + align-items: center; + justify-content: space-between; + gap: var(--space-2); + min-height: var(--row); + border-bottom: thin solid var(--line); +} +.count-row:last-child { + border-bottom: 0; +} +.count-row dt { + font: var(--type-small); + color: var(--muted); +} +.count-row dd { + margin: 0; + font: var(--type-label); + color: var(--text); +} +.slots { + display: grid; + gap: var(--inline-gap); +} +.slots-label { + font: var(--type-caption); + color: var(--muted); +} +.slots .meter { + display: flex; +} +.slots .meter-track { + flex: 1 1 auto; + width: auto; +} +.section-access { + margin: 0; + font: var(--type-small); + color: var(--muted); +} + +/* Needs attention: one row per item, by priority. */ +.attention { + display: grid; + margin: 0; + padding: 0; + list-style: none; + border: thin solid var(--border); + border-radius: var(--radius-md); + background: var(--surface); +} +.attention-item { + display: grid; + grid-template-columns: + var(--space-4) calc(var(--space-12) * 2) minmax(0, 1fr) + auto; + align-items: center; + gap: var(--group-gap); + min-height: var(--row); + padding: 0 var(--group-gap); + border-bottom: thin solid var(--line); +} +.attention-item:last-child { + border-bottom: 0; +} +.attention-icon { + color: var(--muted); +} +.attention-item[data-kind="job"] .attention-icon, +.attention-item[data-kind="approval"] .attention-icon { + color: var(--warning-ink); +} +.attention-item[data-kind="incident"] .attention-icon, +.attention-item[data-kind="run"] .attention-icon { + color: var(--danger-ink); +} +.attention-kind { + font: var(--type-caption); + color: var(--muted); +} +.attention-title { + display: flex; + align-items: center; + min-height: var(--row); + min-width: 0; + font: var(--type-body); + color: var(--text); + overflow-wrap: anywhere; +} +a.attention-title { + color: var(--link); +} +.attention-when { + font: var(--type-small); + color: var(--muted); + font-variant-numeric: tabular-nums; + white-space: nowrap; +} + +/* Loading: blocks shaped like the bands, never shimmering. */ +.skeleton-line { + display: block; + width: calc(var(--space-12) * 2); + height: var(--space-4); + border-radius: var(--radius-xs); + background: var(--skeleton); +} +.skeleton-block { + display: block; + grid-row: auto; + height: calc(var(--control-touch) * 2); + border: 0; + background: var(--skeleton); +} + +/* Narrower panes: three health tiles a row, figures and panels in fewer + columns. */ +@container overview (max-width: 1119px) { + .health-strip { + grid-template-columns: repeat(3, minmax(0, 1fr)); + } + .panels { + grid-template-columns: repeat(2, minmax(0, 1fr)); + } +} +@container overview (max-width: 959px) { + .figures { + grid-template-columns: repeat(3, minmax(0, 1fr)); + } +} +/* Narrow: every band is one column; figures and loops read as labelled + rows, value on the right. */ +@container overview (max-width: 719px) { + .health-strip { + grid-template-columns: repeat(2, minmax(0, 1fr)); + } + .panels { + grid-template-columns: minmax(0, 1fr); + } + .figures { + grid-template-columns: minmax(0, 1fr); + gap: 0; + border: thin solid var(--border); + border-radius: var(--radius-md); + background: var(--surface); + } + .figure { + grid-row: auto; + grid-template-rows: none; + grid-template-columns: minmax(0, 1fr) auto; + grid-template-areas: + "label value" + "detail value"; + align-items: center; + column-gap: var(--space-2); + border: 0; + border-bottom: thin solid var(--line); + border-radius: 0; + background: transparent; + } + .figure:last-child { + border-bottom: 0; + } + .figure-label { + grid-area: label; + font: var(--type-small); + color: var(--text); + } + .figure-value { + grid-area: value; + font: var(--type-title-sm); + } + .figure-detail { + grid-area: detail; + font: var(--type-caption); + } + .attention-item { + grid-template-columns: var(--space-4) minmax(0, 1fr); + grid-template-areas: + "icon kind" + ". title" + ". when"; + row-gap: 0; + padding-block: var(--space-2); + } + .attention-icon { + grid-area: icon; + } + .attention-kind { + grid-area: kind; + } + .attention-title { + grid-area: title; + } + .attention-when { + grid-area: when; + white-space: normal; + } +} +/* A phone: one tile a row, its name, state and Details on one line. */ +@container overview (max-width: 419px) { + .health-strip { + grid-template-columns: minmax(0, 1fr); + } + .health-tile { + grid-row: auto; + grid-template-rows: none; + grid-template-columns: minmax(0, 1fr) auto auto; + grid-template-areas: + "label state details" + "figure figure figure"; + column-gap: var(--space-2); + row-gap: 0; + padding-block: var(--space-1) var(--space-2); + } + .tile-head { + display: contents; + } + .tile-label { + grid-area: label; + } + .tile-state { + grid-area: state; + } + .details-toggle { + grid-area: details; + margin-block: 0; + } + .tile-figure { + grid-area: figure; + } +} + +/* Counts and figures in tabular figures; after every font shorthand above, + which would reset them. */ +.numeric, +.figure-value, +.count-row dd, +.attention-when { + font-variant-numeric: tabular-nums; +} diff --git a/studio/src/app/operate/runs/runs-model.ts b/studio/src/app/operate/runs/runs-model.ts new file mode 100644 index 00000000..a0348f0f --- /dev/null +++ b/studio/src/app/operate/runs/runs-model.ts @@ -0,0 +1,335 @@ +/* +Copyright 2026 Firefly Software Foundation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +Author: Firefly Software Foundation +SPDX-License-Identifier: Apache-2.0 +*/ +// Operate › Runs: the filters a person sets, the address that keeps the +// shareable ones, and the run_summaries.list query each page of results +// uses. A relative time window ("the last 24 hours") stays a window in the +// address and in the page; each chain of pages resolves it to fixed bounds +// once, so Load more continues exactly the list that was first read. Business +// and correlation keys may carry business data: they reach the platform but +// never the address. Pure: the page renders it, tests check it. +import { durationText } from "../../format"; +import { statusLabel, statusTone, type Tone } from "../../status-labels"; +import { + runOrigins, + runStatuses, + runSummaryOrders, + runViewQuery, + terminalRunStatuses, + type RunOrigin, + type RunStatus, + type RunSummary, + type RunSummaryOrder, + type RunSummaryQuery, +} from "../run-contracts"; + +export type RunTimeWindow = + | { kind: "relative"; milliseconds: number } + | { kind: "absolute"; started_after?: string; started_before?: string }; +export type RunsFilters = Omit< + RunSummaryQuery, + "limit" | "cursor" | "started_after" | "started_before" +> & { timeWindow: RunTimeWindow }; + +/** Runs read per page. */ +export const RUNS_PAGE_SIZE = 50; +/** The longest window the platform accepts. */ +export const MAX_RANGE_MS = 400 * 86_400_000; +const DEFAULT_RANGE_MS = 86_400_000; +/** The time ranges a person picks from, besides a custom range. */ +export const rangePresets: { + id: string; + label: string; + milliseconds: number; +}[] = [ + { id: "15m", label: "Last 15 minutes", milliseconds: 15 * 60_000 }, + { id: "1h", label: "Last hour", milliseconds: 3_600_000 }, + { id: "24h", label: "Last 24 hours", milliseconds: DEFAULT_RANGE_MS }, + { id: "7d", label: "Last 7 days", milliseconds: 7 * 86_400_000 }, + { id: "30d", label: "Last 30 days", milliseconds: 30 * 86_400_000 }, +]; + +export function defaultRunsFilters(): RunsFilters { + return { + timeWindow: { kind: "relative", milliseconds: DEFAULT_RANGE_MS }, + order: "started_desc", + include_test: false, + include_archived: false, + status: [], + top_level_only: false, + }; +} + +/** + * The first page's query: a relative window ends now and starts its length + * before; a custom range keeps its exact bounds. + */ +export function resolveRunsQuery( + filters: RunsFilters, + now: Date, +): RunSummaryQuery { + const { timeWindow, ...query } = filters; + if (timeWindow.kind === "absolute") { + return { + ...query, + started_after: timeWindow.started_after, + started_before: timeWindow.started_before, + limit: RUNS_PAGE_SIZE, + }; + } + return { + ...query, + limit: RUNS_PAGE_SIZE, + started_before: now.toISOString(), + started_after: new Date( + now.getTime() - timeWindow.milliseconds, + ).toISOString(), + }; +} + +/** The request for one page of a chain: its frozen query, and its cursor after the first page. */ +export function runsQuery( + query: RunSummaryQuery, + cursor: string | null, +): string { + return runViewQuery({ ...query, ...(cursor ? { cursor } : {}) }); +} + +const units: Record = { + m: 60_000, + h: 3_600_000, + d: 86_400_000, +}; +/** "7d" or "90m": a positive whole duration of 400 days or less. */ +function rangeMilliseconds(value: string): number | null { + const match = /^([1-9][0-9]{0,5})([mhd])$/.exec(value); + if (!match) return null; + const milliseconds = Number(match[1]) * units[match[2]]; + return Number.isFinite(milliseconds) && + milliseconds > 0 && + milliseconds <= MAX_RANGE_MS + ? milliseconds + : null; +} +function rangeId(milliseconds: number): string { + const preset = rangePresets.find( + (item) => item.milliseconds === milliseconds, + ); + if (preset) return preset.id; + for (const unit of ["d", "h", "m"]) + if (milliseconds % units[unit] === 0) + return `${milliseconds / units[unit]}${unit}`; + return `${Math.max(1, Math.round(milliseconds / 60_000))}m`; +} +const instant = + /^\d{4}-\d{2}-\d{2}[Tt]\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:[Zz]|[+-]\d{2}:\d{2})$/; +const uuid = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; +const name = /^[A-Za-z0-9][A-Za-z0-9_.-]{0,199}$/; +const version = /^[0-9A-Za-z][0-9A-Za-z.+-]{0,199}$/; + +/** A custom range from the address, when its bounds are times in order and at most 400 days apart. */ +function absoluteWindow( + after: string | null, + before: string | null, +): RunTimeWindow | null { + if (!after && !before) return null; + if ((after && !instant.test(after)) || (before && !instant.test(before))) + return null; + const from = after ? Date.parse(after) : null; + const to = before ? Date.parse(before) : null; + if ( + (from !== null && !Number.isFinite(from)) || + (to !== null && !Number.isFinite(to)) + ) + return null; + if (from !== null && to !== null && (from >= to || to - from > MAX_RANGE_MS)) + return null; + return { + kind: "absolute", + ...(after ? { started_after: after } : {}), + ...(before ? { started_before: before } : {}), + }; +} + +/** + * The filters an address shares. Business and correlation keys are never + * read from it, and values the platform would refuse are left out. + */ +export function runsFiltersFromQuery(search: string): RunsFilters { + const query = new URLSearchParams(search); + const filters = defaultRunsFilters(); + const flag = (key: string) => { + const value = query.get(key); + return value === "true" ? true : value === "false" ? false : undefined; + }; + const statuses = [...new Set(query.getAll("status"))].filter((value) => + (runStatuses as readonly string[]).includes(value), + ) as RunStatus[]; + filters.status = statuses.slice(0, runStatuses.length); + const workflow = query.get("workflow"); + if (workflow && name.test(workflow)) { + filters.workflow = workflow; + const chosen = query.get("version"); + if (chosen && version.test(chosen)) filters.version = chosen; + } + const origin = query.get("origin"); + if (origin && (runOrigins as readonly string[]).includes(origin)) + filters.origin = origin as RunOrigin; + for (const key of [ + "caller_run_id", + "retried_from_run_id", + "activation_id", + ] as const) { + const value = query.get(key); + if (value && uuid.test(value)) filters[key] = value.toLowerCase(); + } + filters.include_test = flag("include_test") ?? false; + filters.include_archived = flag("include_archived") ?? false; + filters.top_level_only = flag("top_level_only") ?? false; + const incident = flag("has_active_incident"); + if (incident !== undefined) filters.has_active_incident = incident; + const handled = flag("has_handled_errors"); + if (handled !== undefined) filters.has_handled_errors = handled; + const order = query.get("order"); + if (order && (runSummaryOrders as readonly string[]).includes(order)) + filters.order = order as RunSummaryOrder; + const custom = absoluteWindow( + query.get("started_after"), + query.get("started_before"), + ); + const range = rangeMilliseconds(query.get("range") ?? ""); + if (custom) filters.timeWindow = custom; + else if (range !== null) + filters.timeWindow = { kind: "relative", milliseconds: range }; + return filters; +} + +/** + * The address for these filters: only what differs from the defaults, a + * relative window as its length ("range=7d"), a custom range as its exact + * bounds. Business and correlation keys are left out by name. + */ +export function runsFiltersToQuery(filters: RunsFilters): string { + const { + business_key: _business, + correlation_key: _correlation, + timeWindow, + ...shared + } = filters; + const query = new URLSearchParams(); + for (const status of shared.status ?? []) query.append("status", status); + if (shared.workflow) { + query.set("workflow", shared.workflow); + if (shared.version) query.set("version", shared.version); + } + if (shared.origin) query.set("origin", shared.origin); + for (const key of [ + "caller_run_id", + "retried_from_run_id", + "activation_id", + ] as const) + if (shared[key]) query.set(key, shared[key]); + for (const key of [ + "include_test", + "include_archived", + "top_level_only", + ] as const) + if (shared[key]) query.set(key, "true"); + for (const key of ["has_active_incident", "has_handled_errors"] as const) + if (shared[key] !== undefined) query.set(key, String(shared[key])); + if (shared.order && shared.order !== "started_desc") + query.set("order", shared.order); + if (timeWindow.kind === "absolute") { + if (timeWindow.started_after) + query.set("started_after", timeWindow.started_after); + if (timeWindow.started_before) + query.set("started_before", timeWindow.started_before); + } else if (timeWindow.milliseconds !== DEFAULT_RANGE_MS) + query.set("range", rangeId(timeWindow.milliseconds)); + const text = query.toString(); + return text ? `?${text}` : ""; +} + +/** How many filters differ from the defaults; private ones count too. */ +export function runFilterCount(filters: RunsFilters): number { + const defaults = defaultRunsFilters(); + const window = filters.timeWindow; + return [ + (filters.status ?? []).length > 0, + !!filters.workflow, + !!filters.origin, + window.kind !== "relative" || window.milliseconds !== DEFAULT_RANGE_MS, + !!filters.business_key, + !!filters.correlation_key, + filters.include_test !== defaults.include_test, + filters.include_archived !== defaults.include_archived, + filters.top_level_only !== defaults.top_level_only, + filters.has_active_incident !== undefined, + filters.has_handled_errors !== undefined, + !!filters.caller_run_id, + !!filters.retried_from_run_id, + !!filters.activation_id, + (filters.order ?? "started_desc") !== "started_desc", + ].filter(Boolean).length; +} + +/** The run's status pill. A run that handled errors and succeeded says so. */ +export function runOutcome(run: RunSummary): { label: string; tone: Tone } { + if (run.status === "succeeded" && (run.handled_errors ?? 0) > 0) + return { label: "Succeeded with errors", tone: "warning" }; + return { + label: statusLabel("run", run.status), + tone: statusTone("run", run.status), + }; +} + +export function isTerminalRun(run: RunSummary): boolean { + return terminalRunStatuses.includes(run.status); +} + +/** "2 errors handled" on a run that is still going; "" otherwise. */ +export function handledText(run: RunSummary): string { + const count = run.handled_errors ?? 0; + if (!count || isTerminalRun(run)) return ""; + return `${count} ${count === 1 ? "error" : "errors"} handled`; +} + +const origins: Record = { + manual: "Manual", + webhook: "Webhook", + schedule: "Schedule", + broker: "Kafka", + email: "Email", + provider: "App event", + retry: "Retry", + call: "Called by a workflow", + test: "Test", +}; +/** How the run started, in plain words; older runs may not say. */ +export function originLabel(origin: RunOrigin | null): string { + return origin ? origins[origin] : "Unknown"; +} + +/** Its recorded duration, or how long it has run so far while active. */ +export function liveDuration(run: RunSummary, now: Date): string { + if (run.duration_ms !== null) return durationText(run.duration_ms); + if (isTerminalRun(run)) return "—"; + const started = Date.parse(run.started_at); + return Number.isFinite(started) + ? durationText(Math.max(0, now.getTime() - started)) + : "—"; +} diff --git a/studio/src/app/operate/runs/runs-page.ts b/studio/src/app/operate/runs/runs-page.ts new file mode 100644 index 00000000..76934a00 --- /dev/null +++ b/studio/src/app/operate/runs/runs-page.ts @@ -0,0 +1,1757 @@ +/* +Copyright 2026 Firefly Software Foundation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +Author: Firefly Software Foundation +SPDX-License-Identifier: Apache-2.0 +*/ +// Operate › Runs: run summaries (run_summaries.list), newest first, with the +// filters a person shares in the address and the private ones (business and +// correlation keys) kept in the page. Each refresh reads the first page again +// with the time window resolved to now; Load more continues that same chain +// with its frozen bounds. The open run keeps the shell's run detail and gets +// its steps from runs.steps, every 2 s until it finishes. The list refreshes +// every 10 s while Auto-refresh is on and the page is visible. +import { + ChangeDetectionStrategy, + ChangeDetectorRef, + Component, + DoCheck, + HostListener, + Input, + OnDestroy, + OnInit, + inject, +} from "@angular/core"; +import { NgTemplateOutlet } from "@angular/common"; +import { NavigationEnd, Router } from "@angular/router"; +import type { App } from "../../app"; +import { copyText } from "../../connection"; +import { describeError, type PlainError } from "../../errors"; +import { shortId } from "../../format"; +import { Icon } from "../../icon"; +import { ModalSheet, sheetWhen } from "../../modal-sheet"; +import { RunDetail } from "../../operations/run-detail"; +import { statusLabel, toneAttribute } from "../../status-labels"; +import { MAX_PAGES } from "../operate-limits"; +import { environmentOf, listFailure, scopeKeyOf } from "../operate-list"; +import { viewFromPath, viewPath } from "../operate-routes"; +import { OperateState } from "../operate-state"; +import { Poller } from "../operate-store"; +import { timeAbsolute, timeIso, timeRelative } from "../operate-time"; +import { RefreshStatus } from "../refresh-status"; +import { + asRunViewsApi, + isUnavailableRun, + listRunSteps, + listRunSummaries, + runOrigins, + runStatuses, + stepOutput, + terminalRunStatuses, + type RunStatus, + type RunSummary, + type RunSummaryItem, + type RunSummaryPage, + type RunSummaryQuery, + type StepFact, +} from "../run-contracts"; +import { + explainRefusal, + isTerminal, + reasonProblem, + retryInput, +} from "./run-actions"; +import { + MAX_RANGE_MS, + defaultRunsFilters, + handledText, + isTerminalRun, + liveDuration, + originLabel, + rangePresets, + resolveRunsQuery, + runFilterCount, + runOutcome, + runsFiltersFromQuery, + runsFiltersToQuery, + runsQuery, + type RunsFilters, +} from "./runs-model"; +import { + instanceKey, + type RunStepsView, + type StepOutputState, +} from "./step-timeline"; + +type Json = Record; +const isRecord = (value: unknown): value is Json => + !!value && typeof value === "object" && !Array.isArray(value); +/** + * Whether only the person behind the same platform, environment and sign-in + * became known. Runs reads before Studio knows them, with the same session, + * so nothing read belongs to anyone else; any other change of scope does. + */ +function personLearned(before: string, after: string): boolean { + if (!before) return false; + const [environment, person, ended] = JSON.parse(before) as unknown[]; + const [nextEnvironment, nextPerson, nextEnded] = JSON.parse( + after, + ) as unknown[]; + return ( + person === null && + nextPerson !== null && + environment === nextEnvironment && + ended === nextEnded + ); +} +/** The most runs one bulk action takes, one after another. */ +const BULK_LIMIT = 50; +const STEP_PAGE = 200; + +/** The steps of the run that is open, and the outputs a person asked for. */ +interface StepsState { + runId: string; + environment: string; + items: StepFact[]; + complete: boolean; + nextCursor: string | null; + loading: boolean; + error: PlainError | null; + outputs: Map; +} +/** One chain of pages: the query its first page resolved, and where it continues. */ +interface Chain { + generation: number; + environment: string; + query: RunSummaryQuery; + cursor: string | null; +} +interface BulkResult { + id: string; + ok: boolean; + text: string; +} + +@Component({ + selector: "weave-runs-page", + standalone: true, + changeDetection: ChangeDetectionStrategy.Eager, + imports: [ + Icon, + ModalSheet, + NgTemplateOutlet, + OperateState, + RefreshStatus, + RunDetail, + ], + styleUrls: ["../operate.css", "./runs.css"], + template: ` +
+
+

Runs

+

+ Every run in this environment, newest first. Open one to follow its + steps. +

+
+ @if (ready && readable && !forbidden) { +
+ + + @if (host.can("run.start")) { + + } +
+ } +
+ @if (!host.profile) { +
+

Runs appear here once you connect

+

Runs happen on a platform. Connect one to start and follow runs.

+ +
+ } @else if (host.signInEnded || host.platformSignedOut) { +
+

Sign in to see runs

+

+ {{ + host.signInEnded + ? "Studio shows them as soon as you're signed in again." + : "You're signed out of " + host.profile.name + "." + }} +

+ +
+ } @else if (!readable || forbidden) { + + } @else { + + @if (host.runRefused && !forbidden) { + + } + @if (loaded && error) { + + } + @if (selection.size) { +
+ {{ selection.size }} selected + @if (host.can("run.cancel")) { + + } + @if (host.can("run.retry")) { + + } + +
+ } + @if (bulk; as outcome) { +
+
+

{{ outcome.title }}

+ @if (!bulkRunning) { + + } +
+
    + @for (result of outcome.results; track result.id) { +
  • + {{ short(result.id) }}{{ result.text }} +
    • + } +
+
+ } + @if (!loaded && error && !openRunId()) { + + } @else { +
+
+ @if (!loaded && error) { + + } @else if (!loaded) { + + } @else if (!runs.length) { + @if (filterCount()) { + + + + } @else { + + } + } @else { +
+
+
+ + @if (canBulk()) { + + }StatusWorkflowOriginStartedDurationIncidentsBusiness keyRunDetails +
+
+
+ @for (item of runs; track item.id) { + @if (unavailable(item)) { +
+ Unavailable runStudio can't show this run's details.{{ + short(item.id) + }}Studio can't show this run's details.{{ + short(item.id) + }} +
+ } @else { + @let run = summaryOf(item); + @let outcome = outcomeOf(run); +
+ + @if (canBulk()) { + + }{{ outcome.label }} + @if (run.paused) { + Paused + } + @if (run.test) { + Test + } + @if (handled(run)) { + {{ handled(run) }} + }{{ + run.workflow.version + }} + {{ duration(run) }} + @if (run.active_incidents) { + {{ + run.active_incidents + }} + } @else { + — + }{{ + run.business_key ?? "—" + }}{{ short(run.id) }}{{ duration(run) }} + @if (run.active_incidents) { + {{ + incidentsLabel(run.active_incidents) + }} + } + @if (run.business_key) { + {{ run.business_key }} + } + {{ + short(run.id) + }} +
+ } + } +
+
+ @if (canLoadMore) { + + } @else if (atLimit) { +

+ Showing the first {{ maxRuns }} runs. Narrow the filters to + see others. +

+ } + } +
+ @if (openRunId()) { + + } +
+ } + } + + @if (run.retried_from_run_id) { + + } @else if (run.caller) { + + } @else { + {{ originLabel(run.origin) }} + } + + `, +}) +export class RunsPage implements DoCheck, OnInit, OnDestroy { + @Input({ required: true }) host!: App; + private cdr = inject(ChangeDetectorRef); + private router = inject(Router); + private navigated = this.router.events.subscribe((event) => { + if (event instanceof NavigationEnd) this.addressChanged(true); + }); + readonly sheetWhen = sheetWhen; + readonly short = shortId; + readonly iso = timeIso; + readonly absolute = timeAbsolute; + readonly originLabel = originLabel; + readonly handled = handledText; + readonly presets = rangePresets; + readonly origins = runOrigins; + readonly statuses = runStatuses; + readonly bulkLimit = BULK_LIMIT; + readonly maxRuns = MAX_PAGES * 50; + /** The list refreshes by itself every 10 s while this is on. */ + autoRefresh = true; + readonly poller = new Poller( + () => this.refreshFirstPage(), + 10_000, + undefined, + // A refresh reads the first page again: it waits while a person works + // with more pages, a selection or a bulk action. + () => + !this.autoRefresh || + this.pages > 1 || + this.selection.size > 0 || + this.bulkRunning, + ); + /** The open run's detail and steps, every 2 s until the run finishes. */ + readonly detailPoller = new Poller(() => this.pollDetail(), 2_000); + filters: RunsFilters = defaultRunsFilters(); + runs: RunSummaryItem[] = []; + chain: Chain | null = null; + loaded = false; + forbidden = false; + error: PlainError | null = null; + appending = false; + moreFilters = false; + /** Narrow screens fold the filters behind "Filters (n)". */ + filtersOpen = false; + rangeProblem = ""; + selection = new Set(); + bulk: { title: string; results: BulkResult[] } | null = null; + bulkRunning = false; + steps: StepsState | null = null; + private pages = 1; + private generation = 0; + private scopeKey = ""; + private privateTimer: ReturnType | null = null; + /** The filters changed with a run open: the address follows once the run has closed. */ + private syncPending = false; + private stepsView$: { + state: StepsState; + now: number; + view: RunStepsView; + } | null = null; + private detailStatus = ""; + /** The first-page read in flight, which Load more waits for. */ + private refreshing: Promise | null = null; + // Durations of running runs count up between refreshes. + private ticker = setInterval(() => this.cdr.markForCheck(), 1000); + + /** + * Whether to read runs: with the person known, only with run.read; before + * Studio knows them, the platform decides, as it does for every read. + */ + get readable() { + return !this.host.identity || this.host.canAnywhere("run.read"); + } + get ready() { + return ( + !!this.host.profile && + !this.host.signInEnded && + !this.host.platformSignedOut + ); + } + /** An ended sign-in is the shell's to explain, with its own Sign in again. */ + private signInGone(error: unknown) { + const plain = describeError(error); + if (plain.status === 401 && !plain.code.startsWith("WV-STUDIO-")) + this.host.fail(error); + } + /** More runs follow, and Load more may read them. */ + get canLoadMore() { + return !!this.chain?.cursor && this.pages < MAX_PAGES; + } + /** More runs follow beyond the most a list reads. */ + get atLimit() { + return !!this.chain?.cursor && this.pages >= MAX_PAGES; + } + ngOnInit() { + this.poller.start(false); + } + ngOnDestroy() { + this.navigated.unsubscribe(); + clearInterval(this.ticker); + if (this.privateTimer) clearTimeout(this.privateTimer); + this.poller.stop(); + this.detailPoller.stop(); + this.generation++; + } + ngDoCheck() { + const key = scopeKeyOf(this.host); + if (key !== this.scopeKey && personLearned(this.scopeKey, key)) { + // The same session's person became known: the rows, the open run and + // focus stay while the first page is read again as that person. + this.scopeKey = key; + this.generation++; + this.appending = false; + if (environmentOf(this.host) && this.readable) void this.poller.refresh(); + } else if (key !== this.scopeKey) { + // Another platform, environment or person: nothing read before, no key + // typed before and no cursor from before stays. + this.scopeKey = key; + this.generation++; + this.runs = []; + this.chain = null; + this.loaded = false; + this.forbidden = false; + this.error = null; + this.pages = 1; + this.appending = false; + this.selection = new Set(); + this.bulk = null; + this.steps = null; + this.detailStatus = ""; + this.detailPoller.stop(); + this.filters = runsFiltersFromQuery(location.search); + if (this.privateTimer) clearTimeout(this.privateTimer); + this.privateTimer = null; + if (environmentOf(this.host) && this.readable) void this.poller.refresh(); + } + this.followDetail(); + } + + // ---------------------------------------------------------------- list + + /** + * Reads the first page again with the window resolved to now: a new chain + * whose cursor replaces the old one. Answers for an older chain or another + * scope change nothing. + */ + async refreshFirstPage(): Promise { + const read = this.readFirstPage(); + this.refreshing = read; + try { + return await read; + } finally { + if (this.refreshing === read) this.refreshing = null; + } + } + private async readFirstPage(): Promise { + const environment = environmentOf(this.host); + if (!environment || !this.readable) return false; + const generation = ++this.generation; + const key = scopeKeyOf(this.host); + const query = resolveRunsQuery(this.filters, new Date()); + try { + const page = await listRunSummaries(asRunViewsApi(this.host.api), query); + if (generation !== this.generation || key !== scopeKeyOf(this.host)) + return false; + this.runs = page.items; + this.chain = { generation, environment, query, cursor: page.next_cursor }; + this.pages = 1; + this.loaded = true; + this.forbidden = false; + this.error = null; + const listed = new Set(page.items.map((item) => item.id)); + this.selection = new Set( + [...this.selection].filter((id) => listed.has(id)), + ); + return true; + } catch (error) { + if (generation !== this.generation || key !== scopeKeyOf(this.host)) + return false; + // The old chain cannot continue: its rows stay, its cursor does not. + if (this.chain) this.chain = { ...this.chain, cursor: null }; + ({ error: this.error, forbidden: this.forbidden } = listFailure(error)); + this.signInGone(error); + throw error; + } finally { + this.cdr.markForCheck(); + } + } + /** + * Continues the current chain with its own query; never resolves a new + * window. Asked while a refresh is out, it continues the chain that refresh + * starts. + */ + async loadMore() { + if (this.appending) return; + this.appending = true; + const key = scopeKeyOf(this.host); + try { + await this.refreshing?.catch(() => undefined); + const chain = this.chain; + if ( + !chain?.cursor || + chain.generation !== this.generation || + this.pages >= MAX_PAGES || + key !== scopeKeyOf(this.host) + ) + return; + const page = await this.host.api.request( + `${chain.environment}/run-summaries${runsQuery(chain.query, chain.cursor)}`, + ); + if ( + this.chain !== chain || + chain.generation !== this.generation || + key !== scopeKeyOf(this.host) + ) + return; + const seen = new Set(this.runs.map((item) => item.id)); + this.runs = [ + ...this.runs, + ...page.items.filter((item) => !seen.has(item.id)), + ]; + this.chain = { ...chain, cursor: page.next_cursor }; + this.pages++; + this.error = null; + } catch (error) { + if (key === scopeKeyOf(this.host)) + ({ error: this.error, forbidden: this.forbidden } = listFailure(error)); + } finally { + this.appending = false; + this.cdr.markForCheck(); + } + } + /** Refresh: the first page again, from now; selection and extra pages go. */ + refreshNow() { + this.selection = new Set(); + void this.poller.refresh(); + } + setAutoRefresh(on: boolean) { + this.autoRefresh = on; + if (on) void this.poller.refresh(); + } + + // ---------------------------------------------------------------- filters + + setFilter(key: K, value: RunsFilters[K]) { + const next = { ...this.filters, [key]: value }; + if (value === undefined) delete next[key]; + this.apply(next); + } + setWorkflow(name: string) { + const next = { ...this.filters }; + delete next.version; + if (name) next.workflow = name; + else delete next.workflow; + this.apply(next); + } + setHandled(value: string) { + this.setFilter( + "has_handled_errors", + value === "true" ? true : value === "false" ? false : undefined, + ); + } + hasStatus(status: RunStatus) { + return (this.filters.status ?? []).includes(status); + } + toggleStatus(status: RunStatus) { + const current = this.filters.status ?? []; + this.setFilter( + "status", + current.includes(status) + ? current.filter((item) => item !== status) + : runStatuses.filter( + (item) => item === status || current.includes(item), + ), + ); + } + statusWord(status: RunStatus) { + return statusLabel("run", status); + } + rangeId() { + const window = this.filters.timeWindow; + if (window.kind === "absolute") return "custom"; + return ( + rangePresets.find((preset) => preset.milliseconds === window.milliseconds) + ?.id ?? "custom" + ); + } + setRange(id: string) { + this.rangeProblem = ""; + const preset = rangePresets.find((item) => item.id === id); + if (preset) { + this.setFilter("timeWindow", { + kind: "relative", + milliseconds: preset.milliseconds, + }); + return; + } + // A custom range starts as the window it replaces, with fixed bounds. + const resolved = resolveRunsQuery(this.filters, new Date()); + this.setFilter("timeWindow", { + kind: "absolute", + started_after: resolved.started_after, + started_before: resolved.started_before, + }); + } + customFrom() { + const window = this.filters.timeWindow; + return window.kind === "absolute" ? (window.started_after ?? "") : ""; + } + customTo() { + const window = this.filters.timeWindow; + return window.kind === "absolute" ? (window.started_before ?? "") : ""; + } + /** "2026-10-07T12:30" in this computer's time zone, for a datetime-local field. */ + localTime(value: string) { + const date = new Date(value); + if (!value || !Number.isFinite(date.getTime())) return ""; + const local = new Date(date.getTime() - date.getTimezoneOffset() * 60_000); + return local.toISOString().slice(0, 16); + } + setCustom(edge: "from" | "to", value: string) { + const date = value ? new Date(value) : null; + const instant = + date && Number.isFinite(date.getTime()) ? date.toISOString() : undefined; + const window = + this.filters.timeWindow.kind === "absolute" + ? this.filters.timeWindow + : { kind: "absolute" as const }; + const next = { + kind: "absolute" as const, + started_after: edge === "from" ? instant : window.started_after, + started_before: edge === "to" ? instant : window.started_before, + }; + const from = next.started_after ? Date.parse(next.started_after) : null; + const to = next.started_before ? Date.parse(next.started_before) : null; + if (from !== null && to !== null && from >= to) { + this.rangeProblem = "Choose a start before the end."; + return; + } + if (from !== null && to !== null && to - from > MAX_RANGE_MS) { + this.rangeProblem = "Choose a range of 400 days or less."; + return; + } + this.rangeProblem = ""; + this.setFilter("timeWindow", next); + } + /** Business and correlation keys: sent after a pause in typing, never put in the address. */ + setPrivate(key: "business_key" | "correlation_key", value: string) { + const next = { ...this.filters }; + if (value.trim()) next[key] = value; + else delete next[key]; + this.filters = next; + if (this.privateTimer) clearTimeout(this.privateTimer); + this.privateTimer = setTimeout(() => { + this.privateTimer = null; + this.changed(); + }, 400); + } + clearFilters() { + this.rangeProblem = ""; + this.apply(defaultRunsFilters()); + } + clearLink() { + const next = { ...this.filters }; + delete next.caller_run_id; + delete next.retried_from_run_id; + this.apply(next); + } + filterCount() { + return runFilterCount(this.filters); + } + private apply(next: RunsFilters) { + this.filters = next; + void this.syncAddress(); + this.changed(); + } + /** New filters: a new first page, no selection, and the open run closes. */ + private changed() { + this.selection = new Set(); + this.bulk = null; + void this.poller.refresh(); + } + /** Workflow names this person has seen: published versions and listed runs. */ + workflowNames() { + const names = new Set(); + for (const version of this.host.workflowVersions.values()) + if (version.name) names.add(version.name); + for (const item of this.runs) + if (!isUnavailableRun(item)) names.add(item.workflow.name); + if (this.filters.workflow) names.add(this.filters.workflow); + return [...names].sort(); + } + versionsOf(name: string | undefined) { + if (!name) return []; + const versions = new Set(); + for (const version of this.host.workflowVersions.values()) + if (version.name === name && version.version) + versions.add(version.version); + for (const item of this.runs) + if (!isUnavailableRun(item) && item.workflow.name === name) + versions.add(item.workflow.version); + if (this.filters.version) versions.add(this.filters.version); + return [...versions].sort(); + } + + // ---------------------------------------------------------------- address + + @HostListener("window:popstate") locationChanged() { + this.addressChanged(); + } + /** The address moved (Back, Forward, a link): the page follows its filters. */ + private addressChanged(routed = false) { + const route = viewFromPath(location.pathname); + if (route?.view !== "runs") return; + // A run's own address carries no list filters: the list keeps its own. + if (route.id && !this.syncPending) return; + if (this.syncPending) { + // The open run is closing for new filters: once the router has landed + // on the list, the address takes those filters. + if (routed && !route.id) { + this.syncPending = false; + void this.syncAddress(); + } + return; + } + const shared = runsFiltersFromQuery(location.search); + if (runsFiltersToQuery(shared) === runsFiltersToQuery(this.filters)) return; + this.filters = { + ...shared, + ...(this.filters.business_key + ? { business_key: this.filters.business_key } + : {}), + ...(this.filters.correlation_key + ? { correlation_key: this.filters.correlation_key } + : {}), + }; + this.changed(); + } + /** Puts the shared filters in the address; a run open first steps back to the list. */ + private async syncAddress() { + if (viewFromPath(location.pathname)?.id) { + this.syncPending = true; + this.host.closeRecord(); + return; + } + const path = viewPath("runs") + runsFiltersToQuery(this.filters); + if (location.pathname + location.search !== path) + await this.router.navigateByUrl(path, { replaceUrl: true }); + } + + // ---------------------------------------------------------------- run detail + + /** The run whose detail is open, or "". */ + openRunId() { + const record = this.host.view === "runs" ? this.host.selectedRecord : null; + return record && typeof record["id"] === "string" ? record["id"] : ""; + } + openRun(run: RunSummary) { + void this.host.open({ + id: run.id, + workflow: run.workflow, + workflow_name: run.workflow.name, + workflow_version: run.workflow.version, + business_key: run.business_key, + correlation_key: run.correlation_key, + state: { status: run.status }, + }); + } + /** A linked run (the one retried, or the caller) opens in the same detail. */ + openLinked(id: string) { + void this.host.viewRun({ id }); + } + /** Starts or stops following the open run as it changes. */ + private followDetail() { + const id = this.openRunId(); + const environment = environmentOf(this.host); + if (!id || !environment || !this.readable) { + if (this.steps) { + this.steps = null; + this.detailStatus = ""; + this.detailPoller.stop(); + } + return; + } + if (this.steps?.runId === id && this.steps.environment === environment) { + const status = this.runStatus(); + if (status && status !== this.detailStatus) { + this.detailStatus = status; + if (terminalRunStatuses.includes(status as RunStatus)) + this.detailPoller.stop(); + } + return; + } + this.steps = { + runId: id, + environment, + items: [], + complete: true, + nextCursor: null, + loading: true, + error: null, + outputs: new Map(), + }; + this.detailStatus = this.runStatus(); + this.detailPoller.stop(); + // The shell reads the run as it opens; its steps are read here at once, + // and both again every 2 s while it is still going. + void this.readSteps(this.steps).catch(() => undefined); + if (!terminalRunStatuses.includes(this.detailStatus as RunStatus)) + this.detailPoller.start(false); + } + /** The open run's status as its detail last read it. */ + private runStatus(): string { + const record = this.host.selectedRecord ?? {}; + const state = isRecord(record["state"]) ? record["state"] : {}; + return typeof state["status"] === "string" ? state["status"] : ""; + } + /** + * Every 2 s while the run is going: its steps, and the run itself. When the + * run changed, the shell reads its whole detail again, as Refresh does. + */ + private async pollDetail(): Promise { + const state = this.steps; + if (!state) return false; + const [view] = await Promise.all([ + this.host.api.request( + `${state.environment}/runs/${encodeURIComponent(state.runId)}`, + ), + this.readSteps(state), + ]); + if (state !== this.steps || this.openRunId() !== state.runId) return false; + if (JSON.stringify(view) !== JSON.stringify(this.host.selectedRecord)) + await this.host.readDetail(); + if (isTerminal(view)) this.detailPoller.stop(); + this.cdr.markForCheck(); + return true; + } + /** Reads the first page of the open run's steps. */ + private async readSteps(state: StepsState) { + try { + const page = await listRunSteps( + { + environment: state.environment, + request: (path) => this.host.api.request(path), + }, + state.runId, + { limit: STEP_PAGE }, + ); + if (state !== this.steps) return; + state.items = page.items; + state.complete = page.complete; + state.nextCursor = page.next_cursor; + state.error = null; + } catch (error) { + if (state !== this.steps) return; + state.error = describeError(error); + throw error; + } finally { + if (state === this.steps) { + state.loading = false; + this.stepsView$ = null; + this.cdr.markForCheck(); + } + } + } + private async moreSteps(state: StepsState) { + if (!state.nextCursor || state.loading) return; + state.loading = true; + this.stepsView$ = null; + try { + const page = await listRunSteps( + { + environment: state.environment, + request: (path) => this.host.api.request(path), + }, + state.runId, + { limit: STEP_PAGE, cursor: state.nextCursor }, + ); + if (state !== this.steps) return; + state.items = [...state.items, ...page.items]; + state.nextCursor = page.next_cursor; + state.complete = state.complete && page.complete; + } catch (error) { + if (state === this.steps) state.error = describeError(error); + } finally { + if (state === this.steps) { + state.loading = false; + this.stepsView$ = null; + this.cdr.markForCheck(); + } + } + } + /** One step's output, only when a person asks for it. */ + private async readOutput(state: StepsState, item: StepFact) { + const key = instanceKey(item); + state.outputs.set(key, { kind: "loading" }); + this.stepsView$ = null; + try { + let cursor: string | undefined; + for (let page = 0; page < MAX_PAGES; page++) { + const answer = await listRunSteps( + { + environment: state.environment, + request: (path) => this.host.api.request(path), + }, + state.runId, + { + step: item.node_id, + include: "output", + limit: 500, + ...(cursor ? { cursor } : {}), + }, + ); + if (state !== this.steps) return; + const found = answer.items.find( + (fact) => fact.instance_key === item.instance_key, + ); + if (found || !answer.next_cursor) { + state.outputs.set(key, { + kind: "ready", + output: found ? stepOutput(found) : { kind: "none" }, + }); + return; + } + cursor = answer.next_cursor; + } + state.outputs.set(key, { kind: "ready", output: { kind: "none" } }); + } catch (error) { + if (state === this.steps) + state.outputs.set(key, { kind: "error", error: describeError(error) }); + } finally { + if (state === this.steps) { + this.stepsView$ = null; + this.cdr.markForCheck(); + } + } + } + /** What the run detail shows in its Steps section; the same object until something changes. */ + stepsView(): RunStepsView | null { + const state = this.steps; + if (!state) return null; + const second = Math.floor(Date.now() / 1000); + if (this.stepsView$?.state === state && this.stepsView$.now === second) + return this.stepsView$.view; + const view: RunStepsView = { + runId: state.runId, + items: state.items, + complete: state.complete, + nextCursor: state.nextCursor, + loading: state.loading, + error: state.error, + now: new Date(), + outputs: new Map(state.outputs), + loadMore: () => void this.moreSteps(state), + retry: () => { + state.loading = true; + state.error = null; + this.stepsView$ = null; + void this.readSteps(state).catch(() => undefined); + }, + readOutput: (item) => void this.readOutput(state, item), + openRun: (id) => this.openLinked(id), + }; + this.stepsView$ = { state, now: second, view }; + return view; + } + + // ---------------------------------------------------------------- selection and bulk actions + + canBulk() { + return this.host.can("run.cancel") || this.host.can("run.retry"); + } + select(id: string, on: boolean) { + const next = new Set(this.selection); + if (on && next.size < BULK_LIMIT) next.add(id); + else next.delete(id); + this.selection = next; + } + selectable() { + return this.runs + .filter((item) => !isUnavailableRun(item)) + .slice(0, BULK_LIMIT); + } + allSelected() { + const rows = this.selectable(); + return rows.length > 0 && rows.every((item) => this.selection.has(item.id)); + } + selectAll(on: boolean) { + this.selection = on + ? new Set(this.selectable().map((item) => item.id)) + : new Set(); + } + clearSelection() { + this.selection = new Set(); + } + /** Cancel runs: one reason for all, sent one run at a time, each with its own answer. */ + async bulkCancel() { + const ids = [...this.selection]; + if (!ids.length || this.bulkRunning || !this.host.can("run.cancel")) return; + const values = await this.host.dialogs.form({ + title: + ids.length === 1 ? "Cancel this run?" : `Cancel ${ids.length} runs?`, + message: + "Steps that already started may still finish outside Weave. Give a reason for the audit log.", + confirmLabel: ids.length === 1 ? "Cancel run" : "Cancel runs", + cancelLabel: "Keep running", + danger: true, + fields: [ + { + name: "reason", + label: "Reason", + required: true, + multiline: true, + validate: reasonProblem, + }, + ], + }); + if (!values) return; + await this.runBulk( + "Cancel runs", + ids, + async (run, environment) => { + if (isTerminalRun(run)) + return { ok: false, text: "Not canceled: it has already finished." }; + await this.host.safeMutation( + `${environment}/runs/${encodeURIComponent(run.id)}/cancel`, + "POST", + { reason: values["reason"] }, + undefined, + "Cancel run", + ); + return { ok: true, text: "Canceled." }; + }, + "run.cancel", + ); + } + /** Retry runs on the same version, with each run's own input. */ + async bulkRetry() { + const ids = [...this.selection]; + if (!ids.length || this.bulkRunning || !this.host.can("run.retry")) return; + const confirmed = await this.host.dialogs.confirm({ + title: ids.length === 1 ? "Retry this run?" : `Retry ${ids.length} runs?`, + message: + "Each run starts again on the same version, with the input it started with.", + confirmLabel: ids.length === 1 ? "Retry run" : "Retry runs", + cancelLabel: "Keep as is", + }); + if (!confirmed) return; + await this.runBulk( + "Retry runs", + ids, + async (run, environment) => { + if (!isTerminalRun(run)) + return { ok: false, text: "Not retried: it has not finished." }; + const base = `${environment}/runs/${encodeURIComponent(run.id)}`; + const view = await this.host.api.request(base); + const activation = isRecord(view["activation"]) + ? view["activation"] + : {}; + if (typeof activation["id"] !== "string") + return { ok: false, text: "Not retried: its version can't be read." }; + const created = await this.host.safeMutation( + `${base}/retry`, + "POST", + { + activation_id: activation["id"], + input: retryInput(view), + ...(run.business_key ? { business_key: run.business_key } : {}), + ...(run.correlation_key + ? { correlation_key: run.correlation_key } + : {}), + }, + undefined, + "Retry run", + ); + return { ok: true, text: `Retried as run ${shortId(created["id"])}.` }; + }, + "run.retry", + ); + } + private async runBulk( + title: string, + ids: string[], + act: ( + run: RunSummary, + environment: string, + ) => Promise<{ ok: boolean; text: string }>, + capability: string, + ) { + const environment = environmentOf(this.host); + const key = scopeKeyOf(this.host); + const runs = new Map( + this.runs + .filter((item) => !isUnavailableRun(item)) + .map((item) => [item.id, item as RunSummary]), + ); + const results: BulkResult[] = ids.map((id) => ({ + id, + ok: false, + text: "Waiting…", + })); + // The results on screen; a newer bulk action replaces them. + let shown = { title, results: [...results] }; + this.bulk = shown; + this.bulkRunning = true; + this.cdr.markForCheck(); + let stopped = ""; + try { + for (const [index, id] of ids.slice(0, BULK_LIMIT).entries()) { + if (key !== scopeKeyOf(this.host)) + stopped = "Not sent: the workspace changed."; + const run = runs.get(id); + let result: { ok: boolean; text: string }; + if (stopped) result = { ok: false, text: stopped }; + else if (!run) + result = { ok: false, text: "Not sent: it is no longer listed." }; + else + try { + result = await act(run, environment); + } catch (error) { + if (this.host.unknownCommand) { + stopped = + "Not sent: Studio didn't get an answer for an earlier run. Use Check now first."; + result = { + ok: false, + text: "No answer: it may have worked. Use Check now.", + }; + } else + result = { + ok: false, + text: explainRefusal(error, capability).message, + }; + } + results[index] = { id, ...result }; + if (this.bulk === shown) { + shown = { title, results: [...results] }; + this.bulk = shown; + } + this.cdr.markForCheck(); + } + } finally { + this.bulkRunning = false; + this.selection = new Set(); + this.cdr.markForCheck(); + } + if (key === scopeKeyOf(this.host)) void this.poller.refresh(); + } + + // ---------------------------------------------------------------- cells + + unavailable(item: RunSummaryItem) { + return isUnavailableRun(item); + } + summaryOf(item: RunSummaryItem) { + return item as RunSummary; + } + outcomeOf(run: RunSummary) { + return runOutcome(run); + } + tone(tone: Parameters[0]) { + return toneAttribute(tone); + } + relative(value: string) { + return timeRelative(value); + } + duration(run: RunSummary) { + return liveDuration(run, new Date()); + } + incidentsLabel(count: number) { + return `${count} active ${count === 1 ? "incident" : "incidents"}`; + } + async copy(id: string) { + if (await copyText(id)) this.host.notify("Copied the run ID."); + else + this.host.notify("Studio couldn't copy. Select the run ID and copy it."); + } + value(event: Event) { + return (event.target as HTMLInputElement | HTMLSelectElement).value; + } + checked(event: Event) { + return (event.target as HTMLInputElement).checked; + } +} diff --git a/studio/src/app/operate/runs/runs.css b/studio/src/app/operate/runs/runs.css new file mode 100644 index 00000000..be91525f --- /dev/null +++ b/studio/src/app/operate/runs/runs.css @@ -0,0 +1,693 @@ +/* +Copyright 2026 Firefly Software Foundation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +Author: Firefly Software Foundation +SPDX-License-Identifier: Apache-2.0 +*/ +/* Operate › Runs and a run's steps, in design tokens only. A fine pointer on + a wide screen gets 36 px controls, 32 px rows (controls inside a row stay + within it, separator included) and 24/12/4 spacing; a narrow screen or a + touch pointer gets 44 px targets and 16/8 spacing. */ +:host { + --section-gap: var(--space-6); + --group-gap: var(--space-3); + --inline-gap: var(--space-1); + --row: var(--control-sm); + --target: var(--control-md); + --cell: calc(var(--row) - var(--space-1)); + display: block; + min-width: 0; +} +@media (max-width: 767px), (pointer: coarse) { + :host { + --section-gap: var(--space-4); + --group-gap: var(--space-2); + --row: var(--control-touch); + --target: var(--control-touch); + --cell: var(--control-touch); + } +} +weave-operate-state { + display: block; + margin-bottom: var(--group-gap); +} +.numeric-cell { + text-align: right; +} + +/* Heading actions: freshness, Auto-refresh and Start run on one line. */ +.toggle-field { + min-height: var(--target); + font: var(--type-small); + color: var(--text); +} + +/* Filters: labelled fields in one wrapping row, statuses as toggles, the + rest behind More filters. Each change applies at once. */ +.run-filters { + display: grid; + gap: var(--group-gap); + margin-bottom: var(--section-gap); + min-width: 0; +} +.filter-body { + display: grid; + gap: var(--group-gap); + min-width: 0; +} +.filters-toggle { + display: none; + justify-self: start; + min-height: var(--target); +} +/* Narrow screens fold the filters behind "Filters (n)"; open, each field + spans the pane, so every field shares its left and right edge. */ +@media (max-width: 767px) { + .filters-toggle { + display: inline-flex; + } + .run-filters:not(.open) .filter-body { + display: none; + } + .filter-row > .field { + flex: 1 1 100%; + } + .filter-row select, + .filter-row input:not([type="checkbox"]), + .key-field input { + width: 100%; + } +} +.filter-row { + display: flex; + flex-wrap: wrap; + align-items: flex-end; + gap: var(--group-gap); + min-width: 0; +} +.filter-row .field { + gap: var(--inline-gap); + min-width: 0; +} +.filter-row select, +.filter-row input:not([type="checkbox"]) { + min-height: var(--target); + max-width: 100%; +} +.key-field input { + width: calc(var(--space-12) * 4); +} +.status-filter { + display: flex; + flex-wrap: wrap; + gap: var(--inline-gap); + min-width: 0; +} +.status-choice { + min-height: var(--target); + padding-inline: var(--space-3); + font: var(--type-small); + font-weight: 500; + color: var(--muted); +} +.status-choice[aria-pressed="true"] { + border-color: var(--accent); + background: var(--selected); + color: var(--text); + font-weight: 600; +} +.filter-row .checkbox-field { + min-height: var(--target); + font: var(--type-small); + color: var(--text); +} +.filter-row > button.tertiary { + min-height: var(--target); +} +.disclosure-chevron { + transform: rotate(90deg); + transition: transform var(--dur-fast) var(--ease); +} +.disclosure-chevron.open { + transform: rotate(-90deg); +} +.more-filters { + display: flex; + flex-wrap: wrap; + align-items: flex-end; + gap: var(--group-gap) var(--section-gap); + padding: var(--group-gap); + border: thin solid var(--border); + border-radius: var(--radius-md); + background: var(--surface); +} +.more-filters .field { + gap: var(--inline-gap); +} +.more-filters select, +.more-filters input:not([type="checkbox"]) { + min-height: var(--target); +} +.more-checks { + display: flex; + flex-wrap: wrap; + gap: var(--inline-gap) var(--group-gap); +} +.more-checks .checkbox-field { + min-height: var(--target); + font: var(--type-small); +} +.filter-chip { + display: flex; + flex-wrap: wrap; + align-items: center; + gap: var(--space-2); + margin: 0; + font: var(--type-small); + color: var(--muted); +} + +/* Bulk actions on the selected runs, and their result per run. */ +.bulk-bar { + display: flex; + flex-wrap: wrap; + align-items: center; + gap: var(--space-2); + margin-bottom: var(--group-gap); + padding: var(--space-2) var(--group-gap); + border: thin solid var(--accent); + border-radius: var(--radius-md); + background: var(--selected); +} +.bulk-count { + margin-right: auto; + font: var(--type-label); +} +.bulk-results { + display: grid; + gap: var(--space-2); + margin-bottom: var(--group-gap); + padding: var(--group-gap); + border: thin solid var(--border); + border-radius: var(--radius-md); + background: var(--surface); +} +.bulk-results-head { + display: flex; + align-items: center; + justify-content: space-between; + gap: var(--space-2); +} +.bulk-results h2 { + margin: 0; + font: var(--type-label); +} +.bulk-results ul { + display: grid; + margin: 0; + padding: 0; + list-style: none; +} +.bulk-result { + display: grid; + grid-template-columns: var(--space-4) auto minmax(0, 1fr); + align-items: center; + gap: var(--space-2); + min-height: var(--row); + font: var(--type-small); + color: var(--text); +} +.bulk-result[data-ok="true"] weave-icon { + color: var(--success-ink); +} +.bulk-result[data-ok="false"] weave-icon { + color: var(--warning-ink); +} + +/* The list and, beside it, the open run. */ +.runs-layout { + display: grid; + grid-template-columns: minmax(0, 1fr); + align-items: start; + gap: var(--section-gap); + min-width: 0; +} +@media (min-width: 1025px) { + .runs-layout.run-focus { + grid-template-columns: calc(var(--space-10) * 6) minmax(0, 1fr); + gap: var(--space-4); + } +} +.runs-list { + display: grid; + min-width: 0; + container: runs / inline-size; +} +.run-detail-panel { + min-width: 0; +} +.list-limit { + margin: var(--group-gap) 0 0; + font: var(--type-small); + color: var(--muted); +} + +/* The table: one 32 px line per run on a wide pane. Every cell is centred on + the row, and numbers use tabular figures. */ +.runs-table { + --cols: var(--target) minmax(0, 1.2fr) minmax(0, 1.6fr) minmax(0, 1fr) + minmax(0, 0.8fr) minmax(0, 0.7fr) minmax(0, 0.6fr) minmax(0, 1fr) + minmax(0, 0.9fr); +} +.runs-table .table-head, +.runs-table .resource-row { + grid-template-columns: var(--cols); + grid-template-areas: "select status workflow origin started duration incidents key run"; + gap: var(--group-gap); + min-height: var(--row); + padding: 0 var(--group-gap); +} +.runs-table .resource-row { + position: relative; + cursor: pointer; + font: var(--type-small); + color: var(--muted); +} +.runs-table .resource-row.unavailable { + cursor: default; +} +.runs-table .table-head { + min-height: var(--row); +} +.runs-table [role="cell"], +.runs-table [role="columnheader"] { + min-width: 0; +} +.cell-select { + grid-area: select; +} +.cell-status { + grid-area: status; +} +.cell-workflow { + grid-area: workflow; +} +.cell-origin { + grid-area: origin; +} +.cell-started { + grid-area: started; +} +.cell-duration { + grid-area: duration; +} +.cell-incidents { + grid-area: incidents; +} +.cell-key { + grid-area: key; + overflow-wrap: anywhere; +} +.cell-run { + grid-area: run; +} +.cell-meta { + grid-area: meta; + display: none; +} +/* An unavailable run explains itself across the columns it cannot fill. */ +.cell-explain { + grid-column: origin-start / key-end; +} +.select-box { + position: relative; + z-index: 1; + display: inline-flex; + align-items: center; + justify-content: center; + width: var(--target); + height: var(--cell); + cursor: pointer; +} +.select-box input { + width: var(--space-4); + height: var(--space-4); + margin: 0; +} +.cell-status.badges { + display: flex; + flex-wrap: wrap; + align-items: center; + gap: var(--inline-gap); +} +.handled { + font: var(--type-caption); + color: var(--warning-ink); +} +.cell-workflow { + display: flex; + flex-wrap: wrap; + align-items: center; + gap: var(--inline-gap) var(--space-2); +} +.runs-table .row-text { + font: var(--type-label); + color: var(--text); +} +/* The workflow name opens the run; its target covers the whole row. */ +.runs-table .row-open { + min-height: 0; + padding: 0; + border: 0; + border-radius: 0; + background: none; + color: var(--text); + font: var(--type-label); + text-align: left; + white-space: normal; + overflow-wrap: anywhere; +} +.runs-table .row-open:hover { + background: none; +} +.runs-table .row-open::after { + content: ""; + position: absolute; + inset: 0; +} +.runs-table .row-open:focus-visible { + outline: none; +} +.runs-table .row-open:focus-visible::after { + outline: medium solid var(--focus); + outline-offset: calc(var(--inline-gap) * -1); +} +.version-tag { + flex: none; +} +.origin-link, +.copy-id { + position: relative; + z-index: 1; +} +.origin-link { + min-height: var(--cell); + font: var(--type-small); +} +.cell-run { + display: flex; + align-items: center; + gap: var(--inline-gap); +} +.cell-run code { + font: var(--type-mono); + color: var(--text); +} +.runs-table .copy-id { + width: var(--cell); + height: var(--cell); + min-height: var(--cell); +} +.muted { + color: var(--subtle); +} + +/* A narrower pane (a run open beside the list, a small window, a phone): + each run reads as two lines, its workflow and status, then the details. */ +@container runs (max-width: 1099px) { + .runs-table .table-head, + .runs-table .resource-row { + grid-template-columns: var(--target) minmax(0, 1fr); + grid-template-areas: + "select workflow" + ". status" + ". meta"; + row-gap: var(--inline-gap); + padding-block: var(--space-2); + } + .runs-table .table-head { + grid-template-areas: "select workflow"; + padding-block: 0; + } + .runs-table + :is( + .cell-origin, + .cell-started, + .cell-duration, + .cell-incidents, + .cell-key, + .cell-run + ), + .runs-table .table-head .cell-status { + display: none; + } + .runs-table .cell-status:empty { + display: none; + } + .runs-table .cell-meta { + display: flex; + flex-wrap: wrap; + align-items: center; + gap: 0 var(--group-gap); + } + .runs-table .table-head .cell-meta { + display: none; + } + /* The shared narrow table rules hide a third column title; here it is the + workflow's, which names the list. */ + .runs-table .table-head .cell-workflow { + display: flex; + } + .meta-key { + overflow-wrap: anywhere; + } +} +/* With a run open beside it, the list is a column of names and statuses. */ +@media (min-width: 1025px) { + .run-focus .runs-table .resource-row .cell-meta { + display: none; + } + .run-focus .runs-table .resource-row { + grid-template-areas: + "select workflow" + ". status"; + } +} + +/* A run's steps: bars for queue wait and run time, loop iterations under + their step, Input/Output on request, and logs not served yet. */ +.steps { + display: grid; + gap: var(--group-gap); + min-width: 0; + margin: var(--section-gap) 0; + container: steps / inline-size; +} +.steps-head { + display: flex; + flex-wrap: wrap; + align-items: flex-end; + justify-content: space-between; + gap: var(--space-2) var(--group-gap); +} +.steps-head h3 { + margin: 0; + font: var(--type-title-sm); + line-height: var(--target); +} +.steps-tabs { + margin: 0; +} +.steps-tabs button { + min-height: var(--target); + padding-inline: var(--space-2); +} +.steps .status-pill { + margin: 0; +} +.steps-note { + margin: 0; + font: var(--type-small); + color: var(--muted); +} +.steps-loading { + display: grid; + gap: var(--inline-gap); +} +.step-skeleton { + display: block; + height: var(--row); + border-radius: var(--radius-sm); + background: var(--skeleton); +} +.step-groups, +.step-rows, +.step-outputs { + display: grid; + margin: 0; + padding: 0; + list-style: none; +} +.step-groups { + border: thin solid var(--border); + border-radius: var(--radius-md); + background: var(--surface); +} +.step-group { + border-bottom: thin solid var(--line); +} +.step-group:last-child { + border-bottom: 0; +} +.step-group-name { + display: flex; + align-items: center; + gap: var(--space-2); + min-height: var(--row); + margin: 0; + padding: 0 var(--group-gap); + font: var(--type-label); + color: var(--text); +} +.step-group-name .numeric { + font: var(--type-caption); + color: var(--muted); +} +.step-row { + display: grid; + grid-template-columns: + calc(var(--space-12) * 2.5) calc(var(--space-12) * 2) minmax(0, 1fr) + calc(var(--space-12) * 1.5) calc(var(--space-12) * 3); + grid-template-areas: "name status track time link"; + align-items: center; + gap: var(--group-gap); + min-height: var(--row); + padding: 0 var(--group-gap); +} +/* Iterations indent their names, never their bars: every bar shares one scale. */ +.step-row.nested .step-name { + padding-left: var(--space-4); +} +.step-name { + grid-area: name; + min-width: 0; + font: var(--type-small); + color: var(--text); + overflow-wrap: anywhere; +} +.step-row > .status-pill { + grid-area: status; +} +.step-track { + grid-area: track; + position: relative; + height: var(--space-2); + border-radius: var(--radius-xs); + background: var(--hover); +} +.step-wait, +.step-run { + position: absolute; + top: 0; + bottom: 0; +} +.step-wait { + background: var(--border-hover); + border-radius: var(--radius-xs) 0 0 var(--radius-xs); +} +.step-run { + background: var(--series-2); + border-radius: 0 var(--radius-xs) var(--radius-xs) 0; +} +.step-run[data-tone="success"] { + background: var(--success); +} +.step-run[data-tone="danger"] { + background: var(--danger); +} +.step-run[data-tone="warning"] { + background: var(--warning); +} +.step-run.open { + background: var(--live); +} +.step-time { + grid-area: time; + font: var(--type-small); + color: var(--muted); + text-align: right; +} +.step-link { + grid-area: link; +} +.step-link .text-link { + min-height: var(--row); + font: var(--type-small); +} +.step-output { + display: grid; + gap: var(--space-2); + padding: var(--space-2) 0; + border-bottom: thin solid var(--line); +} +.step-output:last-child { + border-bottom: 0; +} +.step-output-head { + display: flex; + align-items: center; + justify-content: space-between; + gap: var(--space-2); + min-height: var(--row); +} +.step-output-head .step-label { + font: var(--type-small); + color: var(--text); +} +.step-output-head small { + margin-left: var(--space-2); + color: var(--muted); +} +.step-output-head button { + min-height: var(--target); +} +.step-output pre { + max-height: calc(var(--space-12) * 5); + margin: 0; + overflow: auto; + font: var(--type-mono); +} +/* In a narrow detail the bar takes its own line under the step. */ +@container steps (max-width: 559px) { + .step-row { + grid-template-columns: minmax(0, 1fr) auto auto; + grid-template-areas: + "name status time" + "track track track" + "link link link"; + row-gap: var(--space-1); + padding-block: var(--space-2); + } + .step-link:empty { + display: none; + } +} + +/* Counts and durations in tabular figures; after every font shorthand above, + which would reset them. */ +.numeric, +.step-time, +.bulk-count { + font-variant-numeric: tabular-nums; +} diff --git a/studio/src/app/operate/runs/step-timeline.ts b/studio/src/app/operate/runs/step-timeline.ts new file mode 100644 index 00000000..9622b761 --- /dev/null +++ b/studio/src/app/operate/runs/step-timeline.ts @@ -0,0 +1,420 @@ +/* +Copyright 2026 Firefly Software Foundation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +Author: Firefly Software Foundation +SPDX-License-Identifier: Apache-2.0 +*/ +// A run's steps from runs.steps: one bar per step instance, showing how long +// it waited in the queue and how long it ran, loop iterations grouped under +// their step, and a link to the run a step called. Outputs are read only when +// the person opens Input/Output and asks for one step; logs are not served +// yet, so the Logs tab says so without asking the platform. +import { + ChangeDetectionStrategy, + Component, + computed, + input, + signal, +} from "@angular/core"; +import type { PlainError } from "../../errors"; +import { durationText, shortId } from "../../format"; +import { statusTone, toneAttribute } from "../../status-labels"; +import { + stepOutput, + type OmissionReason, + type StepFact, + type StepOutput, +} from "../run-contracts"; + +/** Steps grouped by their static step ID, in the order the steps first appear. */ +export function timelineGroups(items: readonly StepFact[]) { + const grouped = new Map(); + for (const item of items) + grouped.set(item.node_id, [...(grouped.get(item.node_id) ?? []), item]); + return [...grouped].map(([node_id, items]) => ({ node_id, items })); +} + +/** "Iteration 2 › 1" from the loop indexes, else the instance key as given. */ +export function instanceLabel(item: StepFact): string { + if (item.iteration.length) return `Iteration ${item.iteration.join(" › ")}`; + return item.instance_key; +} + +/** The time the bars share: the first scheduled or started step to the last end (or now). */ +export function timelineSpan( + items: readonly StepFact[], + now: Date, +): { start: number; end: number } | null { + let start = Infinity; + let end = -Infinity; + for (const item of items) { + const begin = Date.parse(item.scheduled_at ?? item.started_at ?? ""); + if (!Number.isFinite(begin)) continue; + start = Math.min(start, begin); + const finish = item.ended_at ? Date.parse(item.ended_at) : now.getTime(); + end = Math.max(end, Number.isFinite(finish) ? finish : begin); + } + if (!Number.isFinite(start)) return null; + return { start, end: Math.max(end, start + 1) }; +} + +/** One instance's bar, in percent of the span: where it starts, its wait and its run. */ +export function stepBar( + item: StepFact, + span: { start: number; end: number }, + now: Date, +): { offset: number; wait: number; run: number; open: boolean } | null { + const scheduled = Date.parse(item.scheduled_at ?? item.started_at ?? ""); + if (!Number.isFinite(scheduled)) return null; + const started = Date.parse(item.started_at ?? ""); + const open = !item.ended_at; + const ended = open ? now.getTime() : Date.parse(item.ended_at!); + const width = span.end - span.start; + const percent = (value: number) => (Math.max(0, value) / width) * 100; + const runFrom = Number.isFinite(started) ? started : ended; + return { + offset: percent(scheduled - span.start), + wait: percent(Math.min(runFrom, ended) - scheduled), + run: percent(ended - runFrom), + open, + }; +} + +const omissionText: Record = { + classified_secret: "This output is classified as secret.", + classification_unavailable: + "Studio can't tell whether this output is safe to show.", + uncertain_derived: "This output may be derived from protected data.", + confidential_text: "This output holds confidential text.", + resource_limit: "This output is too large to show.", +}; + +/** One step instance's output, as the person asked for it. */ +export type StepOutputState = + | { kind: "loading" } + | { kind: "error"; error: PlainError } + | { kind: "ready"; output: StepOutput }; + +/** What the Runs page gives the run detail for its Steps section. */ +export interface RunStepsView { + runId: string; + items: StepFact[]; + /** false when some steps ran before the platform recorded their timings. */ + complete: boolean; + nextCursor: string | null; + loading: boolean; + error: PlainError | null; + /** The server's clock, for steps still running. */ + now: Date; + outputs: ReadonlyMap; + loadMore(): void; + retry(): void; + readOutput(item: StepFact): void; + openRun(id: string): void; +} + +/** The key an output is kept under. */ +export const instanceKey = (item: StepFact) => + `${item.node_id}\u0000${item.instance_key}`; + +const stepWords: Record = { + scheduled: "Scheduled", + running: "Running", + waiting: "Waiting", + succeeded: "Succeeded", + failed: "Failed", + cancelled: "Canceled", + timed_out: "Timed out", +}; + +type Tab = "timeline" | "io" | "logs"; + +@Component({ + selector: "weave-step-timeline", + standalone: true, + changeDetection: ChangeDetectionStrategy.OnPush, + styleUrls: ["../operate.css", "./runs.css"], + template: `@let v = view(); +
+
+

Steps

+
+ @for (tab of tabs; track tab[0]) { + + } +
+
+
+ @if (selected() === "logs") { +

+ Logs aren't available from this platform yet. Export history keeps + the run's events. +

+ } @else if (v.loading && !v.items.length) { +
+ Loading steps… + @for (row of [0, 1, 2]; track row) { + + } +
+ } @else if (v.error && !v.items.length) { + + } @else if (!v.items.length) { +

No steps recorded yet.

+ } @else { + @if (!v.complete) { +

+ Some steps ran before this platform recorded step timings: they + are listed without times. +

+ } + @if (selected() === "timeline") { +
    + @for (group of groups(); track group.node_id) { +
  1. + @if (group.items.length > 1) { +

    + {{ group.node_id }} + {{ group.items.length }} instances +

    + } +
      + @for (item of group.items; track item.instance_key) { + @let bar = barOf(item); +
    1. + {{ + group.items.length > 1 && item.instance_key + ? label(item) + : item.node_id + }}{{ word(item.status) }} + @if (bar) { + + }{{ took(item) }} + @if (item.child_run_id) { + + } + +
      2. + } +
    +
    2. + } +
+ } @else { +
    + @for ( + item of v.items; + track item.node_id + "|" + item.instance_key + ) { + @let shown = outputOf(item); +
  • +
    + {{ item.node_id }} + @if (label(item)) { + {{ label(item) }} + } + + @if (!shown || shown.kind === "error") { + + } +
    + @if (shown?.kind === "loading") { +

    Loading output…

    + } @else if (shown?.kind === "error") { + + } @else if (shown?.kind === "ready") { + @let output = readyOf(shown); + @if (output.kind === "value") { + 
    {{ pretty(output.value) }}
    + } @else if (output.kind === "omitted") { +

    {{ omitted(output.reason) }}

    + } @else { +

    This step recorded no output.

    + } + } +
    • + } +
+ } + @if (v.nextCursor) { + + } + } +
+
`, +}) +export class StepTimeline { + view = input.required(); + readonly tabs: [Tab, string][] = [ + ["timeline", "Timeline"], + ["io", "Input/Output"], + ["logs", "Logs"], + ]; + readonly selected = signal("timeline"); + readonly short = shortId; + readonly label = instanceLabel; + readonly groups = computed(() => timelineGroups(this.view().items)); + private readonly span = computed(() => + timelineSpan(this.view().items, this.view().now), + ); + barOf(item: StepFact) { + const span = this.span(); + return span ? stepBar(item, span, this.view().now) : null; + } + word(status: string) { + return stepWords[status] ?? status; + } + tone(status: string) { + return toneAttribute( + statusTone("run", status === "scheduled" ? "queued" : status), + ); + } + took(item: StepFact) { + if (item.duration_ms !== null) return durationText(item.duration_ms); + if (!item.started_at || item.ended_at) return "—"; + const started = Date.parse(item.started_at); + return Number.isFinite(started) + ? durationText(Math.max(0, this.view().now.getTime() - started)) + : "—"; + } + timing(item: StepFact) { + const parts = []; + if (item.queue_wait_ms !== null) + parts.push(`waited ${durationText(item.queue_wait_ms)}`); + const took = this.took(item); + if (took !== "—") + parts.push(item.ended_at ? `ran ${took}` : `running for ${took}`); + return parts.length + ? `${item.node_id}: ${parts.join(", ")}` + : `${item.node_id}: no timings recorded`; + } + outputOf(item: StepFact) { + return this.view().outputs.get(instanceKey(item)) ?? null; + } + errorOf(state: StepOutputState) { + return state.kind === "error" ? state.error.message : ""; + } + readyOf(state: StepOutputState): StepOutput { + return state.kind === "ready" ? state.output : { kind: "none" }; + } + omitted(reason: OmissionReason) { + return omissionText[reason] ?? "This output can't be shown."; + } + pretty(value: unknown) { + return JSON.stringify(value, null, 2); + } + /** Arrow keys move between the tabs (WAI-ARIA tabs). */ + move(event: KeyboardEvent) { + const order = this.tabs.map((tab) => tab[0]); + const index = order.indexOf(this.selected()); + const step = + event.key === "ArrowRight" ? 1 : event.key === "ArrowLeft" ? -1 : 0; + if (!step) return; + event.preventDefault(); + const next = order[(index + step + order.length) % order.length]; + this.selected.set(next); + (event.currentTarget as HTMLElement).parentElement + ?.querySelector(`#steps-tab-${next}`) + ?.focus(); + } +} + +/** The output a step's runs.steps?include=output answer carries for it. */ +export function outputFromPage( + item: StepFact, + page: readonly StepFact[], +): StepOutput { + const found = page.find( + (fact) => + fact.node_id === item.node_id && fact.instance_key === item.instance_key, + ); + return found ? stepOutput(found) : { kind: "none" }; +} diff --git a/studio/src/app/operations/records-view.ts b/studio/src/app/operations/records-view.ts index ac903b6a..1af18d91 100644 --- a/studio/src/app/operations/records-view.ts +++ b/studio/src/app/operations/records-view.ts @@ -15,10 +15,11 @@ limitations under the License. Author: Firefly Software Foundation SPDX-License-Identifier: Apache-2.0 */ -// The list pages (Workflows, Runs, My tasks, Email, Connections): -// heading, filters, a table with per-view columns and the detail panel. The -// shell (App) owns the state and the commands; this view renders them and -// loads lazily (@defer), outside the initial bundle. +// The list pages (Workflows, My tasks, Email, Connections): heading, +// filters, a table with per-view columns and the detail panel. The shell +// (App) owns the state and the commands; this view renders them and loads +// lazily (@defer), outside the initial bundle. Runs has its own page under +// Operate. import { ChangeDetectionStrategy, Component, input } from "@angular/core"; import { NgTemplateOutlet } from "@angular/common"; import type { App, View } from "../app"; @@ -32,7 +33,6 @@ import { ConnectionDetail } from "../integrations/connection-detail"; import { NewMenu } from "../templates/new-menu"; import { localKeepLabel, type LocalDraftEntry } from "../local-drafts"; import { isDesktopShell } from "../export-file"; -import { OperateState } from "../operate/operate-state"; import { dateWithRelative, isoTime, @@ -46,7 +46,6 @@ import { toneAttribute, type Tone, } from "../status-labels"; -import { RunDetail } from "./run-detail"; import { columns, contextFacts, @@ -54,8 +53,6 @@ import { rowSecondary, rowStatus, rowTitle, - runDuration, - runStarted, when, type ListView, type Pill, @@ -63,14 +60,12 @@ import { } from "./records-model"; type Json = Record; +/** The lists this view shows; Runs has its own Operate page. */ +type ShownView = Exclude; /** The page's title and one-line purpose. */ -const pages: Record = { +const pages: Record = { workflows: ["Workflows", "Design, publish, and activate your workflows."], - runs: [ - "Runs", - "Follow every run in this workspace and step in when one needs you.", - ], tasks: ["My tasks", "Tasks waiting for a decision from you or your team."], email: [ "Email", @@ -79,11 +74,7 @@ const pages: Record = { connections: ["Connections", ""], }; /** Local mode: what each operations page needs a platform for. */ -const needPlatform: Partial> = { - runs: [ - "Runs appear here once you connect", - "Runs happen on a platform. Connect one to start and follow runs.", - ], +const needPlatform: Partial> = { tasks: [ "Your tasks appear here once you connect", "People get tasks when a run reaches a human task step.", @@ -97,19 +88,12 @@ const needPlatform: Partial> = { "Connect to add the APIs and systems your workflows use.", ], }; -const icons: Record = { +const icons: Record = { workflows: "workflows", - runs: "runs", tasks: "humanTask", email: "email", connections: "connections", }; -const runStatusFilters: [string, string][] = [ - ["", "All"], - ["waiting", "Waiting"], - ["failed", "Failed"], - ["succeeded", "Succeeded"], -]; @Component({ selector: "weave-records-view", @@ -123,9 +107,7 @@ const runStatusFilters: [string, string][] = [ TaskForm, ConnectionDetail, NewMenu, - RunDetail, NgTemplateOutlet, - OperateState, ], styleUrl: "./records-view.css", template: `@let h = host(); @@ -168,10 +150,6 @@ const runStatusFilters: [string, string][] = [ New AI connection } - } @else if (v === "runs" && h.can("run.start")) { - } -
- -
- @for (option of statusFilters; track option[0]) { - - } -
- -
- @if (moreFilters) { -
-
- - -

Matches the exact key.

-
- -
- } - - } @if (v === "workflows") {
}
- @if (v !== "runs") { - - } + @if (v === "tasks") {
- + + @for (name of hints.workflows; track name) { + + } + +
+
+ + + + @for (step of hints.steps; track step) { + + } + +
+
+ + +
+
+ + +
+
+ +
+ {{ countText() }} + @if (filterCount()) { + + }
+ @if (rangeProblem) { + + } @if (error) { } + @if (detailProblem; as failed) { + @if (failed.status === 403) { + + } @else if (failed.status === 404) { + + } @else { + + } + }
- @if (!visible().length) { - @if (filters.status === "active" && !filters.code) { + @if (!incidents.length) { + @if (!filterCount()) { } @else {
- @for (item of visible(); track item.id) { + @for (item of incidents; track item.id) { @if (unavailable(item)) {
Studio can't show this incident. + >{{ + short(item.id) + }}
} @else { @let i = view(item); @@ -271,6 +398,9 @@ const noEvents: RecentEvents = { >{{ label(i) }} + @if (i.test) { + Test + } @if (i.external_effects_may_continue) { }{{ stepOf(i) }}{{ i.workflow?.name ?? "Unknown" }} + @if (i.workflow) { + {{ i.workflow.version }} + }{{ stepOf(i) }} + @if (i.opened_at) { + + @if (i.opened_at_estimated) { + estimated + } + } @else { + Unknown + }{{ short(i.run_id) }} } @else if (nextCursor) { -

- Showing up to 500 incidents. More incidents may exist. +

+ Showing the first 500 incidents. Narrow the filters to see others.

}
@@ -349,8 +510,17 @@ export class IncidentsPage implements DoCheck, OnInit, OnDestroy { if (event instanceof NavigationEnd) this.addressChanged(); }); readonly sheetWhen = sheetWhen; + /** Narrow screens fold the filters behind "Filters (n)". */ + filtersOpen = false; readonly short = shortId; + readonly iso = timeIso; + readonly absolute = timeAbsolute; readonly defaults = defaultIncidentFilters; + readonly orderOptions: [IncidentFilters["order"], string][] = [ + ["opened_desc", "Newest opened"], + ["opened_asc", "Oldest opened"], + ["id", "Incident ID"], + ]; readonly statusOptions: [IncidentFilters["status"], string][] = [ ["active", "Active"], ["resolved", "Resolved"], @@ -374,6 +544,9 @@ export class IncidentsPage implements DoCheck, OnInit, OnDestroy { error: PlainError | null = null; filters: IncidentFilters = defaultIncidentFilters; selected: IncidentView | null = null; + /** Why the incident the address names could not be opened. */ + detailProblem: PlainError | null = null; + rangeProblem = ""; events: RecentEvents = noEvents; resolving = false; /** The incidents whose decision is on its way; one never blocks another. */ @@ -394,7 +567,12 @@ export class IncidentsPage implements DoCheck, OnInit, OnDestroy { return !!this.nextCursor && this.pages < MAX_PAGES; } private scopeKey = ""; + /** Bumped when the platform, environment or person changes. */ private generation = 0; + /** Bumped when the filters change: an answer for older filters is dropped. */ + private listGeneration = 0; + /** The incident the latest direct read is for. */ + private opening = ""; private alive = true; get readable() { @@ -417,48 +595,69 @@ export class IncidentsPage implements DoCheck, OnInit, OnDestroy { if (key === this.scopeKey) return; this.scopeKey = key; this.generation++; + this.listGeneration++; this.incidents = []; this.nextCursor = null; this.loaded = false; this.forbidden = false; this.error = null; this.selected = null; + this.detailProblem = null; + this.opening = ""; this.events = noEvents; this.resolving = false; this.failure = null; this.pending.clear(); this.pages = 1; this.appending = false; + this.rangeProblem = ""; this.filters = incidentFiltersFromQuery(location.search); - if (this.scope && this.host.identity && this.readable) + if (this.scope && this.host.identity && this.readable) { void this.poller.refresh(); + void this.openFromAddress(); + } } @HostListener("window:popstate") locationChanged() { - if (viewFromPath(location.pathname)?.view === "incidents") - this.filters = incidentFiltersFromQuery(location.search); + this.addressChanged(); } - /** The Incidents entry or a link moved the address: the filters follow it. */ + /** The address moved (a link, Back, Forward): filters and the open incident follow. */ private addressChanged() { if (viewFromPath(location.pathname)?.view !== "incidents") return; const filters = incidentFiltersFromQuery(location.search); if ( incidentFiltersToQuery(filters) !== incidentFiltersToQuery(this.filters) - ) + ) { this.filters = filters; + this.changed(); + } + const id = incidentFromQuery(location.search); + if (!id) { + if (this.selected && !this.resolving) this.close(false); + } else if (this.selected?.id !== id) void this.openFromAddress(); } + /** Reads as many pages as are on screen, with the filters' query. */ private async load() { const scope = this.scope; if (!scope || !this.host.identity || !this.readable) return false; const generation = this.generation; + const listGeneration = this.listGeneration; + const filters = this.filters; + const current = () => + generation === this.generation && listGeneration === this.listGeneration; return this.listReads.run(async () => { - if (generation !== this.generation) return false; + if (!current()) return false; try { - const { items, cursor } = await readPages((query) => { - if (generation !== this.generation) - throw Error("List scope changed."); - return this.host.api.request(`${scope}/incidents?${query}`); - }, this.pages); - if (generation !== this.generation) return false; + const items: IncidentRecord[] = []; + let cursor: string | null = null; + let read = 0; + do { + const page: Page = await this.host.api.request( + `${scope}/incidents?${incidentQuery(filters, cursor)}`, + ); + if (!current()) return false; + items.push(...page.items); + cursor = page.next_cursor; + } while (cursor && ++read < this.pages); this.incidents = items; this.nextCursor = cursor; this.loaded = true; @@ -471,7 +670,7 @@ export class IncidentsPage implements DoCheck, OnInit, OnDestroy { this.selected = fresh; return true; } catch (error) { - if (generation !== this.generation) return false; + if (!current()) return false; ({ error: this.error, forbidden: this.forbidden } = listFailure(error)); throw error; } finally { @@ -479,6 +678,7 @@ export class IncidentsPage implements DoCheck, OnInit, OnDestroy { } }); } + /** Continues the list with the same filters. */ async loadMore() { const scope = this.scope; if ( @@ -490,56 +690,157 @@ export class IncidentsPage implements DoCheck, OnInit, OnDestroy { ) return; const generation = this.generation; + const listGeneration = this.listGeneration; + const filters = this.filters; + const current = () => + generation === this.generation && listGeneration === this.listGeneration; this.appending = true; try { await this.listReads.run(async () => { - if (generation !== this.generation || !this.canLoadMore) return; - const page = await readPages( - (query) => this.host.api.request(`${scope}/incidents?${query}`), - 1, - this.nextCursor, + if (!current() || !this.canLoadMore) return; + const page: Page = await this.host.api.request( + `${scope}/incidents?${incidentQuery(filters, this.nextCursor)}`, ); - if (generation !== this.generation) return; - this.incidents = [...this.incidents, ...page.items]; - this.nextCursor = page.cursor; + if (!current()) return; + const seen = new Set(this.incidents.map((item) => item.id)); + this.incidents = [ + ...this.incidents, + ...page.items.filter((item) => !seen.has(item.id)), + ]; + this.nextCursor = page.next_cursor; this.pages++; this.forbidden = false; this.error = null; }); } catch (error) { - if (generation === this.generation) + if (current()) ({ error: this.error, forbidden: this.forbidden } = listFailure(error)); } finally { - if (generation === this.generation) this.appending = false; + if (current()) this.appending = false; if (this.alive) this.cdr.markForCheck(); } } - visible() { - return filterIncidents(this.incidents, this.filters); + suggestions() { + return incidentSuggestions(this.incidents); } - codes() { - return incidentCodes(this.incidents); + filterCount() { + return incidentFilterCount(this.filters); } + /** "3 incidents", or "50 incidents loaded" while more follow. */ countText() { - const count = this.visible().length; - return `${count} ${count === 1 ? "incident" : "incidents"}`; + const count = this.incidents.length; + const noun = count === 1 ? "incident" : "incidents"; + return this.nextCursor ? `${count} ${noun} loaded` : `${count} ${noun}`; } setFilters(change: Partial) { - this.filters = { ...this.filters, ...change } as IncidentFilters; - const path = viewPath("incidents") + incidentFiltersToQuery(this.filters); + const next = { ...this.filters, ...change } as IncidentFilters; + next.code = next.code.trim(); + next.workflow = next.workflow.trim(); + next.nodeId = next.nodeId.trim(); + if (change === this.defaults) this.rangeProblem = ""; + if (incidentFiltersToQuery(next) === incidentFiltersToQuery(this.filters)) + return; + this.filters = next; + this.syncAddress(); + this.changed(); + } + /** A bound of the opening range, from a datetime-local field. */ + setOpened(edge: "openedAfter" | "openedBefore", value: string) { + const date = value ? new Date(value) : null; + const instant = + date && Number.isFinite(date.getTime()) ? date.toISOString() : ""; + const next = { ...this.filters, [edge]: instant }; + const after = next.openedAfter ? Date.parse(next.openedAfter) : null; + const before = next.openedBefore ? Date.parse(next.openedBefore) : null; + if (after !== null && before !== null && after >= before) { + this.rangeProblem = "Choose a start before the end."; + return; + } + if (after !== null && before !== null && before - after > MAX_RANGE_MS) { + this.rangeProblem = "Choose a range of 400 days or less."; + return; + } + this.rangeProblem = ""; + this.setFilters({ [edge]: instant }); + } + /** "2026-10-07T12:30" in this computer's time zone, for a datetime-local field. */ + localTime(value: string) { + const date = new Date(value); + if (!value || !Number.isFinite(date.getTime())) return ""; + const local = new Date(date.getTime() - date.getTimezoneOffset() * 60_000); + return local.toISOString().slice(0, 16); + } + /** New filters: the first page again; Load more pages and older answers go. */ + private changed() { + this.listGeneration++; + this.pages = 1; + this.nextCursor = null; + this.appending = false; + if (this.scope && this.host.identity && this.readable) + void this.poller.refresh(); + } + /** Puts the filters and the open incident in the address. */ + private syncAddress() { + const path = + viewPath("incidents") + + incidentFiltersToQuery(this.filters, this.selected?.id ?? ""); if (location.pathname + location.search !== path) void this.router.navigateByUrl(path, { replaceUrl: true }); } open(incident: IncidentView) { this.selected = incident; + this.detailProblem = null; + this.opening = ""; this.resolving = false; this.failure = null; + this.syncAddress(); void this.loadEvents(incident); } - close() { + close(sync = true) { this.selected = null; + this.opening = ""; this.resolving = false; this.failure = null; + if (sync) this.syncAddress(); + } + /** + * Opens the incident the address names from incidents.read, whether or not + * a loaded page lists it. A refusal or a missing incident is explained + * above the list. + */ + async openFromAddress() { + const scope = this.scope; + const id = incidentFromQuery(location.search); + if (!scope || !id || !this.host.identity || !this.readable) return; + const generation = this.generation; + this.opening = id; + this.detailProblem = null; + try { + const incident = await this.host.api.request( + `${scope}/incidents/${encodeURIComponent(id)}`, + ); + if (generation !== this.generation || this.opening !== id) return; + this.opening = ""; + if (isUnavailableIncident(incident)) { + this.detailProblem = { + message: + "Studio can't show this incident: its run can't be shown here.", + code: "", + status: 0, + }; + return; + } + this.selected = incident; + this.resolving = false; + this.failure = null; + void this.loadEvents(incident); + } catch (error) { + if (generation !== this.generation || this.opening !== id) return; + this.opening = ""; + this.detailProblem = describeError(error); + } finally { + if (this.alive) this.cdr.markForCheck(); + } } openRun(incident: IncidentView) { void this.host.viewRun({ id: incident.run_id }); @@ -633,8 +934,13 @@ export class IncidentsPage implements DoCheck, OnInit, OnDestroy { this.host.notify(`Incident ${where} is resolved.`); if (!samePage()) return; } - this.incidents = this.incidents.map((item) => - item.id === resolved.id ? resolved : item, + // A decided incident the status filter no longer matches leaves the + // list at once, as the platform's next page would leave it out. + const listed = + this.filters.status === "all" || + resolved.status === this.filters.status; + this.incidents = this.incidents.flatMap((item) => + item.id !== resolved.id ? [item] : listed ? [resolved] : [], ); // Not while the person decides on another incident: reading the list // would change the revision that decision is sent with. @@ -672,28 +978,61 @@ export class IncidentsPage implements DoCheck, OnInit, OnDestroy { } } /** - * Refresh incident: reads the incidents again so the dialog decides on the - * current revision. A read that fails, or no longer finds the incident, - * says so in the dialog instead of leaving the old revision to fail again. + * Refresh incident: reads the incident again (incidents.read) so the dialog + * decides on its current revision, whether or not a loaded page lists it. + * A read that fails, or finds it gone, says so in the dialog instead of + * leaving the old revision to fail again. */ async reloadIncident() { const incident = this.selected; - if (!incident) return; + const scope = this.scope; + if (!incident || !scope) return; + const generation = this.generation; this.failure = null; - await this.poller.refresh(); - if (!this.alive || this.selected?.id !== incident.id || !this.resolving) - return; - const fresh = this.incidents.find((item) => item.id === incident.id); - if (this.error) this.failure = this.error; - else if (!fresh) - this.failure = { - message: - "Studio couldn't find this incident when it read the list again. Close this dialog and look for it in the list.", - code: "", - status: 404, - }; - else if (!isUnavailableIncident(fresh)) this.selected = fresh; - this.cdr.markForCheck(); + try { + const fresh = await this.host.api.request( + `${scope}/incidents/${encodeURIComponent(incident.id)}`, + ); + if ( + !this.alive || + generation !== this.generation || + this.selected?.id !== incident.id || + !this.resolving + ) + return; + if (isUnavailableIncident(fresh)) + this.failure = { + message: + "Studio can't show this incident any more. Close this dialog and look for it in the list.", + code: "", + status: 0, + }; + else { + this.selected = fresh; + this.incidents = this.incidents.map((item) => + item.id === fresh.id ? fresh : item, + ); + } + } catch (error) { + if ( + !this.alive || + generation !== this.generation || + this.selected?.id !== incident.id || + !this.resolving + ) + return; + const plain = describeError(error); + this.failure = + plain.status === 404 + ? { + ...plain, + message: + "Studio couldn't find this incident when it read it again. Close this dialog and look for it in the list.", + } + : plain; + } finally { + this.cdr.markForCheck(); + } } unavailable(item: IncidentRecord) { return isUnavailableIncident(item); @@ -710,7 +1049,22 @@ export class IncidentsPage implements DoCheck, OnInit, OnDestroy { tone(incident: IncidentView) { return toneAttribute(incidentTone(incident.status)); } + workflowText(incident: IncidentView) { + return incident.workflow + ? `${incident.workflow.name} ${incident.workflow.version}` + : "Unknown workflow"; + } + /** A relative time, or nothing for a value that isn't a date. */ + relative(value: string) { + return timeRelative(value, ""); + } value(event: Event) { return (event.target as HTMLSelectElement).value as never; } + text(event: Event) { + return (event.target as HTMLInputElement).value; + } + checked(event: Event) { + return (event.target as HTMLInputElement).checked; + } } diff --git a/studio/src/app/operate/incidents/incidents.css b/studio/src/app/operate/incidents/incidents.css index 66124ad5..8766abfa 100644 --- a/studio/src/app/operate/incidents/incidents.css +++ b/studio/src/app/operate/incidents/incidents.css @@ -26,7 +26,8 @@ weave-operate-state { margin-bottom: var(--space-4); } .incident-table { - --cols: minmax(200px, 1fr) 150px minmax(120px, 1fr) 110px; + --cols: minmax(180px, 1.2fr) 150px minmax(120px, 1fr) minmax(110px, 1fr) 120px + 96px; } .incident-table .table-head, .incident-table .resource-row { @@ -50,6 +51,17 @@ weave-operate-state { white-space: normal; overflow-wrap: anywhere; } +/* Workflow and step names stay on one line, with the whole name in a title. */ +.incident-table .cell-line { + display: block; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} +.incident-table .estimated { + display: block; + color: var(--subtle); +} .incident-table .badges { align-items: center; } @@ -57,20 +69,49 @@ weave-operate-state { display: inline-flex; color: var(--warning-ink); } -/* The four columns need about 680 px; the shared table rules that hide a - column title on smaller screens don't apply while all four fit. */ -@container incidents (min-width: 680px) { +.operate-filters input:not([type="checkbox"]), +.operate-filters select { + min-height: var(--control-md); +} +.operate-filters input:not([type="checkbox"], [type="datetime-local"]) { + width: 16ch; +} +@media (max-width: 767px), (pointer: coarse) { + .operate-filters input:not([type="checkbox"]), + .operate-filters select { + min-height: var(--control-touch); + } +} +/* Every column fits from about 900 px; the shared table rules that hide a + column title on smaller screens don't apply while they fit. */ +@container incidents (min-width: 900px) { .incident-table .table-head > [role="columnheader"] { display: block; } } -/* Narrower (an incident open beside the list, or a small screen) rows keep - the incident and its status. */ +/* Narrower: the run moves to the drawer. */ +@container incidents (min-width: 680px) and (max-width: 899px) { + .incident-table { + --cols: minmax(160px, 1.2fr) 140px minmax(110px, 1fr) minmax(100px, 1fr) + 110px; + } + .incident-table [data-wider] { + display: none; + } + .incident-table .table-head > [role="columnheader"]:not([data-wider]) { + display: block; + } +} +/* Narrower still (an incident open beside the list, or a small screen): + rows keep the incident and its status. */ @container incidents (max-width: 679px) { + /* A fixed status column: every row's grid is its own, so an auto width + would put each row's status somewhere else. */ .incident-table { - --cols: minmax(0, 1fr) auto; + --cols: minmax(0, 1fr) 150px; } - .incident-table [data-wide] { + .incident-table [data-wide], + .incident-table [data-wider] { display: none; } .incident-table .table-head > [role="columnheader"]:nth-child(2) { @@ -83,6 +124,21 @@ weave-operate-state { display: none; } } +.list-limit { + margin: var(--space-3) 0 0; + font: var(--type-small); + color: var(--muted); +} +.run-fact { + display: flex; + flex-wrap: wrap; + align-items: center; + gap: var(--space-2); +} +.opened small { + display: block; + margin-top: var(--space-1); +} /* The drawer: the run's last engine events. */ .recent-events { diff --git a/studio/src/app/operate/incidents/resolve-incident-dialog.ts b/studio/src/app/operate/incidents/resolve-incident-dialog.ts index 13cbeb75..9bfc8a65 100644 --- a/studio/src/app/operate/incidents/resolve-incident-dialog.ts +++ b/studio/src/app/operate/incidents/resolve-incident-dialog.ts @@ -43,8 +43,14 @@ import { (dismiss)="cancel.emit()" >

- {{ incident().code }} at {{ incident().node_id || "the run" }}. Check the - other system first: Weave can't look it up for you. + {{ incident().code }} at {{ incident().node_id || "the run" }} + @if ( + incident().instance_key && + incident().instance_key !== incident().node_id + ) { + ({{ incident().instance_key }}) + } + . Check the other system first: Weave can't look it up for you.

({ path: `operate/${page}`, children: [] })), { path: "operate/runs/:id", children: [] }, + // Before the worker detail, so "releases" is never read as a worker ID. + { path: "operate/workers/releases", children: [] }, + { path: "operate/workers/releases/:id", children: [] }, { path: "operate/workers/:id", children: [] }, ...clusterCollections.map((collection) => ({ path: `operate/clusters/${collection}/:id`, diff --git a/studio/src/app/operate/operate.css b/studio/src/app/operate/operate.css index 7b7208eb..53cf12d2 100644 --- a/studio/src/app/operate/operate.css +++ b/studio/src/app/operate/operate.css @@ -122,6 +122,28 @@ SPDX-License-Identifier: Apache-2.0 .operate-filters select { max-width: 100%; } +/* Narrow screens fold the filters behind "Filters (n)"; open, each field + spans the pane, so every field shares its left and right edge. */ +.filters-toggle { + display: none; + min-height: var(--control-touch); + margin-bottom: var(--space-3); +} +@media (max-width: 767px) { + .filters-toggle { + display: inline-flex; + } + .operate-filters[id]:not(.open) { + display: none; + } + .operate-filters[id] > .field { + flex: 1 1 100%; + } + .operate-filters[id] select, + .operate-filters[id] input:not([type="checkbox"]) { + width: 100%; + } +} /* A checkbox among the filters is a target as tall as a control: 36 px with a mouse, 44 px on a narrow screen or a touch pointer. */ .operate-filters .checkbox-field { @@ -174,12 +196,14 @@ SPDX-License-Identifier: Apache-2.0 cell's button stretches over its row, and its focus ring outlines the row. Each page decides how a long name fits. */ .worker-table .resource-row, -.incident-table .resource-row { +.incident-table .resource-row, +.release-table .resource-row { position: relative; cursor: pointer; } .worker-table .row-open, -.incident-table .row-open { +.incident-table .row-open, +.release-table .row-open { display: block; min-height: 0; max-width: 100%; @@ -193,21 +217,25 @@ SPDX-License-Identifier: Apache-2.0 text-align: left; } .worker-table .row-open:hover, -.incident-table .row-open:hover { +.incident-table .row-open:hover, +.release-table .row-open:hover { background: none; } .worker-table .row-open::after, -.incident-table .row-open::after { +.incident-table .row-open::after, +.release-table .row-open::after { content: ""; position: absolute; inset: 0; } .worker-table .row-open:focus-visible, -.incident-table .row-open:focus-visible { +.incident-table .row-open:focus-visible, +.release-table .row-open:focus-visible { outline: none; } .worker-table .row-open:focus-visible::after, -.incident-table .row-open:focus-visible::after { +.incident-table .row-open:focus-visible::after, +.release-table .row-open:focus-visible::after { outline: 2px solid var(--focus); outline-offset: -2px; } diff --git a/studio/src/app/operate/workers/release-detail.ts b/studio/src/app/operate/workers/release-detail.ts new file mode 100644 index 00000000..a9d65a95 --- /dev/null +++ b/studio/src/app/operate/workers/release-detail.ts @@ -0,0 +1,359 @@ +/* +Copyright 2026 Firefly Software Foundation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +Author: Firefly Software Foundation +SPDX-License-Identifier: Apache-2.0 +*/ +// One release: its image and task types (releases.read), and what runs it +// (releases.usage): its workers by state, the activations that pin it and the +// deployment components on targets this person may read. The platform leaves +// out every other target; Studio shows only what it answers. Reads again +// every 10 s while open. +import { + ChangeDetectionStrategy, + ChangeDetectorRef, + Component, + OnChanges, + OnDestroy, + OnInit, + SimpleChanges, + inject, + input, + output, +} from "@angular/core"; +import type { App } from "../../app"; +import { describeError, type PlainError } from "../../errors"; +import { shortId } from "../../format"; +import { Icon } from "../../icon"; +import { MAX_PAGES } from "../operate-limits"; +import { environmentOf, scopeKeyOf } from "../operate-list"; +import { OperateState } from "../operate-state"; +import { Poller } from "../operate-store"; +import { + releaseTaskTypes, + releaseUsageQuery, + shortDigest, + type ReleaseUsage, + type WorkerRelease, +} from "./workers-model"; + +type Deployment = ReleaseUsage["deployments"][number]; + +@Component({ + selector: "weave-release-detail", + standalone: true, + changeDetection: ChangeDetectionStrategy.Eager, + imports: [Icon, OperateState], + styleUrls: ["../operate.css", "./workers.css"], + template: `
+

+ Release + {{ release ? digest(release.image_digest) : short(releaseId()) }} +

+ +
+ @if (problem; as failed) { + @if (failed.status === 403) { + + } @else if (failed.status === 404) { + + } @else { + + } + } @else if (!release) { + + } @else { +
+
Image digest
+
+ {{ release.image_digest }} +
+
Task types
+
{{ types() }}
+ @if (release.credential_capabilities?.length) { +
Uses connections for
+
{{ release.credential_capabilities!.join(", ") }}
+ } +
+

Workers

+ @if (usageProblem; as failed) { + + } @else if (!usage) { + + } @else { +
    + @for (state of states; track state.key) { +
  • + {{ usage.workers[state.key] }}{{ state.label }} +
    • + } +
+
+ +
+
+
Activations that pin it
+
{{ usage.activations }}
+
+

Used by

+ @if (deployments.length) { +
+
+
+ Target › deployment › componentReplicas +
+
+
+ @for (row of deployments; track rowKey(row)) { +
+ {{ path(row) }}{{ + row.replicas + }} +
+ } +
+
+ @if (usageCursor && pages < maxPages) { + + } + } @else { +

+ No deployment on a target you can see runs this release. +

+ } + } +
+ +
+

+ Rolling a release out from Studio comes with change requests. Until + then, update the deployment in Clusters or with the CLI. +

+
+ Technical details +
+
Release ID
+
+ {{ release.id }} +
+
+
+ }`, +}) +export class ReleaseDetail implements OnChanges, OnInit, OnDestroy { + host = input.required(); + releaseId = input.required(); + /** The fleet, narrowed to this release's workers. */ + showWorkers = output(); + closed = output(); + private cdr = inject(ChangeDetectorRef); + readonly short = shortId; + readonly digest = shortDigest; + readonly maxPages = MAX_PAGES; + readonly states = [ + { key: "online", label: "Online" }, + { key: "offline", label: "Offline" }, + { key: "draining", label: "Draining" }, + { key: "revoked", label: "Revoked" }, + ] as const; + readonly poller = new Poller(() => this.read(), 10_000); + release: WorkerRelease | null = null; + usage: ReleaseUsage | null = null; + deployments: Deployment[] = []; + usageCursor: string | null = null; + pages = 1; + appending = false; + problem: PlainError | null = null; + usageProblem: PlainError | null = null; + /** Bumped for another release or scope: older answers change nothing. */ + private generation = 0; + + ngOnInit() { + this.poller.start(false); + } + ngOnDestroy() { + this.poller.stop(); + this.generation++; + } + ngOnChanges(changes: SimpleChanges) { + if (!changes["releaseId"]) return; + this.generation++; + this.release = null; + this.usage = null; + this.deployments = []; + this.usageCursor = null; + this.pages = 1; + this.problem = null; + this.usageProblem = null; + void this.poller.refresh(); + } + /** The release, then its usage: as many deployment pages as are on screen. */ + async read(): Promise { + const host = this.host(); + const scope = environmentOf(host); + const id = this.releaseId(); + if (!scope || !id) return false; + const generation = this.generation; + const key = scopeKeyOf(host); + const current = () => + generation === this.generation && key === scopeKeyOf(host); + const base = `${scope}/worker-releases/${encodeURIComponent(id)}`; + try { + const release = await host.api.request(base); + if (!current()) return false; + this.release = release; + this.problem = null; + } catch (error) { + if (current()) this.problem = describeError(error); + this.cdr.markForCheck(); + throw error; + } + try { + const rows: Deployment[] = []; + let cursor: string | null = null; + let read = 0; + let usage: ReleaseUsage; + do { + usage = await host.api.request( + `${base}/usage?${releaseUsageQuery(cursor)}`, + ); + if (!current()) return false; + rows.push(...usage.deployments); + cursor = usage.next_cursor ?? null; + } while (cursor && ++read < this.pages); + this.usage = usage; + this.deployments = rows; + this.usageCursor = cursor; + this.usageProblem = null; + return true; + } catch (error) { + if (current()) this.usageProblem = describeError(error); + throw error; + } finally { + this.cdr.markForCheck(); + } + } + async moreUsage() { + const host = this.host(); + const scope = environmentOf(host); + const cursor = this.usageCursor; + if (!scope || !cursor || this.appending || this.pages >= MAX_PAGES) return; + const generation = this.generation; + this.appending = true; + try { + const usage = await host.api.request( + `${scope}/worker-releases/${encodeURIComponent(this.releaseId())}/usage?${releaseUsageQuery(cursor)}`, + ); + if (generation !== this.generation) return; + const seen = new Set(this.deployments.map((row) => this.rowKey(row))); + this.deployments = [ + ...this.deployments, + ...usage.deployments.filter((row) => !seen.has(this.rowKey(row))), + ]; + this.usageCursor = usage.next_cursor ?? null; + this.pages++; + } catch (error) { + if (generation === this.generation) + this.usageProblem = describeError(error); + } finally { + if (generation === this.generation) this.appending = false; + this.cdr.markForCheck(); + } + } + types() { + return this.release + ? releaseTaskTypes(this.release).join(", ") || "None" + : ""; + } + /** A state's share of every worker of the release, for its meter. */ + share(count: number) { + const workers = this.usage?.workers; + if (!workers) return 0; + const total = + workers.online + workers.offline + workers.draining + workers.revoked; + return total ? (count / total) * 100 : 0; + } + path(row: Deployment) { + return `${row.target_name} › ${row.deployment_name} › ${row.component}`; + } + rowKey(row: Deployment) { + return `${row.deployment_id}/${row.component}`; + } +} diff --git a/studio/src/app/operate/workers/releases-page.ts b/studio/src/app/operate/workers/releases-page.ts new file mode 100644 index 00000000..5d29f733 --- /dev/null +++ b/studio/src/app/operate/workers/releases-page.ts @@ -0,0 +1,367 @@ +/* +Copyright 2026 Firefly Software Foundation. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +Author: Firefly Software Foundation +SPDX-License-Identifier: Apache-2.0 +*/ +// Operate › Workers › Releases: the worker releases admitted in this +// environment (releases.list) and, for the one open, what runs it. Admitting a +// release stays a CLI step; the page shows the exact command and never posts +// one. Reloads every 10 s while visible. +import { + ChangeDetectionStrategy, + ChangeDetectorRef, + Component, + DoCheck, + Input, + OnDestroy, + OnInit, + inject, +} from "@angular/core"; +import { NavigationEnd, Router } from "@angular/router"; +import type { App } from "../../app"; +import { copyText } from "../../connection"; +import type { PlainError } from "../../errors"; +import { shortId } from "../../format"; +import { Icon } from "../../icon"; +import { ModalSheet, sheetWhen } from "../../modal-sheet"; +import type { Page } from "../../operations/deployment-contracts"; +import { MAX_PAGES } from "../operate-limits"; +import { + ListReads, + environmentOf, + listFailure, + scopeKeyOf, +} from "../operate-list"; +import { OperateState } from "../operate-state"; +import { releasesPath, viewPath, workersRoute } from "../operate-routes"; +import { Poller } from "../operate-store"; +import { RefreshStatus } from "../refresh-status"; +import { ReleaseDetail } from "./release-detail"; +import { + releaseAdmissionCommand, + releaseTaskTypes, + shortDigest, + type WorkerRelease, +} from "./workers-model"; + +@Component({ + selector: "weave-releases-page", + standalone: true, + changeDetection: ChangeDetectionStrategy.Eager, + imports: [Icon, ModalSheet, OperateState, RefreshStatus, ReleaseDetail], + styleUrls: ["../operate.css", "./workers.css"], + template: ` + @if (!host.identity) { + + } @else if (!readable || forbidden) { + + } @else if (!loaded && error) { + + } @else if (!loaded) { + + } @else { +
+ {{ countText() }} + +
+ @if (error) { + + } +
+
+ @if (!releases.length) { + + } @else { +
+
+
+ ReleaseTask typesRelease ID +
+
+
+ @for (release of releases; track release.id) { +
+ {{ typesCount(release) }}{{ typesText(release) }}{{ + short(release.id) + }} +
+ } +
+
+ @if (canLoadMore) { + + } @else if (nextCursor) { +

+ Showing the first 500 releases. +

+ } + } +
+

Admit a release

+

+ Studio doesn't admit releases. Put the release request (its image + digest and capabilities) in release.json and run: +

+
+ {{ command }} +
+
+
+ @if (openId) { + + } +
+ } + `, +}) +export class ReleasesPage implements DoCheck, OnInit, OnDestroy { + @Input({ required: true }) host!: App; + private cdr = inject(ChangeDetectorRef); + private router = inject(Router); + private navigated = this.router.events.subscribe((event) => { + if (event instanceof NavigationEnd) + this.openId = workersRoute(location.pathname).release; + }); + readonly sheetWhen = sheetWhen; + readonly short = shortId; + readonly digest = shortDigest; + readonly command = releaseAdmissionCommand; + readonly poller = new Poller(() => this.load(), 10_000); + releases: WorkerRelease[] = []; + nextCursor: string | null = null; + loaded = false; + forbidden = false; + error: PlainError | null = null; + appending = false; + /** The release whose detail is open ("/operate/workers/releases/ID"). */ + openId = ""; + private pages = 1; + private readonly listReads = new ListReads(); + private scopeKey = ""; + private generation = 0; + + get readable() { + return this.host.canAnywhere("catalog.read"); + } + get canLoadMore() { + return !!this.nextCursor && this.pages < MAX_PAGES; + } + private get scope() { + return environmentOf(this.host); + } + ngOnInit() { + this.openId = workersRoute(location.pathname).release; + this.poller.start(false); + } + ngOnDestroy() { + this.navigated.unsubscribe(); + this.poller.stop(); + this.generation++; + } + ngDoCheck() { + const key = scopeKeyOf(this.host); + if (key === this.scopeKey) return; + this.scopeKey = key; + this.generation++; + this.releases = []; + this.nextCursor = null; + this.loaded = false; + this.forbidden = false; + this.error = null; + this.pages = 1; + this.appending = false; + if (this.scope && this.host.identity && this.readable) + void this.poller.refresh(); + } + private async load() { + const scope = this.scope; + if (!scope || !this.host.identity || !this.readable) return false; + const generation = this.generation; + return this.listReads.run(async () => { + if (generation !== this.generation) return false; + try { + const items: WorkerRelease[] = []; + let cursor: string | null = null; + let read = 0; + do { + const query = new URLSearchParams({ limit: "50" }); + if (cursor) query.set("cursor", cursor); + const page: Page = await this.host.api.request( + `${scope}/worker-releases?${query}`, + ); + if (generation !== this.generation) return false; + items.push(...page.items); + cursor = page.next_cursor; + } while (cursor && ++read < this.pages); + this.releases = items; + this.nextCursor = cursor; + this.loaded = true; + this.forbidden = false; + this.error = null; + return true; + } catch (error) { + if (generation !== this.generation) return false; + ({ error: this.error, forbidden: this.forbidden } = listFailure(error)); + throw error; + } finally { + this.cdr.markForCheck(); + } + }); + } + async loadMore() { + const scope = this.scope; + if (!scope || !this.canLoadMore || this.appending) return; + const generation = this.generation; + this.appending = true; + try { + await this.listReads.run(async () => { + if (generation !== this.generation || !this.canLoadMore) return; + const query = new URLSearchParams({ + limit: "50", + cursor: this.nextCursor!, + }); + const page: Page = await this.host.api.request( + `${scope}/worker-releases?${query}`, + ); + if (generation !== this.generation) return; + const seen = new Set(this.releases.map((item) => item.id)); + this.releases = [ + ...this.releases, + ...page.items.filter((item) => !seen.has(item.id)), + ]; + this.nextCursor = page.next_cursor; + this.pages++; + this.error = null; + }); + } catch (error) { + if (generation === this.generation) + ({ error: this.error, forbidden: this.forbidden } = listFailure(error)); + } finally { + if (generation === this.generation) this.appending = false; + this.cdr.markForCheck(); + } + } + open(id: string) { + this.openId = id; + void this.router.navigateByUrl(releasesPath(id)); + } + close() { + this.openId = ""; + void this.router.navigateByUrl(releasesPath()); + } + /** The fleet, narrowed to the workers that run this release. */ + showWorkers(id: string) { + void this.router.navigateByUrl( + `${viewPath("workers")}?release=${encodeURIComponent(id)}`, + ); + } + async copyCommand() { + if (await copyText(this.command)) this.host.notify("Copied the command."); + else + this.host.notify("Studio couldn't copy. Select the command and copy it."); + } + countText() { + const count = this.releases.length; + const noun = count === 1 ? "release" : "releases"; + return this.nextCursor ? `${count} ${noun} loaded` : `${count} ${noun}`; + } + typesText(release: WorkerRelease) { + return releaseTaskTypes(release).join(", ") || "None"; + } + typesCount(release: WorkerRelease) { + const count = release.capabilities?.length ?? 0; + return `${count} task ${count === 1 ? "type" : "types"}`; + } +} diff --git a/studio/src/app/operate/workers/worker-detail.ts b/studio/src/app/operate/workers/worker-detail.ts index 0a9856a4..3da06c52 100644 --- a/studio/src/app/operate/workers/worker-detail.ts +++ b/studio/src/app/operate/workers/worker-detail.ts @@ -15,8 +15,10 @@ limitations under the License. Author: Firefly Software Foundation SPDX-License-Identifier: Apache-2.0 */ -// One worker: where its contact and load stand, and Drain or Resume. The -// server checks worker.drain and the revision on every command. +// One worker: where its contact and load stand, what it reports about where +// it runs and where the platform places it, and Drain, Resume or Revoke. The +// server checks worker.drain (with the revision) or release.retire on every +// command. Reported values are shown as text, never as links. import { ChangeDetectionStrategy, ChangeDetectorRef, @@ -39,6 +41,9 @@ import { workerBadges, workerLoad, workerPresence, + workerVersion, + workerWhere, + type WorkerIdentity, type WorkerStatus, } from "./workers-model"; @@ -99,6 +104,32 @@ import {
Task types
{{ w.task_types.join(", ") || "None" }}
+
Registered
+
+ @if (w.registered_at) { + + } @else { + Unknown + } +
+ @if (w.draining) { +
Draining since
+
+ @if (w.drained_at) { + + } @else { + Unknown + } +
+ }
Observed
+

Where it runs

+
+
Placement
+
+ @if (w.placement; as placed) { + {{ placed.target_name }} › {{ placed.component }} + } @else { + {{ where(w) }} + } +
+ @if (w.identity; as identity) { +
Host
+
+ @if (identity.host) { + {{ identity.host }} + } @else { + Not reported + } +
+
Runs as
+
{{ runtimeText(identity.runtime, identity.platform_hint) }}
+ @if (version(w)) { +
Version
+
{{ version(w) }}
+ } + @if (identity.os || identity.arch) { +
System
+
{{ systemText(identity.os, identity.arch) }}
+ } + @if (identity.namespace || identity.node) { +
Kubernetes
+
+ {{ kubernetesText(identity.namespace, identity.node) }} +
+ } + @if (labelsOf(identity.labels).length) { +
Labels
+
+
    + @for (label of labelsOf(identity.labels); track label[0]) { +
  • + {{ label[0] }}={{ label[1] }} +
    • + } +
+
+ } + } @else { +
Reported
+
Nothing: this worker doesn't say where it runs.
+ } +
+

+ What a worker reports is shown as it reports it; it never decides what the + worker may do. +

Draining stops new tasks. Running tasks finish or expire; draining doesn't stop the container or cancel them. @@ -121,15 +208,27 @@ import { }

} - @if (canControl(w)) { + @if (canControl(w) || canRevoke(w)) {
- @if (w.draining) { - - } @else { - + } @else { + + } + } + @if (canRevoke(w)) { + }
@@ -147,6 +246,16 @@ import {
Revision
{{ w.revision }}
+ @if (w.placement; as placed) { +
Target ID
+
+ {{ placed.target_id }} +
+
Deployment ID
+
+ {{ placed.deployment_id }} +
+ } `, }) @@ -156,6 +265,8 @@ export class WorkerDetail implements OnChanges, OnDestroy { /** The worker after a command, read again from the platform. */ changed = output(); refresh = output(); + /** The worker was revoked: the list reads it again. */ + revoked = output(); closed = output(); problem: PlainError | null = null; private cdr = inject(ChangeDetectorRef); @@ -165,6 +276,8 @@ export class WorkerDetail implements OnChanges, OnDestroy { readonly relative = timeRelative; readonly badges = workerBadges; readonly tone = toneAttribute; + readonly where = workerWhere; + readonly version = workerVersion; /** The workers whose command is on its way; one worker's never blocks another's. */ private running = new Set(); private alive = true; @@ -206,6 +319,86 @@ export class WorkerDetail implements OnChanges, OnDestroy { host.can("worker.drain", worker.id) ); } + canRevoke(worker: WorkerStatus) { + const host = this.host(); + return ( + !host.signInEnded && + !worker.revoked && + host.can("release.retire", worker.id) + ); + } + runtimeText( + runtime: WorkerIdentity["runtime"], + hint: WorkerIdentity["platform_hint"], + ) { + const kind = { + embedded: "Inside the API", + container: "A container", + process: "A process", + }[runtime]; + return hint === "kubernetes" + ? `${kind} on Kubernetes` + : hint === "docker" + ? `${kind} on Docker` + : kind; + } + systemText(os?: string, arch?: string) { + return [os, arch].filter(Boolean).join(" · "); + } + kubernetesText(namespace?: string, node?: string) { + return [ + namespace ? `Namespace ${namespace}` : "", + node ? `node ${node}` : "", + ] + .filter(Boolean) + .join(", "); + } + labelsOf(labels?: Record) { + return Object.entries(labels ?? {}).sort(([a], [b]) => a.localeCompare(b)); + } + /** Revoke: asks first; the list reads the worker again after. */ + async revoke() { + const host = this.host(); + const worker = this.worker(); + if (this.running.has(worker.id) || !this.canRevoke(worker)) return; + const scope = this.environment(); + const identity = JSON.stringify(host.identity); + const confirmed = await host.dialogs.confirm({ + title: "Revoke this worker?", + message: + "It can't claim tasks again. Tasks it is running finish or expire.", + confirmLabel: "Revoke worker", + cancelLabel: "Keep", + danger: true, + }); + if (!confirmed || !this.current(worker, scope, identity)) return; + this.running.add(worker.id); + this.problem = null; + this.cdr.markForCheck(); + const named = `Worker ${shortId(worker.id)}`; + try { + const path = `${scope}/workers/${encodeURIComponent(worker.id)}`; + await host.api.request(`${path}/revoke`, "POST"); + const read = await host.api.request(path); + if (this.current(worker, scope, identity)) this.changed.emit(read); + this.revoked.emit(); + host.notify(`${named} is revoked.`); + } catch (error) { + const plain = describeError(error); + if (this.current(worker, scope, identity)) + this.problem = + plain.status === 403 + ? { + ...plain, + message: `You need release.retire for this worker. ${plain.message}`, + } + : plain; + else host.notify(`${named} was not revoked. ${plain.message}`); + } finally { + this.running.delete(worker.id); + if (this.alive) this.cdr.markForCheck(); + } + } /** The environment commands go to; empty when there is none. */ private environment() { try { diff --git a/studio/src/app/operate/workers/workers-model.ts b/studio/src/app/operate/workers/workers-model.ts index 098ee15e..5673c700 100644 --- a/studio/src/app/operate/workers/workers-model.ts +++ b/studio/src/app/operate/workers/workers-model.ts @@ -15,9 +15,10 @@ limitations under the License. Author: Firefly Software Foundation SPDX-License-Identifier: Apache-2.0 */ -// What the Workers page says about each worker: presence, claims, load and -// the filters that narrow the loaded list. Pure: the page renders it, tests -// check it. Presence is the platform's observation, not container health. +// What the Workers page says about each worker: presence, claims, load, where +// it runs, and the filters the platform applies to the list. Pure: the page +// renders it, tests check it. Presence is the platform's observation, not +// container health; identity is what the worker reports, for display only. import type { Tone } from "../../status-labels"; /** A worker as the platform reports it (`workers.list`, `workers.read`). */ @@ -37,6 +38,34 @@ export interface WorkerStatus { observed_at: string; active_leases: number; available_capacity: number | null; + /** What the worker reports about where it runs; left out when unknown. */ + identity?: WorkerIdentity; + /** When the platform recorded the registration; left out when unknown. */ + registered_at?: string; + /** Where an observation of a target the reader may see places it. */ + placement?: WorkerPlacement; + /** When it started draining; left out when unknown or not draining. */ + drained_at?: string; +} +/** Where a worker says it runs. Display only: never authority or placement. */ +export interface WorkerIdentity { + host?: string; + runtime: "embedded" | "container" | "process"; + sdk_version?: string; + language_version?: string; + os?: string; + arch?: string; + platform_hint?: "kubernetes" | "docker" | "none"; + namespace?: string; + node?: string; + labels?: Record; +} +/** The target, deployment and component the platform matched the worker to. */ +export interface WorkerPlacement { + target_id: string; + target_name: string; + deployment_id: string; + component: string; } /** A worker the platform can't show. */ export interface UnavailableWorker { @@ -121,84 +150,240 @@ export function workerLoad(worker: WorkerStatus): Load { }; } -/** Filters over the loaded workers; they live in the address. */ +/** + * Where the worker runs: "Target › component" when an observation places it, + * "In API process" for an executor inside the API, "Unmanaged" for a worker + * that reports where it runs but no target the reader may see holds, and + * "Unknown" for one that reports nothing. + */ +export function workerWhere(worker: WorkerStatus): string { + if (worker.placement) + return `${worker.placement.target_name} › ${worker.placement.component}`; + if (!worker.identity) return "Unknown"; + return worker.identity.runtime === "embedded" + ? "In API process" + : "Unmanaged"; +} +/** "SDK 0.1.0a15 · Python 3.12.4", or "" when the worker reports neither. */ +export function workerVersion(worker: WorkerStatus): string { + const identity = worker.identity; + return [ + identity?.sdk_version ? `SDK ${identity.sdk_version}` : "", + identity?.language_version ? `Runtime ${identity.language_version}` : "", + ] + .filter(Boolean) + .join(" · "); +} + +/** + * Filters the platform applies to the list. Every one but the host prefix + * lives in the address; a host prefix may name a machine, so it stays on the + * page and goes when the platform, environment or person changes. + */ export interface WorkerFilters { presence: Presence | ""; - draining: boolean; + /** Draining (true), not draining (false), or either (null). */ + draining: boolean | null; release: string; taskType: string; + target: string; + hostPrefix: string; + /** Show retired workers: revoked and long offline ones too. */ revoked: boolean; + order: WorkerOrder; } +export type WorkerOrder = "last_seen_desc" | "registered_desc"; export const noWorkerFilters: WorkerFilters = { presence: "", - draining: false, + draining: null, release: "", taskType: "", + target: "", + hostPrefix: "", revoked: false, + order: "last_seen_desc", }; -export function filterWorkers( - workers: readonly WorkerRecord[], +/** The `workers.list` query for the filters, from the start or after `cursor`. */ +export function workerQuery( filters: WorkerFilters, - now = Date.now(), -): WorkerRecord[] { - return workers.filter((worker) => { - if (isUnavailableWorker(worker)) return filterCount(filters) === 0; - if (worker.revoked && !filters.revoked) return false; - if (filters.presence && workerPresence(worker, now) !== filters.presence) - return false; - if (filters.draining && !worker.draining) return false; - if (filters.release && worker.release_id !== filters.release) return false; - if (filters.taskType && !worker.task_types.includes(filters.taskType)) - return false; - return true; + cursor: string | null, +): string { + const query = new URLSearchParams({ + limit: "50", + state: filters.revoked ? "all" : "current", + order: filters.order, }); + const presence = { + online: "recent", + offline: "stale", + unknown: "unknown", + } as const; + if (filters.presence) query.set("presence", presence[filters.presence]); + if (filters.draining !== null) + query.set("draining", String(filters.draining)); + for (const [key, value] of [ + ["release_id", filters.release], + ["task_type", filters.taskType], + ["target_id", filters.target], + ["host_prefix", filters.hostPrefix], + ]) + if (value) query.set(key, value); + if (cursor) query.set("cursor", cursor); + return query.toString(); } /** How many filters narrow the list ("Clear filters" shows from one). */ export function filterCount(filters: WorkerFilters) { return [ filters.presence, - filters.draining, + filters.draining !== null, filters.release, filters.taskType, + filters.target, + filters.hostPrefix, filters.revoked, ].filter(Boolean).length; } const presences: readonly string[] = ["online", "offline", "unknown"]; +const orders: readonly string[] = ["last_seen_desc", "registered_desc"]; /** Reads the filters from "?presence=online&draining=true&task_type=…". */ export function filtersFromQuery(search: string): WorkerFilters { const query = new URLSearchParams(search); const presence = query.get("presence") ?? ""; + const draining = query.get("draining"); + const order = query.get("order") ?? ""; return { presence: presences.includes(presence) ? (presence as Presence) : "", - draining: query.get("draining") === "true", + draining: draining === "true" ? true : draining === "false" ? false : null, release: query.get("release") ?? "", taskType: query.get("task_type") ?? "", + target: query.get("target") ?? "", + hostPrefix: "", revoked: query.get("revoked") === "true", + order: orders.includes(order) ? (order as WorkerOrder) : "last_seen_desc", }; } -/** The query for the filters, "" when none is set. */ +/** The address query for the filters, "" when none is set; never the host prefix. */ export function filtersToQuery(filters: WorkerFilters): string { const query = new URLSearchParams(); if (filters.presence) query.set("presence", filters.presence); - if (filters.draining) query.set("draining", "true"); + if (filters.draining !== null) + query.set("draining", String(filters.draining)); if (filters.release) query.set("release", filters.release); if (filters.taskType) query.set("task_type", filters.taskType); + if (filters.target) query.set("target", filters.target); if (filters.revoked) query.set("revoked", "true"); + if (filters.order !== "last_seen_desc") query.set("order", filters.order); const text = query.toString(); return text ? `?${text}` : ""; } -/** The releases and task types the loaded workers report, sorted. */ -export function filterChoices(workers: readonly WorkerRecord[]) { - const releases = new Set(); +/** + * The releases and task types to offer: those the loaded workers report, + * those of the releases read for the environment, and the ones chosen. + */ +export function filterChoices( + workers: readonly WorkerRecord[], + releases: readonly { id: string; task_types: readonly string[] }[] = [], + chosen: Pick = noWorkerFilters, +) { + const ids = new Set(); const taskTypes = new Set(); for (const worker of workers) { if (isUnavailableWorker(worker)) continue; - releases.add(worker.release_id); + ids.add(worker.release_id); for (const type of worker.task_types) taskTypes.add(type); } - return { releases: [...releases].sort(), taskTypes: [...taskTypes].sort() }; + for (const release of releases) { + ids.add(release.id); + for (const type of release.task_types) taskTypes.add(type); + } + if (chosen.release) ids.add(chosen.release); + if (chosen.taskType) taskTypes.add(chosen.taskType); + return { releases: [...ids].sort(), taskTypes: [...taskTypes].sort() }; +} + +/** A worker release as the platform reports it (`releases.list`, `releases.read`). */ +export interface WorkerRelease { + id: string; + image_digest: string; + capabilities: { taskType: string; taskVersion: string }[]; + credential_capabilities?: string[]; + connector_bindings?: unknown[]; +} +/** What runs a release (`releases.usage`): only targets the reader may see. */ +export interface ReleaseUsage { + workers: { + online: number; + offline: number; + draining: number; + revoked: number; + }; + deployments: { + target_id: string; + target_name: string; + deployment_id: string; + deployment_name: string; + component: string; + replicas: number; + }[]; + activations: number; + next_cursor?: string | null; +} +/** The task types a release serves, as workers register them ("echo@1.2.0"). */ +export function releaseTaskTypes(release: WorkerRelease): string[] { + return (release.capabilities ?? []).map( + (capability) => `${capability.taskType}@${capability.taskVersion}`, + ); +} +/** "sha256:3f2a9b1c…": the start of the image digest a release admits. */ +export function shortDigest(digest: string): string { + const [algorithm, value] = digest.includes(":") + ? digest.split(":", 2) + : ["", digest]; + return `${algorithm ? algorithm + ":" : ""}${value.slice(0, 12)}`; +} +/** + * Admitting a release stays a CLI step: the exact command, which reads the + * release request (image digest and capabilities) from a file. + */ +export const releaseAdmissionCommand = + "weave workers releases create --request release.json"; + +/** The release usage query, from the start or after `cursor`. */ +export function releaseUsageQuery(cursor: string | null): string { + const query = new URLSearchParams({ limit: "100" }); + if (cursor) query.set("cursor", cursor); + return query.toString(); +} + +/** The Retire offline workers thresholds, in seconds; 24 hours is the default. */ +export const retireThresholds: { seconds: number; label: string }[] = [ + { seconds: 86_400, label: "24 hours" }, + { seconds: 7 * 86_400, label: "7 days" }, + { seconds: 30 * 86_400, label: "30 days" }, +]; +/** One retirement command revokes at most this many workers. */ +export const RETIRE_BATCH = 100; +/** What a retirement preview or result says. */ +export function retireSummary(result: { + matched: number; + revoked: number; + dry_run: boolean; +}): string { + const workers = (count: number) => + `${count.toLocaleString("en-US")} ${count === 1 ? "worker" : "workers"}`; + if (result.dry_run) { + if (!result.matched) + return "No worker matches: every offline worker holds work or was seen more recently."; + return result.matched > RETIRE_BATCH + ? `${workers(result.matched)} match. One retirement revokes ${RETIRE_BATCH}, oldest first; retire again for the rest.` + : `${workers(result.matched)} ${result.matched === 1 ? "matches" : "match"}. Retiring revokes ${result.matched === 1 ? "it" : "them"}.`; + } + const left = Math.max(0, result.matched - result.revoked); + return left + ? `Revoked ${workers(result.revoked)}. ${workers(left)} still ${left === 1 ? "matches" : "match"}.` + : `Revoked ${workers(result.revoked)}.`; } diff --git a/studio/src/app/operate/workers/workers-page.ts b/studio/src/app/operate/workers/workers-page.ts index 983479ef..001c14d6 100644 --- a/studio/src/app/operate/workers/workers-page.ts +++ b/studio/src/app/operate/workers/workers-page.ts @@ -15,9 +15,13 @@ limitations under the License. Author: Firefly Software Foundation SPDX-License-Identifier: Apache-2.0 */ -// Operate › Workers: every worker in the environment with its presence, -// claims, load and last contact, filters kept in the address, and a detail -// panel with Drain and Resume. Reloads every 10 s while visible. +// Operate › Workers: the fleet (every current worker with its presence, where +// it runs, its release, load and contact) and its releases. The platform +// applies the filters; the address keeps them, except a host prefix, which +// stays on the page. Selected workers drain, resume or are revoked one after +// another, at most 50, each with its own answer. Retire offline workers +// previews what it would revoke before it does. The fleet reloads every 10 s +// while visible; the server checks every command. import { ChangeDetectionStrategy, ChangeDetectorRef, @@ -31,349 +35,713 @@ import { } from "@angular/core"; import { NavigationEnd, Router } from "@angular/router"; import type { App } from "../../app"; +import { Modal } from "../../dialog"; import { describeError, type PlainError } from "../../errors"; import { shortId } from "../../format"; import { Icon } from "../../icon"; import { ModalSheet, sheetWhen } from "../../modal-sheet"; +import type { Page, Target } from "../../operations/deployment-contracts"; import { toneAttribute } from "../../status-labels"; import { MAX_PAGES } from "../operate-limits"; import { ListReads, environmentOf, listFailure, - readPages, scopeKeyOf, } from "../operate-list"; import { OperateState } from "../operate-state"; -import { viewFromPath, viewPath } from "../operate-routes"; +import { + releasesPath, + viewFromPath, + viewPath, + workersRoute, +} from "../operate-routes"; import { Poller } from "../operate-store"; import { timeAbsolute, timeIso, timeRelative } from "../operate-time"; import { RefreshStatus } from "../refresh-status"; +import { ReleasesPage } from "./releases-page"; import { WorkerDetail } from "./worker-detail"; import { + RETIRE_BATCH, filterChoices, filterCount, - filterWorkers, filtersFromQuery, filtersToQuery, isUnavailableWorker, noWorkerFilters, + releaseTaskTypes, + retireSummary, + retireThresholds, + shortDigest, workerBadges, workerLoad, + workerQuery, + workerVersion, + workerWhere, type Presence, type WorkerFilters, type WorkerRecord, + type WorkerRelease, type WorkerStatus, } from "./workers-model"; -// Every worker, revoked and long offline ones included, newest registered first: -// the page filters on its own, and this order does not move rows between polls. -const EVERY_WORKER = "state=all&order=registered_desc"; +/** The most workers one bulk action takes, one after another. */ +const BULK_LIMIT = 50; +type BulkAction = "drain" | "resume" | "revoke"; +interface BulkResult { + id: string; + ok: boolean; + text: string; +} +interface RetireAnswer { + matched: number; + revoked: number; + worker_ids: string[]; +} +/** The Retire offline workers dialog while it is open. */ +interface RetireState { + seconds: number; + /** The last preview (dry run) for these seconds. */ + preview: RetireAnswer | null; + /** The last retirement's answer. */ + result: RetireAnswer | null; + busy: boolean; + problem: PlainError | null; + /** + * The key of a retirement whose answer was lost: sending it again reuses + * the key, so the platform replays the first answer. + */ + pendingKey: string | null; +} @Component({ selector: "weave-workers-page", standalone: true, changeDetection: ChangeDetectionStrategy.Eager, - imports: [Icon, ModalSheet, OperateState, RefreshStatus, WorkerDetail], + imports: [ + Icon, + Modal, + ModalSheet, + OperateState, + RefreshStatus, + ReleasesPage, + WorkerDetail, + ], styleUrls: ["../operate.css", "./workers.css"], template: `

Workers

- Programs that run your worker actions in this environment. + {{ + tab === "releases" + ? "Worker releases in this environment and what runs them." + : "Programs that run your worker actions in this environment." + }}

- @if (readable && loaded) { + @if (tab === "fleet" && readable && loaded) {
+ @if (canRetire()) { + + }
}
- @if (!host.profile) { -
-

Workers appear here once you connect

-

Workers run your worker actions on a platform.

- +
- } @else if (host.signInEnded) { -
-

Sign in to see workers

-

Studio shows them as soon as you're signed in again.

-
- } @else if (!host.identity) { - - } @else if (!readable || forbidden) { - - } @else if (!loaded && error) { - - } @else if (!loaded) { - - } @else { - @let choices = choicesOf(); - - @if (error) { - - } - @if (detailError; as failed) { - @if (failed.error.status === 403) { + @if (error) { - } @else { - } - } - @if (!visible().length) { - @if (workers.length) { - - + + } + @if (canRetire()) { + + } + - - } @else { - +
} - } @else { -
-
-
+
+

{{ outcome.title }}

+ @if (!bulkRunning) { + + } +
+
    + @for (result of outcome.results; track result.id) { +
  • + {{ short(result.id) }}{{ result.text }} +
    • + } +
+
+ } + @if (!workers.length) { + @if (filterCount(filters)) { + -
-
- WorkerStatusReleaseLoadLast contact + + + } @else { + + } + } @else { +
+
+
+
+
+ @if (canSelect()) { + + } + WorkerStatusWhereReleaseVersionLoadLast contactRegistered +
-
-
- @for (worker of visible(); track worker.id) { - @if (unavailable(worker)) { -
- Unavailable workerStudio can't show this worker. -
- } @else { - @let w = status(worker); - @let load = loadOf(w); -
- {{ typesText(w) }} - @for (badge of badges(w); track badge.label) { - {{ badge.label }} + @for (worker of workers; track worker.id) { + @if (unavailable(worker)) { +
+ @if (canSelect()) { + + } + Unavailable workerStudio can't show this worker. +
+ } @else { + @let w = status(worker); + @let load = loadOf(w); +
+ @if (canSelect()) { + + } +
+ role="cell" + data-wide + class="cell-text" + [attr.title]="where(w)" + >{{ where(w) }}{{ + short(w.release_id) + }}{{ typesText(w) }}{{ version(w) || "Unknown" }} + @if (w.last_seen_at) { + + } @else { + Never + } + @if (w.registered_at) { + + } @else { + Unknown + } + +
+ } } - } +
+ @if (canLoadMore) { + + } @else if (nextCursor) { +

+ Showing the first 500 workers. Narrow the filters to see + others. +

+ }
+ @if (selected) { + + }
- @if (selected) { -