From 40d09934855d4c6ae870253567bf4c7ea5fa8897 Mon Sep 17 00:00:00 2001 From: Guillaume Onfroy Date: Thu, 17 Sep 2026 16:48:15 +0200 Subject: [PATCH 1/5] refactor(core)!: make an operation a component `Operation` declared, under `if TYPE_CHECKING`, attributes it does not own (`id`, `kind`, `key`, `relations`, `materializable`, `source`, `partitioning`, `to_spec`, `bound`) and supplied runtime stand-ins for four of them. That block was never a protocol: it was a promise that `Operation` would only ever be mixed into a `Component`, written in the form the type checker accepts. Both implementors already kept it (`Asset`, and `Connection` through `Resource`), and the platform depends on it: events, the executions view and the failed-scope retry walk are all keyed by component row id. `Operation` is now `Component, Workload` and `Asset` a plain subclass of it, so those attributes are inherited and real. `materializable` moves up as a hidden field, and `qualified_key` goes: it returned the bare key as a stand-in for the qualified form `Component` already builds, which the old base order hid and the new one would have let win, silently unqualifying every asset key. Kinds become explicit where derivation no longer reaches: `Operation` declares an empty kind to opt out, and `Asset` declares its own now that `Component` is not one of its direct bases. Guarding derivation on abstractness instead would be wrong, because `Destination` is abstract and is itself a kind. By Digitl --- .../src/interloper/asset/base.py | 6 +- .../src/interloper/operation/base.py | 82 +++++++------------ .../interloper-core/tests/asset/test_base.py | 5 ++ .../tests/connection/test_base.py | 4 + .../interloper-core/tests/dag/test_base.py | 4 +- .../tests/operation/test_base.py | 21 ++++- .../tests/runner/test_state.py | 6 +- .../interloper-db/tests/store/test_runs.py | 3 +- 8 files changed, 70 insertions(+), 61 deletions(-) diff --git a/packages/interloper-core/src/interloper/asset/base.py b/packages/interloper-core/src/interloper/asset/base.py index 117deb9f..f2aa4791 100644 --- a/packages/interloper-core/src/interloper/asset/base.py +++ b/packages/interloper-core/src/interloper/asset/base.py @@ -30,7 +30,6 @@ from interloper.operation import Operation, OperationContext, OperationResult from interloper.partitioning import ( Partition, - PartitionConfig, PartitionWindow, TimePartition, TimePartitionConfig, @@ -99,7 +98,7 @@ def qualified_key(self) -> str: return str(ComponentIdentity(self.source_key or None, self.key)) -class Asset(Component, Operation): +class Asset(Operation): """A data-producing component. Subclass and implement ``data()`` to define an asset. Every parameter of @@ -121,9 +120,9 @@ def revenue(connection: MyConnection, orders: Upstream) -> Any: """ # Definition + kind: ClassVar[str] = "asset" destinations: list[Destination] = Relation("destination", many=True, optional=True) schema: ClassVar[type[Schema] | None] = None - partitioning: ClassVar[PartitionConfig | None] = None internal_fields: ClassVar[frozenset[str]] = frozenset({"normalizer"}) tags: ClassVar[list[str]] = [] @@ -132,7 +131,6 @@ def revenue(connection: MyConnection, orders: Upstream) -> Any: # State dataset: str = Field(default="") default_destination_key: str = Field(default="") - materializable: bool = Field(default=True) materialization_strategy: MaterializationStrategy = SelectField( default=MaterializationStrategy.RECONCILE, label="Materialization Strategy", diff --git a/packages/interloper-core/src/interloper/operation/base.py b/packages/interloper-core/src/interloper/operation/base.py index 629f6cdb..d0a98650 100644 --- a/packages/interloper-core/src/interloper/operation/base.py +++ b/packages/interloper-core/src/interloper/operation/base.py @@ -24,13 +24,16 @@ from dataclasses import dataclass, field from typing import TYPE_CHECKING, Any, ClassVar +from pydantic import Field + +from interloper.component.base import Component from interloper.errors import format_exception +from interloper.partitioning.base import PartitionConfig if TYPE_CHECKING: - from interloper.component.base import Relation + from interloper.component.relation import Relation from interloper.dag.base import DAG - from interloper.partitioning.base import Partition, PartitionConfig, PartitionWindow - from interloper.serializable import SerializationContext, Spec + from interloper.partitioning.base import Partition, PartitionWindow @dataclass @@ -85,67 +88,44 @@ def operations(self) -> list[Operation]: """ -class Operation(Workload): +class Operation(Component, Workload): """A unit of work: the node a DAG orders and a runner drives. - Beyond :meth:`execute` and :meth:`failure`, this class carries the node - protocol the graph machinery reads: defaults that make any operation a - valid DAG node, which ``Asset`` (the graph-structured, partitioned - operation) overrides with its real fields and properties. They are - deliberately plain class attributes, not pydantic fields: an operation - class is usually also a pydantic component, and a field here would leak - into every subclass's config schema. + An operation is a component: the DAG, the runner and the platform all + address it by its component identity, and its events and executions are + keyed by its row id. What it adds to a component is the execution + contract (:meth:`execute`, :meth:`failure`) and the node attributes the + graph machinery reads, which ``Asset`` (the graph-structured, partitioned + operation) narrows with its own fields and properties. + + ``kind`` is deliberately empty: an operation is a contract several kinds + satisfy, not a kind of its own, and declaring it here is what stops + ``Component.__init_subclass__`` deriving one. Each implementor declares + its own. ``capture_traceback`` controls whether a failed execution's traceback is attached to its failure event; off for operations whose raw errors embed secrets (credential exchanges carry them in URLs). """ + kind: ClassVar[str] = "" capture_traceback: ClassVar[bool] = True + partitioning: ClassVar[PartitionConfig | None] = None - # -- Node protocol ----------------------------------------------------- - - if TYPE_CHECKING: - id: str - kind: ClassVar[str] - key: ClassVar[str] - relations: ClassVar[dict[str, Relation]] - materializable: bool - source: Any | None - partitioning: ClassVar[PartitionConfig | None] - - def to_spec(self, *, context: SerializationContext | None = None) -> Spec: - """Serialize this node (see ``Component.to_spec``). - - Args: - context: The state shared with the other roots of one document. - - Returns: - The node's spec. - """ - ... + materializable: bool = Field(default=True, json_schema_extra={"x-hidden": True}) - def bound(self, name: str) -> Any: - """What is bound to one of this node's relations (see ``Component.bound``). - - Args: - name: The relation name as declared on the class. - - Returns: - The bound node(s): a list for a many-valued relation, the - single node or ``None`` otherwise. - """ - ... + @property + def source(self) -> Any: + """The source that owns this node, or ``None`` when nothing does. - materializable = True - relations = {} # noqa: RUF012 - source = None - partitioning = None + Read by the graph and by the event metadata on any node, including + the operations no source owns, which is why it is answered here + rather than only on ``Asset``. - @property - def qualified_key(self) -> str: - """The node's display key; subclasses qualify it (``source.asset``).""" - return self.key + Returns: + The owning source, or ``None``. + """ + return None def operations(self) -> list[Operation]: """An operation is trivially its own workload. diff --git a/packages/interloper-core/tests/asset/test_base.py b/packages/interloper-core/tests/asset/test_base.py index dc2a8afa..8c62e35b 100644 --- a/packages/interloper-core/tests/asset/test_base.py +++ b/packages/interloper-core/tests/asset/test_base.py @@ -160,6 +160,11 @@ def handler(event: Event) -> None: # -- Identity and class metadata ----------------------------------------------- +class TestKind: + def test_asset_declares_its_kind_explicitly(self): + assert il.Asset.__dict__["kind"] == "asset" + + class TestIdentity: def test_key_auto_derived_from_class_name(self): assert FakeAsset.key == "fake_asset" diff --git a/packages/interloper-core/tests/connection/test_base.py b/packages/interloper-core/tests/connection/test_base.py index d8febbc4..e6c1bf54 100644 --- a/packages/interloper-core/tests/connection/test_base.py +++ b/packages/interloper-core/tests/connection/test_base.py @@ -16,6 +16,10 @@ class TestConnection: + def test_kind_survives_operation_becoming_a_component(self): + assert Connection.kind == "connection" + assert Connection.model_fields["materializable"].json_schema_extra == {"x-hidden": True} + def test_definition_without_oauth(self): class Plain(Connection): host: str = "localhost" diff --git a/packages/interloper-core/tests/dag/test_base.py b/packages/interloper-core/tests/dag/test_base.py index dd64d15c..c274db15 100644 --- a/packages/interloper-core/tests/dag/test_base.py +++ b/packages/interloper-core/tests/dag/test_base.py @@ -4,7 +4,7 @@ # ``data()`` methods whose parameter annotations must be real classes (not lazy # strings) for ``Asset._collect`` to infer their relations. -from typing import Any, ClassVar +from typing import Any, ClassVar, cast import pytest @@ -999,7 +999,7 @@ def test_round_trip_from_a_multi_document_file(self, tmp_path): rebuilt = DAG.from_spec_file(file) assert set(rebuilt.operation_map) == set(original.operation_map) revenue = rebuilt.operation_map[finance.revenue.id] - assert revenue.bound("orders").id == shop.orders.id + assert cast("il.Component", revenue.bound("orders")).id == shop.orders.id assert revenue.bound("orders") is rebuilt.operation_map[shop.orders.id] diff --git a/packages/interloper-core/tests/operation/test_base.py b/packages/interloper-core/tests/operation/test_base.py index f3c4debb..08e39449 100644 --- a/packages/interloper-core/tests/operation/test_base.py +++ b/packages/interloper-core/tests/operation/test_base.py @@ -3,11 +3,12 @@ from __future__ import annotations from typing import Any, ClassVar -from uuid import uuid4 +from uuid import UUID, uuid4 import pytest import interloper as il +from interloper.component import Component from interloper.errors import DAGError from interloper.operation import Operation, OperationContext, OperationResult from interloper.runner.results import ExecutionStatus @@ -40,6 +41,22 @@ def test_default_failure_formats_the_error(self): assert failed.error == "ValueError: boom" assert failed.state == {} + def test_an_operation_is_a_component_with_real_identity(self): + operation = _NoopOperation() + assert isinstance(operation, Component) + assert operation.key == "noop_operation" + assert operation.relations == {} + assert str(UUID(operation.id)) == operation.id + assert operation.to_spec() is not None + + def test_operation_declares_no_kind_of_its_own(self): + assert Operation.kind == "" + assert "destination" in il.KINDS + + def test_materializable_is_a_field(self): + assert _NoopOperation().materializable is True + assert _NoopOperation(materializable=False).materializable is False + def test_node_protocol_defaults(self): operation = _NoopOperation() assert operation.materializable is True @@ -90,7 +107,7 @@ def test_non_workload_anchors(self): assert not issubclass(il.KINDS[kind], il.Workload) -class _EffectfulOperation(il.Component, il.Operation): +class _EffectfulOperation(il.Operation): """Test-only operation kind carrying effects and a curated failure.""" capture_traceback: ClassVar[bool] = False diff --git a/packages/interloper-core/tests/runner/test_state.py b/packages/interloper-core/tests/runner/test_state.py index 7e408f18..caca7323 100644 --- a/packages/interloper-core/tests/runner/test_state.py +++ b/packages/interloper-core/tests/runner/test_state.py @@ -127,7 +127,11 @@ def test_a_dependent_of_only_skipped_operations_is_promoted(self, monkeypatch: A # node) must not leave its dependent queued forever. dag = il.DAG(ChainSource(destinations=[il.MemoryDestination()])) root = next(operation for operation in dag.operations if operation.key == "root") - monkeypatch.setattr(type(root), "materializable", property(lambda self: self.key != "root")) + # `materializable` is a field on `Operation`, so there is no class attribute to replace: + # the property is installed, not overridden. + monkeypatch.setattr( + type(root), "materializable", property(lambda self: self.key != "root"), raising=False + ) state = RunState(dag) diff --git a/packages/interloper-db/tests/store/test_runs.py b/packages/interloper-db/tests/store/test_runs.py index d46bc7e3..8793be79 100644 --- a/packages/interloper-db/tests/store/test_runs.py +++ b/packages/interloper-db/tests/store/test_runs.py @@ -25,9 +25,10 @@ _ORG_ID = uuid4() -class FakePlumbing(il.Component, il.Operation): +class FakePlumbing(il.Operation): """Test-only kind whose operation is platform plumbing (non-billable).""" + kind: ClassVar[str] = "fake_plumbing" billable: ClassVar[bool] = False async def execute(self, context: il.OperationContext) -> il.OperationResult: From 1ea6d839dcfa18a9256a24d05f43031aa27e4c1c Mon Sep 17 00:00:00 2001 From: Guillaume Onfroy Date: Thu, 17 Sep 2026 16:48:57 +0200 Subject: [PATCH 2/5] docs: spec the retry system and its prerequisite Two specs and four plans for making failed work heal on its own. The diagnosis is that the execution hierarchy (batch, run, operation, request) equips its levels inconsistently: retry exists at one level and by hand, contention is bounded by an anonymous per-process number that cannot name the resource it protects, and hooks observe attempts rather than verdicts. The retry spec covers attempts and verdicts; capacity and request-level retry are named as the two sibling specs rather than folded in. The prerequisite spec is the refactor this commit's parent implements. By Digitl --- .../2026-09-17-operation-is-a-component.md | 233 +++++ .../plans/2026-09-17-retry-phase-1-core.md | 877 ++++++++++++++++++ .../2026-09-17-retry-phase-2-platform.md | 835 +++++++++++++++++ .../2026-09-17-retry-phase-3-surfaces.md | 320 +++++++ ...6-09-17-operation-is-a-component-design.md | 145 +++ .../specs/2026-09-17-retry-design.md | 454 +++++++++ 6 files changed, 2864 insertions(+) create mode 100644 docs/superpowers/plans/2026-09-17-operation-is-a-component.md create mode 100644 docs/superpowers/plans/2026-09-17-retry-phase-1-core.md create mode 100644 docs/superpowers/plans/2026-09-17-retry-phase-2-platform.md create mode 100644 docs/superpowers/plans/2026-09-17-retry-phase-3-surfaces.md create mode 100644 docs/superpowers/specs/2026-09-17-operation-is-a-component-design.md create mode 100644 docs/superpowers/specs/2026-09-17-retry-design.md diff --git a/docs/superpowers/plans/2026-09-17-operation-is-a-component.md b/docs/superpowers/plans/2026-09-17-operation-is-a-component.md new file mode 100644 index 00000000..aa2ccc22 --- /dev/null +++ b/docs/superpowers/plans/2026-09-17-operation-is-a-component.md @@ -0,0 +1,233 @@ +# Operation is a Component Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Make `Operation` a real `Component` so it stops declaring stand-in attributes it does not own. + +**Architecture:** `Operation` becomes `Component, Workload`, and `Asset` becomes a plain subclass of +`Operation`. The `if TYPE_CHECKING` block and the `relations`/`source` runtime stand-ins are deleted, +because `id`, `kind`, `key`, `relations`, `to_spec()` and `bound()` are then inherited and real. +`materializable` moves up from `Asset` to `Operation`. Kinds need two explicit declarations: +`Operation` declares `kind = ""` so the derivation skips it, and `Asset` declares `kind = "asset"` +because `Component` is no longer one of its direct bases. + +**Tech Stack:** Python 3.10+, pydantic v2, pydantic-settings, pytest, ruff, ty, uv workspace. + +Spec: `docs/superpowers/specs/2026-09-17-operation-is-a-component-design.md`. + +## Global Constraints + +- Line length 120, ruff-formatted. Type-checked with `ty`. +- Google-style docstrings on every module, class, function and method, with every applicable section + (`Args:`, `Returns:`, `Raises:`). See `.claude/rules/python-style.md`. +- Comment sparingly. Never attribute-level comments on pydantic fields. +- Tests mirror the package layout one to one. A test for `src/interloper//.py` lives in + `tests//test_.py`. Never add a standalone `test_.py`. +- Run checks from the repo root: `uv run ruff check`, `uv run ty check`, `uv run pytest`. +- A fresh worktree needs `uv sync --all-packages --all-extras` first, or `ty` and `pytest` fail on + optional imports (suds, facebook_business, OTel exporters). +- **Do not commit without Guillaume asking.** The commit step in each task records the intended + message; run it only when he says so. + +--- + +### Task 1: Operation becomes a Component + +**Files:** +- Modify: `packages/interloper-core/src/interloper/operation/base.py:88-245` +- Modify: `packages/interloper-core/src/interloper/asset/base.py:102-150` +- Test: `packages/interloper-core/tests/operation/test_base.py` +- Test: `packages/interloper-core/tests/asset/test_base.py` +- Test: `packages/interloper-core/tests/connection/test_base.py` + +**Interfaces:** +- Consumes: nothing. +- Produces: `class Operation(Component, Workload)` with a real `id`, `kind`, `key`, `relations`, + `to_spec()` and `bound()`, plus the field `materializable: bool` (default `True`, hidden from the + public schema). `class Asset(Operation)` with `kind: ClassVar[str] = "asset"`. The retry plans + declare `Operation.retry` on this class. + +- [ ] **Step 1: Write the failing tests** + +Add to `packages/interloper-core/tests/operation/test_base.py`: + +```python +def test_operation_is_a_component_with_real_identity(): + class Thing(Operation): + kind: ClassVar[str] = "thing" + + async def execute(self, context: OperationContext) -> OperationResult: + return OperationResult() + + thing = Thing() + assert isinstance(thing, Component) + assert thing.kind == "thing" + assert thing.key == "thing" + assert thing.relations == {} + assert UUID(thing.id) + assert thing.to_spec() is not None + assert thing.operations() == [thing] + + +def test_operation_declares_no_kind_of_its_own(): + assert Operation.kind == "" + assert "destination" in il.KINDS + + +def test_operation_is_materializable_by_default(): + class Thing(Operation): + kind: ClassVar[str] = "thing" + + async def execute(self, context: OperationContext) -> OperationResult: + return OperationResult() + + assert Thing().materializable is True + assert Thing(materializable=False).materializable is False +``` + +Add to `packages/interloper-core/tests/asset/test_base.py`: + +```python +def test_asset_kind_is_declared_explicitly(): + assert Asset.__dict__["kind"] == "asset" +``` + +Add to `packages/interloper-core/tests/connection/test_base.py`: + +```python +def test_connection_keeps_its_kind_and_hides_materializable(): + assert Connection.kind == "connection" + assert Connection.model_fields["materializable"].json_schema_extra == {"x-hidden": True} +``` + +- [ ] **Step 2: Run the tests to verify they fail** + +Run: `uv run pytest packages/interloper-core/tests/operation/test_base.py packages/interloper-core/tests/asset/test_base.py::test_asset_kind_is_declared_explicitly packages/interloper-core/tests/connection/test_base.py::test_connection_keeps_its_kind_and_hides_materializable -v` +Expected: FAIL. `Operation` cannot be instantiated as a component, `Asset.__dict__` has a derived +rather than declared `kind`, and `materializable` is not a field on `Connection`. + +- [ ] **Step 3: Rewrite the Operation class header** + +In `packages/interloper-core/src/interloper/operation/base.py`, change the imports to bring in +`Component` (`from interloper.component.base import Component`) and `Field` from pydantic, drop the +now-unneeded `TYPE_CHECKING` imports of `Relation`, `SerializationContext` and `Spec`, and replace +the class header and the whole node-protocol block: + +```python +class Operation(Component, Workload): + """A unit of work: the node a DAG orders and a runner drives. + + An operation is a component: the DAG, the runner and the platform all + address it by its component identity, and its events, executions and + retries are keyed by its row id. What it adds to a component is the + execution contract (:meth:`execute`, :meth:`failure`) and the node + attributes the graph machinery reads, which ``Asset`` (the + graph-structured, partitioned operation) narrows with its own fields. + + ``capture_traceback`` controls whether a failed execution's traceback + is attached to its failure event; off for operations whose raw errors + embed secrets (credential exchanges carry them in URLs). + """ + + kind: ClassVar[str] = "" + partitioning: ClassVar[PartitionConfig | None] = None + capture_traceback: ClassVar[bool] = True + + materializable: bool = Field(default=True, json_schema_extra={"x-hidden": True}) +``` + +`kind = ""` is the opt-out the derivation in `Component.__init_subclass__` already honours: it only +derives a kind for a direct child that declares none. Do not instead make the derivation skip +abstract classes; `Destination` is abstract and *is* a kind, so that unregisters `destination` and +breaks every relation declared against it. + +Delete the entire `if TYPE_CHECKING:` block (the `id`/`kind`/`key`/`relations`/`materializable`/ +`source`/`partitioning` declarations and the `to_spec`/`bound` stubs) and the four runtime stand-ins +below it (`materializable = True`, `relations = {}`, `source = None`, `partitioning = None`), keeping +`source = None` only if step 5 shows a read that `Component.parent` does not already serve. + +Everything from `qualified_key` downwards is unchanged. + +- [ ] **Step 4: Rewrite the Asset class header** + +In `packages/interloper-core/src/interloper/asset/base.py`, change the header and remove the two +declarations that moved up: + +```python +class Asset(Operation): + ... + kind: ClassVar[str] = "asset" +``` + +Delete `materializable: bool = Field(default=True)` from the `# State` block, and delete +`partitioning: ClassVar[PartitionConfig | None] = None` from the `# Definition` block. Keep the +`Component` import only if it is still used elsewhere in the module; remove it otherwise. + +- [ ] **Step 5: Resolve `source`** + +Run: `uv run grep -rn "\.source" packages/interloper-core/src/interloper/dag packages/interloper-core/src/interloper/runner` + +For each hit, confirm whether it reads an operation's owning source. If every hit is served by +`Asset.source` (assets) and nothing reads `source` on a non-asset operation, delete the `source = +None` stand-in. If any read reaches a `Connection`, keep `source = None` on `Operation` and record it +in the spec's follow-ups instead of changing behaviour here. + +- [ ] **Step 6: Run the tests to verify they pass** + +Run: `uv run pytest packages/interloper-core/tests/operation packages/interloper-core/tests/asset packages/interloper-core/tests/connection -v` +Expected: PASS. + +If `Connection` fails to build its model, the `BaseSettings` plus `BaseModel` diamond is the cause: +`Connection(Resource, Operation)` reaches `Component` through both bases. The MRO linearizes as +`Connection, Resource, BaseSettings, Operation, Component, ..., Workload, object`. Report the exact +pydantic error rather than working around it; this is the one risk the spec flagged as worth proving +before going further. + +- [ ] **Step 7: Commit** + +```bash +git add packages/interloper-core/src/interloper/operation/base.py packages/interloper-core/src/interloper/asset/base.py packages/interloper-core/tests +git commit -m "refactor(core)!: make an operation a component + +By Digitl" +``` + +--- + +### Task 2: Sweep the workspace + +**Files:** +- Modify: whatever the checks surface, across `packages/`. + +**Interfaces:** +- Consumes: Task 1. +- Produces: a green workspace. No new API. + +- [ ] **Step 1: Run the Python checks** + +Run: `uv run ruff check && uv run ty check` +Expected: clean. Likely fallout: imports of `Component` that `asset/base.py` and other modules no +longer need, and `ty` errors where a caller narrowed an `Operation` to reach a component attribute +that is now inherited (those narrowings can be deleted). + +- [ ] **Step 2: Run the full test suite** + +Run: `uv run pytest` +Expected: PASS. Watch `packages/interloper-core/tests/dag`, `tests/runner` and +`packages/interloper-db/tests/store/test_hydration.py`, which are the regression surface for node +attribute reads and for reconstructing an operation from a spec. + +- [ ] **Step 3: Check the catalog still builds** + +Run: `uv run pytest packages/interloper-assets -q` +Expected: PASS. This is the broadest set of real `Asset` subclasses and decorator-built classes, so +it is what proves the inheritance change holds for the shipped definitions. + +- [ ] **Step 4: Commit** + +```bash +git add -A +git commit -m "chore: fix fallout from operation becoming a component + +By Digitl" +``` diff --git a/docs/superpowers/plans/2026-09-17-retry-phase-1-core.md b/docs/superpowers/plans/2026-09-17-retry-phase-1-core.md new file mode 100644 index 00000000..0d34914e --- /dev/null +++ b/docs/superpowers/plans/2026-09-17-retry-phase-1-core.md @@ -0,0 +1,877 @@ +# Retry Phase 1: Core Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** An operation that fails is re-executed in place, within a declared attempt budget, without the run ever seeing the intermediate failure. + +**Architecture:** One flat `RetryPolicy` carries the numbers. It is declared on the component whose +unit it governs: `Operation.retry` for an operation's execution, `Source.retry` as the default for its +assets, `Job.retry` for that job's runs (consumed in phase 2). `Operation.retryable(error)` is the +code-level classifier. The runner wraps its node execution in an attempt loop, emits +`OPERATION_RETRIED` for an attempt that will be retried, and reserves `OPERATION_FAILED` for +exhaustion, so an intermediate attempt is recorded without ever becoming the verdict. + +**Tech Stack:** Python 3.10+, pydantic v2, asyncio, pytest, ruff, ty, uv workspace. + +Spec: `docs/superpowers/specs/2026-09-17-retry-design.md`, sections 3 and 4. +Prerequisite: `docs/superpowers/plans/2026-09-17-operation-is-a-component.md` must be complete. +`Operation` is a `Component` and can carry a pydantic field. + +## Global Constraints + +- Line length 120, ruff-formatted. Type-checked with `ty`. +- Google-style docstrings on every module, class, function and method, with every applicable section + (`Args:`, `Returns:`, `Raises:`). See `.claude/rules/python-style.md`. +- Comment sparingly. Never attribute-level comments on pydantic fields. +- Every concept gets a package with a `base.py`; `__init__.py` re-exports with an explicit `__all__` + and defines nothing. +- Tests mirror the package layout one to one. +- Run checks from the repo root: `uv run ruff check`, `uv run ty check`, `uv run pytest`. +- **Do not commit without Guillaume asking.** The commit step records the intended message. + +--- + +### Task 1: The RetryPolicy type + +**Files:** +- Create: `packages/interloper-core/src/interloper/retry/__init__.py` +- Create: `packages/interloper-core/src/interloper/retry/base.py` +- Create: `packages/interloper-core/tests/retry/__init__.py` +- Create: `packages/interloper-core/tests/retry/test_base.py` +- Modify: `packages/interloper-core/src/interloper/__init__.py` + +**Interfaces:** +- Consumes: nothing. +- Produces: `interloper.RetryPolicy` with fields `max_attempts: int`, `delay: float`, + `backoff: float`, `max_delay: float`, `jitter: float`, and methods `allows(attempt: int) -> bool` + and `delay_before(attempt: int) -> float`. Every later task imports it from `interloper.retry`. + +- [ ] **Step 1: Write the failing test** + +Create `packages/interloper-core/tests/retry/__init__.py` (empty) and +`packages/interloper-core/tests/retry/test_base.py`: + +```python +import pytest + +from interloper.retry import RetryPolicy + + +def test_allows_up_to_max_attempts(): + policy = RetryPolicy(max_attempts=3) + assert policy.allows(1) + assert policy.allows(3) + assert not policy.allows(4) + + +def test_first_attempt_has_no_delay(): + assert RetryPolicy(delay=10.0).delay_before(1) == 0.0 + + +def test_delay_grows_geometrically(): + policy = RetryPolicy(delay=10.0, backoff=2.0, jitter=0.0) + assert policy.delay_before(2) == 10.0 + assert policy.delay_before(3) == 20.0 + assert policy.delay_before(4) == 40.0 + + +def test_delay_is_capped(): + policy = RetryPolicy(delay=10.0, backoff=10.0, max_delay=50.0, jitter=0.0) + assert policy.delay_before(4) == 50.0 + + +def test_jitter_stays_within_its_fraction(): + policy = RetryPolicy(delay=10.0, backoff=1.0, jitter=0.5) + delays = [policy.delay_before(2) for _ in range(200)] + assert all(5.0 <= delay <= 15.0 for delay in delays) + assert len(set(delays)) > 1 + + +def test_max_attempts_is_at_least_one(): + with pytest.raises(ValueError): + RetryPolicy(max_attempts=0) +``` + +- [ ] **Step 2: Run the test to verify it fails** + +Run: `uv run pytest packages/interloper-core/tests/retry/test_base.py -v` +Expected: FAIL with `ModuleNotFoundError: No module named 'interloper.retry'`. + +- [ ] **Step 3: Write the implementation** + +Create `packages/interloper-core/src/interloper/retry/base.py`: + +```python +"""Retry: an attempt budget for a unit of work.""" + +from __future__ import annotations + +import random + +from pydantic import BaseModel, Field + + +class RetryPolicy(BaseModel): + """How many times a unit of work is attempted, and how long between attempts. + + Declared on the component whose unit it governs: an operation's policy + retries that operation's execution, a source's is the default for its + assets, a job's retries that job's runs. The policy carries numbers only; + whether a given error is worth another attempt is behaviour, and lives on + ``Operation.retryable``. + """ + + max_attempts: int = Field( + default=3, + ge=1, + title="Max attempts", + description="Total attempts, including the first", + ) + delay: float = Field( + default=5.0, + ge=0, + title="Delay", + description="Seconds to wait before the second attempt", + ) + backoff: float = Field( + default=2.0, + ge=1, + title="Backoff", + description="Multiplier applied to the delay for each further attempt", + ) + max_delay: float = Field( + default=3600.0, + ge=0, + title="Max delay", + description="Upper bound on any single delay, in seconds", + ) + jitter: float = Field( + default=0.1, + ge=0, + le=1, + title="Jitter", + description="Fraction of the delay spread randomly around it, so attempts do not align", + ) + + def allows(self, attempt: int) -> bool: + """Whether the budget covers an attempt. + + Args: + attempt: The attempt number, counting the first execution as 1. + + Returns: + ``True`` while the attempt is within the budget. + """ + return attempt <= self.max_attempts + + def delay_before(self, attempt: int) -> float: + """How long to wait before an attempt. + + The growth is geometric from the second attempt (the first retry), + capped at ``max_delay``, then spread by ``jitter`` so that operations + failing together do not retry in lockstep. + + Args: + attempt: The attempt number, counting the first execution as 1. + + Returns: + The delay in seconds; ``0.0`` for the first attempt. + """ + if attempt <= 1: + return 0.0 + delay = min(self.delay * self.backoff ** (attempt - 2), self.max_delay) + if not self.jitter: + return delay + spread = delay * self.jitter + return max(0.0, random.uniform(delay - spread, delay + spread)) +``` + +Create `packages/interloper-core/src/interloper/retry/__init__.py`: + +```python +"""Retry policies.""" + +from interloper.retry.base import RetryPolicy + +__all__ = ["RetryPolicy"] +``` + +- [ ] **Step 4: Export it from the package root** + +In `packages/interloper-core/src/interloper/__init__.py`, add the import alongside the other +alphabetically-grouped imports and the name to `__all__`: + +```python +from interloper.retry import RetryPolicy +``` + +```python + "RetryPolicy", +``` + +- [ ] **Step 5: Run the tests to verify they pass** + +Run: `uv run pytest packages/interloper-core/tests/retry/test_base.py -v` +Expected: PASS, 6 tests. + +- [ ] **Step 6: Commit** + +```bash +git add packages/interloper-core/src/interloper/retry packages/interloper-core/tests/retry packages/interloper-core/src/interloper/__init__.py +git commit -m "feat(core): add a retry policy + +By Digitl" +``` + +--- + +### Task 2: Declare the policy on the components + +**Files:** +- Modify: `packages/interloper-core/src/interloper/operation/base.py` +- Modify: `packages/interloper-core/src/interloper/source/base.py` (class body, and `_resolve` at :547) +- Modify: `packages/interloper-core/src/interloper/job/base.py:32-56` +- Test: `packages/interloper-core/tests/operation/test_base.py` +- Test: `packages/interloper-core/tests/source/test_base.py` +- Test: `packages/interloper-core/tests/job/test_base.py` + +**Interfaces:** +- Consumes: `interloper.retry.RetryPolicy` from Task 1. +- Produces: `Operation.retry: RetryPolicy | None` (inherited by `Asset` and `Connection`), + `Operation.retryable(error: Exception) -> bool`, `Source.retry`, `Job.retry`, and the + source-to-asset inheritance in `Source._resolve`. Task 4 reads `operation.retry` and + `operation.retryable`. + +- [ ] **Step 1: Write the failing tests** + +Add to `packages/interloper-core/tests/operation/test_base.py`: + +```python +def test_operation_has_no_retry_policy_by_default(): + class Thing(Operation): + kind: ClassVar[str] = "thing" + + async def execute(self, context: OperationContext) -> OperationResult: + return OperationResult() + + assert Thing().retry is None + + +def test_every_error_is_retryable_by_default(): + class Thing(Operation): + kind: ClassVar[str] = "thing" + + async def execute(self, context: OperationContext) -> OperationResult: + return OperationResult() + + assert Thing().retryable(ValueError("nope")) is True + + +def test_retryable_can_be_narrowed(): + class Picky(Operation): + kind: ClassVar[str] = "picky" + + async def execute(self, context: OperationContext) -> OperationResult: + return OperationResult() + + def retryable(self, error: Exception) -> bool: + return not isinstance(error, ValueError) + + assert Picky().retryable(TypeError()) is True + assert Picky().retryable(ValueError()) is False +``` + +Add to `packages/interloper-core/tests/source/test_base.py`: + +```python +def test_source_retry_policy_fills_its_assets(): + policy = il.RetryPolicy(max_attempts=5) + + @il.asset + def one(**kwargs): ... + + @il.source(retry=policy) + class MySource(il.Source): + assets = (one,) + + source = MySource() + assert source.assets[0].retry == policy + + +def test_an_assets_own_retry_policy_wins(): + source_policy = il.RetryPolicy(max_attempts=5) + asset_policy = il.RetryPolicy(max_attempts=2) + + @il.asset(retry=asset_policy) + def one(**kwargs): ... + + @il.source(retry=source_policy) + class MySource(il.Source): + assets = (one,) + + assert MySource().assets[0].retry == asset_policy +``` + +Match the `@il.source` declaration style already used in that test module; copy the surrounding +tests' shape for how assets are attached rather than inventing one. + +Add to `packages/interloper-core/tests/job/test_base.py`: + +```python +def test_job_carries_a_retry_policy(): + policy = il.RetryPolicy(max_attempts=2) + assert il.Job(name="nightly", retry=policy).retry == policy + assert il.Job(name="nightly").retry is None +``` + +- [ ] **Step 2: Run the tests to verify they fail** + +Run: `uv run pytest packages/interloper-core/tests/operation/test_base.py packages/interloper-core/tests/source/test_base.py packages/interloper-core/tests/job/test_base.py -v -k retry` +Expected: FAIL. `retry` is not a field and `retryable` does not exist. + +- [ ] **Step 3: Add the field and the classifier to Operation** + +In `packages/interloper-core/src/interloper/operation/base.py`, import `RetryPolicy` +(`from interloper.retry import RetryPolicy`) and add the field to the class body next to +`materializable`: + +```python + retry: RetryPolicy | None = Field( + default=None, + title="Retry", + description="Attempt budget for this operation's execution", + ) +``` + +Add the classifier in the `# -- Execution` section, directly above `failure`: + +```python + def retryable(self, error: Exception) -> bool: + """Whether another attempt at this operation is worth making. + + Consulted by the runner before it spends an attempt from the budget. + Override to recognise a permanent error, such as a vendor rejecting a + request that will be rejected identically every time. The default is + permissive: the budget, not the classifier, is what bounds waste. + + Args: + error: The exception :meth:`execute` raised. + + Returns: + ``True`` when the failure may be transient. + """ + return True +``` + +Extend the class docstring with one sentence naming `retry` and `retryable` as the execution +contract's retry half. + +- [ ] **Step 4: Add the field to Source and Job** + +In `packages/interloper-core/src/interloper/source/base.py`, add to the class body: + +```python + retry: RetryPolicy | None = Field( + default=None, + title="Retry", + description="Default attempt budget for this source's assets", + ) +``` + +and one line to `_resolve`, in the per-asset loop alongside the other source-level defaults: + +```python + if asset.retry is None and self.retry is not None: + asset.retry = self.retry +``` + +In `packages/interloper-core/src/interloper/job/base.py`, add to the class body: + +```python + retry: RetryPolicy | None = Field( + default=None, + title="Retry", + description="Attempt budget for this job's runs", + ) +``` + +Import `RetryPolicy` in both modules. + +- [ ] **Step 5: Run the tests to verify they pass** + +Run: `uv run pytest packages/interloper-core/tests/operation packages/interloper-core/tests/source packages/interloper-core/tests/job -v` +Expected: PASS. + +- [ ] **Step 6: Verify the decorators take it for free** + +Run: `uv run python -c "import interloper as il; print(il.asset(lambda **kw: None, retry=il.RetryPolicy(max_attempts=2)).model_fields['retry'])"` +Expected: prints the field. `@il.asset` and `@il.source` split `**overrides` against the class's +ClassVars and `model_fields`, so a new field needs no decorator change. Add `retry` to the +`**overrides` list in both decorators' docstrings. + +- [ ] **Step 7: Commit** + +```bash +git add packages/interloper-core/src/interloper packages/interloper-core/tests +git commit -m "feat(core): declare a retry policy on operations, sources and jobs + +By Digitl" +``` + +--- + +### Task 3: Record attempts in the event stream + +An operation retried in place must not emit `OPERATION_FAILED` for an intermediate attempt, and each +attempt's events need distinct ids: `RunState._operation_event_id` is a uuid5 of +`(run_id, component_id, event_type)`, so a second attempt would collide with the first and dedup away. + +**Files:** +- Modify: `packages/interloper-core/src/interloper/events/types.py:25-29` +- Modify: `packages/interloper-core/src/interloper/events/console.py:24-28` +- Modify: `packages/interloper-core/src/interloper/runner/state.py` (`__init__`, `_operation_event_metadata` at :352, `_operation_event_id` at :373, `_emit_operation_event` at :391, plus the new `mark_retried`) +- Test: `packages/interloper-core/tests/runner/test_state.py` +- Test: `packages/interloper-core/tests/runner/test_state_event_ids.py` + +**Interfaces:** +- Consumes: nothing from earlier tasks. +- Produces: `EventType.OPERATION_RETRIED`; `RunState.attempts: dict[str, int]` (operation id to + current attempt, starting at 1); `RunState.mark_retried(operation, error, *, emit=True)` which + emits the retried event and increments the counter; every operation event carries `attempt` in its + metadata; `_operation_event_id(run_id, component_id, event_type, attempt=1)`. Task 4 calls + `mark_retried`. + +- [ ] **Step 1: Write the failing tests** + +Add to `packages/interloper-core/tests/runner/test_state_event_ids.py`: + +```python +def test_event_id_is_unchanged_for_a_first_attempt(): + first = RunState._operation_event_id("run", "component", EventType.OPERATION_FAILED) + explicit = RunState._operation_event_id("run", "component", EventType.OPERATION_FAILED, attempt=1) + assert first == explicit + + +def test_event_id_differs_per_attempt(): + first = RunState._operation_event_id("run", "component", EventType.OPERATION_STARTED, attempt=1) + second = RunState._operation_event_id("run", "component", EventType.OPERATION_STARTED, attempt=2) + assert first != second +``` + +Add to `packages/interloper-core/tests/runner/test_state.py`, following that module's existing way of +building a `RunState` and capturing emitted events: + +```python +def test_mark_retried_emits_and_advances_the_attempt(state, events): + operation = state.dag.operations[0] + + state.mark_retried(operation, "boom") + + assert state.attempts[operation.id] == 2 + retried = [event for event in events if event.type is EventType.OPERATION_RETRIED] + assert len(retried) == 1 + assert retried[0].metadata["attempt"] == 1 + assert retried[0].metadata["error"] == "boom" + + +def test_events_after_a_retry_carry_the_new_attempt(state, events): + operation = state.dag.operations[0] + + state.mark_retried(operation, "boom") + state.mark_failed(operation, "boom again") + + failed = [event for event in events if event.type is EventType.OPERATION_FAILED] + assert failed[0].metadata["attempt"] == 2 +``` + +- [ ] **Step 2: Run the tests to verify they fail** + +Run: `uv run pytest packages/interloper-core/tests/runner/test_state.py packages/interloper-core/tests/runner/test_state_event_ids.py -v -k "retry or retried or attempt"` +Expected: FAIL. `OPERATION_RETRIED`, `mark_retried` and `attempts` do not exist, and +`_operation_event_id` takes three arguments. + +- [ ] **Step 3: Add the event type** + +In `packages/interloper-core/src/interloper/events/types.py`, in the operation-lifecycle block: + +```python + OPERATION_RETRIED = "operation_retried" +``` + +In `packages/interloper-core/src/interloper/events/console.py`, in the level map: + +```python + EventType.OPERATION_RETRIED: logging.WARNING, +``` + +- [ ] **Step 4: Track attempts in RunState** + +In `packages/interloper-core/src/interloper/runner/state.py`, initialise the counter in `__init__` +next to the `executions` dict: + +```python + self.attempts: dict[str, int] = {operation.id: 1 for operation in dag.operations} +``` + +Add `attempt` to `_operation_event_metadata`, in the dict it builds: + +```python + "attempt": self.attempts[operation.id], +``` + +Thread the attempt through the id derivation: + +```python + @staticmethod + def _operation_event_id(run_id: str, component_id: str, event_type: EventType, attempt: int = 1) -> str: + key = f"{run_id}:{component_id}:{event_type.value}" + if attempt > 1: + key = f"{key}:{attempt}" + return str(uuid.uuid5(RunState._OPERATION_EVENT_NS, key)) +``` + +Extend its docstring with an `attempt` entry and the reason the first attempt is left out of the key: +ids written before retries existed stay identical, so nothing in history re-keys. + +In `_emit_operation_event`, pass the attempt the metadata carries: + +```python + event_id = self._operation_event_id( + run_id=str(self.metadata.get("run_id")), + component_id=str(metadata["component_id"]), + event_type=event_type, + attempt=int(metadata.get("attempt", 1)), + ) +``` + +Match the existing call's argument style in that method rather than the keyword form above if it +differs. + +- [ ] **Step 5: Add mark_retried** + +In the same module, next to `mark_failed`: + +```python + def mark_retried(self, operation: Operation, error: str, *, emit: bool = True) -> None: + """Record a failed attempt that will be retried, and open the next one. + + A retried attempt is not a verdict: the execution keeps its current + status, nothing downstream is canceled, and ``OPERATION_FAILED`` stays + reserved for an exhausted budget. The counter it advances is what makes + the next attempt's events distinct from this one's. + + Args: + operation: The operation whose attempt failed. + error: Error message describing the failed attempt. + emit: Emit ``OPERATION_RETRIED`` on the EventBus. Set to ``False`` + for cross-process runners where the child emits its own events. + """ + attempt = self.attempts[operation.id] + if emit: + self._emit_operation_event( + EventType.OPERATION_RETRIED, + { + **self._operation_event_metadata(operation), + "error": error, + "message": f"Operation '{operation.key}' failed on attempt {attempt}, retrying: {error}", + }, + ) + self.attempts[operation.id] = attempt + 1 +``` + +- [ ] **Step 6: Run the tests to verify they pass** + +Run: `uv run pytest packages/interloper-core/tests/runner -v` +Expected: PASS, including the existing state and event-id suites unchanged. + +- [ ] **Step 7: Commit** + +```bash +git add packages/interloper-core/src/interloper/events packages/interloper-core/src/interloper/runner/state.py packages/interloper-core/tests/runner +git commit -m "feat(core): record operation attempts in the event stream + +By Digitl" +``` + +--- + +### Task 4: The attempt loop in AsyncRunner + +**Files:** +- Modify: `packages/interloper-core/src/interloper/runner/async_runner.py:166-205` +- Test: `packages/interloper-core/tests/runner/test_async_runner.py` + +**Interfaces:** +- Consumes: `RetryPolicy` (Task 1), `Operation.retry` and `Operation.retryable` (Task 2), + `RunState.mark_retried` and `RunState.attempts` (Task 3). +- Produces: an `AsyncRunner` that re-executes a failed node in place within its declared budget. An + operation that declares no policy is attempted once, as today. Task 5 mirrors the loop in + `MultiProcessRunner`. + +- [ ] **Step 1: Write the failing tests** + +Add to `packages/interloper-core/tests/runner/test_async_runner.py`, following that module's existing +way of building a DAG of test assets and collecting events: + +```python +async def test_an_operation_that_heals_on_the_second_attempt_succeeds(): + calls = [] + + @il.asset(retry=il.RetryPolicy(max_attempts=3, delay=0.0, jitter=0.0)) + def flaky(**kwargs): + calls.append(1) + if len(calls) == 1: + raise RuntimeError("transient") + return "ok" + + events = [] + result = await il.AsyncRunner(on_event=events.append).run(il.DAG(flaky())) + + assert result.status is ExecutionStatus.COMPLETED + assert len(calls) == 2 + assert [e.type for e in events if e.type is EventType.OPERATION_RETRIED] + assert not [e.type for e in events if e.type is EventType.OPERATION_FAILED] + + +async def test_an_exhausted_budget_fails_once(): + calls = [] + + @il.asset(retry=il.RetryPolicy(max_attempts=2, delay=0.0, jitter=0.0)) + def broken(**kwargs): + calls.append(1) + raise RuntimeError("permanent") + + events = [] + result = await il.AsyncRunner(on_event=events.append, reraise=False).run(il.DAG(broken())) + + assert result.status is ExecutionStatus.FAILED + assert len(calls) == 2 + assert len([e for e in events if e.type is EventType.OPERATION_RETRIED]) == 1 + assert len([e for e in events if e.type is EventType.OPERATION_FAILED]) == 1 + + +async def test_an_unretryable_error_is_not_retried(): + calls = [] + + class Picky(il.Asset): + retry: il.RetryPolicy | None = il.RetryPolicy(max_attempts=3, delay=0.0, jitter=0.0) + + def data(self, **kwargs): + calls.append(1) + raise ValueError("permanent") + + def retryable(self, error: Exception) -> bool: + return not isinstance(error, ValueError) + + result = await il.AsyncRunner(reraise=False).run(il.DAG(Picky())) + + assert result.status is ExecutionStatus.FAILED + assert len(calls) == 1 + + +async def test_an_operation_without_a_policy_is_attempted_once(): + calls = [] + + @il.asset + def flaky(**kwargs): + calls.append(1) + raise RuntimeError("transient") + + result = await il.AsyncRunner(reraise=False).run(il.DAG(flaky())) + + assert result.status is ExecutionStatus.FAILED + assert len(calls) == 1 + + +async def test_a_retrying_node_does_not_trip_fail_fast(): + downstream_ran = [] + + @il.asset(retry=il.RetryPolicy(max_attempts=2, delay=0.0, jitter=0.0)) + def flaky(**kwargs): + if not downstream_ran: + raise RuntimeError("transient") + return "ok" + + @il.asset + def after(flaky: il.Upstream, **kwargs): + downstream_ran.append(1) + return "ok" + + result = await il.AsyncRunner(fail_fast=True).run(il.DAG(flaky(), after())) + + assert result.status is ExecutionStatus.COMPLETED +``` + +Adapt the asset-construction and DAG-assembly style to whatever the surrounding tests already use. + +- [ ] **Step 2: Run the tests to verify they fail** + +Run: `uv run pytest packages/interloper-core/tests/runner/test_async_runner.py -v -k "attempt or retry or heal or exhaust"` +Expected: FAIL. `AsyncRunner` has no `retry` field and executes each node exactly once. + +- [ ] **Step 3: Wrap the execution in the attempt loop** + +In `packages/interloper-core/src/interloper/runner/async_runner.py`, replace the body of +`_execute_operation` from `self.state.mark_running(operation)` to the end with: + +```python + effective_partition = operation.effective_partition(partition_or_window) + span_attrs = attributes.from_metadata(operation._event_metadata(self.state.metadata, effective_partition)) + context = OperationContext( + partition_or_window=effective_partition, + dag=self.state.dag, + metadata=self.state.metadata, + ) + policy = operation.retry + + while True: + self.state.mark_running(operation) + try: + with tracer().start_as_current_span("interloper.operation.execute", attributes=span_attrs): + result = await operation.execute(context) + except Exception as e: # noqa: BLE001 — every failure becomes the node's record + attempt = self.state.attempts[operation.id] + if policy is not None and policy.allows(attempt + 1) and operation.retryable(e): + self.state.mark_retried(operation, format_exception(e)) + await asyncio.sleep(policy.delay_before(attempt + 1)) + continue + failed = operation.failure(e) + tb = traceback.format_exc() if type(operation).capture_traceback else None + self.state.mark_failed( + operation, failed.error or format_exception(e), tb=tb, effects=failed, exception=e + ) + return None + self.state.mark_completed(operation, effects=result) + return result +``` + +Update the method's docstring to describe the loop: an operation carrying a policy has a failed +attempt recorded as retried and executed again after the backoff when the error is retryable and the +budget allows; an operation carrying none is attempted once; only an exhausted or declined failure +marks the node failed and propagates downstream. + +- [ ] **Step 4: Run the tests to verify they pass** + +Run: `uv run pytest packages/interloper-core/tests/runner/test_async_runner.py -v` +Expected: PASS, including every existing test in the module. + +- [ ] **Step 5: Commit** + +```bash +git add packages/interloper-core/src/interloper/runner packages/interloper-core/tests/runner +git commit -m "feat(core): retry a failed operation in place + +By Digitl" +``` + +--- + +### Task 5: Parity in MultiProcessRunner + +**Files:** +- Modify: `packages/interloper-core/src/interloper/runner/multi_process.py:40-80` +- Test: `packages/interloper-core/tests/runner/test_multi_process.py` + +**Interfaces:** +- Consumes: everything from Tasks 1 to 4. +- Produces: the same retry behaviour in the cross-process runner. No new API. + +- [ ] **Step 1: Confirm the execution paths** + +Run: `uv run grep -rn "operation.execute(" packages/interloper-core/src/interloper/runner/` +Expected: two call sites, `async_runner.py` (done in Task 4) and `multi_process.py`. If a third +appears, it gets the same treatment in this task. + +- [ ] **Step 2: Write the failing test** + +Add to `packages/interloper-core/tests/runner/test_multi_process.py`, following that module's +existing way of declaring picklable module-level assets (a closure will not survive the process +boundary, so the counter has to live in a file or a module-level global that the worker mutates): + +```python +def test_an_operation_that_heals_on_the_second_attempt_succeeds(tmp_path): + marker = tmp_path / "attempts" + + @il.asset(retry=il.RetryPolicy(max_attempts=3, delay=0.0, jitter=0.0)) + def flaky(**kwargs): + attempts = marker.read_text().count("x") if marker.exists() else 0 + marker.write_text("x" * (attempts + 1)) + if attempts == 0: + raise RuntimeError("transient") + return "ok" + + result = il.run(il.MultiProcessRunner().run(il.DAG(flaky()))) + + assert result.status is ExecutionStatus.COMPLETED + assert marker.read_text() == "xx" +``` + +- [ ] **Step 3: Run the test to verify it fails** + +Run: `uv run pytest packages/interloper-core/tests/runner/test_multi_process.py -v -k heal` +Expected: FAIL. The worker executes the operation once and the run fails. + +- [ ] **Step 4: Apply the loop in the worker** + +In `packages/interloper-core/src/interloper/runner/multi_process.py`, wrap the `operation.execute(...)` +call in the same loop, in its synchronous form. The worker process has no shared `RunState`, so it +counts its own attempts and reports the final count back with the outcome; the parent is what emits +the events: + +```python +def _execute_in_worker( + operation: Operation, + context: OperationContext, + policy: RetryPolicy | None, +) -> tuple[Any, int, Exception | None]: + """Execute an operation in the worker process, retrying within its budget. + + The worker owns the attempt loop because only it sees the failures, but it + owns none of the reporting: the parent holds the ``RunState`` and emits + every event, so the attempt count travels back with the outcome. + + Args: + operation: The operation to execute. + context: The facts the execution is scoped to. + policy: The attempt budget, or ``None`` for a single attempt. + + Returns: + The result, the number of attempts made, and the exception that ended + the loop (``None`` on success). + """ + attempt = 1 + while True: + try: + return il.run(operation.execute(context)), attempt, None + except Exception as error: # noqa: BLE001 — every failure becomes the node's record + if policy is None or not policy.allows(attempt + 1) or not operation.retryable(error): + return None, attempt, error + time.sleep(policy.delay_before(attempt + 1)) + attempt += 1 +``` + +Resolve `policy = operation.retry` in the parent, before submitting, and pass it in. +On the way back, call `self.state.mark_retried(operation, format_exception(error))` once per attempt +beyond the first so the parent's counter and events match what the worker did, then +`mark_completed` or `mark_failed` as the outcome dictates. Match the module's existing submit and +result-handling shape (`_handle_completed` at :157, `_handle_flushed` at :186) rather than the sketch +above; the point is the loop, its budget checks and the returned attempt count. + +- [ ] **Step 5: Run the tests to verify they pass** + +Run: `uv run pytest packages/interloper-core/tests/runner -v` +Expected: PASS. + +- [ ] **Step 6: Run the full checks** + +Run: `uv run ruff check && uv run ty check && uv run pytest` +Expected: clean. + +- [ ] **Step 7: Commit** + +```bash +git add packages/interloper-core +git commit -m "feat(core): retry a failed operation in the multi-process runner + +By Digitl" +``` diff --git a/docs/superpowers/plans/2026-09-17-retry-phase-2-platform.md b/docs/superpowers/plans/2026-09-17-retry-phase-2-platform.md new file mode 100644 index 00000000..137012e3 --- /dev/null +++ b/docs/superpowers/plans/2026-09-17-retry-phase-2-platform.md @@ -0,0 +1,835 @@ +# Retry Phase 2: Platform Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** A failed run queues its own next attempt, backfills and hooks report the stack's verdict rather than any one attempt's. + +**Architecture:** There is no instance-wide default in this phase, so a run retries only when its +target job declares a policy; phases 1 and 2 therefore change no behaviour until something declares +one. A run attempt is a `Run` row; a stack is the attempts of one unit of work, keyed by +a new `root_run_id`. `RunStore.complete()` is the single terminal path every failure takes, so it is +where the successor is created, in the same transaction that marks the run failed. That makes "has a +successor" true the instant the run fails, which is what lets the hook evaluator gate on it without +knowing anything about budgets. A `scheduled_for` column holds the backoff, honoured by the queue's +claim. Backfill accounting and the `executions` view both move from counting attempts to reading the +latest one. + +**Tech Stack:** Python 3.10+, SQLModel, SQLAlchemy, Alembic, PostgreSQL, pytest, ruff, ty. + +Spec: `docs/superpowers/specs/2026-09-17-retry-design.md`, sections 5 to 8 and 10. +Prerequisite: `docs/superpowers/plans/2026-09-17-retry-phase-1-core.md` must be complete. +`RetryPolicy` and `Job.retry` exist. + +## Global Constraints + +- Line length 120, ruff-formatted. Type-checked with `ty`. +- Google-style docstrings on every module, class, function and method, with every applicable section. + An Alembic revision's `upgrade`/`downgrade` pair is exempt; the module docstring carries the intent. +- Tests mirror the package layout one to one. +- The migration chain was folded to `001`-`003`, so this is revision `004` with `down_revision = "003"`. +- `env.py` uses `transaction_per_migration` and commits after `SET lock_timeout`; do not add a + `CREATE INDEX CONCURRENTLY` without checking that file first. +- Run checks from the repo root: `uv run ruff check`, `uv run ty check`, `uv run pytest`. +- **Do not commit without Guillaume asking.** The commit step records the intended message. + +--- + +### Task 1: Stack and schedule columns + +**Files:** +- Modify: `packages/interloper-db/src/interloper_db/models/runs.py:52-96` +- Create: `packages/interloper-db/src/interloper_db/migrations/versions/004_run_attempts.py` +- Modify: `packages/interloper-db/src/interloper_db/store/runs.py` (`create` at :51, `create_backfill` at :335, `retry` at :286) +- Modify: `packages/interloper-scheduler/src/interloper_scheduler/cron.py:155-175` +- Test: `packages/interloper-db/tests/store/test_runs.py` + +**Interfaces:** +- Consumes: nothing from phase 1. +- Produces: `Run.root_run_id: UUID` (not null, its own id for a first attempt) and + `Run.scheduled_for: datetime | None`. Every run-creating call site assigns `id=uuid4()` explicitly + so the root can be set in the same statement. Tasks 2 to 5 read both columns. + +- [ ] **Step 1: Write the failing test** + +Add to `packages/interloper-db/tests/store/test_runs.py`: + +```python +def test_a_new_run_is_its_own_stack_root(store, org): + run = store.runs.create(org.id, component_id=None) + + assert run.root_run_id == run.id + assert run.scheduled_for is None + + +def test_backfill_runs_are_each_their_own_root(store, org): + backfill = store.runs.create_backfill(org.id, start_key="2026-01-01", end_key="2026-01-03") + + runs = store.runs.list_all(org.id, backfill_id=backfill.id) + assert {run.root_run_id for run in runs} == {run.id for run in runs} +``` + +Use the fixtures the surrounding tests in that module already use for `store` and `org`. + +- [ ] **Step 2: Run the test to verify it fails** + +Run: `uv run pytest packages/interloper-db/tests/store/test_runs.py -v -k "stack_root or own_root"` +Expected: FAIL with `AttributeError: 'Run' object has no attribute 'root_run_id'`. + +- [ ] **Step 3: Add the columns to the model** + +In `packages/interloper-db/src/interloper_db/models/runs.py`, add to `Run`, after `retry_scope`: + +```python + root_run_id: UUID = SQLField( + sa_column=Column(ForeignKey("runs.id", ondelete="SET NULL"), index=True, nullable=False), + ) + scheduled_for: datetime | None = SQLField(default=None, sa_column=Column(TZDateTime)) +``` + +Extend the class docstring with one paragraph: a run is one attempt, `root_run_id` groups the +attempts of one unit of work into a stack and is the run's own id for a first attempt, and +`scheduled_for` is the earliest instant the queue may claim it. + +- [ ] **Step 4: Assign ids and roots at every creation site** + +`root_run_id` cannot default to the row's own server-generated id, so the id is assigned in Python. +In `packages/interloper-db/src/interloper_db/store/runs.py`, in `create`: + +```python + run_id = uuid4() + db_run = Run( + id=run_id, + root_run_id=run_id, + org_id=org_id, + component_id=component_id, + partition_key=partition_key, + status="queued", + billable=billable, + ) +``` + +In `create_backfill`, inside the per-partition loop: + +```python + run_id = uuid4() + db_run = Run( + id=run_id, + root_run_id=run_id, + org_id=org_id, + component_id=component_id, + backfill_id=db_backfill.id, + partition_key=window.granularity.format(value), + status="queued" if index >= first_queued else "pending", + ) +``` + +In `retry`, the successor joins its predecessor's stack: + +```python + db_run = Run( + id=uuid4(), + root_run_id=src.root_run_id, + org_id=src.org_id, + ... + ) +``` + +In `packages/interloper-scheduler/src/interloper_scheduler/cron.py`, both `Run(...)` constructions +(the per-partition one and the unpartitioned one) get the same `run_id = uuid4()` treatment. + +Add `from uuid import uuid4` where it is missing. + +- [ ] **Step 5: Write the migration** + +Create `packages/interloper-db/src/interloper_db/migrations/versions/004_run_attempts.py`: + +```python +"""Add the run stack and its schedule. + +A run is one attempt. ``root_run_id`` groups the attempts of one unit of work +so a stack is a single indexed predicate rather than a recursive walk, and is +the run's own id for a first attempt. ``scheduled_for`` holds a retry's +backoff: the queue claims a run only once it has passed. + +Existing rows are folded into stacks by walking the ``retry_of`` chains that +manual retries already created. A run whose predecessor was deleted becomes +its own root, which is correct: its lineage is gone. +""" + +from alembic import op + +revision: str = "004" +down_revision: str | None = "003" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.execute("ALTER TABLE runs ADD COLUMN root_run_id uuid REFERENCES runs(id) ON DELETE SET NULL") + op.execute("ALTER TABLE runs ADD COLUMN scheduled_for timestamptz") + op.execute( + """ + WITH RECURSIVE chain AS ( + SELECT id, id AS root FROM runs WHERE retry_of IS NULL + UNION ALL + SELECT r.id, c.root FROM runs r JOIN chain c ON r.retry_of = c.id + ) + UPDATE runs SET root_run_id = chain.root FROM chain WHERE runs.id = chain.id + """ + ) + op.execute("UPDATE runs SET root_run_id = id WHERE root_run_id IS NULL") + op.execute("ALTER TABLE runs ALTER COLUMN root_run_id SET NOT NULL") + op.execute("CREATE INDEX ix_runs_root_run_id ON runs (root_run_id)") + op.execute("CREATE INDEX ix_runs_claim ON runs (status, scheduled_for, created_at)") + + +def downgrade() -> None: + op.execute("DROP INDEX IF EXISTS ix_runs_claim") + op.execute("DROP INDEX IF EXISTS ix_runs_root_run_id") + op.execute("ALTER TABLE runs DROP COLUMN scheduled_for") + op.execute("ALTER TABLE runs DROP COLUMN root_run_id") +``` + +- [ ] **Step 6: Run the tests to verify they pass** + +Run: `uv run pytest packages/interloper-db/tests/store/test_runs.py -v` +Expected: PASS. + +- [ ] **Step 7: Verify the migration applies to an empty and a populated database** + +Run: `make dev-reset` +Expected: the chain migrates to `004` and the seed succeeds. + +Then apply it to a database that already holds runs, which is what exercises the recursive backfill. +Do not run this against the shared dev database if another session may be using it; use a throwaway. + +- [ ] **Step 8: Commit** + +```bash +git add packages/interloper-db packages/interloper-scheduler/src/interloper_scheduler/cron.py +git commit -m "feat(db): give every run a stack root and a schedule + +By Digitl" +``` + +--- + +### Task 2: A failed run queues its next attempt + +**Files:** +- Modify: `packages/interloper-db/src/interloper_db/store/runs.py:245-284` (`complete`) and a new `_plan_retry` +- Test: `packages/interloper-db/tests/store/test_runs.py` + +**Interfaces:** +- Consumes: `Run.root_run_id`/`Run.scheduled_for` (Task 1) and `Job.retry` (phase 1). +- Produces: `RunStore._plan_retry(session, db_run) -> Run | None`, called from `complete` on failure + before `_advance_backfill`. Tasks 4 and 6 rely on the successor existing by the time the + transaction commits. + +- [ ] **Step 1: Write the failing tests** + +Add to `packages/interloper-db/tests/store/test_runs.py`: + +```python +def test_a_failed_run_queues_its_next_attempt(store, org, job_component): + store.components.merge_config(job_component.id, {"retry": {"max_attempts": 2, "delay": 60}}) + run = store.runs.create(org.id, component_id=job_component.id) + + store.runs.complete(run.id, success=False) + + successor = store.runs.list_all(org.id, component_id=job_component.id)[0] + assert successor.id != run.id + assert successor.retry_of == run.id + assert successor.root_run_id == run.root_run_id + assert successor.attempt == 2 + assert successor.retry_scope == "failed" + assert successor.status == "queued" + assert successor.scheduled_for is not None + + +def test_an_exhausted_budget_queues_nothing(store, org, job_component): + store.components.merge_config(job_component.id, {"retry": {"max_attempts": 1}}) + run = store.runs.create(org.id, component_id=job_component.id) + + store.runs.complete(run.id, success=False) + + assert len(store.runs.list_all(org.id, component_id=job_component.id)) == 1 + + +def test_a_successful_run_queues_nothing(store, org, job_component): + store.components.merge_config(job_component.id, {"retry": {"max_attempts": 3}}) + run = store.runs.create(org.id, component_id=job_component.id) + + store.runs.complete(run.id, success=True) + + assert len(store.runs.list_all(org.id, component_id=job_component.id)) == 1 + + +def test_a_run_whose_target_is_gone_queues_nothing(store, org): + run = store.runs.create(org.id, component_id=None) + + store.runs.complete(run.id, success=False) + + assert len(store.runs.list_all(org.id)) == 1 + + +def test_a_backfill_run_keeps_its_backfill_when_it_retries(store, org, job_component): + store.components.merge_config(job_component.id, {"retry": {"max_attempts": 2}}) + backfill = store.runs.create_backfill( + org.id, component_id=job_component.id, start_key="2026-01-01", end_key="2026-01-01" + ) + run = store.runs.list_all(org.id, backfill_id=backfill.id)[0] + + store.runs.complete(run.id, success=False) + + successor = [r for r in store.runs.list_all(org.id, backfill_id=backfill.id) if r.id != run.id][0] + assert successor.backfill_id == backfill.id +``` + +Add a `job_component` fixture to that module if one does not exist, creating a `job`-kind component +row for the org through the same path the surrounding tests use. + +- [ ] **Step 2: Run the tests to verify they fail** + +Run: `uv run pytest packages/interloper-db/tests/store/test_runs.py -v -k "next_attempt or exhausted or queues_nothing or keeps_its_backfill"` +Expected: FAIL. Nothing creates a successor. + +- [ ] **Step 3: Write the retry planner** + +In `packages/interloper-db/src/interloper_db/store/runs.py`, add to `RunStore`, in the internals +section: + +```python + @staticmethod + def _retry_policy(session: Session, db_run: Run) -> il.RetryPolicy | None: + """The run-level policy in force for a run. + + A job's declared policy governs its runs, and nothing else does: a + source's or an asset's own ``retry`` is an operation policy, and + reading it here would apply an operation's budget to whole runs. There + is no instance-wide fallback, so a run whose target declares nothing is + attempted once. The config column is plain JSON, so this is one row + read and no hydration. + + Args: + session: Open session the target row is read through. + db_run: The run whose policy is resolved. + + Returns: + The policy, or ``None`` when the target declares none. + """ + if db_run.component_id is None: + return None + db_component = session.get(Component, db_run.component_id) + if db_component is None or db_component.kind != "job": + return None + declared = (db_component.config or {}).get("retry") + return il.RetryPolicy.model_validate(declared) if declared else None + + def _plan_retry(self, session: Session, db_run: Run) -> Run | None: + """Queue the next attempt of a failed run, when its budget allows one. + + Called from the single terminal path, in the transaction that marks the + run failed, so that a doomed attempt never looks final to anything + reading the table. The successor stays in its predecessor's backfill and + stack, and re-runs only what failed. The quota is deliberately not + checked here: dispatch is the authoritative gate and cancels an + over-quota run at claim time, like any other run. + + Args: + session: Open session the successor is written through. + db_run: The run that just failed. + + Returns: + The queued successor, or ``None`` when nothing is retried. + """ + if db_run.component_id is None: + return None + policy = self._retry_policy(session, db_run) + if policy is None or not policy.allows(db_run.attempt + 1): + return None + + successor = Run( + id=uuid4(), + org_id=db_run.org_id, + component_id=db_run.component_id, + backfill_id=db_run.backfill_id, + partition_key=db_run.partition_key, + status="queued", + scheduled_for=datetime.now(timezone.utc) + + timedelta(seconds=policy.delay_before(db_run.attempt + 1)), + retry_of=db_run.id, + root_run_id=db_run.root_run_id, + attempt=db_run.attempt + 1, + retry_scope="failed", + billable=db_run.billable, + ) + session.add(successor) + session.flush() + logger.info("Queued attempt %d of run stack %s", successor.attempt, successor.root_run_id) + return successor +``` + +Add the imports it needs: `timedelta` and `uuid4`. `interloper as il` is already imported, which is +what `il.RetryPolicy` resolves through. + +- [ ] **Step 4: Call it from the terminal path** + +In `complete`, between the component stamp and the backfill advance: + +```python + if not success: + self._plan_retry(session, db_run) + + if db_run.backfill_id: + self._advance_backfill(session, db_run.backfill_id, failed=not success) +``` + +Extend `complete`'s docstring: on failure it also queues the next attempt when the target's policy +allows one, and it does so before advancing the backfill so that the batch's in-flight count sees the +successor and does not finalize early. + +- [ ] **Step 5: Run the tests to verify they pass** + +Run: `uv run pytest packages/interloper-db/tests/store/test_runs.py -v` +Expected: PASS. + +- [ ] **Step 6: Commit** + +```bash +git add packages/interloper-db +git commit -m "feat(db): queue the next attempt when a run fails + +By Digitl" +``` + +--- + +### Task 3: The queue honours the schedule + +**Files:** +- Modify: `packages/interloper-scheduler/src/interloper_scheduler/queue.py:79-116` +- Test: `packages/interloper-scheduler/tests/test_queue.py` + +**Interfaces:** +- Consumes: `Run.scheduled_for` (Task 1). +- Produces: a claim that skips a run whose `scheduled_for` is still in the future. No new API. + +- [ ] **Step 1: Write the failing test** + +Add to `packages/interloper-scheduler/tests/test_queue.py`, following the module's existing way of +building a controller with a fake launcher: + +```python +def test_a_scheduled_run_is_not_claimed_before_its_time(controller, store, org): + run = store.runs.create(org.id, component_id=None) + with Session(store.engine) as session: + db_run = session.get(Run, run.id) + db_run.scheduled_for = datetime.now(timezone.utc) + timedelta(hours=1) + session.add(db_run) + session.commit() + + assert controller._claim_next() is None + + +def test_a_run_whose_schedule_has_passed_is_claimed(controller, store, org): + run = store.runs.create(org.id, component_id=None) + with Session(store.engine) as session: + db_run = session.get(Run, run.id) + db_run.scheduled_for = datetime.now(timezone.utc) - timedelta(seconds=1) + session.add(db_run) + session.commit() + + assert controller._claim_next() == run.id +``` + +- [ ] **Step 2: Run the tests to verify they fail** + +Run: `uv run pytest packages/interloper-scheduler/tests/test_queue.py -v -k scheduled` +Expected: FAIL. The first test claims the run anyway. + +- [ ] **Step 3: Write the implementation** + +In `packages/interloper-scheduler/src/interloper_scheduler/queue.py`, add the predicate to the claim +statement in `_claim_next`: + +```python + statement = ( + select(Run) + .where(Run.status == "queued") + .where(col(Run.scheduled_for).is_(None) | (col(Run.scheduled_for) <= func.now())) + .order_by(col(Run.created_at).asc()) + .limit(1) + .with_for_update(skip_locked=True) + ) +``` + +Import `func` from sqlalchemy. Extend the method's docstring: a run carrying a schedule is not +claimable until it has passed, which is how a retry's backoff is served without a second status. + +- [ ] **Step 4: Run the tests to verify they pass** + +Run: `uv run pytest packages/interloper-scheduler/tests/test_queue.py -v` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add packages/interloper-scheduler +git commit -m "feat(scheduler): claim a run only once its schedule has passed + +By Digitl" +``` + +--- + +### Task 4: Backfills finalize on stacks + +**Files:** +- Modify: `packages/interloper-db/src/interloper_db/store/runs.py:584-643` (`_advance_backfill`) +- Test: `packages/interloper-db/tests/store/test_runs.py` + +**Interfaces:** +- Consumes: `Run.root_run_id` (Task 1), `_plan_retry` (Task 2). +- Produces: a backfill whose terminal status reads the latest attempt of each stack. No new API. + +- [ ] **Step 1: Write the failing test** + +Add to `packages/interloper-db/tests/store/test_runs.py`: + +```python +def test_a_backfill_healed_by_a_retry_succeeds(store, org, job_component): + store.components.merge_config(job_component.id, {"retry": {"max_attempts": 2, "delay": 0}}) + backfill = store.runs.create_backfill( + org.id, component_id=job_component.id, start_key="2026-01-01", end_key="2026-01-01" + ) + first = store.runs.list_all(org.id, backfill_id=backfill.id)[0] + + store.runs.complete(first.id, success=False) + successor = [r for r in store.runs.list_all(org.id, backfill_id=backfill.id) if r.id != first.id][0] + store.runs.complete(successor.id, success=True) + + assert store.runs.get_backfill(backfill.id).status == "success" + + +def test_a_backfill_whose_stack_exhausts_its_budget_fails(store, org, job_component): + store.components.merge_config(job_component.id, {"retry": {"max_attempts": 2, "delay": 0}}) + backfill = store.runs.create_backfill( + org.id, component_id=job_component.id, start_key="2026-01-01", end_key="2026-01-01" + ) + first = store.runs.list_all(org.id, backfill_id=backfill.id)[0] + + store.runs.complete(first.id, success=False) + successor = [r for r in store.runs.list_all(org.id, backfill_id=backfill.id) if r.id != first.id][0] + store.runs.complete(successor.id, success=False) + + assert store.runs.get_backfill(backfill.id).status == "failed" +``` + +- [ ] **Step 2: Run the tests to verify they fail** + +Run: `uv run pytest packages/interloper-db/tests/store/test_runs.py -v -k backfill_healed` +Expected: FAIL. The healed backfill is marked `failed`, because the first attempt's row still counts. + +- [ ] **Step 3: Write the implementation** + +In `_advance_backfill`, replace the `any_failed` lookup in the finalize branch: + +```python + if in_flight_count == 0 and len(pending_runs) == 0: + latest = ( + select(Run.root_run_id, func.max(col(Run.attempt)).label("attempt")) + .where(Run.backfill_id == backfill_id) + .group_by(col(Run.root_run_id)) + .subquery() + ) + any_failed = session.exec( + select(Run) + .join( + latest, + onclause=(col(Run.root_run_id) == latest.c.root_run_id) + & (col(Run.attempt) == latest.c.attempt), + ) + .where(Run.backfill_id == backfill_id, Run.status == "failed") + ).first() + db_backfill.status = "failed" if any_failed else "success" + db_backfill.completed_at = datetime.now(timezone.utc) + session.add(db_backfill) + return +``` + +Extend the method's docstring to say the verdict reads each stack's latest attempt, so an attempt +that a later one healed no longer condemns the batch. + +Leave the in-flight count and the fail-fast branch alone: a queued successor already counts as +in-flight, which is exactly what keeps the batch open while a retry is pending. + +- [ ] **Step 4: Run the tests to verify they pass** + +Run: `uv run pytest packages/interloper-db/tests/store/test_runs.py -v` +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add packages/interloper-db +git commit -m "fix(db): finalize a backfill on its stacks, not its attempts + +By Digitl" +``` + +--- + +### Task 5: The executions view reads the latest attempt + +**Files:** +- Modify: `packages/interloper-db/src/interloper_db/migrations/versions/004_run_attempts.py` +- Modify: `packages/interloper-db/src/interloper_db/models/runs.py:166-186` (`Execution`) +- Test: `packages/interloper-db/tests/store/test_events.py` + +**Interfaces:** +- Consumes: `EventType.OPERATION_RETRIED` and the `attempt` key in operation event data (phase 1). +- Produces: an `executions` view still keyed by `(run_id, component_id)`, whose `status` is the + latest attempt's and which gains `attempts: int`. `Execution` gains the matching field. Phase 3 + renders it. + +- [ ] **Step 1: Write the failing test** + +Add to `packages/interloper-db/tests/store/test_events.py`, following the module's existing way of +saving events and reading executions back: + +```python +def test_an_operation_that_healed_reads_as_a_success(store, org, run, component): + store.events.save(_operation_event(EventType.OPERATION_FAILED, component, attempt=1), org_id=org.id, run_id=run.id) + store.events.save(_operation_event(EventType.OPERATION_RETRIED, component, attempt=1), org_id=org.id, run_id=run.id) + store.events.save( + _operation_event(EventType.OPERATION_COMPLETED, component, attempt=2), org_id=org.id, run_id=run.id + ) + + executions = store.events.list_executions(run.id) + assert len(executions) == 1 + assert executions[0].status == "success" + assert executions[0].attempts == 2 + + +def test_an_operation_that_exhausted_its_budget_reads_as_a_failure(store, org, run, component): + store.events.save(_operation_event(EventType.OPERATION_RETRIED, component, attempt=1), org_id=org.id, run_id=run.id) + store.events.save(_operation_event(EventType.OPERATION_FAILED, component, attempt=2), org_id=org.id, run_id=run.id) + + executions = store.events.list_executions(run.id) + assert executions[0].status == "failed" + assert executions[0].attempts == 2 +``` + +Write the `_operation_event` helper in the test module: it builds an `il.Event` of the given type +whose metadata carries `component_id`, `component_kind`, `component_key` and `attempt`. + +- [ ] **Step 2: Run the tests to verify they fail** + +Run: `uv run pytest packages/interloper-db/tests/store/test_events.py -v -k "healed or exhausted"` +Expected: FAIL. The healed operation reads `failed`, because the view ranks `operation_failed` above +`operation_completed` regardless of recency, and `attempts` does not exist. + +- [ ] **Step 3: Replace the view in migration 004** + +Add to the `upgrade()` of `004_run_attempts.py`, after the index creation, a +`CREATE OR REPLACE VIEW executions AS ...` that is the migration 002 definition with three changes: + +1. `operation_retried` joins the `event_type` filter in the `WHERE` clause of the `ranked` CTE. +2. The window's `ORDER BY` puts the attempt first, so the latest attempt decides the verdict and the + severity ranking only breaks ties within it: + +```sql + row_number() OVER ( + PARTITION BY e.run_id, e.component_id + ORDER BY + COALESCE((e.data->>'attempt')::int, 1) DESC, + CASE e.event_type + WHEN 'operation_failed' THEN 1 + WHEN 'operation_canceled' THEN 2 + WHEN 'operation_completed' THEN 3 + WHEN 'operation_started' THEN 4 + WHEN 'operation_skipped' THEN 5 + WHEN 'operation_retried' THEN 6 + WHEN 'operation_queued' THEN 7 + END, + e.timestamp DESC + ) AS rn, +``` + +3. A new window carries the attempt count into the outer select: + +```sql + max(COALESCE((e.data->>'attempt')::int, 1)) OVER ( + PARTITION BY e.run_id, e.component_id + ) AS attempts, +``` + +selected as `r.attempts`. `operation_retried` must never win the verdict, which is why it sits below +every terminal type in the severity ranking; it is in the view only so it counts toward `attempts`. + +Add the matching `DROP` and re-create of the 002 definition to `downgrade()`. + +- [ ] **Step 4: Add the column to the read model** + +In `packages/interloper-db/src/interloper_db/models/runs.py`, add to `Execution`: + +```python + attempts: int = 1 +``` + +Extend the class docstring: the status is the latest attempt's, and `attempts` is how many that +operation took. + +- [ ] **Step 5: Run the tests to verify they pass** + +Run: `make dev-reset && uv run pytest packages/interloper-db/tests/store/test_events.py -v` +Expected: PASS. + +- [ ] **Step 6: Verify the failed-scope retry still reads correctly** + +Run: `uv run pytest packages/interloper-scheduler/tests/test_executor.py -v` +Expected: PASS. `RunExecutor._prior_successes` reads `list_executions` and keys by component id; the +view still returns one row per operation, so it is unaffected. If this fails, the view is returning +more than one row per `(run, operation)` and step 3 is wrong. + +- [ ] **Step 7: Commit** + +```bash +git add packages/interloper-db +git commit -m "feat(db): read an execution's verdict from its latest attempt + +By Digitl" +``` + +--- + +### Task 6: Hooks observe verdicts + +**Files:** +- Modify: `packages/interloper-scheduler/src/interloper_scheduler/hooks.py:87-105` (`_tick`) and `:138-170` (`_event_metadata`) +- Test: `packages/interloper-scheduler/tests/test_hooks.py` + +**Interfaces:** +- Consumes: `Run.retry_of`/`Run.root_run_id` (Task 1), `_plan_retry` (Task 2). +- Produces: a sweep that skips a failed run with a successor, and `HookContext.metadata` carrying + `attempt` and `attempts`. No new hook event types. + +- [ ] **Step 1: Write the failing tests** + +Add to `packages/interloper-scheduler/tests/test_hooks.py`, following the module's existing way of +building a hook component and a terminal run: + +```python +def test_a_failed_run_that_will_be_retried_does_not_fire(controller, store, org, hook, job_component): + store.components.merge_config(job_component.id, {"retry": {"max_attempts": 2}}) + run = store.runs.create(org.id, component_id=job_component.id) + store.runs.complete(run.id, success=False) + + controller._tick() + + assert not fired_events(store, run.id) + + +def test_an_exhausted_stack_fires_once(controller, store, org, hook, job_component): + store.components.merge_config(job_component.id, {"retry": {"max_attempts": 1}}) + run = store.runs.create(org.id, component_id=job_component.id) + store.runs.complete(run.id, success=False) + + controller._tick() + + assert len(fired_events(store, run.id)) == 1 + + +def test_a_healed_stack_fires_completed_on_the_successful_attempt(controller, store, org, hook, job_component): + store.components.merge_config(job_component.id, {"retry": {"max_attempts": 2, "delay": 0}}) + first = store.runs.create(org.id, component_id=job_component.id) + store.runs.complete(first.id, success=False) + successor = [r for r in store.runs.list_all(org.id, component_id=job_component.id) if r.id != first.id][0] + store.runs.complete(successor.id, success=True) + + controller._tick() + + assert not fired_events(store, first.id) + assert len(fired_events(store, successor.id)) == 1 + + +def test_the_context_carries_the_stack_position(controller, store, org, hook, job_component): + store.components.merge_config(job_component.id, {"retry": {"max_attempts": 1}}) + run = store.runs.create(org.id, component_id=job_component.id) + store.runs.complete(run.id, success=False) + + controller._tick() + + assert hook.seen_context.metadata["attempt"] == 1 + assert hook.seen_context.metadata["attempts"] == 1 +``` + +Write `fired_events(store, run_id)` in the test module: the `hook_fired` events attached to that run. +The `hook` fixture is a hook component whose `fire` records the context it was handed; a hook +subscribing to `run_completed` as well as `run_failed` is what makes the third test meaningful. + +- [ ] **Step 2: Run the tests to verify they fail** + +Run: `uv run pytest packages/interloper-scheduler/tests/test_hooks.py -v -k "retried or exhausted or healed or stack_position"` +Expected: FAIL. The first test fires a notification for an attempt that will be retried. + +- [ ] **Step 3: Gate the sweep** + +In `packages/interloper-scheduler/src/interloper_scheduler/hooks.py`, add the predicate to `_tick`'s +statement: + +```python + successor = aliased(Run) + has_successor = select(successor.id).where(col(successor.retry_of) == Run.id).exists() + runs = session.exec( + select(Run) + .where(col(Run.status).in_(_TERMINAL_STATUSES)) + .where(col(Run.completed_at) > since) + .where(~((Run.status == "failed") & has_successor)) + .order_by(col(Run.completed_at)) + ).all() +``` + +Import `aliased` from `sqlalchemy.orm`. Extend the module docstring with the rule this implements: a +hook observes a verdict, never an attempt, so a failed run whose next attempt is already queued is +not an outcome and does not fire. Because the successor is created in the same transaction that marks +the run failed, there is no window in which a doomed attempt looks final. + +- [ ] **Step 4: Carry the stack position into the context** + +In `_event_metadata`, add the stack's position to the dict it builds: + +```python + metadata: dict[str, Any] = { + "status": run.status, + "component_name": target.name or target.key, + "component_key": target.key, + "attempt": run.attempt, + "attempts": session.exec( + select(func.count()).select_from(Run).where(Run.root_run_id == run.root_run_id) + ).one(), + } +``` + +Import `func` from sqlalchemy. Extend the method's docstring: a message addressing humans renders the +stack's position, so it can say the work succeeded on the second attempt or failed after three. + +- [ ] **Step 5: Run the tests to verify they pass** + +Run: `uv run pytest packages/interloper-scheduler/tests/test_hooks.py -v` +Expected: PASS. + +- [ ] **Step 6: Run the full checks** + +Run: `uv run ruff check && uv run ty check && uv run pytest` +Expected: clean. + +- [ ] **Step 7: Commit** + +```bash +git add packages/interloper-scheduler +git commit -m "feat(scheduler): fire hooks on a stack's verdict, not on each attempt + +By Digitl" +``` diff --git a/docs/superpowers/plans/2026-09-17-retry-phase-3-surfaces.md b/docs/superpowers/plans/2026-09-17-retry-phase-3-surfaces.md new file mode 100644 index 00000000..07b9431a --- /dev/null +++ b/docs/superpowers/plans/2026-09-17-retry-phase-3-surfaces.md @@ -0,0 +1,320 @@ +# Retry Phase 3: Surfaces Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** A stack reads as one piece of work in the API and the app, so a workload that eventually succeeded looks like a success. + +**Architecture:** The runs listing becomes stack-native: one row per stack carrying its latest +attempt, selected with `DISTINCT ON (root_run_id)` and re-ordered in an outer query so the listing +order is unchanged. A `root_run_id` filter lists one stack's attempts. The app's runs table shows an +`n/N` chip and expands to the attempts; the run detail page navigates the stack. The retry policy +itself needs no UI work: it is a config field on a component, so `SchemaForm` already renders it. + +**Tech Stack:** FastAPI, SQLModel, pydantic v2, Nuxt 4, Vue 3, @nuxt/ui, TypeScript, pytest, vitest. + +Spec: `docs/superpowers/specs/2026-09-17-retry-design.md`, section 9. +Prerequisite: `docs/superpowers/plans/2026-09-17-retry-phase-2-platform.md` must be complete. + +## Global Constraints + +- Python: line length 120, ruff-formatted, `ty`-checked, Google-style docstrings with every + applicable section. +- Frontend: run from `packages/interloper-app/app/`; `pnpm run lint` and `pnpm exec nuxt typecheck`. + Follow that directory's own `AGENTS.md`. +- Tests mirror the package layout one to one. +- Two known `UTable` traps: an inline `:grouping` object literal combined with `v-model:expanded` + causes an auto-reset loop, and `resolveComponent` returns a bare string inside header and cell + render functions (import from `#components` in shared helpers instead). +- **Do not commit without Guillaume asking.** The commit step records the intended message. + +--- + +### Task 1: The API exposes the stack + +**Files:** +- Modify: `packages/interloper-api/src/interloper_api/routes/runs.py:40-90` (`RunResponse`), `:211-260` (`list_runs`) +- Modify: `packages/interloper-db/src/interloper_db/store/runs.py:140-243` (`list_all`, `count`, `_run_filters`) +- Test: `packages/interloper-db/tests/store/test_runs.py` +- Test: `packages/interloper-api/tests/routes/test_runs.py` + +**Interfaces:** +- Consumes: `Run.root_run_id`, `Run.scheduled_for` (phase 2 Task 1). +- Produces: `RunStore.list_all(..., root_run_id: UUID | None = None, stacks: bool = True)` returning + one row per stack when `stacks` is true and every attempt of one stack when `root_run_id` is given; + `RunStore.count` matching it; `RunResponse` carrying `root_run_id`, `scheduled_for` and `attempts`. + Task 2 consumes the response fields. + +- [ ] **Step 1: Write the failing store test** + +Add to `packages/interloper-db/tests/store/test_runs.py`: + +```python +def test_listing_returns_one_row_per_stack(store, org, job_component): + store.components.merge_config(job_component.id, {"retry": {"max_attempts": 3, "delay": 0}}) + first = store.runs.create(org.id, component_id=job_component.id) + store.runs.complete(first.id, success=False) + + runs = store.runs.list_all(org.id, component_id=job_component.id) + + assert len(runs) == 1 + assert runs[0].attempt == 2 + assert store.runs.count(org.id, component_id=job_component.id) == 1 + + +def test_a_stack_can_be_listed_in_full(store, org, job_component): + store.components.merge_config(job_component.id, {"retry": {"max_attempts": 3, "delay": 0}}) + first = store.runs.create(org.id, component_id=job_component.id) + store.runs.complete(first.id, success=False) + + attempts = store.runs.list_all(org.id, root_run_id=first.root_run_id) + + assert [run.attempt for run in attempts] == [2, 1] + + +def test_a_status_filter_reads_the_stacks_verdict(store, org, job_component): + store.components.merge_config(job_component.id, {"retry": {"max_attempts": 3, "delay": 0}}) + first = store.runs.create(org.id, component_id=job_component.id) + store.runs.complete(first.id, success=False) + + assert store.runs.list_all(org.id, status="failed") == [] + assert len(store.runs.list_all(org.id, status="queued")) == 1 +``` + +- [ ] **Step 2: Run the tests to verify they fail** + +Run: `uv run pytest packages/interloper-db/tests/store/test_runs.py -v -k "per_stack or listed_in_full or stacks_verdict"` +Expected: FAIL. Both attempts come back as separate rows. + +- [ ] **Step 3: Make the listing stack-native** + +In `packages/interloper-db/src/interloper_db/store/runs.py`, add `root_run_id: UUID | None = None` +and `stacks: bool = True` to `list_all` and `count`, pass `root_run_id` through `_run_filters` as a +plain equality filter, and select the latest attempt of each stack when `stacks` is true and no +`root_run_id` was given: + +```python + if stacks and root_run_id is None: + latest = ( + select(col(Run.id)) + .where(*filters) + .distinct(col(Run.root_run_id)) + .order_by(col(Run.root_run_id), col(Run.attempt).desc()) + .subquery() + ) + statement = ( + select(Run) + .where(col(Run.id).in_(select(latest.c.id))) + .order_by(col(Run.created_at).desc()) + .offset(offset) + .limit(limit) + .options(*RUN_LOAD_OPTIONS) + ) +``` + +`count` applies the same narrowing before counting. `_run_filters` is shared by both, so build the +filter list once and pass it into the subquery, as the existing code already does. + +Document the behaviour on both methods: a stack is one piece of work, so a listing shows its latest +attempt and every filter, `status` included, applies to that attempt. Passing `root_run_id` lists one +stack's attempts, newest first. + +Order the attempts by `attempt` descending when `root_run_id` is given, so a stack reads newest-first +like the listing does. + +- [ ] **Step 4: Run the store tests to verify they pass** + +Run: `uv run pytest packages/interloper-db/tests/store/test_runs.py -v` +Expected: PASS. + +- [ ] **Step 5: Write the failing API test** + +Add to `packages/interloper-api/tests/routes/test_runs.py`, following the module's existing client +and auth fixtures: + +```python +def test_a_run_response_carries_its_stack(client, org, job_component, store): + store.components.merge_config(job_component.id, {"retry": {"max_attempts": 3, "delay": 0}}) + first = store.runs.create(org.id, component_id=job_component.id) + store.runs.complete(first.id, success=False) + + body = client.get("/runs/").json() + + assert len(body["items"]) == 1 + assert body["items"][0]["root_run_id"] == str(first.root_run_id) + assert body["items"][0]["attempt"] == 2 + assert body["items"][0]["attempts"] == 2 + assert body["items"][0]["scheduled_for"] is not None + + +def test_a_stack_is_listed_by_its_root(client, org, job_component, store): + store.components.merge_config(job_component.id, {"retry": {"max_attempts": 3, "delay": 0}}) + first = store.runs.create(org.id, component_id=job_component.id) + store.runs.complete(first.id, success=False) + + body = client.get(f"/runs/?root_run_id={first.root_run_id}").json() + + assert [item["attempt"] for item in body["items"]] == [2, 1] +``` + +Match the listing response's actual envelope key rather than assuming `items`. + +- [ ] **Step 6: Expose the fields and the filter** + +In `packages/interloper-api/src/interloper_api/routes/runs.py`, add to `RunResponse`: + +```python + root_run_id: UUID + scheduled_for: str | None = None + attempts: int = 1 +``` + +`from_run` sets `root_run_id=run.root_run_id`, `scheduled_for=str(run.scheduled_for) if +run.scheduled_for else None` and `attempts=run.attempt`, since the listing returns the latest attempt +and its number is the stack's total. Extend the class docstring with one sentence: a response is one +attempt, and in a stack-native listing it is the stack's latest, whose `attempt` is therefore the +count of attempts made. + +Add `root_run_id: UUID | None = None` to `list_runs`' query parameters and pass it to `list_all` and +`count`. + +- [ ] **Step 7: Run the API tests to verify they pass** + +Run: `uv run pytest packages/interloper-api/tests/routes/test_runs.py -v` +Expected: PASS. + +- [ ] **Step 8: Commit** + +```bash +git add packages/interloper-db packages/interloper-api +git commit -m "feat(api)!: list runs by stack + +By Digitl" +``` + +--- + +### Task 2: The app reads a stack as one run + +**Files:** +- Modify: `packages/interloper-app/app/app/types/run.ts` +- Modify: `packages/interloper-app/app/app/stores/runs.ts` +- Modify: `packages/interloper-app/app/app/components/executions/RunsTable.vue` +- Modify: `packages/interloper-app/app/app/pages/executions/runs/[run].vue` + +**Interfaces:** +- Consumes: the `root_run_id`, `scheduled_for` and `attempts` fields from Task 1. +- Produces: a runs table whose rows are stacks, and a run detail page that navigates a stack's + attempts. + +- [ ] **Step 1: Extend the type** + +In `packages/interloper-app/app/app/types/run.ts`, add to the `Run` interface: + +```ts + /** The stack this attempt belongs to; its own id for a first attempt. */ + root_run_id: string + /** Earliest instant the queue may claim this run; set on a retry's backoff. */ + scheduled_for: string | null + /** Attempts made in this stack, as of this row. */ + attempts: number +``` + +- [ ] **Step 2: Fetch a stack's attempts** + +In `packages/interloper-app/app/app/stores/runs.ts`, add an action that fetches one stack, following +the store's existing fetch and state conventions: + +```ts +async function fetchStack(rootRunId: string) { + return await api>('/runs/', { query: { root_run_id: rootRunId } }) +} +``` + +Match the store's real helper names and response envelope rather than the sketch above. + +- [ ] **Step 3: Show the stack in the table** + +In `RunsTable.vue`, render an `n/N`-style chip in the status cell when `row.attempts > 1`, reading as +"attempt 2 of 2". Reuse the identifier-chip component the tables already share rather than adding a +bespoke badge, and keep the status badge showing the latest attempt's status, which is what the API +now returns. + +If the row is made expandable to its attempts, note the two `UTable` traps: bind `grouping` to a +`computed`, never an inline object literal, when `v-model:expanded` is also bound, and import +components from `#components` in any shared cell-render helper instead of calling `resolveComponent`. + +- [ ] **Step 4: Navigate the stack on the detail page** + +In `pages/executions/runs/[run].vue`, when the loaded run's `attempts > 1` or its `root_run_id` +differs from its `id`, fetch the stack with the store action from step 2 and render the attempts as +links, marking the one being viewed. Show `scheduled_for` on a queued attempt, so a run waiting out +its backoff reads as scheduled rather than stuck. + +- [ ] **Step 5: Run the frontend checks** + +Run from `packages/interloper-app/app/`: `pnpm run lint && pnpm exec nuxt typecheck` +Expected: clean. + +- [ ] **Step 6: Verify it end to end** + +Stand up a seeded instance on a non-default port so it does not collide with the developer's own: + +Run: `INTERLOPER_SERVER_PORT=3100 make dev-up` + +Give a `demo` job a retry policy through the component form, run it against a target that fails, and +confirm: the runs table shows one row that flips to a second attempt, the detail page lists both +attempts, and the timeline shows the retried operation as a success with two attempts. + +- [ ] **Step 7: Commit** + +```bash +git add packages/interloper-app/app +git commit -m "feat(app): read a run stack as one piece of work + +By Digitl" +``` + +--- + +### Task 3: Document the feature + +**Files:** +- Modify: `docs/guide/` (the page covering runs and scheduling; locate it with the grep in step 1) + +**Interfaces:** +- Consumes: everything from phases 1 to 3. +- Produces: user-facing documentation and the deployment knobs. + +- [ ] **Step 1: Find the pages that describe running and failure** + +Run: `uv run grep -rln "backfill\|run fails\|scheduler" docs/guide docs/extending` +Expected: the pages that need a retry section. + +- [ ] **Step 2: Write the documentation** + +Cover, in the guide's voice: what a stack is and why a failed attempt is not a verdict; where a policy +is declared and what level each declaration governs (an asset or other operation for its own +execution, a source as the default for its assets, a job for its runs); that an automatic retry always +re-runs only what failed while the manual retry endpoint still takes a scope; and that hooks fire on a +stack's verdict, so a healed workload notifies once, as a success. State plainly that there is no +instance-wide default: nothing retries until a component declares a policy. + +Do not document request-level retry. `RESTClient` does not carry a policy yet; that is the third spec, +and describing it here would document a feature that does not exist. + +- [ ] **Step 3: Run the full checks** + +Run: `uv run ruff check && uv run ty check && uv run pytest` +Expected: clean. + +Run from `packages/interloper-app/app/`: `pnpm run lint && pnpm exec nuxt typecheck` +Expected: clean. + +- [ ] **Step 4: Commit** + +```bash +git add docs +git commit -m "docs: document the retry system + +By Digitl" +``` diff --git a/docs/superpowers/specs/2026-09-17-operation-is-a-component-design.md b/docs/superpowers/specs/2026-09-17-operation-is-a-component-design.md new file mode 100644 index 00000000..acc387f6 --- /dev/null +++ b/docs/superpowers/specs/2026-09-17-operation-is-a-component-design.md @@ -0,0 +1,145 @@ +# An operation is a component + +Date: 2026-09-17. Status: approved design, implementation not started. + +Scope: remove the stand-in attribute block from `Operation` by making it what every implementor +already is. Small, self-contained, and a prerequisite that simplifies +`2026-09-17-retry-design.md`: with this landed, `retry` is an ordinary field declared once on +`Operation` instead of a node-protocol declaration plus a component field. + +--- + +## 1. Problem + +`Operation` is a plain ABC that declares, under `if TYPE_CHECKING`, attributes it does not own: +`id`, `kind`, `key`, `relations`, `materializable`, `source`, `partitioning`, `to_spec()` and +`bound()`. Below the block it supplies runtime stand-ins for four of them (`materializable = True`, +`relations = {}`, `source = None`, `partitioning = None`). + +That block is not a protocol. It is a promise that `Operation` will only ever be mixed into a +`Component`, written in the form `ty` accepts. The promise is already kept everywhere: + +- `Asset(Component, Operation)` +- `Connection(Resource, Operation)`, where `Resource(BaseSettings, Component)` + +and the platform depends on it. Events carry `component_id`/`component_kind`/`component_key` per +operation, the `executions` view is keyed by `(run_id, component_id)`, and +`RunExecutor._prior_successes` matches by component row id. There is no non-component operation +anywhere in the system, so the indirection buys a generality nothing uses and costs a block of +attributes that drift. + +A `typing.Protocol` does not fix this. It expresses what the DAG and the runner *read*, which is the +consumer side, and carries neither defaults nor behaviour. `Operation` would keep its ABC for +`effective_partition`, `upstream_relations`, `failure` and `_event_metadata`, and keep the same +stand-ins. Two names for one concept, same duplication. + +--- + +## 2. Decision + +| Topic | Decision | +|---|---| +| Contract | `class Operation(Component, Workload)`. `class Asset(Operation)`. `Connection(Resource, Operation)` is unchanged in form. | +| The block | The whole `if TYPE_CHECKING` declaration and the `relations`/`source` stand-ins are deleted. `id`, `kind`, `key`, `relations`, `to_spec()` and `bound()` are inherited and real. | +| `Workload` | Unchanged. It stays a plain ABC: two members, no stand-ins, and implemented by things that are not operations (`Source`, `Job`). | +| Kinds | `Asset` declares `kind: ClassVar[str] = "asset"` explicitly, as `Connection` already declares its own. Auto-derivation stops reaching it once `Component` is no longer a direct base. | +| Opting out of a kind | `Operation` declares `kind: ClassVar[str] = ""` in its own body, which the existing derivation already honours (`"kind" not in cls.__dict__`). `__init_subclass__` is untouched: abstractness is not the signal, because `Destination` is an abstract class that is itself a kind. | +| Fields that move | `materializable` moves from `Asset` up to `Operation` as the same pydantic field. `partitioning` stays a `ClassVar` defaulting to `None`, declared once on `Operation`. | +| Compatibility | None required. Core and its consumers ship together. | + +--- + +## 3. What moves + +`interloper/operation/base.py` + +```py +class Operation(Component, Workload): + kind: ClassVar[str] = "" + partitioning: ClassVar[PartitionConfig | None] = None + capture_traceback: ClassVar[bool] = True + + materializable: bool = Field(default=True, json_schema_extra={"x-hidden": True}) + + @property + def source(self) -> Any: ... # None; Asset narrows it to its parent + def operations(self) -> list[Operation]: ... + def effective_partition(self, partition_or_window): ... + def upstream_relations(self) -> dict[str, Relation]: ... + def _validate_time_partitioning(self, partitioning, partition_or_window) -> None: ... + def _event_metadata(self, metadata, partition_or_window=None) -> dict[str, Any]: ... + + @abstractmethod + async def execute(self, context: OperationContext) -> OperationResult: ... + def failure(self, error: Exception) -> OperationResult: ... +``` + +`Asset` drops its `materializable` and `partitioning` declarations and gains an explicit `kind`. +Everything else on `Asset` stays where it is. + +`Operation.qualified_key` goes too. It returned the bare `key` as a stand-in for the qualified form +`Component.qualified_key` already builds from a component's identity. Under the old base order +(`Asset(Component, Operation)`) `Component`'s won; under `Asset(Operation)` the stand-in would win and +silently unqualify every asset key. Deleting it is the point of the refactor, and `Connection`, which +has no owner, reads the same either way. + +--- + +## 4. Consequences to handle + +**Kind derivation.** `__init_subclass__` derives `kind` only when `Component` is a direct base +(`any(base is Component for base in cls.__bases__)`) and the class does not declare one itself. Two +effects: + +- `Operation` becomes a direct child and would derive `kind = "operation"`, a kind that does not + exist. It declares `kind: ClassVar[str] = ""` instead, which the derivation already reads as an + opt-out. Guarding on abstractness was tried and is wrong: `Destination` is an abstract class that + is itself a kind, so skipping derivation for abstract classes unregisters `destination` and breaks + every relation declared against it. +- `Asset` stops being a direct child and would silently inherit whatever `Operation` carries. It + declares `kind: ClassVar[str] = "asset"` instead, which is what `Connection` already does. Future + operation kinds declare theirs the same way. + +`KINDS` itself is entry-point driven, so nothing about registration changes. + +**MRO.** `Connection(Resource, Operation)` reaches `Component` through both bases. C3 linearizes it +as `Connection, Resource, BaseSettings, Operation, Component, ..., Workload, object`, which is valid, +but a `BaseSettings` and `BaseModel` diamond is where pydantic is occasionally awkward. This is the +one risk worth proving first, with a throwaway subclass, before touching anything else. + +**Fields land on `Connection`.** Any pydantic field declared on `Operation` is collected for +`Connection` as well as `Asset`, so it enters the connection's config schema, its stored config and +its form. `materializable` therefore carries `x-hidden`, and every later field on `Operation` has to +make the same decision deliberately. This is the real cost of the change and the reason to keep +`Operation`'s field surface minimal. + +**`source` versus `parent`.** The DAG and the run state both read `operation.source` on any node, +including a `Connection` no source owns, so the default stays. It becomes a property returning +`None` rather than a class attribute: pydantic rejects a bare un-annotated attribute, and a +`ClassVar` that `Asset` overrides with a property is an invalid override under `ty`. `Component.parent` +now covers the same ground, and collapsing the two is a follow-up rather than something smuggled in +here. + +--- + +## 5. Testing + +- `tests/operation/test_base.py`: a concrete `Operation` subclass has a real `id`, `key` and + `relations`, serializes through `to_spec()`, and reports `operations() == [self]`. +- `tests/asset/test_base.py`: `Asset.kind == "asset"`, `materializable` still defaults to `True` and + is still honoured by `Source.select`. +- `tests/connection/test_base.py`: `Connection.kind == "connection"`, it constructs from the + environment as before, and `materializable` is hidden from its public schema. +- The existing DAG and runner suites are the regression surface for node-protocol reads and should + pass untouched. + +--- + +## 6. Follow-ups, recorded + +- **`Operation.source` collapses into `Component.parent`**, once it is confirmed that every read of + `operation.source` means "the source that owns this node". +- **Explicit kinds everywhere.** With `Asset` and `Connection` declaring theirs, the auto-derivation + in `__init_subclass__` serves only `Source`, `Job`, `Hook`, `Destination` and `Config`. Dropping it + entirely in favour of an explicit declaration per anchor would remove a piece of magic, and is a + separate, mechanical change. diff --git a/docs/superpowers/specs/2026-09-17-retry-design.md b/docs/superpowers/specs/2026-09-17-retry-design.md new file mode 100644 index 00000000..e22c21b8 --- /dev/null +++ b/docs/superpowers/specs/2026-09-17-retry-design.md @@ -0,0 +1,454 @@ +# Retry: attempts and verdicts across the execution hierarchy + +Date: 2026-09-17. Status: approved design, implementation not started. + +Scope: how work that failed is tried again, at every level of the framework that has a unit of work, +and how the outcome of that work is reported once rather than per attempt. This is the first of three +specs derived from one diagnosis (below). It depends on `2026-09-17-operation-is-a-component-design.md`, +which lands first and is what lets a policy be declared once on `Operation`. It covers the policy +primitive, operation-level retry in +the runner, run-level retry in the platform, stacks, verdict-based hooks, and the surfaces that +render them. Capacity (`Limit`/`Limiter`) is a separate spec and is not designed here. + +--- + +## 1. Diagnosis + +The framework has an execution hierarchy, batch to run to operation to request. Each level is a unit +of work that can fail, contend for a shared resource, and report an outcome. The levels are equipped +inconsistently: + +| level | verdict | attempt | capacity | +|---|---|---|---| +| batch (`Backfill`) | derived, but counts runs | none | `concurrency`, inert for cron backfills | +| run (`Run`) | `status` | manual only, `POST /runs/{id}/retry` | none, the queue drains unbounded | +| operation (`Operation`) | events, via the `executions` view | none | `max_workers`, local and anonymous | +| request | a raised exception | none | none | + +That single inconsistency produces the symptoms: retry exists at one level and by hand, contention is +bounded by an anonymous per-process number that cannot name the resource it protects, and hooks +observe attempts rather than verdicts, so they narrate work in progress instead of reporting +outcomes. + +The plan is to make the hierarchy uniform: every unit of work has a verdict, an attempt budget, and a +capacity, expressed by the same types at every level. Three specs: + +1. **Attempts and verdicts** (this document). +2. **Capacity**: `Limit` declared on the component that is the contended resource, `Limiter` with an + in-process and a store-backed implementation selected by deployment as `Launcher` already is, + absorbing `max_workers` and `Backfill.concurrency`. +3. **Request-level retry** in `RESTClient`, which is a small application of this document's policy + type and a source author's opt-in. + +--- + +## 2. Decisions + +| Topic | Decision | +|---|---| +| Primitive | One flat `RetryPolicy`: numbers only, no per-level nesting, no scope. | +| Level binding | The component a policy is declared on fixes the level, because a component's unit is its level. `Operation.retry` governs that operation's execution, `Source.retry` its assets' operations, `Job.retry` that job's runs, `RESTClient(retry=...)` that client's requests. | +| Contract | `retry` is a field on `Operation`, so the runner reads `operation.retry` on any node without narrowing to `Asset`. `Workload` does not declare it: nothing ever reads a run policy off an instance, and `Operation` extends `Workload`, so one name on both contracts would have to mean two levels at once. | +| Multiplication | Nothing multiplies unless two components deliberately declare at two levels. There is no single number reused across levels. | +| Inheritance | An asset inherits its source's policy through `Source._resolve()`, alongside `dataset`, `normalizer` and `materialization_strategy`. No new cascade mechanism. | +| Run-level reach | A run targeting a source or an asset takes the platform default. Run-level retry is a property of the job; a source's `retry` is an operation policy and is never read at run level. | +| Classification | Behaviour, not config: `Operation.retryable(error)`, next to `Operation.failure()`. Exception types never enter serialized config. | +| Automatic scope | An automatic retry is always failed-scope. `scope` stays an argument of the manual retry endpoint. | +| Attempt identity | A run attempt is a `Run` row. A stack is the attempts of one unit of work, keyed by `root_run_id` (its own id for a first attempt). No new table. | +| Successor creation | In `RunStore.complete()`, in the transaction that marks the run failed, before `_advance_backfill`. | +| Dispatch delay | `Run.scheduled_for`, honoured by the queue claim. Status stays `queued`. | +| Backfills | Successors keep their `backfill_id`. `_advance_backfill` finalizes on stacks, not runs. | +| Verdict rule | A hook observes verdicts, never attempts. A failed run that has a successor does not fire. | +| Intermediate failures | `OPERATION_RETRIED` marks an attempt that will be retried. `OPERATION_FAILED` means exhausted. | +| Executions view | Still one row per `(run, operation)`, the verdict. Ranked by attempt first, severity within the latest attempt, plus an `attempts` column. | +| Defaults | None. A policy applies only where a component declares one; there is no instance-wide fallback. Recorded as a follow-up. | +| Compatibility | None required. All consumers ship together. | + +--- + +## 3. The policy + +`interloper/retry/base.py` + +```py +class RetryPolicy(BaseModel): + """An attempt budget for one unit of work.""" + + max_attempts: int = 3 # total, including the first + delay: float = 5.0 # seconds before the second attempt + backoff: float = 2.0 # multiplier per subsequent attempt + max_delay: float = 3600.0 + jitter: float = 0.1 # fraction of the computed delay, applied symmetrically + + def allows(self, attempt: int) -> bool: ... + def delay_before(self, attempt: int) -> float: ... +``` + +`delay_before(attempt)` is `min(delay * backoff ** (attempt - 2), max_delay)`, jittered; attempt 2 is +the first retry. `allows(attempt)` is `attempt <= max_attempts`. + +Numbers only. The type is serializable, so it renders in `SchemaForm` without bespoke UI and survives +into a spec unchanged. + +### Declaration + +Declared once on the unit it governs, which the prerequisite spec makes a real component: + +```py +class Operation(Component, Workload): + retry: RetryPolicy | None = None # this operation's execution + +class Source(Component, Workload): + retry: RetryPolicy | None = None # default for its assets' operations + +class Job(Component, Workload): + retry: RetryPolicy | None = None # this job's runs +``` + +`Asset` inherits the field from `Operation` and declares nothing. So does `Connection`, where it +governs the renewal operation, which is correct and comes for free. + +`@il.asset(retry=...)` and `@il.source(retry=...)` gain the parameter alongside `partitioning=` and +`normalizer=`. + +`Source._resolve()` gains one line, in the block that already applies source-level defaults to assets +that do not define their own: + +```py +if asset.retry is None and self.retry is not None: + asset.retry = self.retry +``` + +### Resolution + +One step plus the platform default, per level. There is no walk. + +| failing unit | policy | +|---|---| +| request | the policy given to the client at construction; nothing else | +| operation | `operation.retry`, already carrying its source's default for an asset. Nothing else: an operation that declares no policy is attempted once. | +| run | the target's `config.retry` when the target is a job. Nothing else: a run whose target is not a job carrying a policy is attempted once. | + +The run-level policy is read live at completion time, so editing a job's policy takes effect on the +next failure rather than on the next run. + +### Classification + +```py +class Operation: + def retryable(self, error: Exception) -> bool: + """Whether another attempt at this operation is worth making.""" + return True +``` + +Sits next to `failure()`, and is overridden by an operation that can recognise a permanent error. The +default is permissive: the budget, not the classifier, is what bounds waste. + +--- + +## 4. Operation level + +The attempt loop is implemented once on `Runner` and called by both execution paths, +`AsyncRunner._execute_operation` and the multi-process worker function, so the policy is applied +identically everywhere. The multi-process runner calls it inside its worker: `retry` is a field on +the operation itself, so it is already serialized with it and needs no separate channel. + +The loop wraps the existing try/except in `AsyncRunner._execute_operation`: + +- `execute` raises +- the policy is `operation.retry`, so the loop never narrows to `Asset` +- if `operation.retryable(error)` and the policy allows the next attempt: emit `OPERATION_RETRIED` + carrying the attempt number and the error, sleep `delay_before(next)`, execute again +- otherwise: `state.mark_failed(...)` exactly as today + +A retrying node holds its concurrency slot for the duration. The DAG walk, the semaphore and +`fail_fast` are untouched: a node that is still retrying has not failed, so nothing downstream is +cancelled and no fail-fast break is triggered until its budget is exhausted. + +Every operation lifecycle event gains `attempt` in its `data`, defaulting to 1 when absent, so +historical rows read correctly. + +### Event vocabulary + +```py +OPERATION_RETRIED = "operation_retried" +``` + +`OPERATION_FAILED` keeps its meaning and is emitted only on exhaustion. This is the verdict rule one +level down: an intermediate attempt is recorded, and is not the outcome. + +--- + +## 5. Run level + +### Columns + +`Run` gains: + +- `root_run_id: UUID`, not null, its own id for a first attempt. Stack membership is one indexed + predicate with no `COALESCE`. +- `scheduled_for: datetime | None`, the earliest instant the run may be claimed. + +`retry_of`, `attempt` and `retry_scope` already exist and keep their meaning. Run ids are assigned in +Python (`uuid4()`) at every creation site so `root_run_id` can always be set; the server default +stays as a safety net. + +### The terminal path + +`RunStore.complete()` is the single terminal path every failure takes: the executor's verdict, the +executor's exception handler, the queue's launch failure, and the reaper's timeout. Planning the +successor there, rather than in a sweeping controller, is what makes the hook rule race-free: "has a +successor" becomes true in the same transaction that makes the run failed, so there is no window in +which a doomed attempt looks final. + +```py +def complete(self, run_id, *, success): + ... + db_run.status = "success" if success else "failed" + db_run.completed_at = now + UsageLedger(session).settle_run(db_run, success=success) + ... stamp component state ... + if not success: + self._plan_retry(session, db_run) # flushes, so the successor is visible below + if db_run.backfill_id: + self._advance_backfill(session, db_run.backfill_id, failed=not success) + commit(session) +``` + +`_plan_retry` reads the target component row (`config` is plain JSON, so this is one row read and no +hydration), resolves the policy, and inserts the successor when the policy allows the next attempt +and the target still exists: + +```py +Run( + id=uuid4(), + org_id=db_run.org_id, + component_id=db_run.component_id, + backfill_id=db_run.backfill_id, + partition_key=db_run.partition_key, + status="queued", + scheduled_for=now + policy.delay_before(db_run.attempt + 1), + retry_of=db_run.id, + root_run_id=db_run.root_run_id, + attempt=db_run.attempt + 1, + retry_scope="failed", + billable=db_run.billable, +) +``` + +Ordering matters: the successor must exist before `_advance_backfill` runs, so the in-flight count +sees it and the backfill is not finalized prematurely. + +A run whose component was deleted (`component_id` is null) is not retried. A canceled run is not +retried: cancellation does not go through this path. The quota is not checked at creation; dispatch +is the authoritative gate and cancels an over-quota run at claim time, as it does for any run. + +### Dispatch + +`QueueController._claim_next` gains one predicate: + +```sql +WHERE status = 'queued' AND (scheduled_for IS NULL OR scheduled_for <= now()) +ORDER BY created_at ASC +``` + +Status stays `queued`, so nothing else in the pipeline changes, and a delayed run is simply not yet +claimable. + +### Failed scope + +`RunExecutor._prior_successes` already walks the `retry_of` chain and marks previously successful +operations non-materializable, keyed by component row id. It is unchanged: the executions view keeps +one row per `(run, operation)` carrying the latest attempt's verdict, which is exactly what the walk +reads. + +--- + +## 6. Backfills + +A cron `lookback` window is a backfill, so retries have to live inside backfill accounting. This is +the part the earlier manual-retry work deliberately punted, and it cannot be punted. + +- The successor keeps its `backfill_id`. A retry belongs to the batch that produced it. +- `_advance_backfill` finalizes on stacks: the batch is complete when no stack has an attempt in + flight or pending, and it failed only if some stack's latest attempt failed. Attempt 1 dying no + longer condemns a backfill that attempt 2 healed. +- `Backfill.partitions` is unchanged: it counts partitions, and there is still exactly one stack per + partition. + +Progress counts stacks, so a backfill of 7 partitions reports out of 7 however many attempts it took. + +--- + +## 7. Verdicts + +### The executions view + +Today the view ranks `operation_failed` above `operation_completed` regardless of recency, so an +operation that failed and then healed would read as failed. It keeps one row per `(run, operation)`, +the verdict, and changes its ordering: + +```sql +row_number() OVER ( + PARTITION BY e.run_id, e.component_id + ORDER BY + COALESCE((e.data->>'attempt')::int, 1) DESC, + CASE e.event_type ... END, + e.timestamp DESC +) +``` + +plus an `attempts` column, `max(attempt)` over the same partition. `operation_retried` is added to +the view's event filter so it counts toward `attempts` without ever winning the verdict. + +Every consumer keeps working unchanged, `_prior_successes` included. Per-attempt detail stays in +`events`, which is where detail belongs. + +### Hooks + +`HookController` gains one predicate in its sweep, so a failed run with a successor is never +evaluated: + +```sql +AND NOT (runs.status = 'failed' AND EXISTS ( + SELECT 1 FROM runs successor WHERE successor.retry_of = runs.id +)) +``` + +No knowledge of budgets, backoff or policy, and no new hook event types: `run_completed` and +`run_failed` keep their names and now mean the stack's verdict. A stack that succeeds on attempt 3 +fires `run_completed` once, on that attempt. A stack that exhausts its budget fires `run_failed` +once, on the last attempt. + +`HookContext.metadata` gains `attempt` and `attempts` (the stack's total), so a message can say +"succeeded on attempt 2" or "failed after 3 attempts". The firing claim stays `uuid5(hook, run)`: +attempts that do not fire never claim, so no double firing is possible. + +--- + +## 8. Persistence and migration + +Migration 004 (the chain was folded to 001-003): + +1. `ALTER TABLE runs ADD COLUMN root_run_id uuid, ADD COLUMN scheduled_for timestamptz` +2. backfill existing rows: + +```sql +WITH RECURSIVE chain AS ( + SELECT id, id AS root FROM runs WHERE retry_of IS NULL + UNION ALL + SELECT r.id, c.root FROM runs r JOIN chain c ON r.retry_of = c.id +) +UPDATE runs SET root_run_id = chain.root FROM chain WHERE runs.id = chain.id; +``` + +3. `ALTER TABLE runs ALTER COLUMN root_run_id SET NOT NULL` +4. `CREATE INDEX ix_runs_root_run_id ON runs (root_run_id)` +5. an index supporting the claim predicate: `(status, scheduled_for, created_at)` +6. `CREATE OR REPLACE VIEW executions` with the attempt-aware ranking + +Any run row whose `retry_of` points at a deleted run (the FK is `ON DELETE SET NULL`) becomes its own +root, which is correct: its lineage is gone. + +--- + +## 9. Surfaces + +### API + +- `RunResponse` gains `root_run_id`, `scheduled_for`, and `attempts` (the stack's total). +- `GET /runs` becomes stack-native: one row per stack, the latest attempt, selected with + `DISTINCT ON (root_run_id) ... ORDER BY root_run_id, attempt DESC` and re-ordered by `created_at + DESC` in an outer query so the listing order is unchanged. Filters and counts apply to the stack's + verdict, which is what a reader means by "show me failed runs". +- `GET /runs?root_run_id=...` lists the attempts of one stack. +- `POST /runs/{id}/retry` is unchanged and keeps its explicit `scope`. + +### App + +- Runs table: one row per stack, the latest attempt's status, an `n/N` chip when `attempts > 1`, + expandable to the attempts. Watch the known `UTable` traps: an inline `:grouping` literal with + `v-model:expanded` auto-resets, and `resolveComponent` returns a bare string inside header and cell + render functions. +- Run detail: attempt navigation across the stack, and the failed-scope attempts showing which + operations were carried forward as already successful. +- Timeline: an operation with `attempts > 1` reads as its verdict, with its retried attempts visible + from the events it emitted. +- Retry policy needs no bespoke UI: it is a config field on a component, so `SchemaForm` renders it. + +--- + +## 10. No instance-wide default + +Deliberately none. A policy is in force only where a component declares one, so retry is visible at +the thing it retries and nothing acquires attempts by deployment. A deployment-wide default is a real +want, and it is recorded as a follow-up rather than guessed at now: it needs its own answer to how an +instance default interacts with a declared one, and to whether operation and run defaults belong in +one block or with the runner that owns execution. + +The consequence to keep in mind while the two land: nothing retries until a job or an asset says so, +so phases 1 and 2 change no behaviour on their own. + +--- + +## 11. Request level + +Out of scope here beyond the type it shares. The third spec gives `RESTClient` a `retry: +RetryPolicy | None` constructor argument, honoured around the transport call together with +`Retry-After` when the response carries one. A source author opts in where the client is built, +typically the source's `client` cached property, and is free to ignore it and handle vendor +semantics directly. Nothing resolves a request policy from a component. + +--- + +## 12. Testing + +Tests mirror the package layout one to one; no standalone feature files. + +- `interloper-core`: `tests/retry/test_base.py` for the budget and backoff maths, including jitter + bounds and `max_delay` capping. `tests/runner/test_async_runner.py` for the attempt loop: a node + that heals on attempt 2 reports success and emits `OPERATION_RETRIED` then `OPERATION_COMPLETED`; a + node that exhausts its budget emits `OPERATION_FAILED` once; a node whose `retryable()` declines is + not retried; a retrying node does not trigger a fail-fast break. `tests/source/test_base.py` for + the `_resolve` inheritance. +- `interloper-db`: `tests/store/test_runs.py` for successor creation, exhaustion, a deleted target, + the `scheduled_for` stamp, and the claim predicate; stack-based backfill finalization, in + particular a backfill whose attempt 1 failed and attempt 2 succeeded reporting success. +- `interloper-scheduler`: `tests/test_hooks.py` for the gate: a failed run with a successor does not + fire, an exhausted one fires `run_failed` once, a healed stack fires `run_completed` once on the + successful attempt. + +--- + +## 13. Phasing + +1. **Core**: `RetryPolicy`, `Operation.retryable`, the fields on `Operation`/`Source`/`Job`, the + `_resolve` inheritance, the decorator parameters, `OPERATION_RETRIED`, the runner attempt loop. +2. **Platform**: the two columns, migration 004, the executions view, `_plan_retry` in `complete()`, + the claim predicate, stack-based backfill accounting, the hook gate. +3. **Surfaces**: API fields and the stack-native runs listing, then the app. + +Each phase is independently shippable: after phase 1 operations retry in place with nothing else +changed, after phase 2 runs retry and hooks report verdicts, after phase 3 the stack is legible. + +--- + +## 14. Follow-ups, recorded + +- **An instance-wide default.** Dropped from this design on purpose. Whoever picks it up owns two + questions: whether an instance default is a floor, a ceiling or a plain fallback under a declared + policy, and whether the operation-level default belongs in a settings block next to the run one or + on the runner that executes operations, where `max_workers` and `fail_fast` already live. +- **Capacity spec**: `Limit` and `Limiter`, absorbing `max_workers` and `Backfill.concurrency`, with + a store-backed implementation so a limit can be shared across pods. This is the tuning surface for + vendor quotas, and the reason retry is not being asked to solve contention. +- **Cron backfill concurrency**: cron builds its backfill inline and queues every partition at once, + so `Backfill.concurrency` is recorded but inert for exactly the backfills that dominate production. + Folded into the capacity spec rather than fixed on its own. +- **Backfill-level hook events**: stack gating stops a healing attempt from notifying, but a + genuinely broken source still notifies once per partition. `backfill_completed` / `backfill_failed` + would collapse a cron firing into one message. Deliberately deferred until the noise left after + stack gating is known. +- **Run-level classification**: the run level has no exception object, so it retries everything + within its budget. If a class of run failure proves never worth retrying, it needs a verdict + richer than a boolean, which is a change to the terminal path's signature. From b1d03ca6229ec1de6a25132583877840988f7936 Mon Sep 17 00:00:00 2001 From: Guillaume Onfroy Date: Thu, 17 Sep 2026 17:21:50 +0200 Subject: [PATCH 3/5] refactor(core)!: read a node's owner as its parent, not its source `Asset.source` is `cast("Source | None", self.parent)`, so a `source` on `Operation` was a second name for `Component.parent`, living on a contract where it is false for every implementor but one, returning a `None` that is a null object rather than an answer. The same stand-in shape as the `qualified_key` its parent commit removed. Neither generic reader needed source-ness. `DAG.to_spec` reads it to decide what to serialize, which is the ownership rule `Component` already defines, and `RunState` stamps the owner's id into event metadata. Both read `parent` now. `Asset.source` stays: a typed accessor narrowing `parent` to `Source` on the one class where that holds, and part of the authoring surface, since `@il.asset` injects `self.source` into `data()`. The event key stays `source_id`, correct while assets are the only owned operations; renaming it to `parent_id` touches the event stream telemetry reads and is recorded as its own change. By Digitl --- ...6-09-17-operation-is-a-component-design.md | 29 ++++++++++++------- .../src/interloper/dag/base.py | 12 ++++---- .../src/interloper/operation/base.py | 13 --------- .../src/interloper/runner/state.py | 6 ++-- .../interloper-core/tests/dag/test_base.py | 2 +- 5 files changed, 30 insertions(+), 32 deletions(-) diff --git a/docs/superpowers/specs/2026-09-17-operation-is-a-component-design.md b/docs/superpowers/specs/2026-09-17-operation-is-a-component-design.md index acc387f6..378f5488 100644 --- a/docs/superpowers/specs/2026-09-17-operation-is-a-component-design.md +++ b/docs/superpowers/specs/2026-09-17-operation-is-a-component-design.md @@ -61,8 +61,6 @@ class Operation(Component, Workload): materializable: bool = Field(default=True, json_schema_extra={"x-hidden": True}) - @property - def source(self) -> Any: ... # None; Asset narrows it to its parent def operations(self) -> list[Operation]: ... def effective_partition(self, partition_or_window): ... def upstream_relations(self) -> dict[str, Relation]: ... @@ -113,12 +111,20 @@ its form. `materializable` therefore carries `x-hidden`, and every later field o make the same decision deliberately. This is the real cost of the change and the reason to keep `Operation`'s field surface minimal. -**`source` versus `parent`.** The DAG and the run state both read `operation.source` on any node, -including a `Connection` no source owns, so the default stays. It becomes a property returning -`None` rather than a class attribute: pydantic rejects a bare un-annotated attribute, and a -`ClassVar` that `Asset` overrides with a property is an invalid override under `ty`. `Component.parent` -now covers the same ground, and collapsing the two is a follow-up rather than something smuggled in -here. +**`source` collapses into `parent`.** `Asset.source` is literally `cast("Source | None", +self.parent)`, so a `source` on `Operation` is a second name for `Component.parent` living on a +contract where it is false for every implementor but one, with a `None` that is a null object rather +than an answer. The same shape as `qualified_key`, and it goes the same way. + +Both generic readers want ownership, not source-ness. `DAG.to_spec` reads it to decide what to +serialize (no owner means the node's own spec, an owner means the owner's spec once, deduped), which +is the ownership rule `Component` already defines. `RunState._operation_event_metadata` stamps the +owner's id. Both now read `operation.parent`. + +`Asset.source` stays: a typed accessor narrowing `parent` to `Source` on the one class where that +holds, and part of the authoring surface, since `@il.asset` injects `self.source` into `data()`. A +domain alias on the class where it is true is not the smell; hoisting it onto a contract where it is +not is. --- @@ -137,8 +143,11 @@ here. ## 6. Follow-ups, recorded -- **`Operation.source` collapses into `Component.parent`**, once it is confirmed that every read of - `operation.source` means "the source that owns this node". +- **The `source_id` event key.** `RunState` stamps the owner's id under `source_id`, and telemetry + reads it as `interloper.source.id`. That is correct while assets are the only owned operations, but + the key is named for a guarantee the model does not make. Renaming it to `parent_id` is right and + touches the event stream that telemetry and downstream consumers read, so it wants its own change. +- **`DAG.to_spec` duplicates the owner rule** that `Component` serialization already implements. - **Explicit kinds everywhere.** With `Asset` and `Connection` declaring theirs, the auto-derivation in `__init_subclass__` serves only `Source`, `Job`, `Hook`, `Destination` and `Config`. Dropping it entirely in favour of an explicit declaration per anchor would remove a piece of magic, and is a diff --git a/packages/interloper-core/src/interloper/dag/base.py b/packages/interloper-core/src/interloper/dag/base.py index a94ca6ef..3b5ff046 100644 --- a/packages/interloper-core/src/interloper/dag/base.py +++ b/packages/interloper-core/src/interloper/dag/base.py @@ -461,14 +461,14 @@ def to_spec(self) -> DAGSpec: """ context = SerializationContext(cast("list[Component]", self.operations)) items: list[Spec] = [] - written_sources: set[str] = set() + written_owners: set[str] = set() for operation in self.operations: - source = operation.source - if source is None: + owner = operation.parent + if owner is None: items.append(operation.to_spec(context=context)) - elif source.id not in written_sources: - written_sources.add(source.id) - items.append(source.to_spec(context=context)) + elif owner.id not in written_owners: + written_owners.add(owner.id) + items.append(owner.to_spec(context=context)) return DAGSpec(items=items) @classmethod diff --git a/packages/interloper-core/src/interloper/operation/base.py b/packages/interloper-core/src/interloper/operation/base.py index d0a98650..3b156d69 100644 --- a/packages/interloper-core/src/interloper/operation/base.py +++ b/packages/interloper-core/src/interloper/operation/base.py @@ -114,19 +114,6 @@ class Operation(Component, Workload): materializable: bool = Field(default=True, json_schema_extra={"x-hidden": True}) - @property - def source(self) -> Any: - """The source that owns this node, or ``None`` when nothing does. - - Read by the graph and by the event metadata on any node, including - the operations no source owns, which is why it is answered here - rather than only on ``Asset``. - - Returns: - The owning source, or ``None``. - """ - return None - def operations(self) -> list[Operation]: """An operation is trivially its own workload. diff --git a/packages/interloper-core/src/interloper/runner/state.py b/packages/interloper-core/src/interloper/runner/state.py index 947dab40..841aa230 100644 --- a/packages/interloper-core/src/interloper/runner/state.py +++ b/packages/interloper-core/src/interloper/runner/state.py @@ -365,8 +365,10 @@ def _operation_event_metadata(self, operation: Operation) -> dict[str, Any]: "component_key": operation.key, "partition_or_window": str(self.partition_or_window) if self.partition_or_window else None, } - if operation.source is not None: - meta["source_id"] = operation.source.id + # Only assets are owned today, and their owner is their source, so the + # key stays `source_id`; a parent of another kind would need a rename. + if operation.parent is not None: + meta["source_id"] = operation.parent.id return meta @staticmethod diff --git a/packages/interloper-core/tests/dag/test_base.py b/packages/interloper-core/tests/dag/test_base.py index c274db15..6202e8bb 100644 --- a/packages/interloper-core/tests/dag/test_base.py +++ b/packages/interloper-core/tests/dag/test_base.py @@ -641,7 +641,7 @@ def test_bound_upstream_outside_workloads_joins_read_only(self): node = dag.operation_map[fb.campaigns.id] assert node.materializable is False assert node is not fb.campaigns - assert node.source is fb + assert node.parent is fb assert dag.get_predecessors(matcher.campaign_matches.id) == [fb.campaigns.id] def test_unbound_wildcard_binds_every_candidate_in_dag(self): From 76778a1507f872f82257ab1b5005d8008a7ff65d Mon Sep 17 00:00:00 2001 From: Guillaume Onfroy Date: Thu, 17 Sep 2026 17:27:33 +0200 Subject: [PATCH 4/5] refactor(core)!: name the owner's id parent_id in the event stream `source_id` named a guarantee the model does not make: the key carries the id of the component that owns the one an event is about, and only assets happen to be owned by a source today. It is now `parent_id`, and the telemetry attribute `interloper.source.id` is `interloper.parent.id`. All three producers move together (node lifecycle metadata, asset-level metadata, the component log emitter). Renaming only the generic one would have left two keys for the same value, which is worse than a loose name. The same line divides the plumbing as divides `Asset.source` from `Operation.parent`: `EventLogger` is component-generic and takes `parent_id`, while `ExecutionContext` is asset-specific and keeps `source_id`. Events written before this keep the old key. They are history and are not rewritten, so a consumer reading across the boundary sees both, and any saved telemetry query on `interloper.source.id` needs updating. By Digitl --- .../2026-09-17-operation-is-a-component-design.md | 15 +++++++++++---- .../interloper-core/src/interloper/asset/base.py | 2 +- .../src/interloper/asset/context.py | 2 +- .../src/interloper/events/logger.py | 10 +++++----- .../src/interloper/runner/state.py | 4 +--- .../src/interloper/telemetry/attributes.py | 4 ++-- .../interloper-core/tests/asset/test_context.py | 2 +- .../interloper-core/tests/events/test_logger.py | 6 +++--- .../tests/telemetry/test_attributes.py | 4 ++-- packages/interloper-db/tests/store/test_events.py | 4 ++-- 10 files changed, 29 insertions(+), 24 deletions(-) diff --git a/docs/superpowers/specs/2026-09-17-operation-is-a-component-design.md b/docs/superpowers/specs/2026-09-17-operation-is-a-component-design.md index 378f5488..38c9e272 100644 --- a/docs/superpowers/specs/2026-09-17-operation-is-a-component-design.md +++ b/docs/superpowers/specs/2026-09-17-operation-is-a-component-design.md @@ -126,6 +126,17 @@ holds, and part of the authoring surface, since `@il.asset` injects `self.source domain alias on the class where it is true is not the smell; hoisting it onto a contract where it is not is. +**The event key follows.** All three producers of the owner's id (the node lifecycle metadata, the +asset-level metadata and the component log emitter) wrote `source_id`, and the telemetry attribute +was `interloper.source.id`. That named a guarantee the model does not make, so the key is `parent_id` +and the attribute `interloper.parent.id`. Renaming only the generic producer would have left two keys +for one value, which is worse than the name being loose. + +The same line divides the plumbing: `EventLogger`, which is component-generic, takes `parent_id`, +while `ExecutionContext`, which is asset-specific, keeps `source_id` and fills it. Events written +before this carry the old key; they are history and are not rewritten, so anything reading the event +stream over a period spanning it has to accept both. + --- ## 5. Testing @@ -143,10 +154,6 @@ not is. ## 6. Follow-ups, recorded -- **The `source_id` event key.** `RunState` stamps the owner's id under `source_id`, and telemetry - reads it as `interloper.source.id`. That is correct while assets are the only owned operations, but - the key is named for a guarantee the model does not make. Renaming it to `parent_id` is right and - touches the event stream that telemetry and downstream consumers read, so it wants its own change. - **`DAG.to_spec` duplicates the owner rule** that `Component` serialization already implements. - **Explicit kinds everywhere.** With `Asset` and `Connection` declaring theirs, the auto-derivation in `__init_subclass__` serves only `Source`, `Job`, `Hook`, `Destination` and `Config`. Dropping it diff --git a/packages/interloper-core/src/interloper/asset/base.py b/packages/interloper-core/src/interloper/asset/base.py index f2aa4791..71422d9c 100644 --- a/packages/interloper-core/src/interloper/asset/base.py +++ b/packages/interloper-core/src/interloper/asset/base.py @@ -1129,5 +1129,5 @@ def _event_metadata( } source = self.source if source is not None: - base["source_id"] = source.id + base["parent_id"] = source.id return base diff --git a/packages/interloper-core/src/interloper/asset/context.py b/packages/interloper-core/src/interloper/asset/context.py index eb07e771..297ec38b 100644 --- a/packages/interloper-core/src/interloper/asset/context.py +++ b/packages/interloper-core/src/interloper/asset/context.py @@ -62,7 +62,7 @@ def logger(self) -> EventLogger: self._asset_key, self._metadata, component_id=self._asset_id, - source_id=self._source_id, + parent_id=self._source_id, ) return self._logger diff --git a/packages/interloper-core/src/interloper/events/logger.py b/packages/interloper-core/src/interloper/events/logger.py index 86658971..cd0206d5 100644 --- a/packages/interloper-core/src/interloper/events/logger.py +++ b/packages/interloper-core/src/interloper/events/logger.py @@ -28,7 +28,7 @@ def __init__( metadata: dict[str, Any], component_id: str | None = None, component_kind: str = "asset", - source_id: str | None = None, + parent_id: str | None = None, ) -> None: """Initialize the logger. @@ -39,13 +39,13 @@ def __init__( every emitted ``LOG`` event so it can be attributed to the component (e.g. filtered alongside its lifecycle events). component_kind: Kind of the owning component. - source_id: Id of the source the component belongs to, if any. + parent_id: Id of the component that owns this one, if any. """ self._component_key = component_key self._metadata = metadata self._component_id = component_id self._component_kind = component_kind - self._source_id = source_id + self._parent_id = parent_id def _emit(self, level: int, message: str) -> None: """Emit a ``LOG`` event with the given level and message. @@ -64,8 +64,8 @@ def _emit(self, level: int, message: str) -> None: if self._component_id is not None: metadata["component_id"] = self._component_id metadata["component_kind"] = self._component_kind - if self._source_id is not None: - metadata["source_id"] = self._source_id + if self._parent_id is not None: + metadata["parent_id"] = self._parent_id EventBus.emit(EventType.LOG, metadata=metadata) def debug(self, message: str) -> None: diff --git a/packages/interloper-core/src/interloper/runner/state.py b/packages/interloper-core/src/interloper/runner/state.py index 841aa230..3a4601de 100644 --- a/packages/interloper-core/src/interloper/runner/state.py +++ b/packages/interloper-core/src/interloper/runner/state.py @@ -365,10 +365,8 @@ def _operation_event_metadata(self, operation: Operation) -> dict[str, Any]: "component_key": operation.key, "partition_or_window": str(self.partition_or_window) if self.partition_or_window else None, } - # Only assets are owned today, and their owner is their source, so the - # key stays `source_id`; a parent of another kind would need a rename. if operation.parent is not None: - meta["source_id"] = operation.parent.id + meta["parent_id"] = operation.parent.id return meta @staticmethod diff --git a/packages/interloper-core/src/interloper/telemetry/attributes.py b/packages/interloper-core/src/interloper/telemetry/attributes.py index c7394acd..84913144 100644 --- a/packages/interloper-core/src/interloper/telemetry/attributes.py +++ b/packages/interloper-core/src/interloper/telemetry/attributes.py @@ -10,7 +10,7 @@ COMPONENT_KIND = "interloper.component.kind" COMPONENT_KEY = "interloper.component.key" COMPONENT_QUALIFIED_KEY = "interloper.component.qualified_key" -SOURCE_ID = "interloper.source.id" +PARENT_ID = "interloper.parent.id" PARTITION = "interloper.partition" DESTINATION_KEY = "interloper.destination.key" UPSTREAM_KEY = "interloper.upstream.key" @@ -34,7 +34,7 @@ "component_key": COMPONENT_KEY, "qualified_key": COMPONENT_QUALIFIED_KEY, "partition_or_window": PARTITION, - "source_id": SOURCE_ID, + "parent_id": PARENT_ID, "destination_key": DESTINATION_KEY, "org_id": ORG_ID, "target_id": TARGET_ID, diff --git a/packages/interloper-core/tests/asset/test_context.py b/packages/interloper-core/tests/asset/test_context.py index f217d289..4d9bc2a0 100644 --- a/packages/interloper-core/tests/asset/test_context.py +++ b/packages/interloper-core/tests/asset/test_context.py @@ -159,7 +159,7 @@ def test_messages_become_log_events(self) -> None: assert len(logs) == 1 assert logs[0].metadata["message"] == "fetching page 1" assert logs[0].metadata["component_id"] == "a1" - assert logs[0].metadata["source_id"] == "s1" + assert logs[0].metadata["parent_id"] == "s1" assert logs[0].metadata["run_id"] == "r1" diff --git a/packages/interloper-core/tests/events/test_logger.py b/packages/interloper-core/tests/events/test_logger.py index da22602e..751961de 100644 --- a/packages/interloper-core/tests/events/test_logger.py +++ b/packages/interloper-core/tests/events/test_logger.py @@ -27,7 +27,7 @@ def test_log_event_carries_component_and_source_id() -> None: "demo.a", {"run_id": "run-1"}, component_id="asset-1", - source_id="source-1", + parent_id="source-1", ) events = _capture(lambda: logger.info("hello")) @@ -35,7 +35,7 @@ def test_log_event_carries_component_and_source_id() -> None: assert len(events) == 1 meta = events[0].metadata assert meta["component_id"] == "asset-1" - assert meta["source_id"] == "source-1" + assert meta["parent_id"] == "source-1" assert meta["component_key"] == "demo.a" assert meta["message"] == "hello" assert meta["run_id"] == "run-1" @@ -50,4 +50,4 @@ def test_log_event_omits_ids_when_unset() -> None: assert len(events) == 1 meta = events[0].metadata assert "component_id" not in meta - assert "source_id" not in meta + assert "parent_id" not in meta diff --git a/packages/interloper-core/tests/telemetry/test_attributes.py b/packages/interloper-core/tests/telemetry/test_attributes.py index e39126f5..3c8260f8 100644 --- a/packages/interloper-core/tests/telemetry/test_attributes.py +++ b/packages/interloper-core/tests/telemetry/test_attributes.py @@ -15,7 +15,7 @@ def test_maps_event_metadata_keys(self): "component_key": "orders", "qualified_key": "shop.orders", "partition_or_window": "2026-07-01", - "source_id": "s1", + "parent_id": "s1", } ) assert attrs == { @@ -25,7 +25,7 @@ def test_maps_event_metadata_keys(self): attributes.COMPONENT_KEY: "orders", attributes.COMPONENT_QUALIFIED_KEY: "shop.orders", attributes.PARTITION: "2026-07-01", - attributes.SOURCE_ID: "s1", + attributes.PARENT_ID: "s1", } def test_drops_none_and_unknown_keys(self): diff --git a/packages/interloper-db/tests/store/test_events.py b/packages/interloper-db/tests/store/test_events.py index 653e2a7c..ccf67d5d 100644 --- a/packages/interloper-db/tests/store/test_events.py +++ b/packages/interloper-db/tests/store/test_events.py @@ -124,14 +124,14 @@ def test_event_values_spills_unpromoted_metadata_into_data() -> None: "component_id": str(uuid4()), "component_key": "ads", "asset_qualified_key": "facebook.ads", - "source_id": "src-1", + "parent_id": "src-1", "error": "boom", } ), org_id=uuid4(), run_id=None, ) - assert values["data"] == {"asset_qualified_key": "facebook.ads", "source_id": "src-1"} + assert values["data"] == {"asset_qualified_key": "facebook.ads", "parent_id": "src-1"} assert values["error"] == "boom" From e93bd3a83519dde8902202aca3450c07cdcf5a99 Mon Sep 17 00:00:00 2001 From: Guillaume Onfroy Date: Thu, 17 Sep 2026 17:46:27 +0200 Subject: [PATCH 5/5] refactor!: an operation is enabled, not materializable `materializable` named an asset concept (computed and written to a destination) on a flag the whole graph reads: the DAG's generations and satisfied edges, `RunState`'s skipped-versus-queued, the runner's partition preflight, `Source.select`, and the failed-scope retry, which marks any operation that already succeeded, a connection renewal included. For a connection renewal there is nothing to materialize. `enabled` is the word the framework already uses for "this will run", on `Job` ("Job will run on the configured schedule") and `Hook` ("Hook will fire on matching events"). `Operation` is the third: a disabled operation stays in the graph, ordering whatever depends on it, and does not execute. `Asset` adds the half only an asset has, that a disabled one is read-only and its downstreams read its data from its destination. The rename is user-visible: run manifests, the `source(enabled=False)` copy keyword, the docs, the examples, the app helper and the agent tool all move with it. Two defects go with it. The field carried `json_schema_extra={"x-hidden": True}`, which is the wrong mechanism (config fields are hidden through `internal_fields`, applied by `strip_internal_fields`; `x-hidden` is a state convention the app reads for state columns) and the wrong intent, since the app and the agent treat this flag as user-facing. And the pydantic shadowing-warning filter in `asset/base.py` was dead once `Asset` stopped redeclaring the field over `Operation`'s plain default. By Digitl --- docs/extending/operations.md | 2 +- docs/extending/runners.md | 2 +- docs/guide/assets.md | 6 +- docs/guide/backfilling.md | 2 +- docs/guide/cli.md | 2 +- docs/guide/dependencies.md | 8 +- docs/guide/execution.md | 6 +- docs/guide/partitioning.md | 2 +- docs/guide/sources.md | 4 +- docs/guide/specs.md | 8 +- docs/reference/decorators.md | 2 +- docs/reference/events.md | 2 +- examples/campaign_matcher.yaml | 6 +- examples/source.py | 6 +- .../src/interloper_agent/tools/scheduling.py | 8 +- .../interloper-app/app/app/types/component.ts | 6 +- .../tests/test_campaign_matcher.py | 4 +- .../src/interloper/asset/base.py | 18 ++--- .../src/interloper/cli/commands/run.py | 10 +-- .../src/interloper/dag/base.py | 20 ++--- .../src/interloper/operation/base.py | 8 +- .../src/interloper/runner/base.py | 6 +- .../src/interloper/runner/state.py | 2 +- .../src/interloper/source/base.py | 14 ++-- .../interloper-core/tests/asset/test_base.py | 38 +++++----- .../tests/cli/commands/test_run.py | 4 +- .../tests/component/test_decorator.py | 4 +- .../tests/connection/test_base.py | 2 +- .../interloper-core/tests/dag/test_base.py | 74 +++++++++---------- .../tests/operation/test_base.py | 8 +- .../interloper-core/tests/runner/test_base.py | 6 +- .../tests/runner/test_state.py | 10 +-- .../tests/serializable/test_base.py | 2 +- .../interloper-core/tests/source/test_base.py | 24 +++--- .../tests/source/test_decorator.py | 8 +- .../tests/store/test_hydration.py | 8 +- .../src/interloper_docker/runner.py | 6 +- .../src/interloper_k8s/runner.py | 4 +- .../src/interloper_scheduler/executor.py | 2 +- .../tests/test_executor.py | 10 +-- .../skills/interloper-manifest/SKILL.md | 8 +- 41 files changed, 188 insertions(+), 184 deletions(-) diff --git a/docs/extending/operations.md b/docs/extending/operations.md index 9a48b327..d7d67535 100644 --- a/docs/extending/operations.md +++ b/docs/extending/operations.md @@ -37,7 +37,7 @@ plain defaults that make any subclass a valid node; `Asset` overrides them with | Member | Default | Asset | |--------|---------|-------| | `id`, `kind`, `key`, `qualified_key` | from `Component` | qualified with the source key | -| `materializable` | `True` | field | +| `enabled` | `True` | field | | `relations` | `{}` | every relation the class declares, by name | | `upstream_relations()` | `{}` | the relations whose kind is `asset`: the graph's edges | | `bound(name)` | from `Component` | what is bound to one relation | diff --git a/docs/extending/runners.md b/docs/extending/runners.md index eced3592..d6e18399 100644 --- a/docs/extending/runners.md +++ b/docs/extending/runners.md @@ -44,7 +44,7 @@ state.mark_canceled(operation) ``` Completing an operation promotes its dependents to ready; failing one cancels everything -downstream. Non-materializable operations start as `SKIPPED` and count as satisfied +downstream. Non-enabled operations start as `SKIPPED` and count as satisfied predecessors. Each transition emits the matching `OPERATION_*` event with a deterministic id (`emit=False` skips the emission for runners whose child process emits it itself). All mutations happen on the event loop thread, so no locking is needed. diff --git a/docs/guide/assets.md b/docs/guide/assets.md index e30256b1..89dba46b 100644 --- a/docs/guide/assets.md +++ b/docs/guide/assets.md @@ -137,7 +137,7 @@ An instance carries the runtime state a definition does not know about: | `destinations` | Destination instances to write to. A single destination is accepted and wrapped in a list. | | `dataset` | Namespace (schema, folder) the asset materializes into. Defaults to the source's. | | `default_destination_key` | With several destinations, the one downstream readers load from. | -| `materializable` | `False` turns the asset into a read-only dependency: it is skipped by runners but its stored output is still readable. | +| `enabled` | `False` turns the asset into a read-only dependency: it is skipped by runners but its stored output is still readable. | | `materialization_strategy` | How strictly the data is checked against the schema. | | `normalizer` | The normalizer applied before conform. | | `id` | Instance identity, a UUID by default. | @@ -151,7 +151,7 @@ Set them at construction, or derive a reconfigured copy by calling an existing i ```py asset = users(destinations=il.CSVDestination(base_path="./data"), dataset="shop") -read_only = asset(materializable=False) +read_only = asset(enabled=False) strict = asset(materialization_strategy=il.MaterializationStrategy.STRICT) bare = asset(normalizer=None) # None explicitly clears the normalizer ``` @@ -170,7 +170,7 @@ Unknown keyword arguments raise `TypeError` rather than being silently dropped. | Call | Effect | |------|--------| | `asset.run(partition, dag, metadata)` | Execute, normalize, conform. Return the data. Write nothing. | -| `asset.materialize(partition, dag, metadata)` | Everything `run` does, then write to every destination. Returns the data, or `None` when the asset is not materializable. | +| `asset.materialize(partition, dag, metadata)` | Everything `run` does, then write to every destination. Returns the data, or `None` when the asset is disabled. | | `await asset.run_async(...)`, `await asset.materialize_async(...)` | The same, for async callers. | `partition` is required for partitioned assets and ignored (with a warning) for unpartitioned diff --git a/docs/guide/backfilling.md b/docs/guide/backfilling.md index 9d47716a..b6fab35f 100644 --- a/docs/guide/backfilling.md +++ b/docs/guide/backfilling.md @@ -23,7 +23,7 @@ data in place rather than the ancient tail. ## One run for the whole window -When every materializable partitioned asset in the DAG declares `allow_window=True`, pass the +When every enabled partitioned asset in the DAG declares `allow_window=True`, pass the window itself. Each asset receives the range through `context.window` and fetches it in one call; destinations split the write per partition: diff --git a/docs/guide/cli.md b/docs/guide/cli.md index a7e541d8..508aa183 100644 --- a/docs/guide/cli.md +++ b/docs/guide/cli.md @@ -59,7 +59,7 @@ Run and operation events flow through the logging stack, sharing one format and ordinary log lines on stderr: ``` -19:39:52.368 INFO Running DAG with 3 materializable operation(s) (3 total) using AsyncRunner +19:39:52.368 INFO Running DAG with 3 enabled operation(s) (3 total) using AsyncRunner 19:39:52.370 INFO RUN_STARTED - Run started (3 operations) 19:39:52.370 INFO OPERATION_STARTED users Operation 'users' started 19:39:52.371 INFO OPERATION_STARTED orders Operation 'orders' started diff --git a/docs/guide/dependencies.md b/docs/guide/dependencies.md index 52fee895..0e482b5f 100644 --- a/docs/guide/dependencies.md +++ b/docs/guide/dependencies.md @@ -153,11 +153,11 @@ running them, the same explicit-bind pattern as any other relation: ```py matcher.campaign_matches.bind("campaigns", fb.campaigns) -dag = il.DAG(matcher) # fb.campaigns joins the DAG as materializable=False +dag = il.DAG(matcher) # fb.campaigns joins the DAG as enabled=False ``` `examples/campaign_matcher.yaml` is the same wiring written as a manifest: two connectors each -override their `campaigns` asset to `materializable: false`, and `campaign_matches` names both by +override their `campaigns` asset to `enabled: false`, and `campaign_matches` names both by `{ref: ...}` since an asset always travels under its own source. See [Specs](specs.md) for the manifest format. @@ -245,13 +245,13 @@ included. ## Running one asset with its parents -A bound upstream the run does not materialize joins the DAG anyway, as a non-materializable copy +A bound upstream the run does not materialize joins the DAG anyway, as a disabled copy under the same id: it is a live instance with its own destinations, so it can be read without being run. That is the whole mechanism behind running one asset with its parents: ```py mini = dag.mini_dag(fin.revenue.id) -[(op.qualified_key, op.materializable) for op in mini.operations] +[(op.qualified_key, op.enabled) for op in mini.operations] # [("finance.revenue", True), ("shop.orders", False)] ``` diff --git a/docs/guide/execution.md b/docs/guide/execution.md index 499654fa..5c400fd5 100644 --- a/docs/guide/execution.md +++ b/docs/guide/execution.md @@ -26,7 +26,7 @@ dag.get_successors(asset.id) # downstream ids dag.mini_dag(asset.id) # one node plus read-only parents ``` -Non-materializable nodes stay in the graph as dependencies but never execute and never appear +Non-enabled nodes stay in the graph as dependencies but never execute and never appear in the generations. ## Materializing @@ -104,7 +104,7 @@ print(result) `ExecutionInfo` carries `component_id`, `component_key`, `status`, `start_time`, `end_time`, `execution_time`, `error`, `traceback`, and `effects` (what the operation asked the platform to persist). `ExecutionStatus` is `QUEUED`, `READY`, `RUNNING`, `COMPLETED`, `FAILED`, `SKIPPED` -(non-materializable), `CANCELED` (downstream of a failure). +(disabled), `CANCELED` (downstream of a failure). ## Failure handling @@ -114,7 +114,7 @@ traceback, its dependents are canceled, and the run continues or stops depending so events and results are complete before it surfaces. A failure of the walk machinery itself (a deadlock, an invalid graph) raises `RunnerError`. -Before anything executes, the runner validates the scope against every materializable +Before anything executes, the runner validates the scope against every enabled operation: partitioned operations without a scope, windows against operations that forbid them, and time-partition mismatches fail the whole run up front. diff --git a/docs/guide/partitioning.md b/docs/guide/partitioning.md index 88d37377..ec50327a 100644 --- a/docs/guide/partitioning.md +++ b/docs/guide/partitioning.md @@ -113,7 +113,7 @@ Passing a window to a run is then a single execution covering the whole range. A partition still works and is presented to the asset as a one-partition window. Destinations split window writes per partition, so storage looks identical either way. -A run given a window fails as a whole when any materializable partitioned asset in the DAG does +A run given a window fails as a whole when any enabled partitioned asset in the DAG does not allow windows. ## What the context exposes diff --git a/docs/guide/sources.md b/docs/guide/sources.md index 188382d1..6c43ad99 100644 --- a/docs/guide/sources.md +++ b/docs/guide/sources.md @@ -108,10 +108,10 @@ Calling an instance returns a deep copy with overrides applied; the original is ```py staging = source(destinations=il.CSVDestination(base_path="./staging")) -read_only = source(materializable=False) +read_only = source(enabled=False) ``` -Accepted keywords: `dataset`, `default_destination_key`, `materializable` (applied to every +Accepted keywords: `dataset`, `default_destination_key`, `enabled` (applied to every asset), `normalizer`, `materialization_strategy`, and any relation name. A fixed keyword left out means "unchanged"; a relation name passed at all changes it, since `None` there clears the binding. Rebinding a relation also repoints the assets that had received it by trickle, leaving diff --git a/docs/guide/specs.md b/docs/guide/specs.md index ea698608..0cfe94a7 100644 --- a/docs/guide/specs.md +++ b/docs/guide/specs.md @@ -58,11 +58,11 @@ nothing else to ask. In the platform, the store resolves it. A reference nobody A source is the unit of reconstruction. Its spec carries the assets as an **override map** keyed by asset key rather than as individual specs, which keeps the document compact and lets -per-asset state (its own destinations, `materializable`, its bound upstreams) survive: +per-asset state (its own destinations, `enabled`, its bound upstreams) survive: ```py Shop(account_id="act_1").to_spec().init -# {"account_id": "act_1", "assets": {"orders": {"id": "...", "materializable": True}, ...}} +# {"account_id": "act_1", "assets": {"orders": {"id": "...", "enabled": True}, ...}} ``` Reconstruction builds each asset class with its overrides. The map is also the list of assets the @@ -101,7 +101,7 @@ init: assets: campaigns: id: fb-campaigns - materializable: false + enabled: false - key: campaign_matcher init: assets: @@ -141,7 +141,7 @@ dag = il.DAG.from_spec(DAGSpec(**payload)) It is what `MultiProcessRunner` ships to its workers and what `interloper run --format inline` accepts. The override map is built from the DAG's **actual** asset instances, so a parent the run -only reads travels as its own document carrying that one asset with `materializable: false`, +only reads travels as its own document carrying that one asset with `enabled: false`, and stays read-only after the round-trip. ## What makes something serializable diff --git a/docs/reference/decorators.md b/docs/reference/decorators.md index d995556c..0bee10cf 100644 --- a/docs/reference/decorators.md +++ b/docs/reference/decorators.md @@ -126,7 +126,7 @@ Calling an instance returns a copy; omitted keywords mean "unchanged". | `id` | | | `dataset` | `dataset` (re-points assets that inherited the old value) | | `default_destination_key` | `default_destination_key` | -| `materializable` | `materializable` (applied to every asset) | +| `enabled` | `enabled` (applied to every asset) | | `materialization_strategy` | `materialization_strategy` | | `normalizer` (`None` clears) | `normalizer` | | any relation name (replaced; `None` clears) | any relation name (replaced; `None` clears, and repoints what was trickled) | diff --git a/docs/reference/events.md b/docs/reference/events.md index 65ee974b..4e3c4471 100644 --- a/docs/reference/events.md +++ b/docs/reference/events.md @@ -22,7 +22,7 @@ and type. Common metadata: `component_id`, `component_kind`, `component_key`, `s | Type | When | |------|------| -| `operation_queued` | At run start, for every materializable operation. | +| `operation_queued` | At run start, for every enabled operation. | | `operation_started` | The operation was submitted. | | `operation_completed` | `execute()` returned. | | `operation_failed` | `execute()` raised. Adds `error` and, when the operation captures tracebacks, `traceback`. | diff --git a/examples/campaign_matcher.yaml b/examples/campaign_matcher.yaml index f1337e6f..9871e4ac 100644 --- a/examples/campaign_matcher.yaml +++ b/examples/campaign_matcher.yaml @@ -31,11 +31,11 @@ init: access_token: ${FACEBOOK_ADS_ACCESS_TOKEN} app_id: ${FACEBOOK_ADS_APP_ID} app_secret: ${FACEBOOK_ADS_APP_SECRET} - # materializable: false keeps this asset read-only in this job: it is + # enabled: false keeps this asset read-only in this job: it is # never run here, only read back so campaign_matches below can fan it # in alongside the other connector's campaigns. assets: - campaigns: {id: fb-campaigns, materializable: false} + campaigns: {id: fb-campaigns, enabled: false} - key: tiktok_ads init: @@ -45,7 +45,7 @@ init: init: access_token: ${TIKTOK_ADS_ACCESS_TOKEN} assets: - campaigns: {id: tt-campaigns, materializable: false} + campaigns: {id: tt-campaigns, enabled: false} - key: campaign_matcher init: diff --git a/examples/source.py b/examples/source.py index eee3c91d..92b6aebb 100644 --- a/examples/source.py +++ b/examples/source.py @@ -62,10 +62,10 @@ def metrics() -> dict[str, float]: print(f"{a.key}: {a.run()}") print("\n=== Reconfigure ===") - disabled = s(materializable=False) + disabled = s(enabled=False) for a in disabled.assets: - print(f"{a.key}: materializable={a.materializable}") + print(f"{a.key}: enabled={a.enabled}") print("\n=== Original unchanged ===") for a in s.assets: - print(f"{a.key}: materializable={a.materializable}") + print(f"{a.key}: enabled={a.enabled}") diff --git a/packages/interloper-agent/src/interloper_agent/tools/scheduling.py b/packages/interloper-agent/src/interloper_agent/tools/scheduling.py index b3a0fae8..6058a1f0 100644 --- a/packages/interloper-agent/src/interloper_agent/tools/scheduling.py +++ b/packages/interloper-agent/src/interloper_agent/tools/scheduling.py @@ -150,21 +150,21 @@ def trigger_backfill( def toggle_asset( asset_id: str, - materializable: bool, + enabled: bool, tool_context: ToolContext | None = None, ) -> dict[str, Any]: """Enable or disable materialization for an asset. Args: asset_id: UUID of the asset. - materializable: True to enable materialization, false to disable. + enabled: True to enable materialization, false to disable. """ try: store = get_store() aid = UUID(asset_id) asset = store.components.get(aid, kind="asset") - updated = store.components.update(aid, config={**(asset.config or {}), "materializable": materializable}) - action = "enabled" if materializable else "disabled" + updated = store.components.update(aid, config={**(asset.config or {}), "enabled": enabled}) + action = "enabled" if enabled else "disabled" return { "status": "success", "message": f"Asset '{updated.key}' materialization {action}", diff --git a/packages/interloper-app/app/app/types/component.ts b/packages/interloper-app/app/app/types/component.ts index d2b7298a..01c0acd6 100644 --- a/packages/interloper-app/app/app/types/component.ts +++ b/packages/interloper-app/app/app/types/component.ts @@ -93,9 +93,9 @@ export interface ComponentInput { // ─── Helpers ───────────────────────────────────────────────────────── -/** Whether an asset materializes (clients own `config.materializable`). */ -export function materializable(c: ComponentRecord): boolean { - return c.config?.materializable ?? true +/** Whether an operation executes (clients own `config.enabled`). */ +export function enabled(c: ComponentRecord): boolean { + return c.config?.enabled ?? true } /** Relation refs under a given name, e.g. `relationRefs(c, 'destinations')`. */ diff --git a/packages/interloper-assets/tests/test_campaign_matcher.py b/packages/interloper-assets/tests/test_campaign_matcher.py index 4c7a0cb1..80efe35e 100644 --- a/packages/interloper-assets/tests/test_campaign_matcher.py +++ b/packages/interloper-assets/tests/test_campaign_matcher.py @@ -172,7 +172,7 @@ def test_leg_without_data_is_skipped_not_fatal() -> None: fb = _connector("fb_like", ["x"])(destinations=[memory]) tt = _connector("tt_like", ["y"])(destinations=[memory]) matcher = CampaignMatcher(destinations=[memory]) - tt.campaigns.materializable = False # nothing written for tt: its leg reads as None + tt.campaigns.enabled = False # nothing written for tt: its leg reads as None partition = il.TimePartition(dt.date(2026, 9, 1)) il.DAG(fb, tt, matcher).materialize(partition) rows = memory.read(il.IOContext(asset=matcher.campaign_matches, partition_or_window=partition)) @@ -187,7 +187,7 @@ def test_matcher_alone_reads_bound_upstreams_read_only() -> None: matcher = CampaignMatcher(destinations=[memory]) matcher.campaign_matches.bind("campaigns", fb.campaigns) dag = il.DAG(matcher) - assert dag.operation_map[fb.campaigns.id].materializable is False + assert dag.operation_map[fb.campaigns.id].enabled is False dag.materialize(partition) assert len(memory.read(il.IOContext(asset=matcher.campaign_matches, partition_or_window=partition))) == 1 diff --git a/packages/interloper-core/src/interloper/asset/base.py b/packages/interloper-core/src/interloper/asset/base.py index 71422d9c..3005cd60 100644 --- a/packages/interloper-core/src/interloper/asset/base.py +++ b/packages/interloper-core/src/interloper/asset/base.py @@ -59,8 +59,6 @@ warnings.filterwarnings("ignore", message='Field name "schema" in "AssetDefinition"') -# Deliberate: Asset refines the Operation node protocol's plain defaults into real fields. -warnings.filterwarnings("ignore", message='Field name "materializable" in "Asset"') class AssetDefinition(ComponentDefinition): @@ -313,7 +311,7 @@ def __call__( self, *, id: str | None = None, - materializable: bool | None = None, + enabled: bool | None = None, dataset: str | None = None, default_destination_key: str | None = None, materialization_strategy: MaterializationStrategy | None = None, @@ -330,12 +328,12 @@ def __call__( with a ``ConfigError``, since it cannot be left empty. The copy carries this asset's own bindings and parent, so a copy made - to flip one field (the non-materializable parents of a mini-DAG, say) + to flip one field (the disabled parents of a mini-DAG, say) still reads from the same destinations and upstreams. Args: id: New component id for the copy. - materializable: Whether the copy writes to destinations at all. + enabled: Whether the copy executes; a disabled copy is read-only. dataset: Dataset (schema/namespace) the asset materializes into. default_destination_key: When the asset has several destinations, the one downstream assets read it from. @@ -358,8 +356,8 @@ def __call__( overrides: dict[str, Any] = {} if id is not None: overrides["id"] = id - if materializable is not None: - overrides["materializable"] = materializable + if enabled is not None: + overrides["enabled"] = enabled if dataset is not None: overrides["dataset"] = dataset if default_destination_key is not None: @@ -510,7 +508,7 @@ def materialize( metadata: Arbitrary metadata dict (e.g. run_id, backfill_id). Returns: - The execution result, or ``None`` if the asset is not materializable. + The execution result, or ``None`` if the asset is disabled. """ return concurrency.run(self.materialize_async(partition_or_window, dag, metadata)) @@ -528,9 +526,9 @@ async def materialize_async( metadata: Arbitrary metadata dict (e.g. run_id, backfill_id). Returns: - The execution result, or ``None`` if the asset is not materializable. + The execution result, or ``None`` if the asset is disabled. """ - if not self.materializable: + if not self.enabled: return None metadata = metadata or {} diff --git a/packages/interloper-core/src/interloper/cli/commands/run.py b/packages/interloper-core/src/interloper/cli/commands/run.py index 9285ff31..f3be9c95 100644 --- a/packages/interloper-core/src/interloper/cli/commands/run.py +++ b/packages/interloper-core/src/interloper/cli/commands/run.py @@ -222,10 +222,10 @@ def _cmd_run(args: argparse.Namespace) -> None: if args.run_id: metadata["run_id"] = args.run_id - materializable = [operation for operation in dag.operations if operation.materializable] + enabled = [operation for operation in dag.operations if operation.enabled] logger.info( - "Running DAG with %d materializable operation(s) (%d total) using %s", - len(materializable), + "Running DAG with %d enabled operation(s) (%d total) using %s", + len(enabled), len(dag.operations), type(runner).__name__, ) @@ -307,7 +307,7 @@ def _print_plan( runner_name: Class name of the configured runner. name: Run name; omitted from the output when empty. """ - materializable = [operation for operation in dag.operations if operation.materializable] + enabled = [operation for operation in dag.operations if operation.enabled] lines: list[str] = [] if name: lines.append(f"Run: {name}") @@ -315,7 +315,7 @@ def _print_plan( [ f"Runner: {runner_name}", f"Partition: {partition if partition is not None else '(none)'}", - f"Operations: {len(materializable)} materializable / {len(dag.operations)} total", + f"Operations: {len(enabled)} enabled / {len(dag.operations)} total", "", ] ) diff --git a/packages/interloper-core/src/interloper/dag/base.py b/packages/interloper-core/src/interloper/dag/base.py index 3b5ff046..286a48bc 100644 --- a/packages/interloper-core/src/interloper/dag/base.py +++ b/packages/interloper-core/src/interloper/dag/base.py @@ -142,7 +142,7 @@ def _build_graph(self, items: tuple[Workload | type[Workload], ...]) -> None: self.successors[operation.id] = [] for operation in self.operations: - if not operation.materializable: + if not operation.enabled: continue self.predecessors[operation.id] = [] @@ -189,7 +189,7 @@ def _resolve_declared(self) -> None: """ assets = [operation for operation in self.operations if isinstance(operation, Asset)] for asset in assets: - if not asset.materializable: + if not asset.enabled: continue for name, relation in asset.upstream_relations().items(): if asset.bound(name) or not relation.keys: @@ -218,17 +218,17 @@ def _include_read_only_upstreams(self) -> None: A materializing node reads its upstreams through the DAG's own node (see :meth:`~interloper.asset.base.Asset._read_upstreams`), so an upstream nobody in the run materializes still has to be one: it joins - as a non-materializable copy, same id, same bindings, same parent, + as a disabled copy, same id, same bindings, same parent, which the runners skip and the dependent reads. Those copies never execute, so their own upstreams are not pulled in with them. """ for operation in list(self.operations): - if not operation.materializable: + if not operation.enabled: continue for upstream in self._upstream_targets(operation): if upstream.id in self.operation_map: continue - read_only = upstream(materializable=False) + read_only = upstream(enabled=False) self.operations.append(read_only) self.operation_map[read_only.id] = read_only @@ -249,7 +249,7 @@ def _check_relations(self) -> None: """ nodes = {id: node for id, node in self.operation_map.items() if isinstance(node, Component)} for operation in self.operations: - if operation.materializable and isinstance(operation, Component): + if operation.enabled and isinstance(operation, Component): operation.validate_relations(nodes) def _check_circular_dependencies(self) -> None: @@ -321,8 +321,8 @@ def topological_generations(self) -> list[list[Operation]]: Lists are ordered so that all dependencies of a level appear in previous levels (Kahn's algorithm). - Only materializable operations appear in the generations. Edges from - non-materializable operations count as already satisfied, mirroring + Only enabled operations appear in the generations. Edges from + disabled operations count as already satisfied, mirroring the runners, which mark those nodes as skipped (e.g. the parents in a :meth:`mini_dag`). @@ -540,7 +540,7 @@ def mini_dag(self, operation_id: str) -> DAG: """Create a mini-DAG with the target operation and its immediate parents. A DAG over the target alone: its bound upstreams join as - non-materializable copies through the very mechanism any run uses for + disabled copies through the very mechanism any run uses for an upstream it does not materialize (:meth:`_include_read_only_upstreams`), so the parents are there, under their own ids, read instead of executed. @@ -559,4 +559,4 @@ def mini_dag(self, operation_id: str) -> DAG: target = self.operation_map[operation_id] # Only an asset has upstreams to pull in, and only an asset can be re-flagged. - return DAG(target(materializable=True) if isinstance(target, Asset) else target) + return DAG(target(enabled=True) if isinstance(target, Asset) else target) diff --git a/packages/interloper-core/src/interloper/operation/base.py b/packages/interloper-core/src/interloper/operation/base.py index 3b156d69..49d2547a 100644 --- a/packages/interloper-core/src/interloper/operation/base.py +++ b/packages/interloper-core/src/interloper/operation/base.py @@ -103,6 +103,12 @@ class Operation(Component, Workload): ``Component.__init_subclass__`` deriving one. Each implementor declares its own. + ``enabled`` is the same word ``Job`` and ``Hook`` already use for "this + will run": a disabled operation stays in the graph, ordering whatever + depends on it, and does not execute. ``Asset`` adds the half only an + asset has, which is that a disabled one is read-only and its downstreams + resolve its data from its destination instead of recomputing it. + ``capture_traceback`` controls whether a failed execution's traceback is attached to its failure event; off for operations whose raw errors embed secrets (credential exchanges carry them in URLs). @@ -112,7 +118,7 @@ class Operation(Component, Workload): capture_traceback: ClassVar[bool] = True partitioning: ClassVar[PartitionConfig | None] = None - materializable: bool = Field(default=True, json_schema_extra={"x-hidden": True}) + enabled: bool = Field(default=True, description="Operation will execute") def operations(self) -> list[Operation]: """An operation is trivially its own workload. diff --git a/packages/interloper-core/src/interloper/runner/base.py b/packages/interloper-core/src/interloper/runner/base.py index f39437d4..7ba53b4a 100644 --- a/packages/interloper-core/src/interloper/runner/base.py +++ b/packages/interloper-core/src/interloper/runner/base.py @@ -277,7 +277,7 @@ def _preflight_validation( partitioned = sorted( operation.key for operation in dag.operations - if operation.materializable and operation.partitioning is not None + if operation.enabled and operation.partitioning is not None ) if partitioned: raise PartitionError( @@ -289,7 +289,7 @@ def _preflight_validation( unsupported = [ operation.key for operation in dag.operations - if operation.materializable + if operation.enabled and operation.partitioning is not None and not operation.partitioning.allow_window ] @@ -300,5 +300,5 @@ def _preflight_validation( ) for operation in dag.operations: - if operation.materializable and isinstance(operation.partitioning, TimePartitionConfig): + if operation.enabled and isinstance(operation.partitioning, TimePartitionConfig): operation._validate_time_partitioning(operation.partitioning, partition_or_window) diff --git a/packages/interloper-core/src/interloper/runner/state.py b/packages/interloper-core/src/interloper/runner/state.py index 3a4601de..6fb31efb 100644 --- a/packages/interloper-core/src/interloper/runner/state.py +++ b/packages/interloper-core/src/interloper/runner/state.py @@ -322,7 +322,7 @@ def mark_failed( def _initialize_operations(self) -> None: """Initialize all operations as QUEUED, then promote root operations to READY.""" for operation in self.dag.operations: - status = ExecutionStatus.SKIPPED if not operation.materializable else ExecutionStatus.QUEUED + status = ExecutionStatus.SKIPPED if not operation.enabled else ExecutionStatus.QUEUED self.executions[operation.id] = ExecutionInfo( component_id=operation.id, component_key=operation.key, diff --git a/packages/interloper-core/src/interloper/source/base.py b/packages/interloper-core/src/interloper/source/base.py index cecc1870..e676362e 100644 --- a/packages/interloper-core/src/interloper/source/base.py +++ b/packages/interloper-core/src/interloper/source/base.py @@ -242,7 +242,7 @@ def validate_relations(self, nodes: Mapping[str, Component] | None = None) -> No """ super().validate_relations(nodes) for asset in self.assets: - if asset.materializable: + if asset.enabled: asset.validate_relations(nodes) def on_rebind(self, name: str) -> None: @@ -262,7 +262,7 @@ def on_rebind(self, name: str) -> None: self.trickle(destination) def _apply_select(self) -> None: - """Mark assets outside ``select`` as non-materializable. + """Mark assets outside ``select`` as disabled. Unselected assets stay in the list so intra-source dependency wiring can still resolve them by key and their outputs stay readable, but @@ -280,7 +280,7 @@ def _apply_select(self) -> None: if unknown: raise SourceError(f"Source '{self.key}' has no asset(s) {unknown}; available: {sorted(known)}") selected = set(self.select or []) - self.assets = [a if a.key in selected else a(materializable=False) for a in self.assets] + self.assets = [a if a.key in selected else a(enabled=False) for a in self.assets] for asset in self.assets: asset.parent = self @@ -371,7 +371,7 @@ def __call__( *, dataset: str | None = None, default_destination_key: str | None = None, - materializable: bool | None = None, + enabled: bool | None = None, normalizer: Normalizer | None = None, materialization_strategy: MaterializationStrategy | None = None, **relations: Any, @@ -390,7 +390,7 @@ def __call__( dataset are re-pointed; per-asset overrides are preserved. default_destination_key: Replacement key of the destination downstream assets read from. - materializable: Applied to every asset of the copy. + enabled: Applied to every asset of the copy. normalizer: Replacement normalizer for the source. materialization_strategy: Replacement default strategy for the source. @@ -428,8 +428,8 @@ def __call__( copy.dataset = dataset if default_destination_key is not None: copy.default_destination_key = default_destination_key - if materializable is not None: - copy.assets = [asset(materializable=materializable) for asset in copy.assets] + if enabled is not None: + copy.assets = [asset(enabled=enabled) for asset in copy.assets] if normalizer is not None: copy.normalizer = normalizer if materialization_strategy is not None: diff --git a/packages/interloper-core/tests/asset/test_base.py b/packages/interloper-core/tests/asset/test_base.py index 8c62e35b..a4af37dc 100644 --- a/packages/interloper-core/tests/asset/test_base.py +++ b/packages/interloper-core/tests/asset/test_base.py @@ -548,7 +548,7 @@ def handler(event: Event) -> None: EventBus.subscribe(handler) try: - result = DAG(fb(materializable=False), asset).materialize(TimePartition(dt.date(2026, 9, 1))) + result = DAG(fb(enabled=False), asset).materialize(TimePartition(dt.date(2026, 9, 1))) EventBus.flush(timeout=5.0) finally: EventBus.unsubscribe(handler) @@ -556,7 +556,7 @@ def handler(event: Event) -> None: assert result.status is ExecutionStatus.COMPLETED # The leg carries the DAG's own node, the read-only copy of the binding. assert seen["leg"].asset.id == fb.campaigns.id - assert seen["leg"].asset.materializable is False + assert seen["leg"].asset.enabled is False assert seen["leg"].data is None assert any("found no data in upstream" in e.metadata.get("message", "") for e in warnings_seen) @@ -834,8 +834,8 @@ def test_override_id(self): def test_override_dataset(self): assert FakeAsset()(dataset="my_ds").dataset == "my_ds" - def test_override_materializable(self): - assert FakeAsset(materializable=True)(materializable=False).materializable is False + def test_override_enabled(self): + assert FakeAsset(enabled=True)(enabled=False).enabled is False def test_dataset_and_strategy_are_overridable(self): from interloper.normalizer import MaterializationStrategy @@ -857,9 +857,9 @@ def test_normalizer_explicit_none_clears_normalizer(self): assert reconfigured.normalizer is None def test_omitted_fields_preserved(self): - asset = FakeAsset(dataset="original", materializable=False) + asset = FakeAsset(dataset="original", enabled=False) reconfigured = asset(dataset="updated") - assert reconfigured.materializable is False + assert reconfigured.enabled is False def test_the_copy_keeps_the_originals_bindings(self): destination = FakeDestination() @@ -867,7 +867,7 @@ def test_the_copy_keeps_the_originals_bindings(self): asset = FakeSourceOwnedAsset(destinations=[destination]) asset.parent = source - reconfigured = asset(materializable=False) + reconfigured = asset(enabled=False) assert reconfigured.destinations == [destination] assert reconfigured.parent is source @@ -895,11 +895,11 @@ def test_an_unknown_keyword_is_rejected(self): class TestSerialization: def test_standalone_asset_roundtrip(self): - asset = FakeAsset(dataset="ds", materializable=False) + asset = FakeAsset(dataset="ds", enabled=False) restored = Component.from_spec(asset.to_spec()) assert isinstance(restored, FakeAsset) assert restored.dataset == "ds" - assert restored.materializable is False + assert restored.enabled is False def test_asset_with_destination_roundtrip(self): asset = FakeAsset(destinations=[FakeDestination()]) @@ -914,13 +914,13 @@ def test_asset_preserves_instance_id(self): def test_source_owned_asset_roundtrip_preserves_subclass(self): source = FakeParentSource() - asset = FakeSourceOwnedAsset(dataset="override", materializable=False) + asset = FakeSourceOwnedAsset(dataset="override", enabled=False) asset.parent = source restored = FakeSourceOwnedAsset.from_spec(asset.to_spec()) assert isinstance(restored, FakeSourceOwnedAsset) assert restored.dataset == "override" - assert restored.materializable is False + assert restored.enabled is False def test_roundtrip_via_json_string(self): asset = FakeAsset(dataset="ds", default_destination_key="memory") @@ -1357,7 +1357,7 @@ def handler(event: Event) -> None: EventBus.subscribe(handler) try: - result = DAG(one(materializable=False), two(materializable=False), matcher).materialize(partition) + result = DAG(one(enabled=False), two(enabled=False), matcher).materialize(partition) EventBus.flush(timeout=5.0) finally: EventBus.unsubscribe(handler) @@ -1380,7 +1380,7 @@ def read(self, context: il.IOContext) -> Any: matcher = self._matcher()(destinations=[mem], campaigns=[one.campaigns]) partition = TimePartition(dt.date(2026, 1, 1)) - result = DAG(one(materializable=False), matcher).materialize(partition) + result = DAG(one(enabled=False), matcher).materialize(partition) assert result.status is ExecutionStatus.FAILED @@ -1403,7 +1403,7 @@ def single(context: il.ExecutionContext, c: il.Upstream) -> Any: return [{"date": context.partition_date, "ids": [row["id"] for row in c.data]}] asset = single(destinations=[mem], c=one.campaigns) - DAG(one(materializable=False), asset).materialize(partition) + DAG(one(enabled=False), asset).materialize(partition) assert mem.read(il.IOContext(asset=asset, partition_or_window=partition)) == [ {"date": dt.date(2026, 1, 1), "ids": ["fb"]} @@ -1423,7 +1423,7 @@ def lenient(context: il.ExecutionContext, c: il.Upstream | None = None) -> Any: asset = lenient(destinations=[mem], c=one.campaigns) partition = TimePartition(dt.date(2030, 5, 5)) # provider never ran for this day - result = DAG(one(materializable=False), asset).materialize(partition) + result = DAG(one(enabled=False), asset).materialize(partition) assert result.status is ExecutionStatus.COMPLETED assert mem.read(il.IOContext(asset=asset, partition_or_window=partition)) == [ @@ -1438,7 +1438,7 @@ def read(self, context: il.IOContext) -> Any: broken = FbLike(destinations=[Broken()]) asset = lenient(destinations=[mem], c=broken.campaigns) - assert DAG(broken(materializable=False), asset).materialize(partition).status is ExecutionStatus.FAILED + assert DAG(broken(enabled=False), asset).materialize(partition).status is ExecutionStatus.FAILED def test_an_upstream_absent_from_the_dag_is_skipped_with_a_warning(self): mem = il.MemoryDestination() @@ -1480,7 +1480,7 @@ def consumer(context: il.ExecutionContext, c: il.Upstream) -> Any: # pragma: no return [] asset = consumer(destinations=[il.MemoryDestination()], c=one.campaigns) - dag = DAG(one(materializable=False), asset) + dag = DAG(one(enabled=False), asset) with pytest.raises(AssetError, match="No destination found for upstream asset 'campaigns'"): await asset.run_async(TimePartition(dt.date(2026, 1, 1)), dag=dag) @@ -1507,7 +1507,7 @@ def consumer(context: il.ExecutionContext, c: il.Upstream) -> Any: # pragma: no return [] asset = consumer(destinations=[il.MemoryDestination()], c=one.campaigns) - dag = DAG(one(materializable=False), asset) + dag = DAG(one(enabled=False), asset) captured: list[Event] = [] EventBus.subscribe(captured.append) try: @@ -1565,7 +1565,7 @@ class TestNonMaterializableAssets: """Read-only hydration of an upstream dependency.""" async def test_materialize_returns_nothing(self): - asset = FakeAsset(destinations=[il.MemoryDestination()], materializable=False) + asset = FakeAsset(destinations=[il.MemoryDestination()], enabled=False) assert await asset.materialize_async() is None diff --git a/packages/interloper-core/tests/cli/commands/test_run.py b/packages/interloper-core/tests/cli/commands/test_run.py index c03d4ff0..bdec8128 100644 --- a/packages/interloper-core/tests/cli/commands/test_run.py +++ b/packages/interloper-core/tests/cli/commands/test_run.py @@ -93,7 +93,7 @@ def test_dry_run_prints_plan(self, tmp_path: Path, capsys: pytest.CaptureFixture out = capsys.readouterr().out assert "2026-06-01" in out - assert "1 materializable / 1 total" in out + assert "1 enabled / 1 total" in out assert "1. fake_run_source.one" in out def test_source_spec_is_runnable_directly(self, tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: @@ -202,7 +202,7 @@ def test_job_spec_dry_run_prints_plan(self, tmp_path: Path, capsys: pytest.Captu _cmd_run(_args(file=str(spec_file), dry_run=True)) out = capsys.readouterr().out - assert "1 materializable / 1 total" in out + assert "1 enabled / 1 total" in out assert "1. fake_run_source.one" in out def test_job_spec_run_materializes(self, tmp_path: Path) -> None: diff --git a/packages/interloper-core/tests/component/test_decorator.py b/packages/interloper-core/tests/component/test_decorator.py index 225d0e56..7e94d985 100644 --- a/packages/interloper-core/tests/component/test_decorator.py +++ b/packages/interloper-core/tests/component/test_decorator.py @@ -53,9 +53,9 @@ def test_an_unknown_name_raises_listing_the_accepted_names(self): assert "'dataset'" in message def test_the_decorator_reports_an_unknown_name(self): - with pytest.raises(TypeError, match=r"does not accept 'materializable'"): + with pytest.raises(TypeError, match=r"does not accept 'enabled'"): - @il.source(materializable=False) + @il.source(enabled=False) def probe(): return [] diff --git a/packages/interloper-core/tests/connection/test_base.py b/packages/interloper-core/tests/connection/test_base.py index e6c1bf54..9ff9c8c6 100644 --- a/packages/interloper-core/tests/connection/test_base.py +++ b/packages/interloper-core/tests/connection/test_base.py @@ -18,7 +18,7 @@ class TestConnection: def test_kind_survives_operation_becoming_a_component(self): assert Connection.kind == "connection" - assert Connection.model_fields["materializable"].json_schema_extra == {"x-hidden": True} + assert Connection.model_fields["enabled"].default is True def test_definition_without_oauth(self): class Plain(Connection): diff --git a/packages/interloper-core/tests/dag/test_base.py b/packages/interloper-core/tests/dag/test_base.py index 6202e8bb..8e9c351c 100644 --- a/packages/interloper-core/tests/dag/test_base.py +++ b/packages/interloper-core/tests/dag/test_base.py @@ -517,10 +517,10 @@ def test_complex_dag_successors_match_topology(self, dag: il.DAG): assert dag.successors[by_key["f"].id] == [by_key["g"].id] assert dag.successors[by_key["g"].id] == [] - def test_non_materializable_asset_skipped_from_predecessors(self): - # Non-materializable assets are parents, not roots, so their predecessors + def test_non_enabled_asset_skipped_from_predecessors(self): + # Non-enabled assets are parents, not roots, so their predecessors # are not computed (they don't need to execute). - upstream = FakeAsset(materializable=False) + upstream = FakeAsset(enabled=False) downstream = FakeOtherAsset(upstream=upstream) dag = DAG(upstream, downstream) assert upstream.id not in dag.predecessors @@ -530,14 +530,14 @@ def test_a_bound_upstream_outside_the_dag_joins_read_only(self): upstream = FakeAsset() downstream = FakeAssetRequiringFake(upstream=upstream) dag = DAG(downstream) - assert dag.operation_map[upstream.id].materializable is False + assert dag.operation_map[upstream.id].enabled is False assert dag.predecessors[downstream.id] == [upstream.id] def test_an_optional_bound_upstream_outside_the_dag_joins_read_only_too(self): upstream = FakeAsset() asset = FakeAssetOptionallyRequiringFake(upstream=upstream) dag = DAG(asset) - assert dag.operation_map[upstream.id].materializable is False + assert dag.operation_map[upstream.id].enabled is False assert dag.predecessors[asset.id] == [upstream.id] @@ -572,7 +572,7 @@ def test_partitioned_depending_on_non_partitioned_is_allowed(self, dag_mixed: il class TestGranularityAcrossEdges: def test_mixed_granularity_raises_even_for_read_only_upstreams(self): - daily = FakeDaily(materializable=False) + daily = FakeDaily(enabled=False) monthly = FakeMonthly(daily=daily) with pytest.raises(DAGError, match="partitioned by day"): DAG(daily, monthly) @@ -601,7 +601,7 @@ def test_unbound_slot_fails_at_build(self): DAG(finance) def test_unbound_slot_is_ignored_on_read_only_nodes(self): - finance = FakeFinance()(materializable=False) + finance = FakeFinance()(enabled=False) DAG(finance) # no raise def test_many_slot_binds_every_match(self): @@ -639,7 +639,7 @@ def test_bound_upstream_outside_workloads_joins_read_only(self): dag = DAG(matcher) node = dag.operation_map[fb.campaigns.id] - assert node.materializable is False + assert node.enabled is False assert node is not fb.campaigns assert node.parent is fb assert dag.get_predecessors(matcher.campaign_matches.id) == [fb.campaigns.id] @@ -720,12 +720,12 @@ def test_topological_generations_double_source_dag(self, double_source_dag: il.D assert len(levels) == 1 assert len(levels[0]) == 4 - def test_topological_generations_skips_non_materializable_parent(self): + def test_topological_generations_skips_non_enabled_parent(self): # Mini-DAG shape: a skipped parent upstream of a live target. The # parent's edge counts as satisfied, and only the target appears. a = FakeAsset() b = FakeOtherAsset(upstream=a) - dag = DAG(a(materializable=False), b) + dag = DAG(a(enabled=False), b) levels = dag.topological_generations() assert [[asset.id for asset in level] for level in levels] == [[b.id]] @@ -773,7 +773,7 @@ def test_mini_dag_contains_target_and_parents(self): keys = {type(asset).key for asset in mini.operations} assert keys == {"fake_asset", "fake_other_asset"} - def test_mini_dag_marks_parents_non_materializable(self): + def test_mini_dag_marks_parents_non_enabled(self): a = FakeAsset() b = FakeOtherAsset(upstream=a) dag = DAG(a, b) @@ -781,15 +781,15 @@ def test_mini_dag_marks_parents_non_materializable(self): target = next(asset for asset in mini.operations if type(asset).key == "fake_other_asset") parent = next(asset for asset in mini.operations if type(asset).key == "fake_asset") - assert target.materializable is True - assert parent.materializable is False + assert target.enabled is True + assert parent.enabled is False def test_mini_dag_for_root_asset_has_no_parents(self): a = FakeAsset() dag = DAG(a) mini = dag.mini_dag(a.id) assert len(mini.operations) == 1 - assert mini.operations[0].materializable is True + assert mini.operations[0].enabled is True def test_mini_dag_raises_for_unknown_id(self): dag = DAG(FakeAsset()) @@ -804,10 +804,10 @@ def test_mini_dag_for_deep_target_in_complex_dag(self, dag: il.DAG): assert {type(asset).key for asset in mini.operations} == {"b", "c", "e"} target = next(asset for asset in mini.operations if type(asset).key == "e") - assert target.materializable is True + assert target.enabled is True for asset in mini.operations: if type(asset).key != "e": - assert asset.materializable is False + assert asset.enabled is False # -- Serialization round-trip -------------------------------------------------- @@ -891,24 +891,24 @@ def test_roundtrip_preserves_mini_dag_shape(self): restored = DAG.from_spec(mini.to_spec()) assert len(restored.operations) == 2 - restored_flags = [asset.materializable for asset in restored.operations] - assert restored_flags == [asset.materializable for asset in mini.operations] + restored_flags = [asset.enabled for asset in restored.operations] + assert restored_flags == [asset.enabled for asset in mini.operations] - def test_roundtrip_source_mini_dag_preserves_materializable(self): - """Mini-DAG from a source: only the target asset is materializable.""" + def test_roundtrip_source_mini_dag_preserves_enabled(self): + """Mini-DAG from a source: only the target asset is enabled.""" source = FakeSource() dag = DAG(source) by_key = {type(a).key: a for a in dag.operations} # fake_second depends on fake_first; mini_dag for fake_second - # should mark fake_first as non-materializable. + # should mark fake_first as disabled. mini = dag.mini_dag(by_key["fake_second"].id) restored = DAG.from_spec(mini.to_spec()) restored_by_key = {type(a).key: a for a in restored.operations} assert len(restored.operations) == 2 - assert restored_by_key["fake_second"].materializable is True - assert restored_by_key["fake_first"].materializable is False + assert restored_by_key["fake_second"].enabled is True + assert restored_by_key["fake_first"].enabled is False def test_roundtrip_source_mini_dag_excludes_unrelated_assets(self): """Mini-DAG from a source should not include assets that aren't in the subgraph.""" @@ -924,21 +924,21 @@ def test_roundtrip_source_mini_dag_excludes_unrelated_assets(self): assert restored_keys == {"a", "c"} for asset in restored.operations: if type(asset).key == "c": - assert asset.materializable is True + assert asset.enabled is True else: - assert asset.materializable is False + assert asset.enabled is False - def test_roundtrip_source_mini_dag_only_one_materializable(self, dag: il.DAG): - """Every mini-DAG round-trip should have exactly one materializable asset.""" + def test_roundtrip_source_mini_dag_only_one_enabled(self, dag: il.DAG): + """Every mini-DAG round-trip should have exactly one enabled asset.""" for asset in dag.operations: mini = dag.mini_dag(asset.id) restored = DAG.from_spec(mini.to_spec()) - materializable = [a for a in restored.operations if a.materializable] - assert len(materializable) == 1, ( - f"mini_dag for '{type(asset).key}' has {len(materializable)} " - f"materializable assets after round-trip, expected 1" + enabled = [a for a in restored.operations if a.enabled] + assert len(enabled) == 1, ( + f"mini_dag for '{type(asset).key}' has {len(enabled)} " + f"enabled assets after round-trip, expected 1" ) - assert type(materializable[0]).key == type(asset).key + assert type(enabled[0]).key == type(asset).key class TestDAGSpec: @@ -954,9 +954,9 @@ def test_one_item_per_root_and_one_per_read_only_parent(self): fb_item = next(item for item in spec.items if item.id == fb.id) assert fb_item.init is not None assert set(fb_item.init["assets"]) == {"campaigns"} - assert fb_item.init["assets"]["campaigns"]["materializable"] is False + assert fb_item.init["assets"]["campaigns"]["enabled"] is False - def test_round_trip_preserves_operation_map_and_materializable_flags(self): + def test_round_trip_preserves_operation_map_and_enabled_flags(self): fb = FbLike() matcher = Matcher() matcher.campaign_matches.bind("campaigns", fb.campaigns) @@ -966,9 +966,9 @@ def test_round_trip_preserves_operation_map_and_materializable_flags(self): assert set(rebuilt.operation_map) == set(dag.operation_map) for operation_id, operation in dag.operation_map.items(): - assert rebuilt.operation_map[operation_id].materializable == operation.materializable - assert rebuilt.operation_map[matcher.campaign_matches.id].materializable is True - assert rebuilt.operation_map[fb.campaigns.id].materializable is False + assert rebuilt.operation_map[operation_id].enabled == operation.enabled + assert rebuilt.operation_map[matcher.campaign_matches.id].enabled is True + assert rebuilt.operation_map[fb.campaigns.id].enabled is False # FbLike's own other assets, if it had any, would come along under the # same read-only source item; the map staying the same size rules that out. other_fb_assets = {asset.id for asset in fb.assets if asset.id != fb.campaigns.id} diff --git a/packages/interloper-core/tests/operation/test_base.py b/packages/interloper-core/tests/operation/test_base.py index 08e39449..79d92b6a 100644 --- a/packages/interloper-core/tests/operation/test_base.py +++ b/packages/interloper-core/tests/operation/test_base.py @@ -53,13 +53,13 @@ def test_operation_declares_no_kind_of_its_own(self): assert Operation.kind == "" assert "destination" in il.KINDS - def test_materializable_is_a_field(self): - assert _NoopOperation().materializable is True - assert _NoopOperation(materializable=False).materializable is False + def test_enabled_is_a_field(self): + assert _NoopOperation().enabled is True + assert _NoopOperation(enabled=False).enabled is False def test_node_protocol_defaults(self): operation = _NoopOperation() - assert operation.materializable is True + assert operation.enabled is True assert operation.partitioning is None assert operation.effective_partition(None) is None assert type(operation).capture_traceback is True diff --git a/packages/interloper-core/tests/runner/test_base.py b/packages/interloper-core/tests/runner/test_base.py index fe9c80aa..f6513a33 100644 --- a/packages/interloper-core/tests/runner/test_base.py +++ b/packages/interloper-core/tests/runner/test_base.py @@ -241,15 +241,15 @@ def windowed(context: il.ExecutionContext) -> list[dict[str, Any]]: assert result.status is ExecutionStatus.COMPLETED - def test_non_materializable_partitioned_assets_are_exempt(self): - # Upstream dependencies are hydrated read-only (materializable=False); + def test_non_enabled_partitioned_assets_are_exempt(self): + # Upstream dependencies are hydrated read-only (enabled=False); # they must not force a partition onto an otherwise unpartitioned run. @il.asset(partitioning=TimePartitionConfig(column="date")) def upstream() -> list[dict[str, Any]]: return [{"date": "2026-01-01"}] asset = upstream(id="upstream", destinations=[il.MemoryDestination()]) - asset.materializable = False + asset.enabled = False AsyncRunner()._preflight_validation(il.DAG(asset), None) diff --git a/packages/interloper-core/tests/runner/test_state.py b/packages/interloper-core/tests/runner/test_state.py index caca7323..bf335bd8 100644 --- a/packages/interloper-core/tests/runner/test_state.py +++ b/packages/interloper-core/tests/runner/test_state.py @@ -111,7 +111,7 @@ def test_roots_are_ready_and_dependents_queued(self, chain: tuple[RunState, dict assert state.executions[operations["middle"].id].status is ExecutionStatus.QUEUED assert state.executions[operations["leaf"].id].status is ExecutionStatus.QUEUED - def test_non_materializable_operations_are_skipped_from_the_start(self) -> None: + def test_non_enabled_operations_are_skipped_from_the_start(self) -> None: dag = il.DAG(ChainSource(destinations=[il.MemoryDestination()])) state = RunState(dag) skipped = [ @@ -120,17 +120,17 @@ def test_non_materializable_operations_are_skipped_from_the_start(self) -> None: if state.executions[operation.id].status is ExecutionStatus.SKIPPED ] - assert all(not operation.materializable for operation in skipped) + assert all(not operation.enabled for operation in skipped) def test_a_dependent_of_only_skipped_operations_is_promoted(self, monkeypatch: Any) -> None: - # A DAG whose upstream is not materializable (e.g. a destination-only + # A DAG whose upstream is disabled (e.g. a destination-only # node) must not leave its dependent queued forever. dag = il.DAG(ChainSource(destinations=[il.MemoryDestination()])) root = next(operation for operation in dag.operations if operation.key == "root") - # `materializable` is a field on `Operation`, so there is no class attribute to replace: + # `enabled` is a field on `Operation`, so there is no class attribute to replace: # the property is installed, not overridden. monkeypatch.setattr( - type(root), "materializable", property(lambda self: self.key != "root"), raising=False + type(root), "enabled", property(lambda self: self.key != "root"), raising=False ) state = RunState(dag) diff --git a/packages/interloper-core/tests/serializable/test_base.py b/packages/interloper-core/tests/serializable/test_base.py index 0d74e4e6..9c055459 100644 --- a/packages/interloper-core/tests/serializable/test_base.py +++ b/packages/interloper-core/tests/serializable/test_base.py @@ -250,7 +250,7 @@ def test_an_owned_target_is_referenced_when_carried_and_dropped_when_a_closed_co def test_a_closed_context_writes_its_own_copies_of_the_owned_components(self): original = OwningSource().assets[0] - copy = original(materializable=False) + copy = original(enabled=False) assert SerializationContext().carried([original]) == [original] assert SerializationContext([copy]).carried([original]) == [copy] diff --git a/packages/interloper-core/tests/source/test_base.py b/packages/interloper-core/tests/source/test_base.py index 5feccbf1..073ac4c7 100644 --- a/packages/interloper-core/tests/source/test_base.py +++ b/packages/interloper-core/tests/source/test_base.py @@ -556,10 +556,10 @@ def test_unknown_relation_name_is_rejected(self): with pytest.raises(TypeError, match="declares no relation"): FakeSourceWithAssets()(watches=[FakeDestination()]) - def test_materializable_override_propagates_to_assets(self): + def test_enabled_override_propagates_to_assets(self): source = FakeSourceWithAssets() - reconfigured = source(materializable=False) - assert all(a.materializable is False for a in reconfigured.assets) + reconfigured = source(enabled=False) + assert all(a.enabled is False for a in reconfigured.assets) def test_copied_assets_have_source_backref_on_copy(self): source = FakeSourceWithAssets() @@ -633,14 +633,14 @@ def test_source_with_assets_roundtrip_preserves_asset_count(self): def test_source_roundtrip_preserves_per_asset_mutation(self): source = FakeSourceWithAssets() - source.assets[0].materializable = False + source.assets[0].enabled = False source.assets[0].dataset = "custom" restored = FakeSourceWithAssets.from_spec(source.to_spec()) - assert restored.assets[0].materializable is False + assert restored.assets[0].enabled is False assert restored.assets[0].dataset == "custom" # Other asset unchanged - assert restored.assets[1].materializable is True + assert restored.assets[1].enabled is True def test_source_with_destination_roundtrip(self): source = FakeSource(destinations=[FakeDestination()]) @@ -660,14 +660,14 @@ def test_source_preserves_instance_id(self): def test_roundtrip_via_json_string(self): source = FakeSourceWithAssets(dataset="ds") - source.assets[0].materializable = False + source.assets[0].enabled = False spec_json = source.to_spec().model_dump_json() restored = Spec.model_validate_json(spec_json).reconstruct() assert isinstance(restored, FakeSourceWithAssets) assert restored.dataset == "ds" - assert restored.assets[0].materializable is False + assert restored.assets[0].enabled is False class TestSpecRule: @@ -733,12 +733,12 @@ def test_resolve_supplies_a_reference_inside_an_asset_override(self): class TestSelect: """Init-time asset selection via the ``select`` field.""" - def test_unselected_assets_stay_as_non_materializable_deps(self): + def test_unselected_assets_stay_as_non_enabled_deps(self): source = FakeSourceWithAssets(select=["fake_second"]) by_key = {type(a).key: a for a in source.assets} - assert by_key["fake_second"].materializable - assert not by_key["fake_first"].materializable - # The non-materializable sibling stays wired as an upstream. + assert by_key["fake_second"].enabled + assert not by_key["fake_first"].enabled + # The disabled sibling stays wired as an upstream. assert by_key["fake_second"].bound("fake_first") is by_key["fake_first"] def test_selected_assets_keep_their_source(self): diff --git a/packages/interloper-core/tests/source/test_decorator.py b/packages/interloper-core/tests/source/test_decorator.py index c202318a..1c8bedea 100644 --- a/packages/interloper-core/tests/source/test_decorator.py +++ b/packages/interloper-core/tests/source/test_decorator.py @@ -89,12 +89,12 @@ class Probe(il.Source): class TestMaterializable: - """``materializable`` is an asset-level runtime flag, not a source declaration.""" + """``enabled`` is an asset-level runtime flag, not a source declaration.""" def test_not_accepted_by_the_decorator(self): - with pytest.raises(TypeError, match=r"Source does not accept 'materializable'"): + with pytest.raises(TypeError, match=r"Source does not accept 'enabled'"): - @il.source(materializable=False) + @il.source(enabled=False) def probe(): return [] @@ -107,7 +107,7 @@ def probe_asset() -> list[dict[str, Any]]: def probe_source(): return [probe_asset] - assert all(not a.materializable for a in probe_source()(materializable=False).assets) + assert all(not a.enabled for a in probe_source()(enabled=False).assets) class TestFunctionForm: diff --git a/packages/interloper-db/tests/store/test_hydration.py b/packages/interloper-db/tests/store/test_hydration.py index cee63a49..4f53cd91 100644 --- a/packages/interloper-db/tests/store/test_hydration.py +++ b/packages/interloper-db/tests/store/test_hydration.py @@ -299,12 +299,12 @@ def test_load_hydrates_with_stable_ids_and_deps(self, store: Store): def test_source_owned_asset_loads_through_its_parent(self, store: Store): db_source = store.components.create(_ORG, kind="source", key="demo_source", name="Demo") child = _child(db_source, "a") - store.components.update(child.id, config={"materializable": False}) + store.components.update(child.id, config={"enabled": False}) asset = store.components.load(child.id) assert isinstance(asset, il.Asset) assert asset.key == "a" - assert asset.materializable is False + assert asset.enabled is False def test_children_selection_drops_rows_and_relations(self, store: Store): db_source = store.components.create(_ORG, kind="source", key="demo_source", name="Demo") @@ -399,12 +399,12 @@ class TestStandaloneAsset: """Standalone assets hydrate directly through the generic builder.""" def test_create_and_load(self, store: Store): - db_asset = store.components.create(_ORG, kind="asset", key="demo_asset", config={"materializable": False}) + db_asset = store.components.create(_ORG, kind="asset", key="demo_asset", config={"enabled": False}) asset = store.components.load(db_asset.id) assert isinstance(asset, il.Asset) assert asset.key == "demo_asset" assert asset.id == str(db_asset.id) - assert asset.materializable is False + assert asset.enabled is False class TestJobRoundTrip: diff --git a/packages/interloper-docker/src/interloper_docker/runner.py b/packages/interloper-docker/src/interloper_docker/runner.py index 2d790638..972e141c 100644 --- a/packages/interloper-docker/src/interloper_docker/runner.py +++ b/packages/interloper-docker/src/interloper_docker/runner.py @@ -8,7 +8,7 @@ **Real-time events** are streamed via **stderr** using the ``@EVENT:`` prefix (see :class:`~interloper.events.StderrEventHandler`). Events for the target asset are forwarded to the host EventBus; events for -non-materializable parent assets and container-internal ``RUN_*`` events +disabled parent assets and container-internal ``RUN_*`` events are dropped. The host updates internal state with ``emit=False`` to avoid duplicate events. """ @@ -52,7 +52,7 @@ class DockerRunner(SyncRunner): For each asset, constructs a mini-DAG comprising the asset and all its upstream ancestors. The mini-DAG is sent to the container via inline JSON. Inside the container, all non-target assets are marked as - ``materializable=False`` to avoid recomputation while still enabling + ``enabled=False`` to avoid recomputation while still enabling IO-based dependency resolution. Events are emitted by the container process and streamed to the host @@ -349,7 +349,7 @@ def _start_log_streaming(self, container: Container, *, target_component_id: str """Stream events from the container's stderr to the host EventBus. Only events belonging to the **target asset** are forwarded. - Events for non-materializable parent assets in the mini-DAG and + Events for disabled parent assets in the mini-DAG and container-internal ``RUN_*`` events are dropped. Args: diff --git a/packages/interloper-k8s/src/interloper_k8s/runner.py b/packages/interloper-k8s/src/interloper_k8s/runner.py index 79eda11d..e2a876fd 100644 --- a/packages/interloper-k8s/src/interloper_k8s/runner.py +++ b/packages/interloper-k8s/src/interloper_k8s/runner.py @@ -41,7 +41,7 @@ class KubernetesRunner(SyncRunner): For each asset, constructs a mini-DAG comprising the asset and all its upstream ancestors. The mini-DAG is sent to the container via inline JSON. Inside the container, all non-target assets are marked as - ``materializable=False`` to avoid recomputation while still enabling + ``enabled=False`` to avoid recomputation while still enabling IO-based dependency resolution. The blocking Job-polling walk is offloaded to a thread, so the runner @@ -426,7 +426,7 @@ def _start_log_streaming(self, job_name: str, *, target_component_id: str) -> No Only events for the **target asset** are forwarded. Container-internal ``RUN_*`` events and events for - non-materializable parent assets are dropped. + disabled parent assets are dropped. Args: job_name: The K8s Job name to stream logs from. diff --git a/packages/interloper-scheduler/src/interloper_scheduler/executor.py b/packages/interloper-scheduler/src/interloper_scheduler/executor.py index 61970e8a..c4b782e2 100644 --- a/packages/interloper-scheduler/src/interloper_scheduler/executor.py +++ b/packages/interloper-scheduler/src/interloper_scheduler/executor.py @@ -122,7 +122,7 @@ def execute(self, run_id: UUID) -> bool: successes = self._prior_successes(retry_of) for operation in operations: if UUID(operation.id) in successes: - operation.materializable = False + operation.enabled = False dag = il.DAG(*operations) partition = il.TimePartition.from_key(partition_key) if partition_key else None diff --git a/packages/interloper-scheduler/tests/test_executor.py b/packages/interloper-scheduler/tests/test_executor.py index 61f4deca..f4b1e64f 100644 --- a/packages/interloper-scheduler/tests/test_executor.py +++ b/packages/interloper-scheduler/tests/test_executor.py @@ -450,7 +450,7 @@ class TestUpstreamJoinsReadOnly: ``_run_dag`` to inspect what it built. """ - def test_a_bound_upstream_is_joined_and_made_non_materializable( + def test_a_bound_upstream_is_joined_and_made_non_enabled( self, monkeypatch: pytest.MonkeyPatch ) -> None: il.MemoryDestination.clear() @@ -477,14 +477,14 @@ def _capturing_run_dag(self: RunExecutor, dag: il.DAG, *args: Any, **kwargs: Any assert executor.execute(run.id) is True (dag,) = built - assert dag.operation_map[upstream.id].materializable is False + assert dag.operation_map[upstream.id].enabled is False assert dag.predecessors[target.id] == [upstream.id] class TestRetrySkipsPriorSuccesses: """A failed-only retry reads earlier successes instead of recomputing them.""" - def test_a_previously_successful_node_is_made_non_materializable( + def test_a_previously_successful_node_is_made_non_enabled( self, monkeypatch: pytest.MonkeyPatch ) -> None: il.MemoryDestination.clear() @@ -510,7 +510,7 @@ def solo() -> list[dict[str, Any]]: monkeypatch.setattr(executor, "_prior_successes", lambda _retry_of: {component_id}) assert executor.execute(run.id) is True - assert target.materializable is False + assert target.enabled is False def test_a_whole_run_retry_recomputes_everything(self, monkeypatch: pytest.MonkeyPatch) -> None: il.MemoryDestination.clear() @@ -536,4 +536,4 @@ def solo() -> list[dict[str, Any]]: ) assert executor.execute(run.id) is True - assert target.materializable is True + assert target.enabled is True diff --git a/plugins/interloper/skills/interloper-manifest/SKILL.md b/plugins/interloper/skills/interloper-manifest/SKILL.md index 4f762c6e..085b5213 100644 --- a/plugins/interloper/skills/interloper-manifest/SKILL.md +++ b/plugins/interloper/skills/interloper-manifest/SKILL.md @@ -57,7 +57,7 @@ https://docs.interloper.dev/guide/jobs/ ``` Read the dry-run plan: each numbered line is one generation, parallel operations share a - line. A downstream asset on the same line as its upstream, or a `materializable / total` + line. A downstream asset on the same line as its upstream, or a `enabled / total` count that does not add up, means the edge is not wired. A failed run still leaves the completed upstream files under `./data`. @@ -67,9 +67,9 @@ https://docs.interloper.dev/guide/jobs/ ## Rules that are not obvious - **`assets:` is a whitelist.** Reconstruction builds only the assets listed in the map; an - asset left out does not exist in the run, so `assets: {order_stats: {materializable: false}}` + asset left out does not exist in the run, so `assets: {order_stats: {enabled: false}}` removes `orders`, not `order_stats`. Use `select:` to restrict what runs. Use `assets:` only - for per-asset overrides (`id`, `materializable`, `destinations`, a bound upstream) and list + for per-asset overrides (`id`, `enabled`, `destinations`, a bound upstream) and list every asset the run needs. - **One reference rule, no flags.** A component that has a parent (an asset, always under its source) is `{ref: id}` wherever a relation points at it. A component without one (a @@ -115,7 +115,7 @@ https://docs.interloper.dev/guide/jobs/ ## Common mistakes -- `materializable: false` to drop an asset: the map is a whitelist and the other assets vanish. +- `enabled: false` to drop an asset: the map is a whitelist and the other assets vanish. - `destinations: [{ref: out}]` on every target: the job's destinations already cascade; write a target's own only to override them. - Reading `ConfigError: ... is unbound and non-optional` as a bug: it means the job lacks the