feat: additive retries/assertions + schemaVersion MAJOR version-gate

RELEASE_NOTES.md

@@ -1,3 +1,24 @@
+# Release Notes: FlowRunner CLI (Unreleased)
+
+## ⚠️ Cross-app schema gate — severity: HIGH — unknown MAJOR `schemaVersion` is now rejected loudly
+
+**What changed.** The shared `.flow.json` format carries an OPTIONAL top-level `schemaVersion` string `"MAJOR.MINOR"` (absence ⇒ `"1.0"`, HAR `log.version` precedent). The CLI's `FlowMap` parser now **version-gates** on it:
+
+- **Absent / `"1.0"` / any `"1.x"`** ⇒ accepted and run unchanged. An unknown **MINOR** (e.g. `"1.5"`) is **tolerated with a warning**; any unrecognized construct still degrades gracefully (skip-with-warning) exactly as before.
+- **Unknown MAJOR (`>= 2`, e.g. `"2.0"`)** ⇒ **rejected loudly** with a `ValidationError` attributable to `schemaVersion` (naming the offending version), instead of best-effort mis-executing a genuinely newer format against live customer traffic.
+- A non-string value (e.g. integer `2`) is **coerced-and-warned** (`2` ⇒ `2.0`), then gated on its MAJOR like any other value — never a silent crash.
+
+This converts *silent wrong-execution* — the single most damaging failure for a "what you see is what actually ran" demo tool — into a principled, auditable refusal. It is additive and backward-compatible: a golden conformance test (`tests/unit/test_golden_old_flow.py`) proves a real pre-sprint flow parses to an **identical** execution model with no `schemaVersion`, with `"1.0"`, and with an unknown MINOR. See the FlowRunner UI repo's `docs/schema-versioning.md`.
+
+## ✨ Additive request-step fields honored: `retries` and `assertions`
+
+- **`step.retries = {count, delayMs}`** (severity: LOW — additive, opt-in). Per-request retry policy mirroring the FlowRunner UI JS engine: an outer retry loop re-issues the whole request on a non-2xx status **or** a network/fetch error, sleeping `delayMs` between attempts and issuing a fresh request each pass. `count` defaults to `0` ⇒ a single attempt, **IDENTICAL** to prior behavior. A user-requested stop is **never** retried. This wraps — and is orthogonal to — the built-in connection/5xx resilience loop.
+- **`step.assertions[]`** (severity: LOW — additive, diagnostic-only). Declarative assertions evaluated against the response after each request, **reusing the frozen `conditionData` operator vocabulary** (same operators, same coercion, same missing-target handling). Results are recorded into the execution context under `response_<id>_assertions` (a per-assertion `{name, variable, operator, value, passed}` list) and `response_<id>_assertions_passed` (aggregate boolean). Assertions are diagnostic: they **never** change flow control and **never** crash. Unknown operators / missing targets degrade to a FAILED assertion with a warning.
+
+Both fields are ignored by older CLIs (`extra='ignore'`), so files that use them still run everywhere. This is part of the cross-app FlowMap additive-evolution strategy (see the FlowRunner UI repo's `docs/flowmap-evolution.md`).
+
+---
+
 # Release Notes: FlowRunner CLI v1.2.0
 
 ## Highlights

tests/fixtures/golden_old_flow.json

@@ -0,0 +1,94 @@
+{
+    "id": "golden-old-flow-1",
+    "name": "Golden Old Flow (pre-sprint)",
+    "description": "A real pre-sprint flow used as a cross-app conformance anchor. It MUST parse to an identical execution model with no schemaVersion, with \"1.0\", and with an unknown MINOR such as \"1.5\". Do not add schemaVersion to this base file.",
+    "headers": {
+        "Accept": "application/json",
+        "X-Client": "flowrunner-cli"
+    },
+    "staticVars": {
+        "basePath": "/api/v1",
+        "maxItems": 5,
+        "enabled": true
+    },
+    "steps": [
+        {
+            "id": "login",
+            "name": "Authenticate",
+            "type": "request",
+            "method": "POST",
+            "url": "{{basePath}}/login",
+            "headers": {
+                "Content-Type": "application/json"
+            },
+            "body": {
+                "user": "##VAR:string:username##",
+                "pass": "##VAR:string:password##"
+            },
+            "extract": {
+                "token": "body.data.sessionToken",
+                "loginStatus": ".status"
+            },
+            "onFailure": "stop"
+        },
+        {
+            "id": "check-token",
+            "name": "Token present?",
+            "type": "condition",
+            "conditionData": {
+                "variable": "token",
+                "operator": "exists",
+                "value": ""
+            },
+            "then": [
+                {
+                    "id": "list-items",
+                    "name": "List items",
+                    "type": "request",
+                    "method": "GET",
+                    "url": "{{basePath}}/items?limit={{maxItems}}",
+                    "headers": {
+                        "Authorization": "Bearer {{token}}"
+                    },
+                    "extract": {
+                        "items": "body.data.items",
+                        "firstId": "body.data.items[0].id"
+                    },
+                    "onFailure": "continue"
+                },
+                {
+                    "id": "loop-items",
+                    "name": "Iterate items",
+                    "type": "loop",
+                    "source": "{{items}}",
+                    "loopVariable": "item",
+                    "steps": [
+                        {
+                            "id": "fetch-item",
+                            "name": "Fetch item detail",
+                            "type": "request",
+                            "method": "GET",
+                            "url": "{{basePath}}/items/{{item.id}}",
+                            "onFailure": "continue"
+                        }
+                    ]
+                }
+            ],
+            "else": [
+                {
+                    "id": "no-token-transform",
+                    "name": "Record failure marker",
+                    "type": "transform",
+                    "ops": [
+                        {
+                            "op": "to_string",
+                            "set": "authFailed",
+                            "args": ["{{loginStatus}}"],
+                            "options": {}
+                        }
+                    ]
+                }
+            ]
+        }
+    ]
+}

tests/unit/conftest.py

@@ -0,0 +1,19 @@
+"""Shared test bootstrapping for the unit suite.
+
+``flow_runner`` imports ``psutil`` at module load time even though it is not
+used in the exercised code paths. The historical test module stubbed it inline;
+hoisting the stub into a conftest lets every unit test module import
+``flow_runner`` without repeating the shim (and without requiring psutil to be
+installed in the test environment).
+"""
+
+import os
+import sys
+import types
+
+sys.modules.setdefault("psutil", types.ModuleType("psutil"))
+
+# Ensure the repo root is importable regardless of pytest's rootdir/invocation.
+_REPO_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
+if _REPO_ROOT not in sys.path:
+    sys.path.insert(0, _REPO_ROOT)

tests/unit/test_assertions.py

@@ -0,0 +1,186 @@
+"""Tests for additive declarative ``step.assertions`` on request steps.
+
+Assertions reuse the frozen ``conditionData`` operator vocabulary and are
+evaluated against the request result (status/headers/body/extracted vars) after
+the request completes. Pass/fail is recorded into the execution context; unknown
+operators or missing targets degrade to a FAILED assertion with a warning and
+never crash the run.
+"""
+
+import logging
+from typing import Any, Dict
+from unittest.mock import AsyncMock, MagicMock
+
+import pytest
+
+from flow_runner import (
+    Assertion,
+    ContainerConfig,
+    FlowMap,
+    FlowRunner,
+    Metrics,
+    RequestStep,
+    get_value_from_context,
+)
+
+
+@pytest.fixture
+def empty_flow() -> FlowMap:
+    return FlowMap(name="test", steps=[], staticVars={})
+
+
+def make_runner(config: ContainerConfig, flow: FlowMap) -> FlowRunner:
+    metrics = Metrics()
+    metrics.increment = AsyncMock()
+    metrics.record_flow_duration = AsyncMock()
+    runner = FlowRunner(config, flow, metrics)
+    runner.metrics = metrics
+    runner.running = True
+    return runner
+
+
+def _resp(status: int, body):
+    r = AsyncMock()
+    r.status = status
+    r.headers = {"Content-Type": "application/json"}
+    r.json = AsyncMock(return_value=body)
+    r.text = AsyncMock(return_value="{}")
+    r.read = AsyncMock(return_value=b"{}")
+    return r
+
+
+def _session(resp):
+    session = MagicMock()
+    cm = AsyncMock()
+    cm.__aenter__.return_value = resp
+    cm.__aexit__.return_value = AsyncMock()
+    session.request.return_value = cm
+    return session
+
+
+# --- model parsing ----------------------------------------------------------
+
+def test_request_step_assertions_parsed():
+    step = RequestStep.model_validate({
+        "id": "s1", "type": "request", "method": "GET", "url": "/a",
+        "onFailure": "continue",
+        "assertions": [
+            {"name": "ok", "variable": "response_s1_status", "operator": "equals", "value": "200"},
+        ],
+    })
+    assert step.assertions is not None
+    assert isinstance(step.assertions[0], Assertion)
+    assert step.assertions[0].operator == "equals"
+
+
+def test_request_step_assertions_absent_defaults_none():
+    step = RequestStep.model_validate({
+        "id": "s1", "type": "request", "method": "GET", "url": "/a",
+        "onFailure": "continue",
+    })
+    assert step.assertions is None
+
+
+# --- evaluation: pass / fail recorded --------------------------------------
+
+@pytest.mark.asyncio
+async def test_assertions_all_pass_recorded(empty_flow):
+    cfg = ContainerConfig(flow_target_url="http://base.com", sim_users=1)
+    runner = make_runner(cfg, empty_flow)
+    session = _session(_resp(200, {"data": {"ok": True, "count": 5}}))
+
+    step = RequestStep(
+        id="s1", type="request", method="GET", url="/a", onFailure="continue",
+        assertions=[
+            Assertion(name="status ok", variable="response_s1_status", operator="equals", value="200"),
+            Assertion(name="flag true", variable="response_s1_body.data.ok", operator="is_true"),
+            Assertion(name="count > 3", variable="response_s1_body.data.count", operator="greater_than", value="3"),
+        ],
+    )
+    ctx: Dict[str, Any] = {}
+    await runner._execute_request_step(step, session, {}, {}, ctx)
+
+    results = get_value_from_context(ctx, "response_s1_assertions")
+    assert isinstance(results, list) and len(results) == 3
+    assert all(r["passed"] for r in results)
+    assert get_value_from_context(ctx, "response_s1_assertions_passed") is True
+
+
+@pytest.mark.asyncio
+async def test_assertions_failure_recorded(empty_flow):
+    cfg = ContainerConfig(flow_target_url="http://base.com", sim_users=1)
+    runner = make_runner(cfg, empty_flow)
+    session = _session(_resp(500, {"data": {"ok": False}}))
+
+    step = RequestStep(
+        id="s1", type="request", method="GET", url="/a", onFailure="continue",
+        assertions=[
+            Assertion(name="expects 200", variable="response_s1_status", operator="equals", value="200"),
+        ],
+    )
+    ctx: Dict[str, Any] = {}
+    await runner._execute_request_step(step, session, {}, {}, ctx)
+
+    results = get_value_from_context(ctx, "response_s1_assertions")
+    assert len(results) == 1
+    assert results[0]["passed"] is False
+    assert get_value_from_context(ctx, "response_s1_assertions_passed") is False
+
+
+# --- degrade gracefully -----------------------------------------------------
+
+@pytest.mark.asyncio
+async def test_unknown_operator_degrades_without_crash(empty_flow, caplog):
+    cfg = ContainerConfig(flow_target_url="http://base.com", sim_users=1)
+    runner = make_runner(cfg, empty_flow)
+    session = _session(_resp(200, {}))
+
+    step = RequestStep(
+        id="s1", type="request", method="GET", url="/a", onFailure="continue",
+        assertions=[
+            Assertion(name="weird", variable="response_s1_status", operator="frobnicate", value="x"),
+        ],
+    )
+    ctx: Dict[str, Any] = {}
+    with caplog.at_level(logging.WARNING):
+        # Must not raise.
+        await runner._execute_request_step(step, session, {}, {}, ctx)
+
+    results = get_value_from_context(ctx, "response_s1_assertions")
+    assert len(results) == 1
+    # Unknown operator => failed assertion, flagged, run continues.
+    assert results[0]["passed"] is False
+    assert get_value_from_context(ctx, "response_s1_assertions_passed") is False
+
+
+@pytest.mark.asyncio
+async def test_unknown_target_missing_variable_degrades(empty_flow):
+    cfg = ContainerConfig(flow_target_url="http://base.com", sim_users=1)
+    runner = make_runner(cfg, empty_flow)
+    session = _session(_resp(200, {}))
+
+    step = RequestStep(
+        id="s1", type="request", method="GET", url="/a", onFailure="continue",
+        assertions=[
+            Assertion(name="missing exists", variable="response_s1_body.nope.deep", operator="exists"),
+        ],
+    )
+    ctx: Dict[str, Any] = {}
+    await runner._execute_request_step(step, session, {}, {}, ctx)
+    results = get_value_from_context(ctx, "response_s1_assertions")
+    assert results[0]["passed"] is False  # missing target => 'exists' is False
+
+
+@pytest.mark.asyncio
+async def test_no_assertions_records_nothing(empty_flow):
+    cfg = ContainerConfig(flow_target_url="http://base.com", sim_users=1)
+    runner = make_runner(cfg, empty_flow)
+    session = _session(_resp(200, {}))
+
+    step = RequestStep(id="s1", type="request", method="GET", url="/a", onFailure="continue")
+    ctx: Dict[str, Any] = {}
+    await runner._execute_request_step(step, session, {}, {}, ctx)
+
+    from flow_runner import _MISSING
+    assert get_value_from_context(ctx, "response_s1_assertions") is _MISSING
+    assert get_value_from_context(ctx, "response_s1_assertions_passed") is _MISSING