From 755a47a2b46b32bd44b10cbbf3f15a9732a7dc48 Mon Sep 17 00:00:00 2001 From: Durable Workflow Date: Tue, 1 Sep 2026 10:16:54 +0000 Subject: [PATCH] Polish the Python SDK first-run surface --- .github/workflows/ci.yml | 130 +- .github/workflows/docs-pr.yml | 34 +- .github/workflows/docs-visual.yml | 417 ---- .github/workflows/docs.yml | 350 +--- .github/workflows/publish.yml | 175 -- .github/workflows/pypi-project-surface.yml | 66 - CONFORMANCE.md | 15 +- README.md | 757 +------ docs/index.md | 25 +- docs/sdk-reference.md | 785 +++++++ mkdocs.yml | 4 +- pyproject.toml | 11 +- scripts/api_reference_release.py | 223 -- scripts/check-cli-parity.py | 127 -- scripts/check-docs-layout.py | 1838 ++--------------- scripts/check_api_reference_install.py | 541 ----- scripts/check_pypi_project_surface.py | 23 +- scripts/check_release_metadata.py | 3 - scripts/ci/check-docs-release-audit.sh | 342 --- scripts/ci/classify_docs_visual_changes.py | 209 -- scripts/ci/classify_pr_qualification.py | 135 -- .../ci/test-classify-docs-visual-changes.py | 124 -- scripts/ci/validate-release-docs-source.py | 96 - scripts/mkdocs_hooks.py | 24 - scripts/qualify-docs-promotion.py | 559 ----- scripts/release_compatibility.py | 142 +- tests/test_api_reference_install.py | 421 ---- tests/test_ci_checkout.py | 1 - tests/test_ci_integration_endpoint.py | 1 - tests/test_ci_qualification_policy.py | 286 --- tests/test_docs_promotion_qualification.py | 223 -- tests/test_docs_workflow_policy.py | 401 ---- tests/test_pypi_project_surface.py | 55 +- tests/test_release_docs_audit_workflow.py | 172 -- tests/test_release_docs_source.py | 123 -- tests/test_release_metadata.py | 58 +- 36 files changed, 1101 insertions(+), 7795 deletions(-) delete mode 100644 .github/workflows/docs-visual.yml delete mode 100644 .github/workflows/pypi-project-surface.yml create mode 100644 docs/sdk-reference.md delete mode 100644 scripts/api_reference_release.py delete mode 100755 scripts/check-cli-parity.py delete mode 100644 scripts/check_api_reference_install.py delete mode 100755 scripts/ci/check-docs-release-audit.sh delete mode 100644 scripts/ci/classify_docs_visual_changes.py delete mode 100644 scripts/ci/classify_pr_qualification.py delete mode 100644 scripts/ci/test-classify-docs-visual-changes.py delete mode 100644 scripts/ci/validate-release-docs-source.py delete mode 100644 scripts/mkdocs_hooks.py delete mode 100644 scripts/qualify-docs-promotion.py delete mode 100644 tests/test_api_reference_install.py delete mode 100644 tests/test_ci_qualification_policy.py delete mode 100644 tests/test_docs_promotion_qualification.py delete mode 100644 tests/test_docs_workflow_policy.py delete mode 100644 tests/test_release_docs_audit_workflow.py delete mode 100644 tests/test_release_docs_source.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3a1d7e8..4683602 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,6 +10,10 @@ on: permissions: contents: read +concurrency: + group: ci-${{ github.event_name }}-${{ github.event_name == 'pull_request' && github.ref || github.sha }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + jobs: action-policy: name: Central action policy preflight @@ -41,55 +45,8 @@ jobs: --target sdk-python --workflow-directory .github/workflows - qualification-class: - name: Determine qualification class - runs-on: ubuntu-latest - timeout-minutes: 5 - outputs: - classification: ${{ steps.classify.outputs.classification }} - reason: ${{ steps.classify.outputs.reason }} - changed_count: ${{ steps.classify.outputs.changed_count }} - steps: - - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 - with: - fetch-depth: 0 - persist-credentials: false - - name: Resolve changed paths without repository API access - id: classify - env: - SOURCE_BASE_SHA: ${{ github.event.pull_request.base.sha }} - SOURCE_EVENT_NAME: ${{ github.event_name }} - SOURCE_HEAD_SHA: ${{ github.event.pull_request.head.sha }} - run: | - python scripts/ci/classify_pr_qualification.py \ - --root . \ - --event-name "$SOURCE_EVENT_NAME" \ - --base-ref "$SOURCE_BASE_SHA" \ - --head-ref "$SOURCE_HEAD_SHA" \ - --github-output "$GITHUB_OUTPUT" - - qualification-class-report: - name: Qualification class — ${{ needs.qualification-class.outputs.classification }} - needs: qualification-class - runs-on: ubuntu-latest - timeout-minutes: 2 - steps: - - name: Report selected qualification - env: - CHANGED_COUNT: ${{ needs.qualification-class.outputs.changed_count }} - QUALIFICATION_CLASS: ${{ needs.qualification-class.outputs.classification }} - QUALIFICATION_REASON: ${{ needs.qualification-class.outputs.reason }} - run: | - case "$QUALIFICATION_CLASS" in - focused-documentation|complete) ;; - *) exit 1 ;; - esac - echo "::notice title=Qualification class::$QUALIFICATION_CLASS ($QUALIFICATION_REASON; $CHANGED_COUNT changed paths)" - regression-corpus: name: Regression corpus - needs: qualification-class - if: ${{ needs.qualification-class.outputs.classification == 'complete' }} runs-on: ubuntu-latest timeout-minutes: 5 steps: @@ -100,7 +57,7 @@ jobs: - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 with: python-version: "3.12" - - name: Install the official Python binding + - name: Install the SDK run: pip install -e . - name: Require durable replay and codec evidence env: @@ -115,8 +72,6 @@ jobs: run: python scripts/ci/test-validate-regression-corpus.py lint: - needs: qualification-class - if: ${{ needs.qualification-class.outputs.classification == 'complete' }} runs-on: ubuntu-latest timeout-minutes: 10 steps: @@ -132,8 +87,6 @@ jobs: avro-benchmark: name: Avro Value absolute throughput (advisory) - needs: qualification-class - if: ${{ needs.qualification-class.outputs.classification == 'complete' }} runs-on: ubuntu-latest timeout-minutes: 5 steps: @@ -156,8 +109,7 @@ jobs: if-no-files-found: error test: - needs: qualification-class - if: ${{ needs.qualification-class.outputs.classification == 'complete' }} + name: Python ${{ matrix.python-version }} runs-on: ubuntu-latest timeout-minutes: 15 strategy: @@ -173,8 +125,6 @@ jobs: - run: pytest tests/ -m "not integration" -q package: - needs: qualification-class - if: ${{ needs.qualification-class.outputs.classification == 'complete' }} runs-on: ubuntu-latest timeout-minutes: 10 steps: @@ -185,36 +135,16 @@ jobs: with: python-version: "3.12" - run: pip install build twine - - run: sh -n scripts/ci/check-docs-release-audit.sh - run: python -m build - name: Compare built release metadata with the exact source commit run: python scripts/check_release_metadata.py --source-ref "$(git rev-parse HEAD)" --dist dist - run: twine check dist/* - run: python scripts/smoke-built-package.py - cli-parity: - needs: qualification-class - if: ${{ needs.qualification-class.outputs.classification == 'complete' }} - runs-on: ubuntu-latest - timeout-minutes: 5 - steps: - - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 - with: - path: sdk-python - - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 - with: - python-version: "3.12" - - name: Checkout public CLI integration source - run: python sdk-python/scripts/ci/checkout-public-repository.py cli cli - - name: Compare shared control-plane parity fixtures - working-directory: sdk-python - run: python scripts/check-cli-parity.py --cli ../cli - integration: - if: ${{ needs.qualification-class.outputs.classification == 'complete' }} runs-on: ubuntu-latest timeout-minutes: 25 - needs: [qualification-class, lint, test] + needs: [lint, test] steps: - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 with: @@ -222,14 +152,14 @@ jobs: - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 with: python-version: "3.12" - - name: Checkout public Server integration source + - name: Check out public Server integration source run: python sdk-python/scripts/ci/checkout-public-repository.py server server - run: pip install -e '.[dev]' working-directory: sdk-python - name: Configure isolated Docker project working-directory: sdk-python run: python scripts/ci/configure-compose-project.py - - name: Start server stack + - name: Start Server stack working-directory: sdk-python run: | docker compose --project-name "$COMPOSE_PROJECT_NAME" -f docker-compose.test.yml \ @@ -264,14 +194,11 @@ jobs: if: ${{ always() }} needs: - action-policy - - qualification-class - - qualification-class-report - regression-corpus - lint - avro-benchmark - test - package - - cli-parity - integration runs-on: ubuntu-latest timeout-minutes: 2 @@ -281,39 +208,18 @@ jobs: env: ACTION_POLICY_RESULT: ${{ needs.action-policy.result }} run: test "$ACTION_POLICY_RESULT" = success - - name: Require every supported Python and integration cell env: - CLASSIFICATION_RESULT: ${{ needs.qualification-class.result }} - CLASSIFICATION_REPORT_RESULT: ${{ needs.qualification-class-report.result }} - QUALIFICATION_CLASS: ${{ needs.qualification-class.outputs.classification }} - LINT_RESULT: ${{ needs.lint.result }} AVRO_BENCHMARK_RESULT: ${{ needs.avro-benchmark.result }} CORPUS_RESULT: ${{ needs.regression-corpus.result }} - TEST_RESULT: ${{ needs.test.result }} - PACKAGE_RESULT: ${{ needs.package.result }} - PARITY_RESULT: ${{ needs.cli-parity.result }} INTEGRATION_RESULT: ${{ needs.integration.result }} + LINT_RESULT: ${{ needs.lint.result }} + PACKAGE_RESULT: ${{ needs.package.result }} + TEST_RESULT: ${{ needs.test.result }} run: | - test "$CLASSIFICATION_RESULT" = success - test "$CLASSIFICATION_REPORT_RESULT" = success - if [ "$QUALIFICATION_CLASS" = focused-documentation ]; then - test "$LINT_RESULT" = skipped - test "$AVRO_BENCHMARK_RESULT" = skipped - test "$CORPUS_RESULT" = skipped - test "$TEST_RESULT" = skipped - test "$PACKAGE_RESULT" = skipped - test "$PARITY_RESULT" = skipped - test "$INTEGRATION_RESULT" = skipped - echo "Focused documentation qualification selected; Docs PR checks and Public Boundary remain required." - elif [ "$QUALIFICATION_CLASS" = complete ]; then - test "$LINT_RESULT" = success - test "$AVRO_BENCHMARK_RESULT" = success - test "$CORPUS_RESULT" = success - test "$TEST_RESULT" = success - test "$PACKAGE_RESULT" = success - test "$PARITY_RESULT" = success - test "$INTEGRATION_RESULT" = success - else - exit 1 - fi + test "$AVRO_BENCHMARK_RESULT" = success + test "$CORPUS_RESULT" = success + test "$INTEGRATION_RESULT" = success + test "$LINT_RESULT" = success + test "$PACKAGE_RESULT" = success + test "$TEST_RESULT" = success diff --git a/.github/workflows/docs-pr.yml b/.github/workflows/docs-pr.yml index d6cfbf6..c4f9f23 100644 --- a/.github/workflows/docs-pr.yml +++ b/.github/workflows/docs-pr.yml @@ -9,20 +9,11 @@ on: - 'overrides/**' - 'mkdocs.yml' - 'pyproject.toml' - - 'scripts/ci/classify_docs_visual_changes.py' - - 'scripts/ci/test-classify-docs-visual-changes.py' - - 'scripts/ci/validate-release-docs-source.py' - - 'scripts/api_reference_release.py' - - 'scripts/check_api_reference_install.py' - - 'scripts/release_compatibility.py' - 'scripts/check-docs-analytics.py' - 'scripts/check-docs-layout.py' - 'scripts/docstring_cross_references.py' - - 'scripts/mkdocs_hooks.py' - - 'scripts/qualify-docs-promotion.py' - - '.github/workflows/docs.yml' - '.github/workflows/docs-pr.yml' - - '.github/workflows/docs-visual.yml' + - '.github/workflows/docs.yml' permissions: contents: read @@ -33,34 +24,21 @@ concurrency: jobs: validate: - name: Strict documentation build and rendered layout + name: Build and check the developer portal runs-on: ubuntu-latest + timeout-minutes: 15 steps: - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 - - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 with: python-version: "3.12" - - - name: Install package + docs deps + - name: Install package and documentation dependencies run: pip install -e '.[docs]' - - - name: Install browser for responsive layout checks + - name: Install Chromium run: python -m playwright install --with-deps chromium - - - name: Build site + - name: Build and check the portal run: | - python scripts/ci/test-classify-docs-visual-changes.py mkdocs build --strict python scripts/docstring_cross_references.py --site site - python scripts/check_api_reference_install.py --site site python scripts/check-docs-analytics.py site python scripts/check-docs-layout.py site - - visual-evidence: - name: Supported viewport interaction evidence - uses: ./.github/workflows/docs-visual.yml # local - with: - source_base_sha: ${{ github.event.pull_request.base.sha }} - permissions: - contents: read diff --git a/.github/workflows/docs-visual.yml b/.github/workflows/docs-visual.yml deleted file mode 100644 index d62a1bd..0000000 --- a/.github/workflows/docs-visual.yml +++ /dev/null @@ -1,417 +0,0 @@ -name: Python documentation visual evidence - -on: - workflow_call: - inputs: - source_base_sha: - description: Candidate comparison revision; an unavailable revision fails safe to all tracked files. - required: false - type: string - source_ref: - description: Immutable candidate revision; the calling workflow's commit is the default. - required: false - type: string - -permissions: - contents: read - -jobs: - visual-evidence: - if: github.api_url == 'https://api.github.com' - runs-on: ubuntu-latest - timeout-minutes: 20 - steps: - - name: Check out candidate source - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 - with: - fetch-depth: 0 - path: candidate - persist-credentials: false - ref: ${{ inputs.source_ref || github.sha }} - repository: ${{ github.repository }} - - - name: Check out visual evidence controller - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 - with: - path: visual-controller - persist-credentials: false - ref: 0421c2e3a78ba4ca2adfe118e57db88d2264a62b - repository: durable-workflow/.github - - - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 - with: - python-version: "3.12" - - - uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6 - with: - node-version: "24" - - - name: Classify candidate changes - id: classify - env: - SOURCE_BASE_SHA: ${{ inputs.source_base_sha }} - SOURCE_REPOSITORY: ${{ github.repository }} - run: | - test "$SOURCE_REPOSITORY" = durable-workflow/sdk-python - classification_args=( - --root "$GITHUB_WORKSPACE/candidate" - --github-output "$GITHUB_OUTPUT" - ) - if [[ "$SOURCE_BASE_SHA" =~ ^[0-9a-f]{40}$ ]] && \ - git -C candidate cat-file -e "${SOURCE_BASE_SHA}^{commit}" 2>/dev/null; then - classification_args+=(--base-ref "$SOURCE_BASE_SHA") - else - mapfile -d '' candidate_files < <(git -C candidate ls-files -z) - for candidate_file in "${candidate_files[@]}"; do - classification_args+=(--changed-file "$candidate_file") - done - fi - printf 'source_revision=%s\n' "$(git -C candidate rev-parse HEAD)" >> "$GITHUB_OUTPUT" - python candidate/scripts/ci/classify_docs_visual_changes.py \ - "${classification_args[@]}" > visual-classification.json - - - name: Install documentation dependencies - if: steps.classify.outputs.required == 'true' - working-directory: candidate - run: pip install -e '.[docs]' - - - name: Install browser for navigation interaction checks - if: steps.classify.outputs.required == 'true' - run: python -m playwright install --with-deps chromium - - - name: Install pinned visual capture runtime - if: steps.classify.outputs.required == 'true' - run: npm ci --prefix visual-controller - - - name: Build candidate reference - if: steps.classify.outputs.required == 'true' - working-directory: candidate - run: | - mkdocs build --strict - python scripts/docstring_cross_references.py --site site - python scripts/check-docs-analytics.py site - - - name: Verify responsive navigation breakpoint transition - if: steps.classify.outputs.required == 'true' && steps.classify.outputs.navigation == 'true' - run: | - mkdir -p visual-review - python candidate/scripts/check-docs-layout.py candidate/site \ - --navigation-transition-only \ - --transition-evidence visual-review/navigation-breakpoint-transition.json - - - name: Verify nested reference navigation reachability - if: steps.classify.outputs.required == 'true' - env: - SOURCE_REVISION: ${{ steps.classify.outputs.source_revision }} - run: | - mkdir -p visual-review - python candidate/scripts/check-docs-layout.py candidate/site \ - --nested-navigation-only \ - --nested-navigation-evidence visual-review/nested-navigation-keyboard.json \ - --source-revision "$SOURCE_REVISION" - - - name: Capture candidate interaction states - if: steps.classify.outputs.required == 'true' - env: - NAVIGATION_REQUIRED: ${{ steps.classify.outputs.navigation }} - SEARCH_REQUIRED: ${{ steps.classify.outputs.search }} - SOURCE_REVISION: ${{ steps.classify.outputs.source_revision }} - TMPDIR: ${{ runner.temp }}/python-docs-visual-${{ github.run_id }}-${{ github.run_attempt }} - run: | - mkdir -p "$TMPDIR" visual-review - cp visual-classification.json visual-review/classification.json - python3 -m http.server 4173 --bind 127.0.0.1 --directory candidate/site \ - >"$RUNNER_TEMP/python-docs-preview.log" 2>&1 & - preview_pid=$! - trap 'kill "$preview_pid" 2>/dev/null || true' EXIT - python - <<'PY' - import time - import urllib.request - - for _ in range(50): - try: - with urllib.request.urlopen("http://127.0.0.1:4173/", timeout=1) as response: - if response.status == 200: - break - except OSError: - time.sleep(0.1) - else: - raise SystemExit("candidate documentation preview did not become ready") - PY - capture() { - local state=$1 width=$2 height=$3 - shift 3 - local capture_args=() - local capture_url=http://127.0.0.1:4173/ - if [ "$width" = 640 ] && [ "$height" = 360 ] && [ "$state" = default ]; then - capture_args+=(--full-page) - fi - if [ "$state" = search-populated ]; then - capture_url="${capture_url}?q=workflow" - fi - node visual-controller/scripts/pipeline_visual_capture.mjs \ - --url "$capture_url" \ - --surface python-sdk-reference --state "$state" \ - --width "$width" --height "$height" \ - --source-repository durable-workflow/sdk-python \ - --source-revision "$SOURCE_REVISION" \ - --screenshot "visual-review/${state}-${width}x${height}.png" \ - --report "visual-review/${state}-${width}x${height}.json" \ - --manifest visual-review/manifest.json "${capture_args[@]}" "$@" - } - capture_nested() { - local route=$1 position=$2 state=$3 width=$4 height=$5 - shift 5 - local capture_args=() - if [ "$width" = 640 ] && [ "$height" = 360 ] && [ "$state" = default ]; then - capture_args+=(--full-page) - fi - node visual-controller/scripts/pipeline_visual_capture.mjs \ - --url "http://127.0.0.1:4173/reference/$route/" \ - --surface "python-sdk-$route-reference" --state "$state" \ - --width "$width" --height "$height" \ - --source-repository durable-workflow/sdk-python \ - --source-revision "$SOURCE_REVISION" \ - --screenshot "visual-review/nested-${position}-${route}-${state}-${width}x${height}.png" \ - --report "visual-review/nested-${position}-${route}-${state}-${width}x${height}.json" \ - --manifest visual-review/manifest.json "${capture_args[@]}" "$@" - } - for viewport in 1440x920 768x1024 390x844 640x360; do - width=${viewport%x*} - height=${viewport#*x} - capture default "$width" "$height" - if [ "$NAVIGATION_REQUIRED" = true ] && [ "$width" -lt 960 ]; then - capture navigation-open "$width" "$height" \ - --state-scope responsive \ - --click ".md-header__button[for='__drawer']" - fi - for reference in first:client middle:serializer final:testing; do - position=${reference%%:*} - route=${reference#*:} - capture_nested "$route" "$position" default "$width" "$height" - if [ "$width" -lt 960 ]; then - capture_nested "$route" "$position" navigation-open "$width" "$height" \ - --state-scope responsive \ - --click ".md-header__button[for='__drawer']" - fi - done - if [ "$SEARCH_REQUIRED" = true ]; then - if [ "$width" -ge 960 ]; then - search_selector=.md-search__input - else - search_selector=".md-header__button[for='__search']" - fi - capture search-open "$width" "$height" --click "$search_selector" - capture search-populated "$width" "$height" - fi - done - python - <<'PY' - import json - import os - from pathlib import Path - from urllib.parse import urlsplit - - evidence_root = Path("visual-review") - keyboard_path = evidence_root / "nested-navigation-keyboard.json" - keyboard = json.loads(keyboard_path.read_text(encoding="utf-8")) - expected_source = { - "repository": "durable-workflow/sdk-python", - "revision": os.environ["SOURCE_REVISION"], - } - expected_routes = { - "/reference/client/": { - "active": "Client", - "position": "first", - "surface": "python-sdk-client-reference", - }, - "/reference/serializer/": { - "active": "Serializer", - "position": "middle", - "surface": "python-sdk-serializer-reference", - }, - "/reference/testing/": { - "active": "Testing", - "position": "final", - "surface": "python-sdk-testing-reference", - }, - } - - def validate_reference_panel(panel, expected_active, label): - if not isinstance(panel, dict): - raise ValueError(f"{label} has no measured reference panel") - try: - active = panel["active"] - drawer = panel["drawer"] - reference_list = panel["list"] - active_top = float(active["top"]) - active_bottom = float(active["bottom"]) - list_top = float(reference_list["top"]) - list_bottom = float(reference_list["bottom"]) - drawer_bottom = float(drawer["bottom"]) - client_height = float(reference_list["clientHeight"]) - except (KeyError, TypeError, ValueError) as error: - raise ValueError(f"{label} has malformed reference panel geometry") from error - if active.get("text") != expected_active: - raise ValueError(f"{label} does not identify its active destination") - if active_top < list_top - 1 or active_bottom > list_bottom + 1: - raise ValueError(f"{label} clips its active destination") - if abs(list_bottom - drawer_bottom) > 1 or client_height < 200: - raise ValueError(f"{label} does not use the available drawer height") - if reference_list.get("overflowY") != "auto": - raise ValueError(f"{label} does not expose the qualified nested-list viewport") - - if ( - keyboard.get("schema") != "durable-workflow.python-docs.nested-navigation/v3" - or keyboard.get("outcome") != "pass" - or set(keyboard.get("routes", [])) != set(expected_routes) - ): - raise SystemExit("nested reference keyboard evidence did not qualify the required route positions") - if keyboard.get("source") != expected_source: - raise SystemExit("nested reference keyboard evidence is not bound to the captured source") - regression = keyboard.get("regression", {}) - three_row_fixture = regression.get("three_row_list", {}) - if ( - three_row_fixture.get("visible_rows") != 3 - or three_row_fixture.get("geometry") != "affected fixture rejected" - or three_row_fixture.get("interaction") != "affected fixture rejected" - ): - raise SystemExit("nested reference evidence did not exercise the three-row clipped-list fixture") - try: - validate_reference_panel( - three_row_fixture.get("reference_panel"), - expected_routes["/reference/testing/"]["active"], - "three-row clipped-list fixture", - ) - except ValueError: - pass - else: - raise SystemExit("retained visual-evidence validation accepted the three-row clipped-list fixture") - keyboard_viewports = { - (entry["route"], entry["viewport"]["width"], entry["viewport"]["height"]): entry - for entry in keyboard.get("viewports", []) - } - - reports = sorted(Path("visual-review").glob("nested-*-*x*.json")) - if len(reports) != 21: - raise SystemExit(f"nested reference evidence is incomplete: {[path.name for path in reports]}") - viewport_states = ( - (1440, 920, "default"), - (768, 1024, "default"), - (768, 1024, "navigation-open"), - (390, 844, "default"), - (390, 844, "navigation-open"), - (640, 360, "default"), - (640, 360, "navigation-open"), - ) - expected_states = { - (route, width, height, state) - for route in expected_routes - for width, height, state in viewport_states - } - observed_states = set() - qualified_states = [] - for report in reports: - captured = json.loads(report.read_text(encoding="utf-8")) - state = captured.get("state") - viewport = captured.get("viewport", {}) - viewport_key = (viewport.get("width"), viewport.get("height")) - page_route = urlsplit(captured.get("page_url", "")).path - route_contract = expected_routes.get(page_route) - expected_full_page = viewport_key == (640, 360) and state == "default" - if ( - route_contract is None - or captured.get("surface") != route_contract["surface"] - or captured.get("source") != expected_source - or captured.get("full_page") is not expected_full_page - ): - raise SystemExit(f"{report.name} is not a faithful source-bound nested viewport capture") - keyboard_entry = keyboard_viewports.get((page_route, *viewport_key), {}) - if keyboard_entry.get("position") != route_contract["position"]: - raise SystemExit(f"{report.name} does not match its required route position") - keyboard_state = keyboard_entry.get("states", {}).get(state) - if keyboard_state is None: - raise SystemExit(f"{report.name} has no matching keyboard state evidence") - if keyboard_state.get("pointer_unreachable") != 0: - raise SystemExit(f"{report.name} keyboard fixture reported unreachable pointer controls") - active_keyboard_controls = keyboard_state.get("active_keyboard_controls") - if state == "navigation-open" and not active_keyboard_controls: - raise SystemExit(f"{report.name} has no active drawer keyboard traversal evidence") - reference_panel = keyboard_state.get("reference_panel") - if state == "navigation-open": - try: - validate_reference_panel( - reference_panel, - route_contract["active"], - report.name, - ) - except ValueError as error: - raise SystemExit(str(error)) from error - geometry = captured["geometry"] - if geometry["unreachable_controls"]: - raise SystemExit( - f"{report.name} has unreachable controls: {geometry['unreachable_controls']}" - ) - if state == "navigation-open" and geometry.get("interactive_control_count") != len( - active_keyboard_controls - ): - raise SystemExit(f"{report.name} pointer and keyboard control sets do not match") - observed_states.add((page_route, *viewport_key, state)) - qualified_states.append( - { - "capture_report": report.name, - "position": route_contract["position"], - "route": page_route, - "state": state, - "viewport": viewport, - "full_page": captured["full_page"], - "pointer_unreachable_controls": geometry["unreachable_controls"], - "active_keyboard_controls": active_keyboard_controls or [], - "reference_panel": reference_panel, - } - ) - if observed_states != expected_states: - raise SystemExit(f"nested reference state matrix does not match: {sorted(observed_states)}") - - qualification = { - "schema": "durable-workflow.python-docs.nested-navigation-qualification/v3", - "outcome": "pass", - "source": expected_source, - "routes": list(expected_routes), - "states": qualified_states, - "regression": regression, - } - (evidence_root / "nested-navigation-qualification.json").write_text( - f"{json.dumps(qualification, indent=2)}\n", - encoding="utf-8", - ) - PY - - - name: Validate candidate evidence - if: steps.classify.outputs.required == 'true' - env: - SEARCH_REQUIRED: ${{ steps.classify.outputs.search }} - run: | - classification_root="$RUNNER_TEMP/python-docs-visual-classification" - mkdir -p "$classification_root" - validation_args=( - --root "$classification_root" - --manifest "$GITHUB_WORKSPACE/visual-review/manifest.json" - ) - printf '%s\n' '.documentation-surface { display: block; }' \ - > "$classification_root/documentation.css" - validation_args+=(--changed-file documentation.css) - if [ "$SEARCH_REQUIRED" = true ]; then - printf '%s\n' '' \ - > "$classification_root/search.html" - validation_args+=(--changed-file search.html) - fi - python visual-controller/scripts/visual_evidence.py validate "${validation_args[@]}" - - - name: Retain candidate visual evidence - if: always() && steps.classify.outputs.required == 'true' - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - if-no-files-found: error - name: python-docs-visual-${{ steps.classify.outputs.source_revision }} - path: visual-review - retention-days: 30 diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 8816342..a521643 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -1,10 +1,4 @@ -name: Docs deployment - -run-name: >- - ${{ github.event_name == 'workflow_dispatch' - && format('Deploy Python release docs {0}@{1} from release run {2}.{3}', - inputs.release_version, inputs.release_source_sha, inputs.release_run_id, inputs.release_run_attempt) - || format('Deploy Python docs from main@{0}', github.sha) }} +name: Python developer portal on: push: @@ -15,364 +9,64 @@ on: - 'overrides/**' - 'mkdocs.yml' - 'pyproject.toml' - - 'scripts/ci/classify_docs_visual_changes.py' - - 'scripts/ci/test-classify-docs-visual-changes.py' - - 'scripts/ci/validate-release-docs-source.py' - - 'scripts/api_reference_release.py' - - 'scripts/check_api_reference_install.py' - - 'scripts/release_compatibility.py' - 'scripts/check-docs-analytics.py' - 'scripts/check-docs-layout.py' - 'scripts/docstring_cross_references.py' - - 'scripts/mkdocs_hooks.py' - - 'scripts/qualify-docs-promotion.py' - '.github/workflows/docs.yml' - - '.github/workflows/docs-pr.yml' - - '.github/workflows/docs-visual.yml' workflow_dispatch: - inputs: - release_version: - description: Exact public PyPI version and immutable source tag. - required: true - type: string - release_source_sha: - description: Exact release source commit to render and audit. - required: true - type: string - release_parent_sha: - description: Exact first parent of the release source commit. - required: true - type: string - release_run_id: - description: Authenticated publication workflow run requesting this deployment. - required: true - type: string - release_run_attempt: - description: Exact attempt of the publication workflow run. - required: true - type: string permissions: contents: read concurrency: - group: docs-deployment + group: python-developer-portal cancel-in-progress: false jobs: build: runs-on: ubuntu-latest - outputs: - release_ready: ${{ steps.public_install.outputs.release_ready }} - release_version: ${{ steps.source.outputs.release_version }} - source_base_revision: ${{ steps.source.outputs.base_revision }} - source_revision: ${{ steps.source.outputs.revision }} + timeout-minutes: 15 steps: - - name: Checkout the trusted main-context workflow controller - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 - with: - persist-credentials: false - path: controller - ref: ${{ github.sha }} - - - name: Checkout the exact documentation source - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 - with: - fetch-depth: 0 - persist-credentials: false - path: candidate - ref: ${{ github.event_name == 'workflow_dispatch' && inputs.release_source_sha || github.sha }} - + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 with: python-version: "3.12" - - - name: Validate and record the deployment source identity - id: source - env: - EVENT_BEFORE: ${{ github.event.before }} - EVENT_NAME: ${{ github.event_name }} - RELEASE_PARENT_SHA: ${{ inputs.release_parent_sha }} - RELEASE_RUN_ATTEMPT: ${{ inputs.release_run_attempt }} - RELEASE_RUN_ID: ${{ inputs.release_run_id }} - RELEASE_SOURCE_SHA: ${{ inputs.release_source_sha }} - RELEASE_VERSION: ${{ inputs.release_version }} - WORKFLOW_REF: ${{ github.ref }} - WORKFLOW_SHA: ${{ github.sha }} - run: | - set -euo pipefail - if [ "$WORKFLOW_REF" != refs/heads/main ]; then - printf 'docs deployment workflow must execute from refs/heads/main, got %s\n' "$WORKFLOW_REF" >&2 - exit 1 - fi - observed_source="$(git -C candidate rev-parse HEAD)" - if [ "$EVENT_NAME" = workflow_dispatch ]; then - for identity in "$RELEASE_RUN_ID" "$RELEASE_RUN_ATTEMPT"; do - if [[ ! "$identity" =~ ^[1-9][0-9]*$ ]]; then - printf 'release workflow run identity is invalid\n' >&2 - exit 1 - fi - done - python controller/scripts/ci/validate-release-docs-source.py \ - --repo-root candidate \ - --source-sha "$RELEASE_SOURCE_SHA" \ - --parent-sha "$RELEASE_PARENT_SHA" \ - --release-version "$RELEASE_VERSION" - source_revision="$RELEASE_SOURCE_SHA" - base_revision="$RELEASE_PARENT_SHA" - release_version="$RELEASE_VERSION" - elif [ "$EVENT_NAME" = push ]; then - if [ "$observed_source" != "$WORKFLOW_SHA" ]; then - printf 'main docs checkout is %s, expected event commit %s\n' "$observed_source" "$WORKFLOW_SHA" >&2 - exit 1 - fi - source_revision="$observed_source" - base_revision="$EVENT_BEFORE" - release_version="$(python -c \ - 'import tomllib; print(tomllib.load(open("candidate/pyproject.toml", "rb"))["project"]["version"])')" - else - printf 'docs deployment does not accept event %s\n' "$EVENT_NAME" >&2 - exit 1 - fi - if [[ ! "$source_revision" =~ ^[0-9a-f]{40}$ ]] || - [[ ! "$base_revision" =~ ^[0-9a-f]{40}$ ]]; then - printf 'docs deployment source identities must be exact Git object IDs\n' >&2 - exit 1 - fi - { - printf 'revision=%s\n' "$source_revision" - printf 'base_revision=%s\n' "$base_revision" - printf 'release_version=%s\n' "$release_version" - } >> "$GITHUB_OUTPUT" - - - name: Install package + docs deps - working-directory: candidate + - name: Install package and documentation dependencies run: pip install -e '.[docs]' - - - name: Install browser for responsive layout checks - working-directory: candidate + - name: Install Chromium run: python -m playwright install --with-deps chromium - - - name: Build site - working-directory: candidate + - name: Build and check the portal env: CLOUDFLARE_WEB_ANALYTICS_TOKEN: ${{ vars.CLOUDFLARE_WEB_ANALYTICS_TOKEN }} - SOURCE_REVISION: ${{ steps.source.outputs.revision }} run: | - python scripts/ci/test-classify-docs-visual-changes.py + if ! printf '%s' "$CLOUDFLARE_WEB_ANALYTICS_TOKEN" | grep -Eq '^[a-f0-9]{32}$'; then + echo 'CLOUDFLARE_WEB_ANALYTICS_TOKEN must be the canonical 32-character site token' >&2 + exit 1 + fi + sed -i "s/__CLOUDFLARE_WEB_ANALYTICS_TOKEN__/$CLOUDFLARE_WEB_ANALYTICS_TOKEN/" \ + docs/javascripts/analytics.js mkdocs build --strict python scripts/docstring_cross_references.py --site site - python scripts/check_api_reference_install.py --site site - python scripts/check-docs-analytics.py site + python scripts/check-docs-analytics.py site --require-token python scripts/check-docs-layout.py site - if [ "$GITHUB_SERVER_URL" = https://github.com ]; then - if ! printf '%s' "$CLOUDFLARE_WEB_ANALYTICS_TOKEN" | grep -Eq '^[a-f0-9]{32}$'; then - echo 'CLOUDFLARE_WEB_ANALYTICS_TOKEN must be the canonical 32-character site token' >&2 - exit 1 - fi - sed -i "s/__CLOUDFLARE_WEB_ANALYTICS_TOKEN__/$CLOUDFLARE_WEB_ANALYTICS_TOKEN/" \ - docs/javascripts/analytics.js - mkdocs build --strict - python scripts/docstring_cross_references.py --site site - python scripts/check_api_reference_install.py --site site - python scripts/check-docs-analytics.py site --require-token - fi - python scripts/check_api_reference_install.py \ - --site site \ - --source-revision "$SOURCE_REVISION" - - - name: Verify the rendered command against public PyPI - id: public_install - if: ${{ github.server_url == 'https://github.com' }} - working-directory: candidate - env: - PUBLISHED_RELEASE: ${{ github.event_name == 'workflow_dispatch' }} - run: | - arguments=(--site site --install) - if [ "$PUBLISHED_RELEASE" = true ]; then - python scripts/check_api_reference_install.py "${arguments[@]}" - else - set +e - python scripts/check_api_reference_install.py \ - "${arguments[@]}" \ - --install-attempts 1 \ - --install-retry-sleep 0 \ - --unavailable-exit-code 75 - status="$?" - set -e - if [ "$status" -eq 75 ]; then - printf 'release_ready=false\n' >> "$GITHUB_OUTPUT" - printf '::notice title=API reference preserved::The exact SDK release is not yet public on PyPI.\n' - exit 0 - fi - if [ "$status" -ne 0 ]; then - exit "$status" - fi - fi - printf 'release_ready=true\n' >> "$GITHUB_OUTPUT" - - - name: Upload GitHub Pages artifact - if: >- - github.server_url == 'https://github.com' && - steps.public_install.outputs.release_ready == 'true' - uses: actions/upload-pages-artifact@fc324d3547104276b827a68afc52ff2a11cc49c9 # v5 + test -s site/index.html + test "$(cat site/CNAME)" = "python.durable-workflow.com" + - uses: actions/upload-pages-artifact@fc324d3547104276b827a68afc52ff2a11cc49c9 # v5 + if: github.server_url == 'https://github.com' with: - path: candidate/site - - visual-evidence: - needs: build - uses: ./.github/workflows/docs-visual.yml # local - with: - source_base_sha: ${{ needs.build.outputs.source_base_revision }} - source_ref: ${{ needs.build.outputs.source_revision }} - permissions: - contents: read + path: site deploy: - needs: [build, visual-evidence] - if: >- - github.server_url == 'https://github.com' && - github.ref == 'refs/heads/main' && - needs.build.outputs.release_ready == 'true' && - (github.event_name == 'push' || github.event_name == 'workflow_dispatch') + if: ${{ github.server_url == 'https://github.com' && github.ref == 'refs/heads/main' }} + needs: build runs-on: ubuntu-latest - outputs: - page_url: ${{ steps.deployment.outputs.page_url }} - source_revision: ${{ steps.evidence.outputs.source_revision }} permissions: contents: read id-token: write pages: write environment: name: github-pages - url: ${{ steps.deployment.outputs.page_url }} + url: https://python.durable-workflow.com/ steps: - id: deployment uses: actions/deploy-pages@cd2ce8fcbc39b97be8ca5fce6e763baed58fa128 # v5 - - name: Record the deployed public source revision - id: evidence - env: - DEPLOYED_REVISION: ${{ needs.build.outputs.source_revision }} - DEPLOYMENT_URL: ${{ steps.deployment.outputs.page_url }} - RELEASE_VERSION: ${{ needs.build.outputs.release_version }} - run: | - printf 'source_revision=%s\n' "$DEPLOYED_REVISION" >> "$GITHUB_OUTPUT" - { - printf '## Python API reference deployed\n\n' - printf -- '- SDK version: `%s`\n' "$RELEASE_VERSION" - printf -- '- Source revision: `%s`\n' "$DEPLOYED_REVISION" - printf -- '- Public URL: %s\n' "$DEPLOYMENT_URL" - } >> "$GITHUB_STEP_SUMMARY" - - qualify-promotion: - needs: [build, deploy] - if: ${{ needs.deploy.result == 'success' }} - runs-on: ubuntu-latest - permissions: - contents: read - steps: - - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 - with: - persist-credentials: false - ref: ${{ needs.build.outputs.source_revision }} - - - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 - with: - python-version: "3.12" - - - name: Install browser qualification dependencies - run: | - pip install -e '.[docs]' - python -m playwright install --with-deps chromium - - - name: Qualify deployed landing and promotion transport - run: >- - python scripts/qualify-docs-promotion.py - --source-revision "${{ needs.build.outputs.source_revision }}" - --evidence-directory deployed-visual - - - name: Retain deployed landing evidence - if: always() - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: python-docs-deployed-${{ needs.build.outputs.source_revision }} - path: deployed-visual - if-no-files-found: warn - retention-days: 30 - - audit-release: - needs: [build, deploy] - if: ${{ github.event_name == 'workflow_dispatch' }} - runs-on: ubuntu-latest - steps: - - name: Checkout the trusted main-context workflow controller - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 - with: - persist-credentials: false - path: controller - ref: ${{ github.sha }} - - - name: Checkout the exact audited release source - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 - with: - persist-credentials: false - path: candidate - ref: ${{ needs.build.outputs.source_revision }} - - - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 - with: - python-version: "3.12" - - - name: Build and exercise the API-reference install path - working-directory: candidate - run: | - pip install -e '.[docs]' - mkdocs build --strict - python scripts/docstring_cross_references.py --site site - python ../controller/scripts/check_api_reference_install.py \ - --repo-root . \ - --site site \ - --install - - - name: Verify the deployed API-reference release record - working-directory: candidate - env: - DEPLOYED_REVISION: ${{ needs.deploy.outputs.source_revision }} - EXPECTED_REVISION: ${{ needs.build.outputs.source_revision }} - RELEASE_VERSION: ${{ needs.build.outputs.release_version }} - run: | - if [ "$DEPLOYED_REVISION" != "$EXPECTED_REVISION" ]; then - printf 'deployed revision %s does not match release revision %s\n' \ - "$DEPLOYED_REVISION" "$EXPECTED_REVISION" >&2 - exit 1 - fi - python ../controller/scripts/check_api_reference_install.py \ - --repo-root . \ - --site site \ - --source-revision "$EXPECTED_REVISION" \ - --verify-deployed-url https://python.durable-workflow.com/release-audit.json - { - printf '## Python API reference release audit passed\n\n' - printf -- '- SDK release: `%s`\n' "$RELEASE_VERSION" - printf -- '- Source revision: `%s`\n' "$EXPECTED_REVISION" - printf -- '- Release evidence: https://python.durable-workflow.com/release-audit.json\n' - } >> "$GITHUB_STEP_SUMMARY" - - - name: Verify live docs release audit after PyPI publish - env: - DOCS_RELEASE_AUDIT_ARTIFACT: sdk-python - DOCS_RELEASE_AUDIT_VERSION: ${{ needs.build.outputs.release_version }} - DOCS_RELEASE_AUDIT_EVIDENCE: docs-release-audit-evidence.json - DOCS_RELEASE_AUDIT_HANDOFF: docs-release-audit-handoff.json - DOCS_RELEASE_AUDIT_ENFORCEMENT: advisory - run: controller/scripts/ci/check-docs-release-audit.sh - - - name: Upload docs release audit evidence - if: always() - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: docs-release-audit-evidence-${{ needs.build.outputs.release_version }} - path: | - docs-release-audit-evidence.json - docs-release-audit-handoff.json - if-no-files-found: warn diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 0d450be..91bd577 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -252,178 +252,3 @@ jobs: fi gh release create "$RELEASE_TAG" "${arguments[@]}" fi - - deploy-and-audit-api-reference: - needs: [build, publish] - if: >- - (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/')) || - (github.event_name == 'workflow_dispatch' && - github.ref == 'refs/heads/main' && inputs.publish) - runs-on: ubuntu-latest - permissions: - actions: write - contents: read - steps: - - name: Dispatch the main-context release docs workflow - env: - GH_TOKEN: ${{ github.token }} - RELEASE_PARENT_SHA: ${{ needs.build.outputs.release_base_commit }} - RELEASE_RUN_ATTEMPT: ${{ github.run_attempt }} - RELEASE_RUN_ID: ${{ github.run_id }} - RELEASE_SOURCE_SHA: ${{ needs.build.outputs.release_commit }} - RELEASE_VERSION: ${{ needs.build.outputs.release_tag }} - run: | - set -euo pipefail - expected_title="Deploy Python release docs ${RELEASE_VERSION}@${RELEASE_SOURCE_SHA} from release run ${RELEASE_RUN_ID}.${RELEASE_RUN_ATTEMPT}" - existing_runs="$(gh run list \ - --repo "$GITHUB_REPOSITORY" \ - --workflow docs.yml \ - --event workflow_dispatch \ - --branch main \ - --limit 100 \ - --json databaseId,displayTitle,headBranch)" - if jq -e --arg title "$expected_title" \ - 'any(.[]; .displayTitle == $title and .headBranch == "main")' \ - <<<"$existing_runs" >/dev/null; then - printf 'a release docs run already exists for this release run identity\n' >&2 - exit 1 - fi - gh workflow run docs.yml \ - --repo "$GITHUB_REPOSITORY" \ - --ref main \ - -f release_version="$RELEASE_VERSION" \ - -f release_source_sha="$RELEASE_SOURCE_SHA" \ - -f release_parent_sha="$RELEASE_PARENT_SHA" \ - -f release_run_id="$RELEASE_RUN_ID" \ - -f release_run_attempt="$RELEASE_RUN_ATTEMPT" - printf 'expected_title=%s\n' "$expected_title" >> "$GITHUB_ENV" - - - name: Wait for the exact docs deployment and audit - env: - GH_TOKEN: ${{ github.token }} - RELEASE_SOURCE_SHA: ${{ needs.build.outputs.release_commit }} - RELEASE_VERSION: ${{ needs.build.outputs.release_tag }} - run: | - set -euo pipefail - docs_run_id='' - for attempt in {1..60}; do - runs="$(gh run list \ - --repo "$GITHUB_REPOSITORY" \ - --workflow docs.yml \ - --event workflow_dispatch \ - --branch main \ - --limit 100 \ - --json databaseId,displayTitle,headBranch)" - docs_run_id="$(jq -r --arg title "$expected_title" \ - '[.[] | select(.displayTitle == $title and .headBranch == "main")] | max_by(.databaseId) | .databaseId // empty' \ - <<<"$runs")" - if [ -n "$docs_run_id" ]; then - break - fi - sleep 5 - done - if [ -z "$docs_run_id" ]; then - printf 'the authenticated main-context docs run was not created\n' >&2 - exit 1 - fi - gh run watch "$docs_run_id" \ - --repo "$GITHUB_REPOSITORY" \ - --exit-status \ - --interval 10 - details="$(gh run view "$docs_run_id" \ - --repo "$GITHUB_REPOSITORY" \ - --json conclusion,displayTitle,event,headBranch,url,workflowName)" - jq -e --arg title "$expected_title" ' - .conclusion == "success" and - .displayTitle == $title and - .event == "workflow_dispatch" and - .headBranch == "main" and - .workflowName == "Docs deployment" - ' <<<"$details" >/dev/null - docs_url="$(jq -r '.url' <<<"$details")" - { - printf '## Python API reference publication verified\n\n' - printf -- '- SDK release: `%s`\n' "$RELEASE_VERSION" - printf -- '- Source revision: `%s`\n' "$RELEASE_SOURCE_SHA" - printf -- '- Main-context deployment and audit: %s\n' "$docs_url" - printf -- '- Release evidence: https://python.durable-workflow.com/release-audit.json\n' - } >> "$GITHUB_STEP_SUMMARY" - - publish-test: - needs: build - runs-on: ubuntu-latest - if: >- - github.event_name == 'workflow_dispatch' && - github.ref == 'refs/heads/main' && - !inputs.dry_run && !inputs.publish - environment: test-pypi - permissions: - actions: read - contents: read - steps: - - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 - with: - fetch-depth: 0 - persist-credentials: false - ref: ${{ github.sha }} - - - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 - with: - artifact-ids: ${{ needs.build.outputs.artifact-id }} - digest-mismatch: error - github-token: ${{ github.token }} - path: isolated-python-dist - repository: ${{ github.repository }} - run-id: ${{ needs.build.outputs.source-run-id }} - - - name: Validate the exact producer artifact before use - env: - ARTIFACT_DIRECTORY: isolated-python-dist - EXPECTED_ARTIFACT_DIGEST: ${{ needs.build.outputs.artifact-digest }} - EXPECTED_ARTIFACT_ID: ${{ needs.build.outputs.artifact-id }} - EXPECTED_SOURCE_RUN_ATTEMPT: ${{ needs.build.outputs.source-run-attempt }} - EXPECTED_SOURCE_RUN_ID: ${{ needs.build.outputs.source-run-id }} - run: | - set -euo pipefail - if [[ ! "$EXPECTED_ARTIFACT_DIGEST" =~ ^[0-9a-f]{64}$ ]]; then - printf 'producer artifact digest is not an exact SHA-256 digest\n' >&2 - exit 1 - fi - for identity in "$EXPECTED_ARTIFACT_ID" "$EXPECTED_SOURCE_RUN_ID" "$EXPECTED_SOURCE_RUN_ATTEMPT"; do - if [[ ! "$identity" =~ ^[1-9][0-9]*$ ]]; then - printf 'producer artifact identity is invalid\n' >&2 - exit 1 - fi - done - if [[ ! "$ARTIFACT_DIRECTORY" =~ ^isolated-[a-z0-9][a-z0-9._-]*$ ]]; then - printf 'artifact validation directory is unsafe\n' >&2 - exit 1 - fi - mapfile -d '' entries < <( - /usr/bin/find "$ARTIFACT_DIRECTORY" -mindepth 1 -maxdepth 1 -print0 - ) - if [ "${#entries[@]}" -ne 1 ] || [ ! -f "${entries[0]}" ] || [ -L "${entries[0]}" ]; then - printf 'artifact handoff must contain exactly one regular file\n' >&2 - exit 1 - fi - observed_digest="$(/usr/bin/sha256sum "${entries[0]}")" - observed_digest="${observed_digest%% *}" - if [ "$observed_digest" != "$EXPECTED_ARTIFACT_DIGEST" ]; then - printf 'artifact digest mismatch: expected %s, got %s\n' \ - "$EXPECTED_ARTIFACT_DIGEST" "$observed_digest" >&2 - exit 1 - fi - printf 'validated artifact %s from run %s attempt %s\n' \ - "$EXPECTED_ARTIFACT_ID" "$EXPECTED_SOURCE_RUN_ID" "$EXPECTED_SOURCE_RUN_ATTEMPT" - - - name: Extract the validated TestPyPI handoff - run: | - mkdir dist - tar -xf isolated-python-dist/dist-handoff.tar -C dist - - - name: Publish to TestPyPI - uses: pypa/gh-action-pypi-publish@ba38be9e461d3875417946c167d0b5f3d385a247 # release/v1 - with: - password: ${{ secrets.TEST_PYPI_TOKEN }} - repository-url: https://test.pypi.org/legacy/ - print-hash: true diff --git a/.github/workflows/pypi-project-surface.yml b/.github/workflows/pypi-project-surface.yml deleted file mode 100644 index 0996f28..0000000 --- a/.github/workflows/pypi-project-surface.yml +++ /dev/null @@ -1,66 +0,0 @@ -name: PyPI project surface - -on: - push: - branches: [main] - paths: - - '.github/workflows/pypi-project-surface.yml' - - 'README.md' - - 'pyproject.toml' - - 'scripts/check_pypi_project_surface.py' - - 'scripts/check_release_metadata.py' - - 'scripts/release_compatibility.py' - - 'tests/test_pypi_project_surface.py' - - 'tests/test_release_metadata.py' - schedule: - - cron: '17 7 * * 1' - workflow_dispatch: - -permissions: - contents: read - -jobs: - audit: - runs-on: ubuntu-latest - timeout-minutes: 10 - steps: - - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 - with: - fetch-depth: 0 - persist-credentials: false - - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 - with: - python-version: "3.12" - - name: Install local qualification dependencies - if: github.event_name == 'push' - run: pip install -e '.[dev]' - - name: Test the project-surface audit implementation - if: github.event_name == 'push' - run: pytest tests/test_pypi_project_surface.py tests/test_release_metadata.py -q - - name: Validate current source metadata without querying PyPI - if: github.event_name == 'push' - run: python scripts/check_pypi_project_surface.py --source-ref HEAD --source-only - - name: Resolve the newest immutable Python release-candidate source - if: github.event_name != 'push' - run: | - set -euo pipefail - source_ref="$(git tag -l '2.0.0-rc.*' --sort=-version:refname | head -n 1)" - if [ -z "$source_ref" ]; then - printf 'no Python release-candidate tag is available for public qualification\n' >&2 - exit 1 - fi - printf 'source_ref=%s\n' "$source_ref" >> "$GITHUB_ENV" - - name: Qualify authoritative root metadata and package resolution - if: github.event_name != 'push' - run: >- - python scripts/check_pypi_project_surface.py - --source-ref "$source_ref" - --attempts 3 - --interval-seconds 10 - --evidence pypi-project-surface-evidence.json - - name: Upload project-surface evidence - if: github.event_name != 'push' - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: pypi-project-surface-evidence - path: pypi-project-surface-evidence.json diff --git a/CONFORMANCE.md b/CONFORMANCE.md index 0b6f80c..d31651c 100644 --- a/CONFORMANCE.md +++ b/CONFORMANCE.md @@ -23,7 +23,7 @@ The Python SDK claims two targets from the suite's matrix: | Category | Source path | Status | | --- | --- | --- | -| `control_plane_request_response` | `tests/fixtures/control-plane/` | stable, parity-shared with `cli` | +| `control_plane_request_response` | `tests/fixtures/control-plane/` | stable request/response contract coverage | | `signal_query_runtime_contract` | `tests/test_signals.py`, `tests/test_queries.py`, `tests/test_worker.py` and the public scenario manifest at | stable, parity-shared with PHP worker, CLI, and server routes | | `search_attribute_runtime_contract` | public scenario manifest at | stable, parity-shared with PHP worker, CLI, Waterline, and server query behavior | | `namespace_runtime_contract` | public scenario manifest at | stable, suite v12 runtime coverage for namespace isolation and SDK namespace selection | @@ -43,13 +43,10 @@ The fixtures in this repo are exercised today by: - `tests/test_worker.py` - `tests/test_replay.py` - `tests/test_golden_history_replay.py` -- `scripts/check-cli-parity.py` - `durable-workflow-python-conformance --manifest` -- the `cli-parity` job in `.github/workflows/ci.yml` -These are the per-repo gates that already enforce the contract; the -public conformance harness, when it lands, will read the same fixtures -from this repo's declared paths. +These per-repo gates enforce the SDK contract, while the public conformance +harness exercises cross-SDK behavior against published artifacts. ## Published-artifact Python parity contract @@ -188,7 +185,5 @@ no test in this repo notices. - Compatibility authority: -- Polyglot parity doc: - -- Existing per-repo gates: `tests/test_control_plane_parity_fixtures.py`, - `tests/test_history_event_contract.py`, `scripts/check-cli-parity.py`. +- Existing per-repo gates: `tests/test_control_plane_parity_fixtures.py` and + `tests/test_history_event_contract.py`. diff --git a/README.md b/README.md index 04b27b7..2ce8503 100644 --- a/README.md +++ b/README.md @@ -1,30 +1,23 @@ -# Durable Workflow (Python SDK) +# Durable Workflow Python SDK -A Python SDK for the [Durable Workflow server](https://github.com/durable-workflow/server). Speaks the server's language-neutral HTTP/JSON worker protocol — no PHP runtime required. +[![CI](https://github.com/durable-workflow/sdk-python/actions/workflows/ci.yml/badge.svg?branch=main)](https://github.com/durable-workflow/sdk-python/actions/workflows/ci.yml) +[![PyPI](https://img.shields.io/pypi/v/durable-workflow.svg)](https://pypi.org/project/durable-workflow/) +[![Python](https://img.shields.io/pypi/pyversions/durable-workflow.svg)](https://pypi.org/project/durable-workflow/) +[![License](https://img.shields.io/github/license/durable-workflow/sdk-python.svg)](LICENSE) -Status: **Stable 2.0**. Core features include workflows, activities, -schedules, signals, timers, child workflows, continue-as-new, side effects, -version markers, worker-applied accepted updates, replay verification, the -in-process `WorkflowEnvironment` test harness, and invocable activity carriers. - -Python workers execute server-routed query tasks after the Server advertises the query-tasks capability through cluster discovery. +Build durable Python workflows and activities against [Durable Workflow +Cloud](https://cloud.durable-workflow.com/) or a +[self-hosted Server](https://github.com/durable-workflow/server). The SDK uses +the same language-neutral runtime protocol as the first-party PHP and Rust +SDKs. ## Install ```bash -curl -fsSL https://durable-workflow.com/install-sdk.sh | sh -s -- python +pip install durable-workflow ``` -The versionless resolver reads the last passing public quickstart contract and -invokes pip with the qualified SDK identity. The -[API reference](https://python.durable-workflow.com/) resolves the matching -Server image from that same contract. - -Or for development: - -```bash -pip install -e '.[dev]' -``` +Python 3.10 or newer is required. ## Quickstart @@ -71,716 +64,74 @@ if __name__ == "__main__": asyncio.run(main()) ``` -Pass the server or managed-runtime base URL to `Client`; the SDK appends its -own `/api` routes. For example, use `http://server:8080`, not -`http://server:8080/api`. Managed-runtime paths such as -`https://cloud.example/api/runtime/v1/namespaces/acme` are valid as written. - -A workflow ID is the durable identity of an instance, not a per-attempt request -ID. Starting the same ID again with the default `reject` duplicate policy raises -the typed `WorkflowAlreadyStarted` exception. Generate a unique ID for each new -instance, as the quickstart does. For an intentionally idempotent start, catch -`WorkflowAlreadyStarted` and reconnect with -`client.get_workflow_handle(workflow_id)`; choose `allow` or -`terminate_existing` only when creating another run or replacing the current -instance is the intended behavior. - -For a fuller deployable example, see -[`examples/order_processing`](examples/order_processing), which runs a -multi-activity order workflow against a local server with Docker Compose. - -## Schedule visibility and paging - -`list_schedules()` returns one typed `ScheduleList` page. Status and workflow -type are exact server-side filters; the visibility query uses the server's -documented equality-predicate grammar. All filters combine with AND semantics. - -```python -page = await client.list_schedules( - status="active", - workflow_type="orders.rollup", - query='Region = "eu" AND Priority = 2', - page_size=25, -) - -while page.next_page_token is not None: - page = await client.list_schedules( - status="active", - workflow_type="orders.rollup", - query='Region = "eu" AND Priority = 2', - page_size=25, - next_page_token=page.next_page_token, - ) -``` - -Continuation tokens are opaque. Reuse them unchanged with the same namespace, -status, workflow type, and query; `None` terminates traversal. Invalid filters -and malformed, mismatched, cross-namespace, or stale tokens raise -`ScheduleListError`, which retains `status`, `reason()`, `field`, `errors`, -`last_safe_cursor`, and the complete server response in `body`. - -## Retry policy scopes - -Retry and timeout settings are scoped to the layer where you configure them: - -- `TransportRetryPolicy` on `Client(...)` retries SDK HTTP requests only. It handles transient connection failures, request timeouts, 5xx responses, and 429 rate limits. It does not retry workflow code, activity code, child workflows, or failed workflow runs. -- `ActivityRetryPolicy` on `ctx.schedule_activity(...)` is recorded into durable history with that activity command. It controls server-side attempts for that one activity execution. -- `ChildWorkflowRetryPolicy` on `ctx.start_child_workflow(...)` is recorded with that child-start command. It controls server-side attempts for that child workflow execution. -- `non_retryable_error_types` belongs to durable activity/child retry policies. `non_retryable=True` on an activity failure bypasses the activity retry budget and surfaces the failure to the workflow. - -Timeout names are also layer-specific. `start_to_close_timeout` limits one activity attempt, `schedule_to_start_timeout` limits queue wait before an activity starts, `schedule_to_close_timeout` limits the whole activity execution including retries, and `heartbeat_timeout` limits the gap between activity heartbeats. For child workflows, `execution_timeout_seconds` covers the overall child workflow execution and `run_timeout_seconds` covers one run. - -## Activity failure payloads - -When replay raises `ActivityFailed`, the top-level attributes expose the -stable cross-language fields: `activity_type`, `failure_category`, -`exception_type`, `message`, `non_retryable`, and `code`. The -`exception_payload` dictionary is filtered to language-neutral keys such as -`type`, `message`, `details`, `details_payload_codec`, and `non_retryable`. -Runtime diagnostics like PHP or Python exception classes, source file paths, -line numbers, and traces are not included by default unless the history event -contains an explicit `diagnostics` or `runtime_diagnostics` envelope. - -## Activity retries and timeouts - -Configure per-call activity retries and deadlines from workflow code: - -```python -from durable_workflow import ActivityRetryPolicy - -result = yield ctx.schedule_activity( - "charge-card", - [order], - retry_policy=ActivityRetryPolicy( - max_attempts=4, - initial_interval_seconds=1, - backoff_coefficient=2, - maximum_interval_seconds=30, - non_retryable_error_types=["ValidationError"], - ), - start_to_close_timeout=120, - schedule_to_close_timeout=300, - heartbeat_timeout=15, -) -``` - -Child workflow starts use the same retry policy shape and workflow-level -execution/run timeout names: - -```python -from durable_workflow import ChildWorkflowRetryPolicy - -receipt = yield ctx.start_child_workflow( - "payment.child", - [order], - retry_policy=ChildWorkflowRetryPolicy( - max_attempts=3, - initial_interval_seconds=2, - backoff_coefficient=2, - non_retryable_error_types=["ValidationError"], - ), - execution_timeout_seconds=600, - run_timeout_seconds=120, -) -``` - -## Deterministic parallel groups - -Yield a list to schedule one durable parallel barrier. Lists can nest and mix -activities, child workflows, and timers. The worker flattens only the Server -commands, records a stable full `parallel_group_path` on every leaf, and -returns results in the original nested input shape regardless of terminal -delivery order: - -```python -results = yield [ - ctx.schedule_activity("load-profile", [customer_id]), - [ - ctx.start_child_workflow("quote-shipping", [customer_id]), - ctx.start_timer(5), - ], -] -profile, (shipping, _) = results -``` - -One failed activity or child is thrown at the list-yield point by durable input -position. Already recorded sibling completions remain replayable; late and -exact duplicate terminal deliveries do not change the selected result. - -## Saga compensation - -`ctx.saga()` registers ordinary activity commands as compensations and runs -them sequentially in reverse registration order after failure or cooperative -cancellation: - -```python -def forward(saga): - flight = yield ctx.schedule_activity("trip.reserve-flight", []) - saga.add_compensation("trip.cancel-flight", [flight]) - - hotel = yield ctx.schedule_activity("trip.reserve-hotel", []) - saga.add_compensation("trip.cancel-hotel", [hotel]) - - ctx.throw_if_cancellation_requested() - yield ctx.schedule_activity("trip.charge", []) - return {"status": "booked"} - -return (yield from ctx.saga().run(forward)) -``` - -Compensation stops at its first failure. `SagaCompensationFailed` retains the -initiating failure, compensation failure, activity type, and deterministic -registration order as structured fields. - -## Nexus service calls - -Workflow code can call a registered Nexus service operation through -`WorkflowContext.call_nexus_service(...)`. The worker executes the service -operation through the service-catalog API, records the response or typed -failure as a durable side-effect marker, and resumes replay from that marker -on subsequent workflow tasks. - -```python -from durable_workflow import NexusOperationFailed - -try: - result = yield ctx.call_nexus_service( - "greeter", - "shared", - "greet", - ["Ada"], - service_sdk_language="workflow-php", - ) - print(result.service_call_id, result.result) -except NexusOperationFailed as exc: - print(exc.service_call_id, exc.service_error_type, exc.typed_error_message) -``` - -The SDK assigns a deterministic idempotency key when one is not provided and -attaches the caller workflow instance id, caller run id, `sdk-python` caller -language, target service language, operation name, request payload, -service-call id, response or failure surface, and optional artifact metadata -to the recorded result. - -## Workflow signals, queries, and updates - -Signals mutate workflow state during replay: - -```python -@workflow.defn(name="approval") -class ApprovalWorkflow: - def __init__(self) -> None: - self.approved = False - - @workflow.signal("approve") - def approve(self, by: str) -> None: - self.approved = True - - @workflow.query("status") - def status(self) -> dict: - return {"approved": self.approved} - - @workflow.update("set_approval") - def set_approval(self, approved: bool) -> dict: - self.approved = approved - return {"approved": self.approved} - - @set_approval.validator - def validate_set_approval(self, approved: bool) -> None: - if not isinstance(approved, bool): - raise ValueError("approved must be boolean") -``` - -The Python SDK records query and update receiver metadata on workflow classes. -Python workers poll server-routed query tasks, replay workflow state, execute -the declared query handler, and complete or fail each task back to the Server. -The Server must advertise -`worker_protocol.server_capabilities.query_tasks: true` from -`GET /api/cluster/info`; workers advertise `query_tasks` at registration only -after that discovery succeeds. `Client.query_workflow()` checks the same -manifest before sending a query and raises `RuntimeCapabilityUnsupported` or -`RuntimeDiscoveryUnavailable` with remediation when the route cannot be used. - -Python workers advertise declared update validators and evaluate them on a -dedicated synchronous validation task before the Server records an accepted -update. Validation replays the authoritative workflow state without committing -commands or invoking the update handler. A validator-bearing worker refuses to -register unless Server discovery advertises the exact pre-accept validation -contract, so `wait_for="accepted"` means the declared validator has approved the -update. Rejections raise `UpdateRejected`; worker loss, timeout, incompatible -workers, and unsupported capability paths raise `UpdateValidationFailed` with -the Server's typed reason and retryability. `wait_for="completed"` additionally -waits for the accepted update handler to reach its terminal outcome. - -Malformed signal and query payloads are reported as typed client errors with -the server's documented reason and status preserved: - -```python -from durable_workflow import Client, QueryFailed, SignalFailed - -client = Client("http://localhost:8080") - -try: - await client.signal_workflow("counter-1", "increment", args=["not-an-int"]) -except SignalFailed as exc: - assert exc.reason == "invalid_signal_arguments" - assert exc.status == 422 - assert exc.validation_errors is not None - -try: - await client.query_workflow("counter-1", "current-at", args=["not-an-int"]) -except QueryFailed as exc: - assert exc.reason == "invalid_query_arguments" - assert exc.status == 422 - assert exc.validation_errors is not None -``` - -Use `yield ctx.wait_condition(lambda: self.approved, key="approved", -timeout=30)` to wait for signal- or update-mutated workflow state without -polling timers by hand. The SDK sends a stable predicate fingerprint with the -durable wait command and rejects replay if history records a different wait -key or predicate fingerprint, so condition changes fail visibly instead of -silently resolving a different wait. - -Workers fingerprint registered workflow class definitions and advertise those -fingerprints during registration. Re-registering the same `worker_id` with a -changed class body for an already advertised workflow type raises immediately; -restart the worker process with a new id before serving changed workflow code. - -Workers also advertise their local workflow and activity concurrency limits -during registration. Tune `max_concurrent_workflow_tasks` and -`max_concurrent_activity_tasks` on `Worker(...)` to align local semaphores with -the server's task-queue admission and operator visibility surfaces. Use -`Client.list_task_queues()` or `Client.describe_task_queue("orders")` to read -the server-side workflow, activity, and query-task admission status before -tuning those local limits: - -```python -queues = await client.list_task_queues() -for queue in queues.task_queues: - workflow_admission = queue.admission.workflow_tasks if queue.admission else None - print(queue.name, workflow_admission.status if workflow_admission else "unknown") -``` - -The workflow and activity admission objects expose both queue-level and -namespace-level server budgets, including active lease caps and per-minute -dispatch-rate limits, so automation can detect whether local worker slots, -queue caps, namespace caps, or downstream dispatch budget groups are -constraining throughput. - -## Replay captured histories - -Use `Replayer` to debug a captured history without connecting to a live server: - -```python -from durable_workflow import Replayer - -replayer = Replayer(workflows=[ApprovalWorkflow]) -outcome = replayer.replay(history_export) - -for command in outcome.commands: - print(command) -``` - -`history_export` can be the server's event list or a dictionary with an -`events` key. When the history contains a `WorkflowStarted` event, the replayer -infers the workflow type and input from that event; otherwise pass -`workflow_type=` and `start_input=` explicitly. The returned `ReplayOutcome` -contains the commands the workflow would emit next, including determinism -failures surfaced as workflow failure commands. - -For CI and operator replay gates, the package also installs offline -verification commands: - -```bash -durable-workflow-replay-verify tests/fixtures/golden_history \ - --workflows my_app.workflows:all_workflows \ - --output replay-report.json - -durable-workflow-replay-verify exported-history-bundles \ - --simulate-bundles \ - --output replay-simulation.json - -durable-workflow-history-bundle-verify exported-history-bundles/run-001.json \ - --output integrity-report.json -``` - -`durable-workflow-replay-verify` emits the same verdict and -`promotion_decision` vocabulary as the platform replay contract. Golden-history -mode replays cross-runtime fixtures against registered workflow classes; -`--simulate-bundles` integrity-checks every exported history bundle in a -directory and reports missing bundle evidence as a blocking result. Because -bundle simulation does not execute workflow code in Python, a clean -integrity-only simulation recommends `review_before_promote` rather than -`safe_to_promote`. - -## Python conformance gate - -The package includes the Python SDK published-artifact parity contract used by -host conformance runners: - -```bash -durable-workflow-python-conformance --manifest --pretty -durable-workflow-python-conformance --host-evidence --pretty -durable-workflow-python-conformance --compose host-evidence.json --pretty > python-conformance-result.json -durable-workflow-python-conformance --evaluate python-conformance-result.json --pretty -``` - -The evaluator rejects smoke-only evidence. A passing record must include the -official CLI install/start/result path, cold first-user setup, concrete -artifact versions, protocol traces, a no-PHP-assumption audit, and the complete -Python capability table. Host runners can feed their raw published-artifact -observations to `--compose`; omitted parity cells become explicit -`not_covered` entries so the gate reports the remaining scenario or capability -instead of accepting a smoke-only result. The composer accepts canonical -snake_case IDs and runbook-style hyphenated IDs such as `server-up` and -`result-returned`, nested runner tables, resolved artifact/source aliases, -boolean `passed` cells, nested protocol trace planes, and no-PHP audit check -aliases. CLI result-path evidence should come from the actual published -commands that return terminal workflow output: `workflow:start --wait`, -`workflow:describe`, or `workflow:show-run --follow`. - -## External payload storage - -Large payload transport is automatic when the namespace runtime advertises the -authenticated external-payload capability. `Client` keeps small Avro payloads -inline, uploads larger encoded bytes through the runtime URL, and sends only an -opaque runtime-owned reference. Incoming references are fetched with the same -namespace and role credential, then size and SHA-256 are verified before Avro -decode. The bounded verified-byte cache reduces repeated replay fetches and -never deletes runtime-owned objects. - -Managed Cloud applications do not configure a bucket, container, provider SDK, -provider credential, or provider URI parser. The ordinary client configuration -is sufficient for client operations and workers: +Pass the Server origin to `Client` without a trailing `/api`. For Cloud, pass +the complete namespace runtime URL exactly as provisioned. Cloud client and +worker processes use separate runtime credentials: ```python -from durable_workflow import Client - client = Client( - "https://runtime.example", - token=runtime_role_credential, - namespace="billing", -) -``` - -The local filesystem, S3, GCS, and Azure Blob drivers remain available only as -explicit self-hosted integrations for runtimes that advertise acceptance of -direct provider references. Selecting one requires passing an -`external_storage` instance yourself; namespace discovery never constructs a -provider driver from the runtime's backing-storage identity. Those adapters -are not the managed Cloud contract and the SDK does not install their provider -libraries. - -## Features - -- **Async-first**: Built on `httpx` and `asyncio` -- **Type-safe**: Full type hints, passes `mypy --strict` -- **Polyglot**: Works alongside PHP workers on the same task queue -- **HTTP/JSON protocol**: No gRPC, no protobuf dependencies -- **Codec envelopes**: Avro is the sole workflow payload codec; JSON remains the HTTP document transport -- **External payload references**: automatic runtime-mediated upload/fetch with opaque references, typed failures, integrity verification, and a bounded cache; direct provider drivers remain explicit self-hosted integrations -- **Payload-size warnings**: Structured warnings before oversized workflow, activity, schedule, signal, update, query, or search-attribute payloads reach the server -- **Workflow definition guard**: Worker registration refuses same-id hot reloads when a workflow class definition changed -- **Deterministic workflow helpers**: `ctx.now()`, `ctx.random()`, `ctx.uuid4()`, and `ctx.uuid7()` replay from workflow state -- **Worker interceptors**: Typed hooks around workflow tasks, activity calls, and query tasks for tracing, logging, and custom metrics -- **Metrics hooks**: Pluggable counters and histograms, with an optional Prometheus adapter - -## Payload-size warnings - -The SDK logs a structured warning before an encoded payload reaches 80% of the -default 2 MiB server payload limit. Warnings include context such as -`workflow_id`, `workflow_type`, `activity_name`, `schedule_id`, `signal_name`, -`update_name`, `query_name`, `payload_size`, `threshold_bytes`, and -`limit_bytes` when those fields are known at the call site. - -Tune or disable the warning threshold on the client: - -```python -client = Client( - "https://workflow.example.internal", - payload_size_limit_bytes=4 * 1024 * 1024, - payload_size_warning_threshold_percent=75, -) - -quiet_client = Client( - "https://workflow.example.internal", - payload_size_warnings=False, -) -``` - -## Avro payload type boundaries - -The default Avro codec uses the fixed recursive -`durable_workflow.protocol.Value` schema and standard single-object framing. -It preserves `None`, booleans, signed 64-bit integers, finite doubles, bytes, -UTF-8 strings, lists, and dictionaries with string keys as distinct branches. -Unknown schema fingerprints fail with `unsupported_payload_schema`; the codec -never guesses or silently falls back to JSON. - -Class-carrying values are not encoded with type metadata. Convert pydantic -models, attrs classes, dataclasses, pendulum values, `datetime` / `date` / -`time`, `UUID`, `Decimal`, and plain `Enum` values to explicit dictionaries or -scalars before passing them to the SDK. `IntEnum` and `StrEnum` encode because -they are Python scalar subclasses selected as Avro `LongValue` and -`StringValue`, but they decode as `int` and `str`. -`OrderedDict` decodes as a plain `dict`. - -Use `to_avro_payload_value(...)` when a rich value should enter durable -history through the default Avro envelope: - -```python -from dataclasses import dataclass -from datetime import datetime, timezone -from decimal import Decimal -from enum import Enum -from uuid import UUID - -from durable_workflow import Client, to_avro_payload_value - - -class OrderStatus(Enum): - PENDING = "pending" - - -@dataclass -class OrderInput: - order_id: UUID - placed_at: datetime - amount: Decimal - status: OrderStatus - - -order = OrderInput( - order_id=UUID("12345678-1234-5678-1234-567812345678"), - placed_at=datetime.now(timezone.utc), - amount=Decimal("10.25"), - status=OrderStatus.PENDING, -) - -client = Client("http://server:8080", token="dev-token-123") -await client.start_workflow( - "order-workflow", - task_queue="orders", - workflow_id="order-123", - input=[to_avro_payload_value(order)], -) -``` - -The helper also accepts pydantic-style models with `model_dump(mode="json")` -and attrs-style classes. Rebuild domain objects explicitly inside workflows or -activities, for example `OrderInput(order_id=UUID(data["order_id"]), ...)`. -Adapter output is part of the durable history contract, so changing that shape -is a workflow compatibility change. - -## Authentication - -For local servers that use one shared bearer token, pass `token=`: - -```python -client = Client("http://server:8080", token="shared-token", namespace="default") -``` - -For production servers with role-scoped tokens, keep worker and control -credentials in separate processes. A worker process needs only its worker -credential; the SDK uses it for cluster discovery, registration, polling, -heartbeats, and graceful deregistration: - -```python -worker_client = Client( - "https://workflow.example.internal", - worker_token="worker-token", - namespace="orders", -) -worker = Worker(worker_client, task_queue="orders", workflows=[OrderWorkflow]) -``` - -A control process uses only its operator or admin credential: - -```python -control_client = Client( - "https://workflow.example.internal", - control_token="operator-token", - namespace="orders", -) -handle = await control_client.start_workflow( - "order-workflow", - task_queue="orders", - workflow_id="order-123", + runtime_url, + control_token=client_token, + worker_token=worker_token, + namespace=namespace, ) ``` -Create one client per namespace when your deployment issues namespace-scoped -tokens. The SDK sends the configured token as `Authorization: Bearer ...` and -the namespace as `X-Namespace` on every request. Scoped credentials are never -substituted across roles: `worker_token` authorizes only worker-plane requests, -and `control_token` authorizes only control-plane requests. Cluster discovery is -the explicit exception because the server permits both roles to inspect its -compatibility manifest. A client configured with only the opposite role's token -still fails before transport for actual worker or control operations; use -`token` when one shared credential intentionally authorizes both planes. +Keep the client token in application processes and the worker token in worker +processes when deploying them separately. -## Metrics +## Capabilities -Pass a recorder to `Client(metrics=...)` or `Worker(metrics=...)` to collect request, poll, and task metrics. The SDK ships a no-op default, an `InMemoryMetrics` recorder for tests or custom exporter loops, and `PrometheusMetrics` for deployments that install the optional extra: +- Workflows, activities, child workflows, timers, and continue-as-new +- Signals, queries, validated updates, schedules, and message streams +- Activity retries, timeouts, cancellation, and heartbeats +- Deterministic parallel work, side effects, version markers, and sagas +- Replay verification and an in-process workflow test environment +- Avro payloads, external payload storage, metrics, and interceptors -```bash -pip install 'durable-workflow[prometheus]' -``` - -```python -from durable_workflow import Client, PrometheusMetrics - -metrics = PrometheusMetrics() -client = Client("http://server:8080", token="dev-token-123", metrics=metrics) -``` - -Custom recorders implement `increment(name, value=1.0, tags=None)` and `record(name, value, tags=None)`. - -## Worker interceptors - -Use `Worker(interceptors=[...])` when instrumentation needs the task payload, -result, or exception around worker execution instead of only aggregate counters. -Interceptors run in list order; the first interceptor is the outermost wrapper. - -```python -from durable_workflow import ( - ActivityInterceptorContext, - ActivityHandler, - PassthroughWorkerInterceptor, - Worker, -) - -class LoggingInterceptor(PassthroughWorkerInterceptor): - async def execute_activity( - self, - context: ActivityInterceptorContext, - next: ActivityHandler, - ) -> object: - print("activity started", context.activity_type) - try: - result = await next(context) - except Exception: - print("activity failed", context.activity_type) - raise - print("activity completed", context.activity_type) - return result - -worker = Worker( - client, - task_queue="python-workers", - workflows=[GreeterWorkflow], - activities=[greet], - interceptors=[LoggingInterceptor()], -) -``` +See the [capability matrix](https://durable-workflow.com/docs/2.0/capabilities/) +for the complete cross-SDK contract. ## Documentation -Full documentation is available at -[durable-workflow.github.io/docs/2.0/polyglot/python](https://durable-workflow.github.io/docs/2.0/polyglot/python): - -- [Python SDK guide](https://durable-workflow.com/docs/2.0/polyglot/python) -- [API reference](https://python.durable-workflow.com/) +- [Python SDK portal and API reference](https://python.durable-workflow.com/) +- [Python SDK guide](https://durable-workflow.com/docs/2.0/polyglot/python/) +- [Complete SDK reference](docs/sdk-reference.md) +- [Runnable examples](examples/) +- [Symmetric SDK playground](https://github.com/durable-workflow/sample-app#symmetric-sdk-playground) -## Requirements +## Runtime choices -- Python ≥ 3.10 -- A running [Durable Workflow server](https://github.com/durable-workflow/server) +Use [Durable Workflow Cloud](https://cloud.durable-workflow.com/early-access) +for a managed namespace, or run the published +[`durableworkflow/server`](https://hub.docker.com/r/durableworkflow/server) +image yourself. Workflow and activity type names, task queues, and payloads are +portable between both runtime choices. ## Compatibility -The exact SDK and qualified Server versions are published together in the -[API reference](https://python.durable-workflow.com/). Their release identities -advance independently; matching RC sequence numbers are not required. The -server must advertise these protocol manifests from `GET /api/cluster/info`: - -- `control_plane.version: "2"` -- `control_plane.request_contract.schema: durable-workflow.v2.control-plane-request.contract` version `1` -- `auth_composition_contract.schema: durable-workflow.v2.auth-composition.contract` version `1` -- worker_protocol.version: >=1.19,<2.0 -- `worker_protocol.external_task_input_contract.schema: durable-workflow.v2.external-task-input.contract` version `1` -- `worker_protocol.external_task_result_contract.schema: durable-workflow.v2.external-task-result.contract` version `1` - -The top-level server `version` is build identity only. The worker checks these -protocol manifests at startup and fails closed when compatibility is missing, -unknown, or undiscoverable. - -Carriers and support tooling can validate `auth_composition_contract` with -`parse_auth_composition_contract()` before resolving connection, namespace, -token, TLS, profile, and redacted effective-configuration diagnostics. - -External task carriers can validate fixture artifacts from -`worker_protocol.external_task_input_contract.fixtures` with -`parse_external_task_input_artifact()` and parse leased task envelopes with -`parse_external_task_input()`. - -They can also validate result fixture artifacts from -`worker_protocol.external_task_result_contract.fixtures` with -`parse_external_task_result_artifact()` and parse result envelopes with -`parse_external_task_result()`. The result parser exposes stable carrier -decisions for success, retryability, malformed output, cancellation, deadline -exceeded, handler crash, decode failure, and unsupported payload -codec/reference states without treating stderr as a machine signal. - -Invocable activity carriers can use `InvocableActivityHandler` as a reference -adapter for HTTP or serverless runtimes. It accepts the same external-task input -envelope, invokes a registered activity handler, and returns the same -external-task result envelope while rejecting workflow-task inputs: - -```python -from durable_workflow import InvocableActivityHandler - -adapter = InvocableActivityHandler({"billing.charge-card": charge_card}) -result_envelope = await adapter.handle(request_json) -``` - -Bridge adapters can hand bounded webhook ingress into the server through -`Client.send_webhook_bridge_event()`. The method returns the server's typed -bridge outcome for accepted, duplicate, and rejected events, including -machine-readable HTTP 422 rejection outcomes: - -```python -outcome = await client.send_webhook_bridge_event( - "pagerduty", - action="signal_workflow", - idempotency_key="pagerduty-event-3003", - target={"workflow_id": "wf-remediation-42", "signal_name": "incident_escalated"}, - input={"severity": "critical", "service": "checkout"}, - correlation={"provider": "pagerduty", "event_type": "incident.triggered"}, -) - -if outcome.accepted: - print(outcome.workflow_id, outcome.control_plane_outcome) -else: - print(outcome.outcome, outcome.reason) -``` +Stable `2.x` SDK releases follow semantic versioning and negotiate runtime +capabilities with Server at startup. Use stable `2.x` SDK and Server channels +for new applications. The [compatibility guide](https://durable-workflow.com/docs/2.0/compatibility/) +documents protocol and upgrade guarantees. ## Development ```bash -# Install dev dependencies pip install -e '.[dev]' - -# Run tests -pytest - -# Run integration tests (requires Docker) -pytest -m integration - -# Type check +ruff check src/ tests/ mypy src/durable_workflow/ +pytest tests/ -m "not integration" +``` -# Lint -ruff check src/ tests/ +Integration tests use Docker: -# Preview the API reference site locally -pip install -e '.[docs]' -mkdocs serve +```bash +docker compose -f docker-compose.test.yml up -d --build --wait +pytest tests/integration/ -v +docker compose -f docker-compose.test.yml down -v ``` -The API reference is published to [python.durable-workflow.com](https://python.durable-workflow.com/) and rebuilt automatically on push to `main`. - ## License -MIT +[MIT](LICENSE) diff --git a/docs/index.md b/docs/index.md index f78e910..704a256 100644 --- a/docs/index.md +++ b/docs/index.md @@ -34,11 +34,11 @@ namespace. ## Install ```bash -curl -fsSL https://durable-workflow.com/install-sdk.sh | sh -s -- python +pip install durable-workflow ``` -The versionless resolver reads the public quickstart contract and invokes pip -with its qualified SDK identity. +Use a virtual environment and lock the resolved package version with the rest +of your application dependencies.

View on PyPISource on GitHub

@@ -93,17 +93,15 @@ endpoint, credentials, and operating boundary change. ## Run your first local workflow -This source-free path resolves the compatibility-qualified Server image from -the same public quickstart contract as the SDK installer, then runs one Python -file containing an activity, workflow, worker, and client. +This source-free path runs one Python file containing an activity, workflow, +worker, and client against the stable Server channel. ### 1. Start Server -Docker keeps this first run local. Resolve the Server image without copying a -release version into the page: +Docker keeps this first run local. Select the stable 2.x Server channel: ```bash -{{ durable_workflow_server_image_resolver }} +export DW_SERVER_IMAGE='durableworkflow/server:2' ``` Then bootstrap and start that qualified image: @@ -257,12 +255,9 @@ python greeter.py ## Versioning - - -The SDK installer and Server image resolver both read the public quickstart -contract. Neither command stores a release-candidate sequence number in this -page. Lock the resolved package and Server image digest in your application -when you need reproducible builds. +Stable 2.x SDK releases follow semantic versioning and negotiate runtime +capabilities with Server at startup. Lock the resolved Python package version +and Server image digest in production builds. diff --git a/docs/sdk-reference.md b/docs/sdk-reference.md new file mode 100644 index 0000000..97aa1e7 --- /dev/null +++ b/docs/sdk-reference.md @@ -0,0 +1,785 @@ +# Python SDK reference + +This extended guide covers the complete public surface of the Durable Workflow +Python SDK for [Cloud](https://cloud.durable-workflow.com/) and +[self-hosted Server](https://github.com/durable-workflow/server). Start with +the [SDK portal](index.md) for the shortest runnable path. + +Status: **Stable 2.0**. Core features include workflows, activities, +schedules, signals, timers, child workflows, continue-as-new, side effects, +version markers, worker-applied accepted updates, replay verification, the +in-process `WorkflowEnvironment` test harness, and invocable activity carriers. + +Python workers execute server-routed query tasks after the Server advertises the query-tasks capability through cluster discovery. + +## Install + +```bash +pip install durable-workflow +``` + +Use a virtual environment and lock the resolved package version with the rest +of your application dependencies. + +Or for development: + +```bash +pip install -e '.[dev]' +``` + +## Quickstart + +```python +import asyncio +from uuid import uuid4 + +from durable_workflow import Client, Worker, workflow, activity + +@activity.defn(name="greet") +def greet(name: str) -> str: + return f"hello, {name}" + +@workflow.defn(name="greeter") +class GreeterWorkflow: + def run(self, ctx, name): + result = yield ctx.schedule_activity("greet", [name]) + return result + +async def main(): + workflow_id = f"greet-{uuid4().hex}" + async with Client( + "http://server:8080", + token="dev-token-123", + namespace="default", + ) as client: + worker = Worker( + client, + task_queue="python-workers", + workflows=[GreeterWorkflow], + activities=[greet], + ) + handle = await client.start_workflow( + workflow_type="greeter", + workflow_id=workflow_id, + task_queue="python-workers", + input=["world"], + ) + await worker.run_until(workflow_id=workflow_id, timeout=30.0) + result = await client.get_result(handle) + print(result) # "hello, world" + +if __name__ == "__main__": + asyncio.run(main()) +``` + +Pass the server or managed-runtime base URL to `Client`; the SDK appends its +own `/api` routes. For example, use `http://server:8080`, not +`http://server:8080/api`. Managed-runtime paths such as +`https://cloud.example/api/runtime/v1/namespaces/acme` are valid as written. + +A workflow ID is the durable identity of an instance, not a per-attempt request +ID. Starting the same ID again with the default `reject` duplicate policy raises +the typed `WorkflowAlreadyStarted` exception. Generate a unique ID for each new +instance, as the quickstart does. For an intentionally idempotent start, catch +`WorkflowAlreadyStarted` and reconnect with +`client.get_workflow_handle(workflow_id)`; choose `allow` or +`terminate_existing` only when creating another run or replacing the current +instance is the intended behavior. + +For a fuller deployable example, see +[`examples/order_processing`](https://github.com/durable-workflow/sdk-python/tree/main/examples/order_processing), which runs a +multi-activity order workflow against a local server with Docker Compose. + +## Schedule visibility and paging + +`list_schedules()` returns one typed `ScheduleList` page. Status and workflow +type are exact server-side filters; the visibility query uses the server's +documented equality-predicate grammar. All filters combine with AND semantics. + +```python +page = await client.list_schedules( + status="active", + workflow_type="orders.rollup", + query='Region = "eu" AND Priority = 2', + page_size=25, +) + +while page.next_page_token is not None: + page = await client.list_schedules( + status="active", + workflow_type="orders.rollup", + query='Region = "eu" AND Priority = 2', + page_size=25, + next_page_token=page.next_page_token, + ) +``` + +Continuation tokens are opaque. Reuse them unchanged with the same namespace, +status, workflow type, and query; `None` terminates traversal. Invalid filters +and malformed, mismatched, cross-namespace, or stale tokens raise +`ScheduleListError`, which retains `status`, `reason()`, `field`, `errors`, +`last_safe_cursor`, and the complete server response in `body`. + +## Retry policy scopes + +Retry and timeout settings are scoped to the layer where you configure them: + +- `TransportRetryPolicy` on `Client(...)` retries SDK HTTP requests only. It handles transient connection failures, request timeouts, 5xx responses, and 429 rate limits. It does not retry workflow code, activity code, child workflows, or failed workflow runs. +- `ActivityRetryPolicy` on `ctx.schedule_activity(...)` is recorded into durable history with that activity command. It controls server-side attempts for that one activity execution. +- `ChildWorkflowRetryPolicy` on `ctx.start_child_workflow(...)` is recorded with that child-start command. It controls server-side attempts for that child workflow execution. +- `non_retryable_error_types` belongs to durable activity/child retry policies. `non_retryable=True` on an activity failure bypasses the activity retry budget and surfaces the failure to the workflow. + +Timeout names are also layer-specific. `start_to_close_timeout` limits one activity attempt, `schedule_to_start_timeout` limits queue wait before an activity starts, `schedule_to_close_timeout` limits the whole activity execution including retries, and `heartbeat_timeout` limits the gap between activity heartbeats. For child workflows, `execution_timeout_seconds` covers the overall child workflow execution and `run_timeout_seconds` covers one run. + +## Activity failure payloads + +When replay raises `ActivityFailed`, the top-level attributes expose the +stable cross-language fields: `activity_type`, `failure_category`, +`exception_type`, `message`, `non_retryable`, and `code`. The +`exception_payload` dictionary is filtered to language-neutral keys such as +`type`, `message`, `details`, `details_payload_codec`, and `non_retryable`. +Runtime diagnostics like PHP or Python exception classes, source file paths, +line numbers, and traces are not included by default unless the history event +contains an explicit `diagnostics` or `runtime_diagnostics` envelope. + +## Activity retries and timeouts + +Configure per-call activity retries and deadlines from workflow code: + +```python +from durable_workflow import ActivityRetryPolicy + +result = yield ctx.schedule_activity( + "charge-card", + [order], + retry_policy=ActivityRetryPolicy( + max_attempts=4, + initial_interval_seconds=1, + backoff_coefficient=2, + maximum_interval_seconds=30, + non_retryable_error_types=["ValidationError"], + ), + start_to_close_timeout=120, + schedule_to_close_timeout=300, + heartbeat_timeout=15, +) +``` + +Child workflow starts use the same retry policy shape and workflow-level +execution/run timeout names: + +```python +from durable_workflow import ChildWorkflowRetryPolicy + +receipt = yield ctx.start_child_workflow( + "payment.child", + [order], + retry_policy=ChildWorkflowRetryPolicy( + max_attempts=3, + initial_interval_seconds=2, + backoff_coefficient=2, + non_retryable_error_types=["ValidationError"], + ), + execution_timeout_seconds=600, + run_timeout_seconds=120, +) +``` + +## Deterministic parallel groups + +Yield a list to schedule one durable parallel barrier. Lists can nest and mix +activities, child workflows, and timers. The worker flattens only the Server +commands, records a stable full `parallel_group_path` on every leaf, and +returns results in the original nested input shape regardless of terminal +delivery order: + +```python +results = yield [ + ctx.schedule_activity("load-profile", [customer_id]), + [ + ctx.start_child_workflow("quote-shipping", [customer_id]), + ctx.start_timer(5), + ], +] +profile, (shipping, _) = results +``` + +One failed activity or child is thrown at the list-yield point by durable input +position. Already recorded sibling completions remain replayable; late and +exact duplicate terminal deliveries do not change the selected result. + +## Saga compensation + +`ctx.saga()` registers ordinary activity commands as compensations and runs +them sequentially in reverse registration order after failure or cooperative +cancellation: + +```python +def forward(saga): + flight = yield ctx.schedule_activity("trip.reserve-flight", []) + saga.add_compensation("trip.cancel-flight", [flight]) + + hotel = yield ctx.schedule_activity("trip.reserve-hotel", []) + saga.add_compensation("trip.cancel-hotel", [hotel]) + + ctx.throw_if_cancellation_requested() + yield ctx.schedule_activity("trip.charge", []) + return {"status": "booked"} + +return (yield from ctx.saga().run(forward)) +``` + +Compensation stops at its first failure. `SagaCompensationFailed` retains the +initiating failure, compensation failure, activity type, and deterministic +registration order as structured fields. + +## Nexus service calls + +Workflow code can call a registered Nexus service operation through +`WorkflowContext.call_nexus_service(...)`. The worker executes the service +operation through the service-catalog API, records the response or typed +failure as a durable side-effect marker, and resumes replay from that marker +on subsequent workflow tasks. + +```python +from durable_workflow import NexusOperationFailed + +try: + result = yield ctx.call_nexus_service( + "greeter", + "shared", + "greet", + ["Ada"], + service_sdk_language="workflow-php", + ) + print(result.service_call_id, result.result) +except NexusOperationFailed as exc: + print(exc.service_call_id, exc.service_error_type, exc.typed_error_message) +``` + +The SDK assigns a deterministic idempotency key when one is not provided and +attaches the caller workflow instance id, caller run id, `sdk-python` caller +language, target service language, operation name, request payload, +service-call id, response or failure surface, and optional artifact metadata +to the recorded result. + +## Workflow signals, queries, and updates + +Signals mutate workflow state during replay: + +```python +@workflow.defn(name="approval") +class ApprovalWorkflow: + def __init__(self) -> None: + self.approved = False + + @workflow.signal("approve") + def approve(self, by: str) -> None: + self.approved = True + + @workflow.query("status") + def status(self) -> dict: + return {"approved": self.approved} + + @workflow.update("set_approval") + def set_approval(self, approved: bool) -> dict: + self.approved = approved + return {"approved": self.approved} + + @set_approval.validator + def validate_set_approval(self, approved: bool) -> None: + if not isinstance(approved, bool): + raise ValueError("approved must be boolean") +``` + +The Python SDK records query and update receiver metadata on workflow classes. +Python workers poll server-routed query tasks, replay workflow state, execute +the declared query handler, and complete or fail each task back to the Server. +The Server must advertise +`worker_protocol.server_capabilities.query_tasks: true` from +`GET /api/cluster/info`; workers advertise `query_tasks` at registration only +after that discovery succeeds. `Client.query_workflow()` checks the same +manifest before sending a query and raises `RuntimeCapabilityUnsupported` or +`RuntimeDiscoveryUnavailable` with remediation when the route cannot be used. + +Python workers advertise declared update validators and evaluate them on a +dedicated synchronous validation task before the Server records an accepted +update. Validation replays the authoritative workflow state without committing +commands or invoking the update handler. A validator-bearing worker refuses to +register unless Server discovery advertises the exact pre-accept validation +contract, so `wait_for="accepted"` means the declared validator has approved the +update. Rejections raise `UpdateRejected`; worker loss, timeout, incompatible +workers, and unsupported capability paths raise `UpdateValidationFailed` with +the Server's typed reason and retryability. `wait_for="completed"` additionally +waits for the accepted update handler to reach its terminal outcome. + +Malformed signal and query payloads are reported as typed client errors with +the server's documented reason and status preserved: + +```python +from durable_workflow import Client, QueryFailed, SignalFailed + +client = Client("http://localhost:8080") + +try: + await client.signal_workflow("counter-1", "increment", args=["not-an-int"]) +except SignalFailed as exc: + assert exc.reason == "invalid_signal_arguments" + assert exc.status == 422 + assert exc.validation_errors is not None + +try: + await client.query_workflow("counter-1", "current-at", args=["not-an-int"]) +except QueryFailed as exc: + assert exc.reason == "invalid_query_arguments" + assert exc.status == 422 + assert exc.validation_errors is not None +``` + +Use `yield ctx.wait_condition(lambda: self.approved, key="approved", +timeout=30)` to wait for signal- or update-mutated workflow state without +polling timers by hand. The SDK sends a stable predicate fingerprint with the +durable wait command and rejects replay if history records a different wait +key or predicate fingerprint, so condition changes fail visibly instead of +silently resolving a different wait. + +Workers fingerprint registered workflow class definitions and advertise those +fingerprints during registration. Re-registering the same `worker_id` with a +changed class body for an already advertised workflow type raises immediately; +restart the worker process with a new id before serving changed workflow code. + +Workers also advertise their local workflow and activity concurrency limits +during registration. Tune `max_concurrent_workflow_tasks` and +`max_concurrent_activity_tasks` on `Worker(...)` to align local semaphores with +the server's task-queue admission and operator visibility surfaces. Use +`Client.list_task_queues()` or `Client.describe_task_queue("orders")` to read +the server-side workflow, activity, and query-task admission status before +tuning those local limits: + +```python +queues = await client.list_task_queues() +for queue in queues.task_queues: + workflow_admission = queue.admission.workflow_tasks if queue.admission else None + print(queue.name, workflow_admission.status if workflow_admission else "unknown") +``` + +The workflow and activity admission objects expose both queue-level and +namespace-level server budgets, including active lease caps and per-minute +dispatch-rate limits, so automation can detect whether local worker slots, +queue caps, namespace caps, or downstream dispatch budget groups are +constraining throughput. + +## Replay captured histories + +Use `Replayer` to debug a captured history without connecting to a live server: + +```python +from durable_workflow import Replayer + +replayer = Replayer(workflows=[ApprovalWorkflow]) +outcome = replayer.replay(history_export) + +for command in outcome.commands: + print(command) +``` + +`history_export` can be the server's event list or a dictionary with an +`events` key. When the history contains a `WorkflowStarted` event, the replayer +infers the workflow type and input from that event; otherwise pass +`workflow_type=` and `start_input=` explicitly. The returned `ReplayOutcome` +contains the commands the workflow would emit next, including determinism +failures surfaced as workflow failure commands. + +For CI and operator replay gates, the package also installs offline +verification commands: + +```bash +durable-workflow-replay-verify tests/fixtures/golden_history \ + --workflows my_app.workflows:all_workflows \ + --output replay-report.json + +durable-workflow-replay-verify exported-history-bundles \ + --simulate-bundles \ + --output replay-simulation.json + +durable-workflow-history-bundle-verify exported-history-bundles/run-001.json \ + --output integrity-report.json +``` + +`durable-workflow-replay-verify` emits the same verdict and +`promotion_decision` vocabulary as the platform replay contract. Golden-history +mode replays cross-runtime fixtures against registered workflow classes; +`--simulate-bundles` integrity-checks every exported history bundle in a +directory and reports missing bundle evidence as a blocking result. Because +bundle simulation does not execute workflow code in Python, a clean +integrity-only simulation recommends `review_before_promote` rather than +`safe_to_promote`. + +## Python conformance gate + +The package includes the Python SDK published-artifact parity contract used by +host conformance runners: + +```bash +durable-workflow-python-conformance --manifest --pretty +durable-workflow-python-conformance --host-evidence --pretty +durable-workflow-python-conformance --compose host-evidence.json --pretty > python-conformance-result.json +durable-workflow-python-conformance --evaluate python-conformance-result.json --pretty +``` + +The evaluator rejects smoke-only evidence. A passing record must include the +official CLI install/start/result path, cold first-user setup, concrete +artifact versions, protocol traces, a no-PHP-assumption audit, and the complete +Python capability table. Host runners can feed their raw published-artifact +observations to `--compose`; omitted parity cells become explicit +`not_covered` entries so the gate reports the remaining scenario or capability +instead of accepting a smoke-only result. The composer accepts canonical +snake_case IDs and runbook-style hyphenated IDs such as `server-up` and +`result-returned`, nested runner tables, resolved artifact/source aliases, +boolean `passed` cells, nested protocol trace planes, and no-PHP audit check +aliases. CLI result-path evidence should come from the actual published +commands that return terminal workflow output: `workflow:start --wait`, +`workflow:describe`, or `workflow:show-run --follow`. + +## External payload storage + +Large payload transport is automatic when the namespace runtime advertises the +authenticated external-payload capability. `Client` keeps small Avro payloads +inline, uploads larger encoded bytes through the runtime URL, and sends only an +opaque runtime-owned reference. Incoming references are fetched with the same +namespace and role credential, then size and SHA-256 are verified before Avro +decode. The bounded verified-byte cache reduces repeated replay fetches and +never deletes runtime-owned objects. + +Managed Cloud applications do not configure a bucket, container, provider SDK, +provider credential, or provider URI parser. The ordinary client configuration +is sufficient for client operations and workers: + +```python +from durable_workflow import Client + +client = Client( + "https://runtime.example", + token=runtime_role_credential, + namespace="billing", +) +``` + +The local filesystem, S3, GCS, and Azure Blob drivers remain available only as +explicit self-hosted integrations for runtimes that advertise acceptance of +direct provider references. Selecting one requires passing an +`external_storage` instance yourself; namespace discovery never constructs a +provider driver from the runtime's backing-storage identity. Those adapters +are not the managed Cloud contract and the SDK does not install their provider +libraries. + +## Features + +- **Async-first**: Built on `httpx` and `asyncio` +- **Type-safe**: Full type hints, passes `mypy --strict` +- **Polyglot**: Works alongside PHP workers on the same task queue +- **HTTP/JSON protocol**: No gRPC, no protobuf dependencies +- **Codec envelopes**: Avro is the sole workflow payload codec; JSON remains the HTTP document transport +- **External payload references**: automatic runtime-mediated upload/fetch with opaque references, typed failures, integrity verification, and a bounded cache; direct provider drivers remain explicit self-hosted integrations +- **Payload-size warnings**: Structured warnings before oversized workflow, activity, schedule, signal, update, query, or search-attribute payloads reach the server +- **Workflow definition guard**: Worker registration refuses same-id hot reloads when a workflow class definition changed +- **Deterministic workflow helpers**: `ctx.now()`, `ctx.random()`, `ctx.uuid4()`, and `ctx.uuid7()` replay from workflow state +- **Worker interceptors**: Typed hooks around workflow tasks, activity calls, and query tasks for tracing, logging, and custom metrics +- **Metrics hooks**: Pluggable counters and histograms, with an optional Prometheus adapter + +## Payload-size warnings + +The SDK logs a structured warning before an encoded payload reaches 80% of the +default 2 MiB server payload limit. Warnings include context such as +`workflow_id`, `workflow_type`, `activity_name`, `schedule_id`, `signal_name`, +`update_name`, `query_name`, `payload_size`, `threshold_bytes`, and +`limit_bytes` when those fields are known at the call site. + +Tune or disable the warning threshold on the client: + +```python +client = Client( + "https://workflow.example.internal", + payload_size_limit_bytes=4 * 1024 * 1024, + payload_size_warning_threshold_percent=75, +) + +quiet_client = Client( + "https://workflow.example.internal", + payload_size_warnings=False, +) +``` + +## Avro payload type boundaries + +The default Avro codec uses the fixed recursive +`durable_workflow.protocol.Value` schema and standard single-object framing. +It preserves `None`, booleans, signed 64-bit integers, finite doubles, bytes, +UTF-8 strings, lists, and dictionaries with string keys as distinct branches. +Unknown schema fingerprints fail with `unsupported_payload_schema`; the codec +never guesses or silently falls back to JSON. + +Class-carrying values are not encoded with type metadata. Convert pydantic +models, attrs classes, dataclasses, pendulum values, `datetime` / `date` / +`time`, `UUID`, `Decimal`, and plain `Enum` values to explicit dictionaries or +scalars before passing them to the SDK. `IntEnum` and `StrEnum` encode because +they are Python scalar subclasses selected as Avro `LongValue` and +`StringValue`, but they decode as `int` and `str`. +`OrderedDict` decodes as a plain `dict`. + +Use `to_avro_payload_value(...)` when a rich value should enter durable +history through the default Avro envelope: + +```python +from dataclasses import dataclass +from datetime import datetime, timezone +from decimal import Decimal +from enum import Enum +from uuid import UUID + +from durable_workflow import Client, to_avro_payload_value + + +class OrderStatus(Enum): + PENDING = "pending" + + +@dataclass +class OrderInput: + order_id: UUID + placed_at: datetime + amount: Decimal + status: OrderStatus + + +order = OrderInput( + order_id=UUID("12345678-1234-5678-1234-567812345678"), + placed_at=datetime.now(timezone.utc), + amount=Decimal("10.25"), + status=OrderStatus.PENDING, +) + +client = Client("http://server:8080", token="dev-token-123") +await client.start_workflow( + "order-workflow", + task_queue="orders", + workflow_id="order-123", + input=[to_avro_payload_value(order)], +) +``` + +The helper also accepts pydantic-style models with `model_dump(mode="json")` +and attrs-style classes. Rebuild domain objects explicitly inside workflows or +activities, for example `OrderInput(order_id=UUID(data["order_id"]), ...)`. +Adapter output is part of the durable history contract, so changing that shape +is a workflow compatibility change. + +## Authentication + +For local servers that use one shared bearer token, pass `token=`: + +```python +client = Client("http://server:8080", token="shared-token", namespace="default") +``` + +For production servers with role-scoped tokens, keep worker and control +credentials in separate processes. A worker process needs only its worker +credential; the SDK uses it for cluster discovery, registration, polling, +heartbeats, and graceful deregistration: + +```python +worker_client = Client( + "https://workflow.example.internal", + worker_token="worker-token", + namespace="orders", +) +worker = Worker(worker_client, task_queue="orders", workflows=[OrderWorkflow]) +``` + +A control process uses only its operator or admin credential: + +```python +control_client = Client( + "https://workflow.example.internal", + control_token="operator-token", + namespace="orders", +) +handle = await control_client.start_workflow( + "order-workflow", + task_queue="orders", + workflow_id="order-123", +) +``` + +Create one client per namespace when your deployment issues namespace-scoped +tokens. The SDK sends the configured token as `Authorization: Bearer ...` and +the namespace as `X-Namespace` on every request. Scoped credentials are never +substituted across roles: `worker_token` authorizes only worker-plane requests, +and `control_token` authorizes only control-plane requests. Cluster discovery is +the explicit exception because the server permits both roles to inspect its +compatibility manifest. A client configured with only the opposite role's token +still fails before transport for actual worker or control operations; use +`token` when one shared credential intentionally authorizes both planes. + +## Metrics + +Pass a recorder to `Client(metrics=...)` or `Worker(metrics=...)` to collect request, poll, and task metrics. The SDK ships a no-op default, an `InMemoryMetrics` recorder for tests or custom exporter loops, and `PrometheusMetrics` for deployments that install the optional extra: + +```bash +pip install 'durable-workflow[prometheus]' +``` + +```python +from durable_workflow import Client, PrometheusMetrics + +metrics = PrometheusMetrics() +client = Client("http://server:8080", token="dev-token-123", metrics=metrics) +``` + +Custom recorders implement `increment(name, value=1.0, tags=None)` and `record(name, value, tags=None)`. + +## Worker interceptors + +Use `Worker(interceptors=[...])` when instrumentation needs the task payload, +result, or exception around worker execution instead of only aggregate counters. +Interceptors run in list order; the first interceptor is the outermost wrapper. + +```python +from durable_workflow import ( + ActivityInterceptorContext, + ActivityHandler, + PassthroughWorkerInterceptor, + Worker, +) + +class LoggingInterceptor(PassthroughWorkerInterceptor): + async def execute_activity( + self, + context: ActivityInterceptorContext, + next: ActivityHandler, + ) -> object: + print("activity started", context.activity_type) + try: + result = await next(context) + except Exception: + print("activity failed", context.activity_type) + raise + print("activity completed", context.activity_type) + return result + +worker = Worker( + client, + task_queue="python-workers", + workflows=[GreeterWorkflow], + activities=[greet], + interceptors=[LoggingInterceptor()], +) +``` + +## Documentation + +Full documentation is available at: + +- [Python SDK guide](https://durable-workflow.com/docs/2.0/polyglot/python) +- [API reference](https://python.durable-workflow.com/) + +## Requirements + +- Python ≥ 3.10 +- A running [Durable Workflow server](https://github.com/durable-workflow/server) + +## Compatibility + +Stable `2.x` releases follow semantic versioning. The SDK discovers runtime +capabilities at startup, and the server must advertise these protocol manifests +from `GET /api/cluster/info`: + +- `control_plane.version: "2"` +- `control_plane.request_contract.schema: durable-workflow.v2.control-plane-request.contract` version `1` +- `auth_composition_contract.schema: durable-workflow.v2.auth-composition.contract` version `1` +- `worker_protocol.version: >=1.19,<2.0` +- `worker_protocol.external_task_input_contract.schema: durable-workflow.v2.external-task-input.contract` version `1` +- `worker_protocol.external_task_result_contract.schema: durable-workflow.v2.external-task-result.contract` version `1` + +The top-level server `version` is build identity only. The worker checks these +protocol manifests at startup and fails closed when compatibility is missing, +unknown, or undiscoverable. + +Carriers and support tooling can validate `auth_composition_contract` with +`parse_auth_composition_contract()` before resolving connection, namespace, +token, TLS, profile, and redacted effective-configuration diagnostics. + +External task carriers can validate fixture artifacts from +`worker_protocol.external_task_input_contract.fixtures` with +`parse_external_task_input_artifact()` and parse leased task envelopes with +`parse_external_task_input()`. + +They can also validate result fixture artifacts from +`worker_protocol.external_task_result_contract.fixtures` with +`parse_external_task_result_artifact()` and parse result envelopes with +`parse_external_task_result()`. The result parser exposes stable carrier +decisions for success, retryability, malformed output, cancellation, deadline +exceeded, handler crash, decode failure, and unsupported payload +codec/reference states without treating stderr as a machine signal. + +Invocable activity carriers can use `InvocableActivityHandler` as a reference +adapter for HTTP or serverless runtimes. It accepts the same external-task input +envelope, invokes a registered activity handler, and returns the same +external-task result envelope while rejecting workflow-task inputs: + +```python +from durable_workflow import InvocableActivityHandler + +adapter = InvocableActivityHandler({"billing.charge-card": charge_card}) +result_envelope = await adapter.handle(request_json) +``` + +Bridge adapters can hand bounded webhook ingress into the server through +`Client.send_webhook_bridge_event()`. The method returns the server's typed +bridge outcome for accepted, duplicate, and rejected events, including +machine-readable HTTP 422 rejection outcomes: + +```python +outcome = await client.send_webhook_bridge_event( + "pagerduty", + action="signal_workflow", + idempotency_key="pagerduty-event-3003", + target={"workflow_id": "wf-remediation-42", "signal_name": "incident_escalated"}, + input={"severity": "critical", "service": "checkout"}, + correlation={"provider": "pagerduty", "event_type": "incident.triggered"}, +) + +if outcome.accepted: + print(outcome.workflow_id, outcome.control_plane_outcome) +else: + print(outcome.outcome, outcome.reason) +``` + +## Development + +```bash +# Install dev dependencies +pip install -e '.[dev]' + +# Run tests +pytest + +# Run integration tests (requires Docker) +pytest -m integration + +# Type check +mypy src/durable_workflow/ + +# Lint +ruff check src/ tests/ + +# Preview the API reference site locally +pip install -e '.[docs]' +mkdocs serve +``` + +The API reference is published to [python.durable-workflow.com](https://python.durable-workflow.com/) and rebuilt automatically on push to `main`. + +## License + +MIT diff --git a/mkdocs.yml b/mkdocs.yml index d3b9414..5ee47ac 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -5,9 +5,6 @@ repo_url: https://github.com/durable-workflow/sdk-python repo_name: durable-workflow/sdk-python edit_uri: edit/main/docs/ -hooks: - - scripts/mkdocs_hooks.py - theme: name: material custom_dir: overrides @@ -78,6 +75,7 @@ markdown_extensions: nav: - Start here: index.md + - SDK reference: sdk-reference.md - SDK guide: https://durable-workflow.com/docs/2.0/polyglot/python/ - PyPI: https://pypi.org/project/durable-workflow/ - API reference: diff --git a/pyproject.toml b/pyproject.toml index 58bc234..5b5eb83 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta" [project] name = "durable-workflow" version = "2.0.0" -description = "Python SDK for the Durable Workflow 2.0 platform" +description = "Python client and worker SDK for Durable Workflow Cloud and self-hosted Server" readme = "README.md" requires-python = ">=3.10" license = "MIT" @@ -14,10 +14,13 @@ authors = [ { name = "Durable Workflow Contributors" }, ] keywords = [ + "cloud", + "durable-execution", "workflow", "durable", "orchestration", - "temporal", + "python", + "sdk", "saga", ] classifiers = [ @@ -55,8 +58,8 @@ docs = [ ] [project.urls] -Homepage = "https://github.com/durable-workflow/sdk-python" -Documentation = "https://durable-workflow.github.io/docs/2.0/polyglot/python" +Homepage = "https://python.durable-workflow.com/" +Documentation = "https://python.durable-workflow.com/" Repository = "https://github.com/durable-workflow/sdk-python" Issues = "https://github.com/durable-workflow/sdk-python/issues" diff --git a/scripts/api_reference_release.py b/scripts/api_reference_release.py deleted file mode 100644 index 96beb9e..0000000 --- a/scripts/api_reference_release.py +++ /dev/null @@ -1,223 +0,0 @@ -"""Release identity used to render the Python API reference.""" - -from __future__ import annotations - -import json -import re -from dataclasses import dataclass -from pathlib import Path -from typing import Any - -try: - from scripts.release_compatibility import ( - WorkerProtocolRequirement, - declared_runtime_protocol_version, - validate_readme_compatibility, - worker_protocol_requirement, - ) -except ModuleNotFoundError as error: # pragma: no cover - direct command-line execution - if error.name != "scripts": - raise - import sys - - sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) - from scripts.release_compatibility import ( - WorkerProtocolRequirement, - declared_runtime_protocol_version, - validate_readme_compatibility, - worker_protocol_requirement, - ) - -try: - import tomllib # type: ignore[import-not-found] -except ModuleNotFoundError: # pragma: no cover - exercised by the Python 3.10 CI cell - import tomli as tomllib # type: ignore[import-not-found] - - -SERVER_IMAGE_RESOLVER_TOKEN = "{{ durable_workflow_server_image_resolver }}" -RELEASE_EVIDENCE_FILENAME = "release-audit.json" -RELEASE_EVIDENCE_SCHEMA = "durable-workflow.python-api-reference.release" -QUICKSTART_CONTRACT_SCHEMA = "durable-workflow.docs.v2.quickstart-execution-contract" -QUICKSTART_CONTRACT_URL = "https://durable-workflow.com/quickstart-execution-contract.json" -RELEASE_VERSION_PATTERN = r"[0-9]+\.[0-9]+\.[0-9]+(?:-(?:alpha|beta|rc)\.[0-9]+)?" -SUPPORTED_PRERELEASE_INSTALL_COMMAND = "curl -fsSL https://durable-workflow.com/install-sdk.sh | sh -s -- python" -SUPPORTED_SERVER_IMAGE_RESOLVER_COMMAND = rf'''export DW_SERVER_IMAGE="$( - curl -fsSL "${{DURABLE_WORKFLOW_QUICKSTART_CONTRACT_URL:-{QUICKSTART_CONTRACT_URL}}}" | - python -c 'import json, re, sys -contract = json.load(sys.stdin) -if contract.get("schema") != "{QUICKSTART_CONTRACT_SCHEMA}": - raise SystemExit("The public quickstart contract has an unsupported schema.") -server = contract.get("artifacts", {{}}).get("server", {{}}) -version = server.get("version") -image = server.get("image") -reference = server.get("reference") -if not isinstance(version, str) or re.fullmatch(r"{RELEASE_VERSION_PATTERN}", version) is None: - raise SystemExit("The public quickstart contract has an invalid Server release.") -if not isinstance(image, str) or reference != f"{{image}}:{{version}}": - raise SystemExit("The public quickstart contract has an invalid Server reference.") -print(reference)' -)"''' - - -@dataclass(frozen=True) -class ReleaseIdentity: - package: str - version: str - registry_version: str - server_version: str - worker_protocol_version: str - - @property - def exact_requirement(self) -> str: - return f"{self.package}=={self.registry_version}" - - @property - def server_worker_protocols(self) -> WorkerProtocolRequirement: - return worker_protocol_requirement(self.worker_protocol_version) - - -@dataclass(frozen=True) -class QualifiedOnboarding: - sdk_version: str - sdk_registry_version: str - server_version: str - server_reference: str - - -def normalize_registry_version(version: str) -> str: - match = re.fullmatch( - r"(?P[0-9]+\.[0-9]+\.[0-9]+)(?:-(?P