From bbbea285c8ba7079f183c1407bc479f4af9ec7ce Mon Sep 17 00:00:00 2001 From: Martin Acosta Date: Sat, 17 Jan 2026 12:50:25 -0300 Subject: [PATCH 01/78] Implement nested steps feature in Duron - Introduced the ability to create child steps within parent steps, allowing for better tracking and management of workflows. - Updated the StepHandlerContext to include parentStepId and stepId for improved context handling. - Added UnhandledChildStepsError to manage cases where parent steps complete without awaiting child steps. - Enhanced documentation in CLAUDE.md to reflect the new nested steps functionality and its usage. - Updated database schema to support parent-child relationships in job steps. - Modified various components and tests to accommodate the new nested steps feature. --- CLAUDE.md | 54 +- bun.lock | 38 +- .../src/components/timeline.tsx | 75 +- packages/duron-dashboard/src/dev.tsx | 3 +- .../duron-dashboard/src/styles/globals.css | 114 +- .../duron-dashboard/src/views/step-list.tsx | 93 +- .../migration.sql | 2 + .../snapshot.json | 988 ++++++++++++++++++ packages/duron/package.json | 6 +- packages/duron/src/action.ts | 33 + packages/duron/src/adapters/postgres/base.ts | 9 +- .../duron/src/adapters/postgres/schema.ts | 2 + packages/duron/src/adapters/schemas.ts | 3 + packages/duron/src/errors.ts | 46 +- packages/duron/src/index.ts | 2 +- packages/duron/src/step-manager.ts | 140 ++- packages/duron/test/adapter.test.ts | 8 + packages/duron/test/nested-steps.test.ts | 529 ++++++++++ packages/shared-actions/index.ts | 348 ++++++ 19 files changed, 2397 insertions(+), 96 deletions(-) create mode 100644 packages/duron/migrations/postgres/20260117142756_pretty_vulcan/migration.sql create mode 100644 packages/duron/migrations/postgres/20260117142756_pretty_vulcan/snapshot.json create mode 100644 packages/duron/test/nested-steps.test.ts diff --git a/CLAUDE.md b/CLAUDE.md index beff57c..bf032b1 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -12,6 +12,7 @@ Duron is a modern, type-safe background job processing system built with TypeScr - **Type-Safe Actions** - Define actions with Zod schemas for input/output validation - **Step-Based Execution** - Break down complex workflows into manageable, retryable steps +- **Nested Steps** - Steps can create child steps with proper parent-child tracking and abort signal propagation - **Intelligent Retry Logic** - Configurable exponential backoff with per-action and per-step options - **Flexible Sync Patterns** - Pull, push, hybrid, or manual job fetching - **Advanced Concurrency Control** - Per-action, per-group, and dynamic concurrency limits @@ -198,6 +199,49 @@ const sendEmail = defineAction()({ }) ``` +### Nested Steps + +Steps can create child steps using the `step()` method available in the step handler context. Child steps share abort signals with their parent and are tracked with `parentStepId` in the database. + +```typescript +const processOrder = defineAction()({ + name: 'process-order', + input: z.object({ orderId: z.string() }), + output: z.object({ success: z.boolean() }), + handler: async (ctx) => { + const result = await ctx.step('process', async ({ step, signal, stepId }) => { + // stepId is available for the current step + console.log('Processing step:', stepId) + + // Create child steps - they inherit the parent's abort signal + const validation = await step('validate', async ({ parentStepId }) => { + // parentStepId links back to the 'process' step + return { valid: true } + }) + + // Child steps can also be nested further + const payment = await step('charge', async ({ step: nestedStep }) => { + const auth = await nestedStep('authorize', async () => { + return { authCode: '123' } + }) + return { charged: true, authCode: auth.authCode } + }) + + return { success: validation.valid && payment.charged } + }) + + return result + }, +}) +``` + +**Important:** All child steps MUST be awaited before the parent step returns. If a parent step completes with unawaited children, Duron will: +1. Abort all pending child steps +2. Wait for them to settle +3. Throw an `UnhandledChildStepsError` + +This prevents orphaned processes and ensures proper async patterns. + ### Creating a Client ```typescript @@ -340,6 +384,7 @@ Uses Bun's bundler mode with: | `packages/duron/src/server.ts` | REST API server | | `packages/duron/src/adapters/adapter.ts` | Base adapter class | | `packages/duron/src/adapters/postgres/` | PostgreSQL adapter | +| `packages/duron/src/step-manager.ts` | Step execution and nested step handling | | `packages/duron-dashboard/src/DuronDashboard.tsx` | Dashboard root | | `packages/duron-dashboard/src/views/` | Dashboard pages | | `packages/examples/basic/start.ts` | Basic example | @@ -368,15 +413,22 @@ Uses Bun's bundler mode with: ## Error Handling - Use `NonRetriableError` for errors that should not be retried +- Use `UnhandledChildStepsError` is thrown when parent steps complete with unawaited children - Steps have built-in retry logic with exponential backoff - Jobs have timeout/expiration settings ```typescript -import { NonRetriableError } from 'duron' +import { NonRetriableError, UnhandledChildStepsError } from 'duron' +// For errors that should not be retried if (!apiKey) { throw new NonRetriableError('API key is required') } + +// UnhandledChildStepsError is thrown automatically when: +// - A parent step returns before all child steps are awaited +// - The parent step's callback completes but children are still pending +// This error is non-retriable and will fail the entire job ``` ## Environment Variables diff --git a/bun.lock b/bun.lock index 1ed9a91..3559d4a 100644 --- a/bun.lock +++ b/bun.lock @@ -46,7 +46,7 @@ "name": "duron", "version": "0.2.2", "dependencies": { - "elysia": "^1.4.16", + "elysia": "^1.4.22", "fastq": "^1.19.1", "jose": "^6.1.2", "p-retry": "^7.1.0", @@ -59,15 +59,15 @@ "@electric-sql/pglite": "^0.3.14", "@types/bun": "latest", "@types/node": "^24.0.15", - "drizzle-kit": "^1.0.0-beta.2-b782ae1", - "drizzle-orm": "^1.0.0-beta.2-b782ae1", + "drizzle-kit": "^1.0.0-beta.11-05230d9", + "drizzle-orm": "^1.0.0-beta.11-05230d9", "postgres": "^3.4.7", "typescript": "^5.6.3", }, }, "packages/duron-dashboard": { "name": "duron-dashboard", - "version": "0.1.1", + "version": "0.2.0", "dependencies": { "@dnd-kit/core": "^6.3.1", "@dnd-kit/modifiers": "^9.0.0", @@ -696,7 +696,7 @@ "@types/babel__traverse": ["@types/babel__traverse@7.28.0", "", { "dependencies": { "@babel/types": "^7.28.2" } }, "sha512-8PvcXf70gTDZBgt9ptxJ8elBeBjcLOAcOtoO/mPJjtji1+CdGbHgm77om1GrsPxsiE+uXIpNSK64UYaIwQXd4Q=="], - "@types/bun": ["@types/bun@1.3.3", "", { "dependencies": { "bun-types": "1.3.3" } }, "sha512-ogrKbJ2X5N0kWLLFKeytG0eHDleBYtngtlbu9cyBKFtNL3cnpDZkNdQj8flVf6WTZUX5ulI9AY1oa7ljhSrp+g=="], + "@types/bun": ["@types/bun@1.3.6", "", { "dependencies": { "bun-types": "1.3.6" } }, "sha512-uWCv6FO/8LcpREhenN1d1b6fcspAB+cefwD7uti8C8VffIv0Um08TKMn98FynpTiU38+y2dUO55T11NgDt8VAA=="], "@types/debug": ["@types/debug@4.1.12", "", { "dependencies": { "@types/ms": "*" } }, "sha512-vIChWdVG3LG1SMxEvI/AK+FWJthlrqlTu7fbrlywTkkaONwk/UAGaULXRlf8vkzFBLVm0zkMdCquhL5aOjhXPQ=="], @@ -808,7 +808,7 @@ "bun-plugin-tailwind": ["bun-plugin-tailwind@0.1.2", "", { "peerDependencies": { "bun": ">=1.0.0" } }, "sha512-41jNC1tZRSK3s1o7pTNrLuQG8kL/0vR/JgiTmZAJ1eHwe0w5j6HFPKeqEk0WAD13jfrUC7+ULuewFBBCoADPpg=="], - "bun-types": ["bun-types@1.3.3", "", { "dependencies": { "@types/node": "*" } }, "sha512-z3Xwlg7j2l9JY27x5Qn3Wlyos8YAp0kKRlrePAOjgjMGS5IG6E7Jnlx736vH9UVI4wUICwwhC9anYL++XeOgTQ=="], + "bun-types": ["bun-types@1.3.6", "", { "dependencies": { "@types/node": "*" } }, "sha512-OlFwHcnNV99r//9v5IIOgQ9Uk37gZqrNMCcqEaExdkVq3Avwqok1bJFmvGMCkCE0FqzdY8VMOZpfpR3lwI+CsQ=="], "bundle-name": ["bundle-name@4.1.0", "", { "dependencies": { "run-applescript": "^7.0.0" } }, "sha512-tjwM5exMg6BGRI+kNmTntNsvdZS1X8BFYS6tnJ2hdH0kVxM6/eVZ2xy+FqStSWvYmtfFMDLIxurorHwDKfDz5Q=="], @@ -924,9 +924,9 @@ "drange": ["drange@1.1.1", "", {}, "sha512-pYxfDYpued//QpnLIm4Avk7rsNtAtQkUES2cwAYSvD/wd2pKD71gN2Ebj3e7klzXwjocvE8c5vx/1fxwpqmSxA=="], - "drizzle-kit": ["drizzle-kit@1.0.0-beta.2-b782ae1", "", { "dependencies": { "@drizzle-team/brocli": "^0.11.0", "@js-temporal/polyfill": "^0.5.1", "esbuild": "^0.25.10", "esbuild-register": "^3.6.0" }, "bin": { "drizzle-kit": "bin.cjs" } }, "sha512-ui8nESGtCC82HgsLlsPEvvzyAwwr9EDOhb4DqoY//0n1nJ99YDLhnyS8sJw7c23uCaDZtMQLmU6lIHjzdn8hyQ=="], + "drizzle-kit": ["drizzle-kit@1.0.0-beta.11-05230d9", "", { "dependencies": { "@drizzle-team/brocli": "^0.11.0", "@js-temporal/polyfill": "^0.5.1", "esbuild": "^0.25.10", "tsx": "^4.20.6" }, "bin": { "drizzle-kit": "bin.cjs" } }, "sha512-dIv1Ql5SOJsGE/6mzfUk2k8iQFQtP6Buq7JlRew/Mx1uOOkIgyPT6xAUsSbyXqjqiDQl9k2S7iXtKaOs3Cs4aw=="], - "drizzle-orm": ["drizzle-orm@1.0.0-beta.2-b782ae1", "", { "peerDependencies": { "@aws-sdk/client-rds-data": ">=3", "@cloudflare/workers-types": ">=4", "@electric-sql/pglite": ">=0.2.0", "@libsql/client": ">=0.10.0", "@libsql/client-wasm": ">=0.10.0", "@neondatabase/serverless": ">=0.10.0", "@op-engineering/op-sqlite": ">=2", "@opentelemetry/api": "^1.4.1", "@planetscale/database": ">=1.13", "@prisma/client": "*", "@sqlitecloud/drivers": ">=1.0.653", "@tidbcloud/serverless": "*", "@tursodatabase/database": ">=0.2.1", "@tursodatabase/database-common": ">=0.2.1", "@tursodatabase/database-wasm": ">=0.2.1", "@types/better-sqlite3": "*", "@types/mssql": "^9.1.4", "@types/pg": "*", "@types/sql.js": "*", "@upstash/redis": ">=1.34.7", "@vercel/postgres": ">=0.8.0", "@xata.io/client": "*", "better-sqlite3": ">=9.3.0", "bun-types": "*", "expo-sqlite": ">=14.0.0", "gel": ">=2", "mssql": "^11.0.1", "mysql2": ">=2", "pg": ">=8", "postgres": ">=3", "sql.js": ">=1", "sqlite3": ">=5" }, "optionalPeers": ["@aws-sdk/client-rds-data", "@cloudflare/workers-types", "@electric-sql/pglite", "@libsql/client", "@libsql/client-wasm", "@neondatabase/serverless", "@op-engineering/op-sqlite", "@opentelemetry/api", "@planetscale/database", "@prisma/client", "@sqlitecloud/drivers", "@tidbcloud/serverless", "@tursodatabase/database", "@tursodatabase/database-common", "@tursodatabase/database-wasm", "@types/better-sqlite3", "@types/pg", "@types/sql.js", "@upstash/redis", "@vercel/postgres", "@xata.io/client", "better-sqlite3", "bun-types", "expo-sqlite", "gel", "mysql2", "pg", "postgres", "sql.js", "sqlite3"] }, "sha512-nD4C80uF1gFLA6Bl+n3Vl1TL5ofdFnIcKkDir7xxj5U5bfUlPtVBHtwjFGaUx57ib6yF6S8YT+/J0Zm/zjRBoQ=="], + "drizzle-orm": ["drizzle-orm@1.0.0-beta.11-05230d9", "", { "peerDependencies": { "@aws-sdk/client-rds-data": ">=3", "@cloudflare/workers-types": ">=4", "@effect/sql": "^0.48.5", "@effect/sql-pg": "^0.49.7", "@electric-sql/pglite": ">=0.2.0", "@libsql/client": ">=0.10.0", "@libsql/client-wasm": ">=0.10.0", "@neondatabase/serverless": ">=0.10.0", "@op-engineering/op-sqlite": ">=2", "@opentelemetry/api": "^1.4.1", "@planetscale/database": ">=1.13", "@prisma/client": "*", "@sqlitecloud/drivers": ">=1.0.653", "@tidbcloud/serverless": "*", "@tursodatabase/database": ">=0.2.1", "@tursodatabase/database-common": ">=0.2.1", "@tursodatabase/database-wasm": ">=0.2.1", "@types/better-sqlite3": "*", "@types/mssql": "^9.1.4", "@types/pg": "*", "@types/sql.js": "*", "@upstash/redis": ">=1.34.7", "@vercel/postgres": ">=0.8.0", "@xata.io/client": "*", "better-sqlite3": ">=9.3.0", "bun-types": "*", "expo-sqlite": ">=14.0.0", "gel": ">=2", "mssql": "^11.0.1", "mysql2": ">=2", "pg": ">=8", "postgres": ">=3", "sql.js": ">=1", "sqlite3": ">=5" }, "optionalPeers": ["@aws-sdk/client-rds-data", "@cloudflare/workers-types", "@effect/sql", "@effect/sql-pg", "@electric-sql/pglite", "@libsql/client", "@libsql/client-wasm", "@neondatabase/serverless", "@op-engineering/op-sqlite", "@opentelemetry/api", "@planetscale/database", "@prisma/client", "@sqlitecloud/drivers", "@tidbcloud/serverless", "@tursodatabase/database", "@tursodatabase/database-common", "@tursodatabase/database-wasm", "@types/better-sqlite3", "@types/pg", "@types/sql.js", "@upstash/redis", "@vercel/postgres", "@xata.io/client", "better-sqlite3", "bun-types", "expo-sqlite", "gel", "mysql2", "pg", "postgres", "sql.js", "sqlite3"] }, "sha512-y6i3AgHqeT5Eu/+7hsF1AWyPZQTxiBx1UOy4snEqljW7Fbpp/fO2zmWYFPvjngwOLq9prrbrn36AwCoTR+J2Bg=="], "duron": ["duron@workspace:packages/duron"], @@ -938,7 +938,7 @@ "electron-to-chromium": ["electron-to-chromium@1.5.262", "", {}, "sha512-NlAsMteRHek05jRUxUR0a5jpjYq9ykk6+kO0yRaMi5moe7u0fVIOeQ3Y30A8dIiWFBNUoQGi1ljb1i5VtS9WQQ=="], - "elysia": ["elysia@1.4.16", "", { "dependencies": { "cookie": "^1.0.2", "exact-mirror": "0.2.3", "fast-decode-uri-component": "^1.0.1", "memoirist": "^0.4.0" }, "peerDependencies": { "@sinclair/typebox": ">= 0.34.0 < 1", "@types/bun": ">= 1.2.0", "file-type": ">= 20.0.0", "openapi-types": ">= 12.0.0", "typescript": ">= 5.0.0" }, "optionalPeers": ["@types/bun", "typescript"] }, "sha512-KZtKN160/bdWVKg2hEgyoNXY8jRRquc+m6PboyisaLZL891I+Ufb7Ja6lDAD7vMQur8sLEWIcidZOzj5lWw9UA=="], + "elysia": ["elysia@1.4.22", "", { "dependencies": { "cookie": "^1.1.1", "exact-mirror": "^0.2.6", "fast-decode-uri-component": "^1.0.1", "memoirist": "^0.4.0" }, "peerDependencies": { "@sinclair/typebox": ">= 0.34.0 < 1", "@types/bun": ">= 1.2.0", "file-type": ">= 20.0.0", "openapi-types": ">= 12.0.0", "typescript": ">= 5.0.0" }, "optionalPeers": ["@types/bun", "typescript"] }, "sha512-Q90VCb1RVFxnFaRV0FDoSylESQQLWgLHFmWciQJdX9h3b2cSasji9KWEUvaJuy/L9ciAGg4RAhUVfsXHg5K2RQ=="], "emoji-regex": ["emoji-regex@9.2.2", "", {}, "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg=="], @@ -954,8 +954,6 @@ "esbuild": ["esbuild@0.27.0", "", { "optionalDependencies": { "@esbuild/aix-ppc64": "0.27.0", "@esbuild/android-arm": "0.27.0", "@esbuild/android-arm64": "0.27.0", "@esbuild/android-x64": "0.27.0", "@esbuild/darwin-arm64": "0.27.0", "@esbuild/darwin-x64": "0.27.0", "@esbuild/freebsd-arm64": "0.27.0", "@esbuild/freebsd-x64": "0.27.0", "@esbuild/linux-arm": "0.27.0", "@esbuild/linux-arm64": "0.27.0", "@esbuild/linux-ia32": "0.27.0", "@esbuild/linux-loong64": "0.27.0", "@esbuild/linux-mips64el": "0.27.0", "@esbuild/linux-ppc64": "0.27.0", "@esbuild/linux-riscv64": "0.27.0", "@esbuild/linux-s390x": "0.27.0", "@esbuild/linux-x64": "0.27.0", "@esbuild/netbsd-arm64": "0.27.0", "@esbuild/netbsd-x64": "0.27.0", "@esbuild/openbsd-arm64": "0.27.0", "@esbuild/openbsd-x64": "0.27.0", "@esbuild/openharmony-arm64": "0.27.0", "@esbuild/sunos-x64": "0.27.0", "@esbuild/win32-arm64": "0.27.0", "@esbuild/win32-ia32": "0.27.0", "@esbuild/win32-x64": "0.27.0" }, "bin": { "esbuild": "bin/esbuild" } }, "sha512-jd0f4NHbD6cALCyGElNpGAOtWxSq46l9X/sWB0Nzd5er4Kz2YTm+Vl0qKFT9KUJvD8+fiO8AvoHhFvEatfVixA=="], - "esbuild-register": ["esbuild-register@3.6.0", "", { "dependencies": { "debug": "^4.3.4" }, "peerDependencies": { "esbuild": ">=0.12 <1" } }, "sha512-H2/S7Pm8a9CL1uhp9OvjwrBh5Pvx0H8qVOxNu8Wed9Y7qv56MPtq+GGM8RJpq6glYJn9Wspr8uw7l55uyinNeg=="], - "escalade": ["escalade@3.2.0", "", {}, "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA=="], "escape-string-regexp": ["escape-string-regexp@5.0.0", "", {}, "sha512-/veY75JbMK4j1yjvuUxuVsiS/hr/4iHs9FTT6cgTexxdE0Ly/glccBAkloH/DofkjRbZU3bnoj38mOmhkZ0lHw=="], @@ -984,7 +982,7 @@ "eventsource-parser": ["eventsource-parser@3.0.6", "", {}, "sha512-Vo1ab+QXPzZ4tCa8SwIHJFaSzy4R6SHf7BY79rFBDf0idraZWAkYrDjDj8uWaSm3S2TK+hJ7/t1CEmZ7jXw+pg=="], - "exact-mirror": ["exact-mirror@0.2.3", "", { "peerDependencies": { "@sinclair/typebox": "^0.34.15" }, "optionalPeers": ["@sinclair/typebox"] }, "sha512-aLdARfO0W0ntufjDyytUJQMbNXoB9g+BbA8KcgIq4XOOTYRw48yUGON/Pr64iDrYNZKcKvKbqE0MPW56FF2BXA=="], + "exact-mirror": ["exact-mirror@0.2.6", "", { "peerDependencies": { "@sinclair/typebox": "^0.34.15" }, "optionalPeers": ["@sinclair/typebox"] }, "sha512-7s059UIx9/tnOKSySzUk5cPGkoILhTE4p6ncf6uIPaQ+9aRBQzQjc9+q85l51+oZ+P6aBxh084pD0CzBQPcFUA=="], "examples": ["examples@workspace:packages/examples"], @@ -1722,6 +1720,12 @@ "drizzle-kit/esbuild": ["esbuild@0.25.12", "", { "optionalDependencies": { "@esbuild/aix-ppc64": "0.25.12", "@esbuild/android-arm": "0.25.12", "@esbuild/android-arm64": "0.25.12", "@esbuild/android-x64": "0.25.12", "@esbuild/darwin-arm64": "0.25.12", "@esbuild/darwin-x64": "0.25.12", "@esbuild/freebsd-arm64": "0.25.12", "@esbuild/freebsd-x64": "0.25.12", "@esbuild/linux-arm": "0.25.12", "@esbuild/linux-arm64": "0.25.12", "@esbuild/linux-ia32": "0.25.12", "@esbuild/linux-loong64": "0.25.12", "@esbuild/linux-mips64el": "0.25.12", "@esbuild/linux-ppc64": "0.25.12", "@esbuild/linux-riscv64": "0.25.12", "@esbuild/linux-s390x": "0.25.12", "@esbuild/linux-x64": "0.25.12", "@esbuild/netbsd-arm64": "0.25.12", "@esbuild/netbsd-x64": "0.25.12", "@esbuild/openbsd-arm64": "0.25.12", "@esbuild/openbsd-x64": "0.25.12", "@esbuild/openharmony-arm64": "0.25.12", "@esbuild/sunos-x64": "0.25.12", "@esbuild/win32-arm64": "0.25.12", "@esbuild/win32-ia32": "0.25.12", "@esbuild/win32-x64": "0.25.12" }, "bin": { "esbuild": "bin/esbuild" } }, "sha512-bbPBYYrtZbkt6Os6FiTLCTFxvq4tt3JKall1vRwshA3fdVztsLAatFaZobhkBC8/BrPetoa0oksYoKXoG4ryJg=="], + "duron-dashboard/@types/bun": ["@types/bun@1.3.3", "", { "dependencies": { "bun-types": "1.3.3" } }, "sha512-ogrKbJ2X5N0kWLLFKeytG0eHDleBYtngtlbu9cyBKFtNL3cnpDZkNdQj8flVf6WTZUX5ulI9AY1oa7ljhSrp+g=="], + + "duron-dashboard/elysia": ["elysia@1.4.16", "", { "dependencies": { "cookie": "^1.0.2", "exact-mirror": "0.2.3", "fast-decode-uri-component": "^1.0.1", "memoirist": "^0.4.0" }, "peerDependencies": { "@sinclair/typebox": ">= 0.34.0 < 1", "@types/bun": ">= 1.2.0", "file-type": ">= 20.0.0", "openapi-types": ">= 12.0.0", "typescript": ">= 5.0.0" }, "optionalPeers": ["@types/bun", "typescript"] }, "sha512-KZtKN160/bdWVKg2hEgyoNXY8jRRquc+m6PboyisaLZL891I+Ufb7Ja6lDAD7vMQur8sLEWIcidZOzj5lWw9UA=="], + + "examples/@types/bun": ["@types/bun@1.3.3", "", { "dependencies": { "bun-types": "1.3.3" } }, "sha512-ogrKbJ2X5N0kWLLFKeytG0eHDleBYtngtlbu9cyBKFtNL3cnpDZkNdQj8flVf6WTZUX5ulI9AY1oa7ljhSrp+g=="], + "fumadocs-ui/fumadocs-core": ["fumadocs-core@16.1.0", "", { "dependencies": { "@formatjs/intl-localematcher": "^0.6.2", "@orama/orama": "^3.1.16", "@shikijs/rehype": "^3.15.0", "@shikijs/transformers": "^3.15.0", "estree-util-value-to-estree": "^3.5.0", "github-slugger": "^2.0.0", "hast-util-to-estree": "^3.1.3", "hast-util-to-jsx-runtime": "^2.3.6", "image-size": "^2.0.2", "negotiator": "^1.0.0", "npm-to-yarn": "^3.0.1", "path-to-regexp": "^8.3.0", "remark": "^15.0.1", "remark-gfm": "^4.0.1", "remark-rehype": "^11.1.2", "scroll-into-view-if-needed": "^3.1.0", "shiki": "^3.15.0", "unist-util-visit": "^5.0.0" }, "peerDependencies": { "@mixedbread/sdk": "^0.19.0", "@orama/core": "1.x.x", "@tanstack/react-router": "1.x.x", "@types/react": "*", "algoliasearch": "5.x.x", "lucide-react": "*", "next": "16.x.x", "react": "^19.2.0", "react-dom": "^19.2.0", "react-router": "7.x.x", "waku": "^0.26.0 || ^0.27.0" }, "optionalPeers": ["@mixedbread/sdk", "@orama/core", "@tanstack/react-router", "@types/react", "algoliasearch", "lucide-react", "next", "react", "react-dom", "react-router", "waku"] }, "sha512-5pbO2bOGc/xlb2yLQSy6Oag8mvD5CNf5HzQIG80HjZzLXYWEOHW8yovRKnWKRF9gAibn6WHnbssj3YPAlitV/A=="], "htmlparser2/entities": ["entities@6.0.1", "", {}, "sha512-aN97NXWF6AWBTahfVOIrB/NShkzi5H7F9r1s9mD3cDj4Ko5f2qhhVoYMibXF7GlLveb/D2ioWay8lxI97Ven3g=="], @@ -1750,6 +1754,8 @@ "serve-handler/path-to-regexp": ["path-to-regexp@3.3.0", "", {}, "sha512-qyCH421YQPS2WFDxDjftfc1ZR5WKQzVzqsp4n9M2kQhVOo/ByahFoUNJfl58kOcEGfQ//7weFTDhm+ss8Ecxgw=="], + "shared-actions/@types/bun": ["@types/bun@1.3.3", "", { "dependencies": { "bun-types": "1.3.3" } }, "sha512-ogrKbJ2X5N0kWLLFKeytG0eHDleBYtngtlbu9cyBKFtNL3cnpDZkNdQj8flVf6WTZUX5ulI9AY1oa7ljhSrp+g=="], + "solid-js/seroval": ["seroval@1.3.2", "", {}, "sha512-RbcPH1n5cfwKrru7v7+zrZvjLurgHhGyso3HTyGtRivGWgYjbOmGuivCQaORNELjNONoK35nj28EoWul9sb1zQ=="], "solid-js/seroval-plugins": ["seroval-plugins@1.3.3", "", { "peerDependencies": { "seroval": "^1.0" } }, "sha512-16OL3NnUBw8JG1jBLUoZJsLnQq0n5Ua6aHalhJK4fMQkz1lqR7Osz1sA30trBtd9VUDc2NgkuRCn8+/pBwqZ+w=="], @@ -1820,6 +1826,14 @@ "drizzle-kit/esbuild/@esbuild/win32-x64": ["@esbuild/win32-x64@0.25.12", "", { "os": "win32", "cpu": "x64" }, "sha512-alJC0uCZpTFrSL0CCDjcgleBXPnCrEAhTBILpeAp7M/OFgoqtAetfBzX0xM00MUsVVPpVjlPuMbREqnZCXaTnA=="], + "duron-dashboard/@types/bun/bun-types": ["bun-types@1.3.3", "", { "dependencies": { "@types/node": "*" } }, "sha512-z3Xwlg7j2l9JY27x5Qn3Wlyos8YAp0kKRlrePAOjgjMGS5IG6E7Jnlx736vH9UVI4wUICwwhC9anYL++XeOgTQ=="], + + "duron-dashboard/elysia/exact-mirror": ["exact-mirror@0.2.3", "", { "peerDependencies": { "@sinclair/typebox": "^0.34.15" }, "optionalPeers": ["@sinclair/typebox"] }, "sha512-aLdARfO0W0ntufjDyytUJQMbNXoB9g+BbA8KcgIq4XOOTYRw48yUGON/Pr64iDrYNZKcKvKbqE0MPW56FF2BXA=="], + + "examples/@types/bun/bun-types": ["bun-types@1.3.3", "", { "dependencies": { "@types/node": "*" } }, "sha512-z3Xwlg7j2l9JY27x5Qn3Wlyos8YAp0kKRlrePAOjgjMGS5IG6E7Jnlx736vH9UVI4wUICwwhC9anYL++XeOgTQ=="], + + "shared-actions/@types/bun/bun-types": ["bun-types@1.3.3", "", { "dependencies": { "@types/node": "*" } }, "sha512-z3Xwlg7j2l9JY27x5Qn3Wlyos8YAp0kKRlrePAOjgjMGS5IG6E7Jnlx736vH9UVI4wUICwwhC9anYL++XeOgTQ=="], + "vite/esbuild/@esbuild/aix-ppc64": ["@esbuild/aix-ppc64@0.25.12", "", { "os": "aix", "cpu": "ppc64" }, "sha512-Hhmwd6CInZ3dwpuGTF8fJG6yoWmsToE+vYgD4nytZVxcu1ulHpUQRAB1UJ8+N1Am3Mz4+xOByoQoSZf4D+CpkA=="], "vite/esbuild/@esbuild/android-arm": ["@esbuild/android-arm@0.25.12", "", { "os": "android", "cpu": "arm" }, "sha512-VJ+sKvNA/GE7Ccacc9Cha7bpS8nyzVv0jdVgwNDaR4gDMC/2TTRc33Ip8qrNYUcpkOHUT5OZ0bUcNNVZQ9RLlg=="], diff --git a/packages/duron-dashboard/src/components/timeline.tsx b/packages/duron-dashboard/src/components/timeline.tsx index bed23f9..6a33c79 100644 --- a/packages/duron-dashboard/src/components/timeline.tsx +++ b/packages/duron-dashboard/src/components/timeline.tsx @@ -8,6 +8,9 @@ import { Tooltip, TooltipContent, TooltipTrigger } from '@/components/ui/tooltip import type { Job, JobStep } from '@/lib/api' import { calculateDurationSeconds, formatDurationSeconds } from '@/lib/duration' +// Step with optional parentStepId field +type StepWithParent = Omit & { parentStepId?: string | null } + interface TimelineItem { id: string name: string @@ -16,21 +19,78 @@ interface TimelineItem { finishedAt: Date | string | number | null | undefined status: string level: number + parentStepId?: string | null } interface TimelineProps { job: Job | null - steps: Omit[] + steps: StepWithParent[] selectedStepId?: string | null onStepSelect?: (stepId: string) => void } +interface StepNode { + step: StepWithParent + children: StepNode[] + depth: number +} + +/** + * Build a tree structure from flat steps list using parentStepId + */ +function buildStepTree(steps: StepWithParent[]): StepNode[] { + const stepMap = new Map() + const rootNodes: StepNode[] = [] + + // First pass: create nodes for all steps + for (const step of steps) { + stepMap.set(step.id, { step, children: [], depth: 0 }) + } + + // Second pass: build parent-child relationships + for (const step of steps) { + const node = stepMap.get(step.id)! + const parentStepId = step.parentStepId + + if (parentStepId && stepMap.has(parentStepId)) { + const parentNode = stepMap.get(parentStepId)! + parentNode.children.push(node) + node.depth = parentNode.depth + 1 + } else { + // Root step (no parent or parent not in current view) + rootNodes.push(node) + } + } + + return rootNodes +} + +/** + * Flatten tree back to ordered list with depth info for rendering + */ +function flattenStepTree(nodes: StepNode[]): Array<{ step: StepWithParent; depth: number }> { + const result: Array<{ step: StepWithParent; depth: number }> = [] + + function traverse(node: StepNode) { + result.push({ step: node.step, depth: node.depth }) + for (const child of node.children) { + traverse(child) + } + } + + for (const node of nodes) { + traverse(node) + } + + return result +} + const ROW_HEIGHT = 48 export function Timeline({ job, steps, selectedStepId, onStepSelect }: TimelineProps) { const parentRef = useRef(null) - // Build timeline items from job and steps + // Build timeline items from job and steps with proper nesting const timelineItems = useMemo(() => { if (!job) { return [] @@ -49,14 +109,18 @@ export function Timeline({ job, steps, selectedStepId, onStepSelect }: TimelineP level: 0, }) - // Add steps as children + // Sort steps by startedAt for initial ordering const sortedSteps = [...steps].sort((a, b) => { const aStart = a.startedAt ? new Date(a.startedAt).getTime() : 0 const bStart = b.startedAt ? new Date(b.startedAt).getTime() : 0 return aStart - bStart }) - sortedSteps.forEach((step) => { + // Build tree structure and flatten with proper depth + const tree = buildStepTree(sortedSteps) + const orderedSteps = flattenStepTree(tree) + + orderedSteps.forEach(({ step, depth }) => { items.push({ id: step.id, name: step.name, @@ -64,7 +128,8 @@ export function Timeline({ job, steps, selectedStepId, onStepSelect }: TimelineP startedAt: step.startedAt, finishedAt: step.finishedAt, status: step.status, - level: 1, + level: depth + 1, // +1 because job is at level 0 + parentStepId: step.parentStepId, }) }) diff --git a/packages/duron-dashboard/src/dev.tsx b/packages/duron-dashboard/src/dev.tsx index 0bce3ea..ac7201d 100644 --- a/packages/duron-dashboard/src/dev.tsx +++ b/packages/duron-dashboard/src/dev.tsx @@ -1,6 +1,6 @@ import { serve } from 'bun' -import { getWeather, openaiChat, sendEmail, variables } from '@shared-actions/index' +import { getWeather, openaiChat, processOrder, sendEmail, variables } from '@shared-actions/index' import { postgresAdapter } from 'duron/adapters/postgres/postgres' import { createServer, duron } from 'duron/index' @@ -17,6 +17,7 @@ const client = duron({ sendEmail, openaiChat, getWeather, + processOrder, }, variables, logger: 'info', diff --git a/packages/duron-dashboard/src/styles/globals.css b/packages/duron-dashboard/src/styles/globals.css index db3828f..77802d2 100644 --- a/packages/duron-dashboard/src/styles/globals.css +++ b/packages/duron-dashboard/src/styles/globals.css @@ -4,59 +4,73 @@ @custom-variant dark (&:is(.dark *)); :root { - --background: oklch(95.78% 0.006 264.5); - --foreground: oklch(43.18% 0.043 279.3); - --muted: oklch(91.88% 0.007 268.5); - --muted-foreground: oklch(40.50% 0.023 265.7); - --popover: oklch(93.35% 0.009 264.5); - --popover-foreground: oklch(34.64% 0.031 279.7); - --card: oklch(94.22% 0.007 260.7); - --card-foreground: oklch(38.93% 0.038 278.5); - --border: oklch(91.88% 0.007 268.5); - --input: oklch(89.47% 0.008 271.3); - --primary: oklch(55.47% 0.250 297.0); - --primary-foreground: oklch(100.00% 0.000 89.9); - --secondary: oklch(77.13% 0.056 305.4); - --secondary-foreground: oklch(24.58% 0.045 303.3); - --accent: oklch(83.15% 0.024 264.4); - --accent-foreground: oklch(30.49% 0.031 263.9); - --destructive: oklch(48.29% 0.188 29.3); - --destructive-foreground: oklch(96.83% 0.014 17.4); - --ring: oklch(55.47% 0.250 297.0); - --chart-1: oklch(55.47% 0.250 297.0); - --chart-2: oklch(77.13% 0.056 305.4); - --chart-3: oklch(83.15% 0.024 264.4); - --chart-4: oklch(79.96% 0.050 305.2); - --chart-5: oklch(55.33% 0.256 296.4); - --radius: 0.5rem; + --radius: 0.65rem; + --background: oklch(1 0 0); + --foreground: oklch(0.145 0 0); + --card: oklch(1 0 0); + --card-foreground: oklch(0.145 0 0); + --popover: oklch(1 0 0); + --popover-foreground: oklch(0.145 0 0); + --primary: oklch(0.205 0 0); + --primary-foreground: oklch(0.985 0 0); + --secondary: oklch(0.97 0 0); + --secondary-foreground: oklch(0.205 0 0); + --muted: oklch(0.97 0 0); + --muted-foreground: oklch(0.556 0 0); + --accent: oklch(0.97 0 0); + --accent-foreground: oklch(0.205 0 0); + --destructive: oklch(0.577 0.245 27.325); + --border: oklch(0.922 0 0); + --input: oklch(0.922 0 0); + --ring: oklch(0.708 0 0); + --chart-1: oklch(0.646 0.222 41.116); + --chart-2: oklch(0.6 0.118 184.704); + --chart-3: oklch(0.398 0.07 227.392); + --chart-4: oklch(0.828 0.189 84.429); + --chart-5: oklch(0.769 0.188 70.08); + --radius: 0.625rem; + --sidebar: oklch(0.985 0 0); + --sidebar-foreground: oklch(0.145 0 0); + --sidebar-primary: oklch(0.205 0 0); + --sidebar-primary-foreground: oklch(0.985 0 0); + --sidebar-accent: oklch(0.97 0 0); + --sidebar-accent-foreground: oklch(0.205 0 0); + --sidebar-border: oklch(0.922 0 0); + --sidebar-ring: oklch(0.708 0 0); } .dark { - --background: oklch(32.91% 0.032 274.8); - --foreground: oklch(86.46% 0.051 273.1); - --muted: oklch(37.38% 0.023 275.0); - --muted-foreground: oklch(80.88% 0.016 273.8); - --popover: oklch(30.04% 0.030 273.6); - --popover-foreground: oklch(96.82% 0.012 276.1); - --card: oklch(30.89% 0.031 274.2); - --card-foreground: oklch(91.72% 0.031 271.7); - --border: oklch(38.52% 0.019 277.3); - --input: oklch(41.42% 0.019 273.5); - --primary: oklch(76.48% 0.111 311.7); - --primary-foreground: oklch(24.97% 0.089 309.1); - --secondary: oklch(34.18% 0.070 311.0); - --secondary-foreground: oklch(86.66% 0.035 312.4); - --accent: oklch(45.57% 0.050 274.1); - --accent-foreground: oklch(98.27% 0.003 286.4); - --destructive: oklch(64.41% 0.232 28.6); - --destructive-foreground: oklch(100.00% 0.000 89.9); - --ring: oklch(76.48% 0.111 311.7); - --chart-1: oklch(76.48% 0.111 311.7); - --chart-2: oklch(34.18% 0.070 311.0); - --chart-3: oklch(45.57% 0.050 274.1); - --chart-4: oklch(36.70% 0.078 310.9); - --chart-5: oklch(76.30% 0.117 312.0); - --radius: 0.5rem; + --background: oklch(0.145 0 0); + --foreground: oklch(0.985 0 0); + --card: oklch(0.205 0 0); + --card-foreground: oklch(0.985 0 0); + --popover: oklch(0.205 0 0); + --popover-foreground: oklch(0.985 0 0); + --primary: oklch(0.922 0 0); + --primary-foreground: oklch(0.205 0 0); + --secondary: oklch(0.269 0 0); + --secondary-foreground: oklch(0.985 0 0); + --muted: oklch(0.269 0 0); + --muted-foreground: oklch(0.708 0 0); + --accent: oklch(0.269 0 0); + --accent-foreground: oklch(0.985 0 0); + --destructive: oklch(0.704 0.191 22.216); + --border: oklch(1 0 0 / 10%); + --input: oklch(1 0 0 / 15%); + --ring: oklch(0.556 0 0); + --chart-1: oklch(0.488 0.243 264.376); + --chart-2: oklch(0.696 0.17 162.48); + --chart-3: oklch(0.769 0.188 70.08); + --chart-4: oklch(0.627 0.265 303.9); + --chart-5: oklch(0.645 0.246 16.439); + --sidebar: oklch(0.205 0 0); + --sidebar-foreground: oklch(0.985 0 0); + --sidebar-primary: oklch(0.488 0.243 264.376); + --sidebar-primary-foreground: oklch(0.985 0 0); + --sidebar-accent: oklch(0.269 0 0); + --sidebar-accent-foreground: oklch(0.985 0 0); + --sidebar-border: oklch(1 0 0 / 10%); + --sidebar-ring: oklch(0.556 0 0); } @theme inline { diff --git a/packages/duron-dashboard/src/views/step-list.tsx b/packages/duron-dashboard/src/views/step-list.tsx index e11d061..88b97f2 100644 --- a/packages/duron-dashboard/src/views/step-list.tsx +++ b/packages/duron-dashboard/src/views/step-list.tsx @@ -1,7 +1,7 @@ 'use client' -import { Clock, Search } from 'lucide-react' -import { useCallback, useState } from 'react' +import { ChevronRight, Clock, Search } from 'lucide-react' +import { useCallback, useMemo, useState } from 'react' import { TimelineModal } from '@/components/timeline-modal' import { Accordion, AccordionContent, AccordionItem, AccordionTrigger } from '@/components/ui/accordion' @@ -10,8 +10,12 @@ import { Input } from '@/components/ui/input' import { ScrollArea } from '@/components/ui/scroll-area' import { useDebouncedCallback } from '@/hooks/use-debounced-callback' import { useStepsPolling } from '@/hooks/use-steps-polling' -import { useJobSteps } from '@/lib/api' +import { type GetJobStepsResponse, useJobSteps } from '@/lib/api' import { BadgeStatus } from '../components/badge-status' + +// Step type from the API response (without output field) +type JobStepWithoutOutput = GetJobStepsResponse['steps'][number] & { parentStepId?: string | null } + import { StepDetailsContent } from './step-details-content' interface StepListProps { @@ -20,6 +24,62 @@ interface StepListProps { onStepSelect: (stepId: string) => void } +interface StepNode { + step: JobStepWithoutOutput + children: StepNode[] + depth: number +} + +/** + * Build a tree structure from flat steps list using parentStepId + */ +function buildStepTree(steps: JobStepWithoutOutput[]): StepNode[] { + const stepMap = new Map() + const rootNodes: StepNode[] = [] + + // First pass: create nodes for all steps + for (const step of steps) { + stepMap.set(step.id, { step, children: [], depth: 0 }) + } + + // Second pass: build parent-child relationships + for (const step of steps) { + const node = stepMap.get(step.id)! + const parentStepId = (step as any).parentStepId as string | null + + if (parentStepId && stepMap.has(parentStepId)) { + const parentNode = stepMap.get(parentStepId)! + parentNode.children.push(node) + node.depth = parentNode.depth + 1 + } else { + // Root step (no parent or parent not in current view) + rootNodes.push(node) + } + } + + return rootNodes +} + +/** + * Flatten tree back to ordered list with depth info for rendering + */ +function flattenStepTree(nodes: StepNode[]): Array<{ step: JobStepWithoutOutput; depth: number }> { + const result: Array<{ step: JobStepWithoutOutput; depth: number }> = [] + + function traverse(node: StepNode) { + result.push({ step: node.step, depth: node.depth }) + for (const child of node.children) { + traverse(child) + } + } + + for (const node of nodes) { + traverse(node) + } + + return result +} + export function StepList({ jobId, selectedStepId, onStepSelect }: StepListProps) { const [inputValue, setInputValue] = useState('') const [searchTerm, setSearchTerm] = useState('') @@ -50,14 +110,21 @@ export function StepList({ jobId, selectedStepId, onStepSelect }: StepListProps) // Enable polling for step updates useStepsPolling(jobId, true) + const steps = stepsData?.steps ?? [] + + // Build tree structure and flatten for rendering with depth info + const orderedSteps = useMemo(() => { + if (steps.length === 0) return [] + const tree = buildStepTree(steps) + return flattenStepTree(tree) + }, [steps]) + if (!jobId) { return (
Select a job to view steps
) } - const steps = stepsData?.steps ?? [] - return ( <>
@@ -91,7 +158,7 @@ export function StepList({ jobId, selectedStepId, onStepSelect }: StepListProps)
{stepsLoading ? (
Loading steps...
- ) : steps.length === 0 ? ( + ) : orderedSteps.length === 0 ? (
{inputValue ? 'No steps found matching your search' : 'No steps found'}
@@ -102,13 +169,23 @@ export function StepList({ jobId, selectedStepId, onStepSelect }: StepListProps) value={selectedStepId || undefined} onValueChange={onStepSelect} > - {steps.map((step, index) => { + {orderedSteps.map(({ step, depth }, index) => { const stepNumber = (page - 1) * pageSize + index + 1 + const isNested = depth > 0 + // Calculate left padding based on depth (16px per level) + const paddingLeft = depth * 16 + return ( - +
+ {isNested && } #{stepNumber} {step.name}
diff --git a/packages/duron/migrations/postgres/20260117142756_pretty_vulcan/migration.sql b/packages/duron/migrations/postgres/20260117142756_pretty_vulcan/migration.sql new file mode 100644 index 0000000..32f82a7 --- /dev/null +++ b/packages/duron/migrations/postgres/20260117142756_pretty_vulcan/migration.sql @@ -0,0 +1,2 @@ +ALTER TABLE "duron"."job_steps" ADD COLUMN "parent_step_id" uuid;--> statement-breakpoint +CREATE INDEX "idx_job_steps_parent_step_id" ON "duron"."job_steps" ("parent_step_id"); \ No newline at end of file diff --git a/packages/duron/migrations/postgres/20260117142756_pretty_vulcan/snapshot.json b/packages/duron/migrations/postgres/20260117142756_pretty_vulcan/snapshot.json new file mode 100644 index 0000000..4db21a3 --- /dev/null +++ b/packages/duron/migrations/postgres/20260117142756_pretty_vulcan/snapshot.json @@ -0,0 +1,988 @@ +{ + "version": "8", + "dialect": "postgres", + "id": "25beb926-865c-41eb-8525-bbc1e0d3a19c", + "prevIds": [ + "e32fddc5-6d55-4c79-87b4-13d3a01b7d09" + ], + "ddl": [ + { + "name": "duron", + "entityType": "schemas" + }, + { + "isRlsEnabled": false, + "name": "job_steps", + "entityType": "tables", + "schema": "duron" + }, + { + "isRlsEnabled": false, + "name": "jobs", + "entityType": "tables", + "schema": "duron" + }, + { + "type": "uuid", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "gen_random_uuid()", + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "duron", + "table": "job_steps" + }, + { + "type": "uuid", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "job_id", + "entityType": "columns", + "schema": "duron", + "table": "job_steps" + }, + { + "type": "uuid", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "parent_step_id", + "entityType": "columns", + "schema": "duron", + "table": "job_steps" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "name", + "entityType": "columns", + "schema": "duron", + "table": "job_steps" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'active'", + "generated": null, + "identity": null, + "name": "status", + "entityType": "columns", + "schema": "duron", + "table": "job_steps" + }, + { + "type": "jsonb", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "output", + "entityType": "columns", + "schema": "duron", + "table": "job_steps" + }, + { + "type": "jsonb", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "error", + "entityType": "columns", + "schema": "duron", + "table": "job_steps" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "started_at", + "entityType": "columns", + "schema": "duron", + "table": "job_steps" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "finished_at", + "entityType": "columns", + "schema": "duron", + "table": "job_steps" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "timeout_ms", + "entityType": "columns", + "schema": "duron", + "table": "job_steps" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "expires_at", + "entityType": "columns", + "schema": "duron", + "table": "job_steps" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "0", + "generated": null, + "identity": null, + "name": "retries_limit", + "entityType": "columns", + "schema": "duron", + "table": "job_steps" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "0", + "generated": null, + "identity": null, + "name": "retries_count", + "entityType": "columns", + "schema": "duron", + "table": "job_steps" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "delayed_ms", + "entityType": "columns", + "schema": "duron", + "table": "job_steps" + }, + { + "type": "jsonb", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'{}'", + "generated": null, + "identity": null, + "name": "history_failed_attempts", + "entityType": "columns", + "schema": "duron", + "table": "job_steps" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "duron", + "table": "job_steps" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "duron", + "table": "job_steps" + }, + { + "type": "uuid", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "gen_random_uuid()", + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "duron", + "table": "jobs" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "action_name", + "entityType": "columns", + "schema": "duron", + "table": "jobs" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "group_key", + "entityType": "columns", + "schema": "duron", + "table": "jobs" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'created'", + "generated": null, + "identity": null, + "name": "status", + "entityType": "columns", + "schema": "duron", + "table": "jobs" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "checksum", + "entityType": "columns", + "schema": "duron", + "table": "jobs" + }, + { + "type": "jsonb", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'{}'", + "generated": null, + "identity": null, + "name": "input", + "entityType": "columns", + "schema": "duron", + "table": "jobs" + }, + { + "type": "jsonb", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "output", + "entityType": "columns", + "schema": "duron", + "table": "jobs" + }, + { + "type": "jsonb", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "error", + "entityType": "columns", + "schema": "duron", + "table": "jobs" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "timeout_ms", + "entityType": "columns", + "schema": "duron", + "table": "jobs" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "expires_at", + "entityType": "columns", + "schema": "duron", + "table": "jobs" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "started_at", + "entityType": "columns", + "schema": "duron", + "table": "jobs" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "finished_at", + "entityType": "columns", + "schema": "duron", + "table": "jobs" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "client_id", + "entityType": "columns", + "schema": "duron", + "table": "jobs" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "10", + "generated": null, + "identity": null, + "name": "concurrency_limit", + "entityType": "columns", + "schema": "duron", + "table": "jobs" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "duron", + "table": "jobs" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "duron", + "table": "jobs" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "job_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_job_steps_job_id", + "entityType": "indexes", + "schema": "duron", + "table": "job_steps" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "status", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_job_steps_status", + "entityType": "indexes", + "schema": "duron", + "table": "job_steps" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "name", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_job_steps_name", + "entityType": "indexes", + "schema": "duron", + "table": "job_steps" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "expires_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_job_steps_expires_at", + "entityType": "indexes", + "schema": "duron", + "table": "job_steps" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "parent_step_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_job_steps_parent_step_id", + "entityType": "indexes", + "schema": "duron", + "table": "job_steps" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "job_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "status", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_job_steps_job_status", + "entityType": "indexes", + "schema": "duron", + "table": "job_steps" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "job_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "name", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_job_steps_job_name", + "entityType": "indexes", + "schema": "duron", + "table": "job_steps" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "to_tsvector('english', \"output\"::text)", + "isExpression": true, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "gin", + "concurrently": false, + "name": "idx_job_steps_output_fts", + "entityType": "indexes", + "schema": "duron", + "table": "job_steps" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "action_name", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_jobs_action_name", + "entityType": "indexes", + "schema": "duron", + "table": "jobs" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "status", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_jobs_status", + "entityType": "indexes", + "schema": "duron", + "table": "jobs" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "group_key", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_jobs_group_key", + "entityType": "indexes", + "schema": "duron", + "table": "jobs" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "started_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_jobs_started_at", + "entityType": "indexes", + "schema": "duron", + "table": "jobs" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "finished_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_jobs_finished_at", + "entityType": "indexes", + "schema": "duron", + "table": "jobs" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "expires_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_jobs_expires_at", + "entityType": "indexes", + "schema": "duron", + "table": "jobs" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "client_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_jobs_client_id", + "entityType": "indexes", + "schema": "duron", + "table": "jobs" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "checksum", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_jobs_checksum", + "entityType": "indexes", + "schema": "duron", + "table": "jobs" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "concurrency_limit", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_jobs_concurrency_limit", + "entityType": "indexes", + "schema": "duron", + "table": "jobs" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "action_name", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "status", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_jobs_action_status", + "entityType": "indexes", + "schema": "duron", + "table": "jobs" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "action_name", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "group_key", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_jobs_action_group", + "entityType": "indexes", + "schema": "duron", + "table": "jobs" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "to_tsvector('english', \"input\"::text)", + "isExpression": true, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "gin", + "concurrently": false, + "name": "idx_jobs_input_fts", + "entityType": "indexes", + "schema": "duron", + "table": "jobs" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "to_tsvector('english', \"output\"::text)", + "isExpression": true, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "gin", + "concurrently": false, + "name": "idx_jobs_output_fts", + "entityType": "indexes", + "schema": "duron", + "table": "jobs" + }, + { + "nameExplicit": false, + "columns": [ + "job_id" + ], + "schemaTo": "duron", + "tableTo": "jobs", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "job_steps_job_id_jobs_id_fkey", + "entityType": "fks", + "schema": "duron", + "table": "job_steps" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "job_steps_pkey", + "schema": "duron", + "table": "job_steps", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "jobs_pkey", + "schema": "duron", + "table": "jobs", + "entityType": "pks" + }, + { + "nameExplicit": true, + "columns": [ + "job_id", + "name" + ], + "nullsNotDistinct": false, + "name": "unique_job_step_name", + "entityType": "uniques", + "schema": "duron", + "table": "job_steps" + }, + { + "value": "\"status\" IN ('active','completed','failed','cancelled')", + "name": "job_steps_status_check", + "entityType": "checks", + "schema": "duron", + "table": "job_steps" + }, + { + "value": "\"status\" IN ('created','active','completed','failed','cancelled')", + "name": "jobs_status_check", + "entityType": "checks", + "schema": "duron", + "table": "jobs" + } + ], + "renames": [] +} \ No newline at end of file diff --git a/packages/duron/package.json b/packages/duron/package.json index 1ec91d1..25dcb46 100644 --- a/packages/duron/package.json +++ b/packages/duron/package.json @@ -69,13 +69,13 @@ "@electric-sql/pglite": "^0.3.14", "@types/bun": "latest", "@types/node": "^24.0.15", - "drizzle-kit": "^1.0.0-beta.2-b782ae1", - "drizzle-orm": "^1.0.0-beta.2-b782ae1", + "drizzle-kit": "^1.0.0-beta.11-05230d9", + "drizzle-orm": "^1.0.0-beta.11-05230d9", "postgres": "^3.4.7", "typescript": "^5.6.3" }, "dependencies": { - "elysia": "^1.4.16", + "elysia": "^1.4.22", "fastq": "^1.19.1", "jose": "^6.1.2", "p-retry": "^7.1.0", diff --git a/packages/duron/src/action.ts b/packages/duron/src/action.ts index 4164fa2..b79aa52 100644 --- a/packages/duron/src/action.ts +++ b/packages/duron/src/action.ts @@ -21,7 +21,40 @@ export interface ActionHandlerContext( + name: string, + cb: (ctx: StepHandlerContext) => Promise, + options?: z.input, + ) => Promise } export interface ConcurrencyHandlerContext> { diff --git a/packages/duron/src/adapters/postgres/base.ts b/packages/duron/src/adapters/postgres/base.ts index 46c21b7..10e8ad5 100644 --- a/packages/duron/src/adapters/postgres/base.ts +++ b/packages/duron/src/adapters/postgres/base.ts @@ -1,5 +1,5 @@ import { and, asc, between, desc, eq, gt, gte, ilike, inArray, isNull, ne, or, sql } from 'drizzle-orm' -import type { PgColumn, PgDatabase } from 'drizzle-orm/pg-core' +import type { PgAsyncDatabase, PgColumn } from 'drizzle-orm/pg-core' import { JOB_STATUS_ACTIVE, @@ -47,7 +47,7 @@ type Schema = ReturnType // Re-export types for backward compatibility export type { Job, JobStep } from '../adapter.js' -type DrizzleDatabase = PgDatabase +type DrizzleDatabase = PgAsyncDatabase export interface AdapterOptions { connection: Connection @@ -570,6 +570,7 @@ export class PostgresBaseAdapter e name, timeoutMs, retriesLimit, + parentStepId, }: CreateOrRecoverJobStepOptions): Promise { type StepResult = CreateOrRecoverJobStepResult @@ -591,6 +592,7 @@ export class PostgresBaseAdapter e upserted_step AS ( INSERT INTO ${this.tables.jobStepsTable} ( job_id, + parent_step_id, name, timeout_ms, retries_limit, @@ -602,6 +604,7 @@ export class PostgresBaseAdapter e ) SELECT ${jobId}, + ${parentStepId}, ${name}, ${timeoutMs}, ${retriesLimit}, @@ -863,6 +866,7 @@ export class PostgresBaseAdapter e .select({ id: jobStepsTable.id, jobId: jobStepsTable.job_id, + parentStepId: jobStepsTable.parent_step_id, name: jobStepsTable.name, status: jobStepsTable.status, error: jobStepsTable.error, @@ -1054,6 +1058,7 @@ export class PostgresBaseAdapter e .select({ id: this.tables.jobStepsTable.id, jobId: this.tables.jobStepsTable.job_id, + parentStepId: this.tables.jobStepsTable.parent_step_id, name: this.tables.jobStepsTable.name, output: this.tables.jobStepsTable.output, status: this.tables.jobStepsTable.status, diff --git a/packages/duron/src/adapters/postgres/schema.ts b/packages/duron/src/adapters/postgres/schema.ts index bd777c6..a9b39b2 100644 --- a/packages/duron/src/adapters/postgres/schema.ts +++ b/packages/duron/src/adapters/postgres/schema.ts @@ -66,6 +66,7 @@ export default function createSchema(schemaName: string) { job_id: uuid('job_id') .notNull() .references(() => jobsTable.id, { onDelete: 'cascade' }), + parent_step_id: uuid('parent_step_id'), name: text('name').notNull(), status: text('status').$type().notNull().default(STEP_STATUS_ACTIVE), output: jsonb('output'), @@ -98,6 +99,7 @@ export default function createSchema(schemaName: string) { index('idx_job_steps_status').on(table.status), index('idx_job_steps_name').on(table.name), index('idx_job_steps_expires_at').on(table.expires_at), + index('idx_job_steps_parent_step_id').on(table.parent_step_id), // Composite indexes index('idx_job_steps_job_status').on(table.job_id, table.status), index('idx_job_steps_job_name').on(table.job_id, table.name), diff --git a/packages/duron/src/adapters/schemas.ts b/packages/duron/src/adapters/schemas.ts index 5a0a768..74a1b9b 100644 --- a/packages/duron/src/adapters/schemas.ts +++ b/packages/duron/src/adapters/schemas.ts @@ -55,6 +55,7 @@ export const JobSchema = z.object({ export const JobStepSchema = z.object({ id: z.string(), jobId: z.string(), + parentStepId: z.string().nullable().default(null), name: z.string(), output: z.any().nullable().default(null), status: StepStatusSchema, @@ -190,6 +191,8 @@ export const DeleteJobsOptionsSchema = GetJobsOptionsSchema.optional() export const CreateOrRecoverJobStepOptionsSchema = z.object({ /** The ID of the job this step belongs to */ jobId: z.string(), + /** The ID of the parent step (null for root steps) */ + parentStepId: z.string().nullable().default(null), /** The name of the step */ name: z.string(), /** Timeout in milliseconds for the step */ diff --git a/packages/duron/src/errors.ts b/packages/duron/src/errors.ts index 4ad8f91..b621e2e 100644 --- a/packages/duron/src/errors.ts +++ b/packages/duron/src/errors.ts @@ -126,6 +126,38 @@ export class ActionCancelError extends DuronError { } } +/** + * Error thrown when a parent step completes with unhandled (non-awaited) child steps. + * + * This error indicates a bug in the action handler where child steps were started + * but not properly awaited. All child steps must be awaited before the parent returns. + */ +export class UnhandledChildStepsError extends NonRetriableError { + /** + * The name of the parent step that completed with unhandled children. + */ + public readonly stepName: string + + /** + * The number of unhandled child steps. + */ + public readonly pendingCount: number + + /** + * Create a new UnhandledChildStepsError. + * + * @param stepName - The name of the parent step + * @param pendingCount - The number of unhandled child steps + */ + constructor(stepName: string, pendingCount: number) { + super( + `Parent step "${stepName}" completed with ${pendingCount} unhandled child step(s). All child steps must be awaited before the parent returns.`, + ) + this.stepName = stepName + this.pendingCount = pendingCount + } +} + /** * Checks if an error is a DuronError instance. */ @@ -137,7 +169,19 @@ export function isDuronError(error: unknown): error is DuronError { * Checks if an error is a NonRetriableError instance. */ export function isNonRetriableError(error: unknown): error is NonRetriableError { - return error instanceof NonRetriableError || error instanceof ActionCancelError || error instanceof ActionTimeoutError + return ( + error instanceof NonRetriableError || + error instanceof ActionCancelError || + error instanceof ActionTimeoutError || + error instanceof UnhandledChildStepsError + ) +} + +/** + * Checks if an error is an UnhandledChildStepsError instance. + */ +export function isUnhandledChildStepsError(error: unknown): error is UnhandledChildStepsError { + return error instanceof UnhandledChildStepsError } /** diff --git a/packages/duron/src/index.ts b/packages/duron/src/index.ts index d26cc04..8263b23 100644 --- a/packages/duron/src/index.ts +++ b/packages/duron/src/index.ts @@ -3,7 +3,7 @@ import { Client, type ClientOptions } from './client.js' export { defineAction } from './action.js' export * from './constants.js' -export { NonRetriableError } from './errors.js' +export { NonRetriableError, UnhandledChildStepsError } from './errors.js' export * from './server.js' export const duron = < diff --git a/packages/duron/src/step-manager.ts b/packages/duron/src/step-manager.ts index f01b369..6b77ad3 100644 --- a/packages/duron/src/step-manager.ts +++ b/packages/duron/src/step-manager.ts @@ -19,6 +19,7 @@ import { StepAlreadyExecutedError, StepTimeoutError, serializeError, + UnhandledChildStepsError, } from './errors.js' import pRetry from './utils/p-retry.js' import waitForAbort from './utils/wait-for-abort.js' @@ -28,6 +29,7 @@ export interface TaskStep { cb: (ctx: StepHandlerContext) => Promise options: StepOptions abortSignal: AbortSignal + parentStepId: string | null } /** @@ -61,16 +63,24 @@ export class StepStore { * @param name - The name of the step * @param timeoutMs - Timeout in milliseconds for the step * @param retriesLimit - Maximum number of retries for the step + * @param parentStepId - The ID of the parent step (null for root steps) * @returns Promise resolving to the created step ID * @throws Error if step creation fails */ - async getOrCreate(jobId: string, name: string, timeoutMs: number, retriesLimit: number) { + async getOrCreate( + jobId: string, + name: string, + timeoutMs: number, + retriesLimit: number, + parentStepId: string | null = null, + ) { try { return await this.#adapter.createOrRecoverJobStep({ jobId, name, timeoutMs, retriesLimit, + parentStepId, }) } catch (error) { throw new NonRetriableError(`Failed to get or create step "${name}" for job "${jobId}"`, { cause: error }) @@ -151,7 +161,7 @@ export class StepManager { throw new StepAlreadyExecutedError(task.name, this.#jobId, this.#actionName) } this.#historySteps.add(task.name) - return this.#executeStep(task.name, task.cb, task.options, task.abortSignal) + return this.#executeStep(task.name, task.cb, task.options, task.abortSignal, task.parentStepId) }, options.concurrencyLimit) } @@ -206,9 +216,11 @@ export class StepManager { * @param cb - The step handler function * @param options - Step options including concurrency, retry, and expire settings * @param abortSignal - Abort signal for cancelling the step + * @param parentStepId - The ID of the parent step (null for root steps) * @returns Promise resolving to the step result * @throws StepTimeoutError if the step times out * @throws StepCancelError if the step is cancelled + * @throws UnhandledChildStepsError if child steps are not awaited * @throws Error if the step fails */ async #executeStep( @@ -216,6 +228,7 @@ export class StepManager { cb: (ctx: StepHandlerContext) => Promise, options: StepOptions, abortSignal: AbortSignal, + parentStepId: string | null, ): Promise { const expire = options.expire const retryOptions = options.retry @@ -227,8 +240,8 @@ export class StepManager { throw new ActionCancelError(this.#actionName, this.#jobId, { cause: 'step cancelled before create step' }) } - // Create step record - const newStep = await this.#stepStore.getOrCreate(this.#jobId, name, expire, retryOptions.limit) + // Create step record with parentStepId + const newStep = await this.#stepStore.getOrCreate(this.#jobId, name, expire, retryOptions.limit, parentStepId) if (!newStep) { throw new NonRetriableError( `Failed to create step "${name}" for job "${this.#jobId}" action "${this.#actionName}"`, @@ -245,7 +258,7 @@ export class StepManager { if (step.status === STEP_STATUS_COMPLETED) { // this is how we recover a completed step this.#logger.debug( - { jobId: this.#jobId, actionName: this.#actionName, stepName: name, stepId: step.id }, + { jobId: this.#jobId, actionName: this.#actionName, stepName: name, stepId: step.id, parentStepId }, '[StepManager] Step recovered (already completed)', ) return step.output as TResult @@ -265,11 +278,12 @@ export class StepManager { // Log step start this.#logger.debug( - { jobId: this.#jobId, actionName: this.#actionName, stepName: name, stepId: step.id }, + { jobId: this.#jobId, actionName: this.#actionName, stepName: name, stepId: step.id, parentStepId }, '[StepManager] Step started executing', ) } + // Create abort controller for this step's timeout const stepAbortController = new AbortController() const timeoutId = setTimeout(() => { const timeoutError = new StepTimeoutError(name, this.#jobId, expire) @@ -278,18 +292,78 @@ export class StepManager { timeoutId?.unref?.() - // Combine abort signals - const signal = AbortSignal.any([abortSignal, stepAbortController.signal]) + // Combine abort signals: parent chain + this step's timeout + const stepSignal = AbortSignal.any([abortSignal, stepAbortController.signal]) + + // Track child steps for enforcement + interface TrackedChildStep { + promise: Promise + settled: boolean + } + const childSteps: TrackedChildStep[] = [] + + // Create abort controller for child steps (used when parent returns with pending children) + const childAbortController = new AbortController() + const childSignal = AbortSignal.any([stepSignal, childAbortController.signal]) + + // Create StepHandlerContext with nested step support + const stepContext: StepHandlerContext = { + signal: stepSignal, + stepId: step.id, + parentStepId, + step: ( + childName: string, + childCb: (ctx: StepHandlerContext) => Promise, + childOptions: z.input = {}, + ): Promise => { + const parsedChildOptions = StepOptionsSchema.parse({ + ...options, // Inherit parent step options as defaults + ...childOptions, + }) + + // Push child step with this step as parent + const childPromise = this.push({ + name: childName, + cb: childCb, + options: parsedChildOptions, + abortSignal: childSignal, // Child uses composed signal + parentStepId: step!.id, // This step is the parent + }) + + // Track the child promise + const trackedChild: TrackedChildStep = { + promise: childPromise, + settled: false, + } + childSteps.push(trackedChild) + + // Mark as settled when done (success or failure) + // Use .then/.catch instead of .finally to properly handle rejections + childPromise + .then(() => { + trackedChild.settled = true + }) + .catch(() => { + trackedChild.settled = true + // Swallow the error here - it will be re-thrown to the caller via the returned promise + }) + + return childPromise + }, + } try { // Race between abort signal and callback execution - const abortPromise = waitForAbort(signal) - const callbackPromise = cb({ signal }) + const abortPromise = waitForAbort(stepSignal) + const callbackPromise = cb(stepContext) let result: any = null + let aborted = false await Promise.race([ - abortPromise.promise, + abortPromise.promise.then(() => { + aborted = true + }), callbackPromise .then((res) => { if (res !== undefined && res !== null) { @@ -301,6 +375,41 @@ export class StepManager { }), ]) + // If aborted, wait for child steps to settle before propagating + if (aborted) { + // Wait for all child steps to settle (they'll be aborted via signal propagation) + if (childSteps.length > 0) { + await Promise.allSettled(childSteps.map((c) => c.promise)) + } + // Re-throw the abort reason + throw stepSignal.reason + } + + // After parent callback returns, check for pending children + const unsettledChildren = childSteps.filter((c) => !c.settled) + if (unsettledChildren.length > 0) { + this.#logger.warn( + { + jobId: this.#jobId, + actionName: this.#actionName, + stepName: name, + stepId: step.id, + pendingCount: unsettledChildren.length, + }, + '[StepManager] Parent step completed with unhandled child steps - aborting children', + ) + + // Abort all pending children + const unhandledError = new UnhandledChildStepsError(name, unsettledChildren.length) + childAbortController.abort(unhandledError) + + // Wait for all children to settle (they'll reject with cancellation) + await Promise.allSettled(unsettledChildren.map((c) => c.promise)) + + // Now throw the error + throw unhandledError + } + // Update step as completed const completed = await this.#stepStore.updateStatus(step.id, 'completed', result) if (!completed) { @@ -452,6 +561,7 @@ class ActionContext { const step = await adapter.createOrRecoverJobStep({ jobId, + parentStepId: null, name: 'step-1', timeoutMs: 3000, retriesLimit: 3, @@ -486,6 +487,7 @@ function runAdapterTests(adapterFactory: AdapterFactory) { // Create a step first const step1 = await adapter.createOrRecoverJobStep({ jobId, + parentStepId: null, name: 'step-1', timeoutMs: 3000, retriesLimit: 3, @@ -494,6 +496,7 @@ function runAdapterTests(adapterFactory: AdapterFactory) { // Create it again - should recover the existing step const step2 = await adapter.createOrRecoverJobStep({ jobId, + parentStepId: null, name: 'step-1', timeoutMs: 5000, retriesLimit: 5, @@ -509,6 +512,7 @@ function runAdapterTests(adapterFactory: AdapterFactory) { it('should complete a job step', async () => { const step = await adapter.createOrRecoverJobStep({ jobId, + parentStepId: null, name: 'step-1', timeoutMs: 3000, retriesLimit: 3, @@ -529,6 +533,7 @@ function runAdapterTests(adapterFactory: AdapterFactory) { it('should fail a job step', async () => { const step = await adapter.createOrRecoverJobStep({ jobId, + parentStepId: null, name: 'step-1', timeoutMs: 3000, retriesLimit: 3, @@ -549,6 +554,7 @@ function runAdapterTests(adapterFactory: AdapterFactory) { it('should delay a job step', async () => { const step = await adapter.createOrRecoverJobStep({ jobId, + parentStepId: null, name: 'step-1', timeoutMs: 3000, retriesLimit: 3, @@ -572,6 +578,7 @@ function runAdapterTests(adapterFactory: AdapterFactory) { it('should cancel a job step', async () => { const step = await adapter.createOrRecoverJobStep({ jobId, + parentStepId: null, name: 'step-1', timeoutMs: 3000, retriesLimit: 3, @@ -710,6 +717,7 @@ function runAdapterTests(adapterFactory: AdapterFactory) { for (let i = 0; i < 5; i++) { await adapter.createOrRecoverJobStep({ jobId: jobId!, + parentStepId: null, name: `step-${i}`, timeoutMs: 3000, retriesLimit: 3, diff --git a/packages/duron/test/nested-steps.test.ts b/packages/duron/test/nested-steps.test.ts new file mode 100644 index 0000000..39de38e --- /dev/null +++ b/packages/duron/test/nested-steps.test.ts @@ -0,0 +1,529 @@ +import { afterEach, beforeEach, describe, expect, it } from 'bun:test' + +import { z } from 'zod' + +import { defineAction } from '../src/action.js' +import { Client } from '../src/client.js' +import { JOB_STATUS_COMPLETED, JOB_STATUS_FAILED, STEP_STATUS_CANCELLED, STEP_STATUS_FAILED } from '../src/constants.js' +import { type Adapter, type AdapterFactory, pgliteFactory, postgresFactory } from './adapters.js' +import { expectToBeDefined } from './asserts.js' + +// ============================================================================= +// Test Actions for Nested Steps +// ============================================================================= + +/** + * Action with basic nested steps - child step awaited properly + */ +const nestedStepAction = defineAction()({ + name: 'nested-step-action', + version: '1.0.0', + input: z.object({ + depth: z.number().default(1), + }), + output: z.object({ + results: z.array(z.string()), + }), + handler: async (ctx) => { + const results: string[] = [] + + await ctx.step('parent-step', async (stepCtx) => { + results.push('parent-started') + + // Create a nested child step using stepCtx.step() + await stepCtx.step('child-step', async (childStepCtx) => { + results.push('child-started') + results.push(`child-stepId: ${childStepCtx.stepId}`) + results.push(`child-parentStepId: ${childStepCtx.parentStepId}`) + results.push('child-completed') + return { childResult: true } + }) + + results.push('parent-completed') + return { parentResult: true } + }) + + return { results } + }, +}) + +/** + * Action with deeply nested steps (3+ levels) + */ +const deeplyNestedAction = defineAction()({ + name: 'deeply-nested-action', + version: '1.0.0', + input: z.object({}), + output: z.object({ + depth: z.number(), + stepIds: z.array(z.string()), + }), + handler: async (ctx) => { + const stepIds: string[] = [] + + await ctx.step('level-1', async (level1Ctx) => { + stepIds.push(level1Ctx.stepId) + + await level1Ctx.step('level-2', async (level2Ctx) => { + stepIds.push(level2Ctx.stepId) + + await level2Ctx.step('level-3', async (level3Ctx) => { + stepIds.push(level3Ctx.stepId) + + await level3Ctx.step('level-4', async (level4Ctx) => { + stepIds.push(level4Ctx.stepId) + return { deepest: true } + }) + + return { level: 3 } + }) + + return { level: 2 } + }) + + return { level: 1 } + }) + + return { depth: 4, stepIds } + }, +}) + +/** + * Action with concurrent child steps under same parent + */ +const concurrentChildrenAction = defineAction()({ + name: 'concurrent-children-action', + version: '1.0.0', + input: z.object({ + childCount: z.number().default(3), + }), + output: z.object({ + childResults: z.array(z.number()), + }), + handler: async (ctx) => { + const childResults: number[] = [] + + await ctx.step('parent-step', async (stepCtx) => { + // Create multiple child steps concurrently + const promises = Array.from({ length: ctx.input.childCount }, (_, i) => + stepCtx.step(`child-${i}`, async () => { + await new Promise((resolve) => setTimeout(resolve, 10)) + return { index: i } + }), + ) + + // Await all children + const results = await Promise.all(promises) + childResults.push(...results.map((r) => r.index)) + + return { done: true } + }) + + return { childResults } + }, +}) + +/** + * Action that does NOT await child step - should fail with UnhandledChildStepsError + */ +const unawaitedChildAction = defineAction()({ + name: 'unawaited-child-action', + version: '1.0.0', + input: z.object({}), + output: z.object({ + result: z.string(), + }), + handler: async (ctx) => { + await ctx.step('parent-step', async (stepCtx) => { + // BAD: Start child step but don't await it + stepCtx.step('orphaned-child', async () => { + await new Promise((resolve) => setTimeout(resolve, 500)) + return { orphaned: true } + }) + + // Parent returns immediately without waiting for child + return { parentDone: true } + }) + + return { result: 'should-not-reach' } + }, +}) + +/** + * Action where parent step times out - children should be aborted + */ +const parentTimeoutAction = defineAction()({ + name: 'parent-timeout-action', + version: '1.0.0', + input: z.object({}), + output: z.object({ + result: z.string(), + }), + steps: { + concurrency: 10, + retry: { limit: 0, factor: 2, minTimeout: 1000, maxTimeout: 30000 }, + expire: 5000, // Default timeout + }, + handler: async (ctx) => { + await ctx.step( + 'parent-step', + async (stepCtx) => { + // Create child step first, then parent times out + const childPromise = stepCtx.step( + 'slow-child', + async () => { + await new Promise((resolve) => setTimeout(resolve, 2000)) + return { slow: true } + }, + { expire: 5000 }, + ) + + // Wait a bit then the parent timeout kicks in + await childPromise + + return { done: true } + }, + { expire: 100 }, // Parent times out in 100ms + ) + + return { result: 'should-not-reach' } + }, +}) + +/** + * Action with nested step that verifies parentStepId chain + */ +const parentStepIdVerificationAction = defineAction()({ + name: 'parent-step-id-verification', + version: '1.0.0', + input: z.object({}), + output: z.object({ + parentChain: z.array( + z.object({ + stepId: z.string(), + parentStepId: z.string().nullable(), + }), + ), + }), + handler: async (ctx) => { + const parentChain: Array<{ stepId: string; parentStepId: string | null }> = [] + + await ctx.step('root-step', async (rootCtx) => { + parentChain.push({ + stepId: rootCtx.stepId, + parentStepId: rootCtx.parentStepId, + }) + + await rootCtx.step('nested-step', async (nestedCtx) => { + parentChain.push({ + stepId: nestedCtx.stepId, + parentStepId: nestedCtx.parentStepId, + }) + + await nestedCtx.step('deep-step', async (deepCtx) => { + parentChain.push({ + stepId: deepCtx.stepId, + parentStepId: deepCtx.parentStepId, + }) + return {} + }) + + return {} + }) + + return {} + }) + + return { parentChain } + }, +}) + +// ============================================================================= +// Test Suite +// ============================================================================= + +function runNestedStepsTests(adapterFactory: AdapterFactory) { + describe(`Nested Steps Tests with ${adapterFactory.name}`, () => { + let client: Client< + { + nestedStepAction: typeof nestedStepAction + deeplyNestedAction: typeof deeplyNestedAction + concurrentChildrenAction: typeof concurrentChildrenAction + unawaitedChildAction: typeof unawaitedChildAction + parentTimeoutAction: typeof parentTimeoutAction + parentStepIdVerificationAction: typeof parentStepIdVerificationAction + }, + Record + > + let database: Adapter + let deleteDb: () => Promise + + beforeEach( + async () => { + const adapterInstance = await adapterFactory.create({}) + database = adapterInstance.adapter + deleteDb = adapterInstance.deleteDb + + client = new Client({ + database, + actions: { + nestedStepAction, + deeplyNestedAction, + concurrentChildrenAction, + unawaitedChildAction, + parentTimeoutAction, + parentStepIdVerificationAction, + }, + syncPattern: false, + recoverJobsOnStart: false, + logger: 'error', + }) + }, + { timeout: 60_000 }, + ) + + afterEach(async () => { + if (client) { + await client.stop() + } + if (deleteDb) { + await deleteDb() + } + }) + + describe('Basic Nested Steps', () => { + it('should execute nested steps with proper awaiting', async () => { + const jobId = await client.runAction('nestedStepAction', { depth: 1 }) + await client.fetch({ batchSize: 10 }) + + const job = await client.waitForJob(jobId, { timeout: 5000 }) + expectToBeDefined(job) + expect(job.status).toBe(JOB_STATUS_COMPLETED) + + const output = job.output as { results: string[] } + expect(output.results).toContain('parent-started') + expect(output.results).toContain('child-started') + expect(output.results).toContain('child-completed') + expect(output.results).toContain('parent-completed') + + // Verify order: parent starts, child executes, parent completes + const parentStartIdx = output.results.indexOf('parent-started') + const childStartIdx = output.results.indexOf('child-started') + const childCompleteIdx = output.results.indexOf('child-completed') + const parentCompleteIdx = output.results.indexOf('parent-completed') + + expect(parentStartIdx).toBeLessThan(childStartIdx) + expect(childStartIdx).toBeLessThan(childCompleteIdx) + expect(childCompleteIdx).toBeLessThan(parentCompleteIdx) + }) + + it('should provide stepId and parentStepId in step context', async () => { + const jobId = await client.runAction('nestedStepAction', { depth: 1 }) + await client.fetch({ batchSize: 10 }) + + const job = await client.waitForJob(jobId, { timeout: 5000 }) + expectToBeDefined(job) + expect(job.status).toBe(JOB_STATUS_COMPLETED) + + const output = job.output as { results: string[] } + + // Find stepId and parentStepId entries + const stepIdEntry = output.results.find((r) => r.startsWith('child-stepId:')) + const parentStepIdEntry = output.results.find((r) => r.startsWith('child-parentStepId:')) + + expect(stepIdEntry).toBeDefined() + expect(parentStepIdEntry).toBeDefined() + + // Extract values + const childStepId = stepIdEntry?.split(': ')[1] + const parentStepId = parentStepIdEntry?.split(': ')[1] + + // Child should have a stepId + expect(childStepId).toBeTruthy() + expect(childStepId).not.toBe('undefined') + expect(childStepId).not.toBe('null') + + // Child should have a parentStepId (the parent step's ID) + expect(parentStepId).toBeTruthy() + expect(parentStepId).not.toBe('undefined') + expect(parentStepId).not.toBe('null') + }) + + it('should store parentStepId in database', async () => { + const jobId = await client.runAction('nestedStepAction', { depth: 1 }) + await client.fetch({ batchSize: 10 }) + + const job = await client.waitForJob(jobId, { timeout: 5000 }) + expectToBeDefined(job) + + // Get all steps for the job + const stepsResult = await client.getJobSteps({ jobId, pageSize: 100 }) + const steps = stepsResult.steps + + expect(steps.length).toBe(2) // parent + child + + const parentStep = steps.find((s) => s.name === 'parent-step') + const childStep = steps.find((s) => s.name === 'child-step') + + expectToBeDefined(parentStep) + expectToBeDefined(childStep) + + // Parent step should have no parentStepId (it's a root step) + expect((parentStep as any).parentStepId).toBeNull() + + // Child step should have parentStepId pointing to parent + expect((childStep as any).parentStepId).toBe(parentStep.id) + }) + }) + + describe('Deep Nesting', () => { + it('should support deeply nested steps (4 levels)', async () => { + const jobId = await client.runAction('deeplyNestedAction', {}) + await client.fetch({ batchSize: 10 }) + + const job = await client.waitForJob(jobId, { timeout: 5000 }) + expectToBeDefined(job) + expect(job.status).toBe(JOB_STATUS_COMPLETED) + + const output = job.output as { depth: number; stepIds: string[] } + expect(output.depth).toBe(4) + expect(output.stepIds.length).toBe(4) + + // All step IDs should be unique + const uniqueIds = new Set(output.stepIds) + expect(uniqueIds.size).toBe(4) + }) + + it('should maintain correct parent chain in deep nesting', async () => { + const jobId = await client.runAction('parentStepIdVerificationAction', {}) + await client.fetch({ batchSize: 10 }) + + const job = await client.waitForJob(jobId, { timeout: 5000 }) + expectToBeDefined(job) + expect(job.status).toBe(JOB_STATUS_COMPLETED) + + const output = job.output as { + parentChain: Array<{ stepId: string; parentStepId: string | null }> + } + + expect(output.parentChain.length).toBe(3) + + // Root step should have null parentStepId + expect(output.parentChain[0]!.parentStepId).toBeNull() + + // Nested step should have root's stepId as parentStepId + expect(output.parentChain[1]!.parentStepId).toBe(output.parentChain[0]!.stepId) + + // Deep step should have nested's stepId as parentStepId + expect(output.parentChain[2]!.parentStepId).toBe(output.parentChain[1]!.stepId) + }) + }) + + describe('Concurrent Children', () => { + it('should handle multiple concurrent child steps under same parent', async () => { + const jobId = await client.runAction('concurrentChildrenAction', { childCount: 5 }) + await client.fetch({ batchSize: 10 }) + + const job = await client.waitForJob(jobId, { timeout: 10000 }) + expectToBeDefined(job) + expect(job.status).toBe(JOB_STATUS_COMPLETED) + + const output = job.output as { childResults: number[] } + expect(output.childResults.sort()).toEqual([0, 1, 2, 3, 4]) + + // Verify all steps exist in database + const stepsResult = await client.getJobSteps({ jobId, pageSize: 100 }) + expect(stepsResult.steps.length).toBe(6) // 1 parent + 5 children + + const parentStep = stepsResult.steps.find((s) => s.name === 'parent-step') + expectToBeDefined(parentStep) + + // All child steps should have same parentStepId + const childSteps = stepsResult.steps.filter((s) => s.name.startsWith('child-')) + expect(childSteps.length).toBe(5) + childSteps.forEach((child) => { + expect((child as any).parentStepId).toBe(parentStep.id) + }) + }) + }) + + describe('Unhandled Child Steps', () => { + it('should fail action when child step is not awaited', async () => { + const jobId = await client.runAction('unawaitedChildAction', {}) + await client.fetch({ batchSize: 10 }) + + const job = await client.waitForJob(jobId, { timeout: 5000 }) + expectToBeDefined(job) + expect(job.status).toBe(JOB_STATUS_FAILED) + + // Error should mention unhandled child steps + expect(job.error).toBeDefined() + expect(job.error.name).toBe('UnhandledChildStepsError') + + // The orphaned child step should be cancelled + const stepsResult = await client.getJobSteps({ jobId, pageSize: 100 }) + const orphanedChild = stepsResult.steps.find((s) => s.name === 'orphaned-child') + expectToBeDefined(orphanedChild) + expect(orphanedChild.status).toBe(STEP_STATUS_CANCELLED) + }) + }) + + describe('Abort Propagation', () => { + it('should abort child steps when parent times out', async () => { + const jobId = await client.runAction('parentTimeoutAction', {}) + await client.fetch({ batchSize: 10 }) + + const job = await client.waitForJob(jobId, { timeout: 5000 }) + expectToBeDefined(job) + expect(job.status).toBe(JOB_STATUS_FAILED) + + // Parent should have timed out + expect(job.error).toBeDefined() + expect(job.error.name).toBe('StepTimeoutError') + + // Get steps + const stepsResult = await client.getJobSteps({ jobId, pageSize: 100 }) + + // Parent step should be present and failed + const parentStep = stepsResult.steps.find((s) => s.name === 'parent-step') + expectToBeDefined(parentStep) + expect(parentStep.status).toBe(STEP_STATUS_FAILED) + + // Child step may or may not exist depending on timing + // If it does exist, it should be cancelled or failed + const childStep = stepsResult.steps.find((s) => s.name === 'slow-child') + if (childStep) { + expect(childStep.status === STEP_STATUS_CANCELLED || childStep.status === STEP_STATUS_FAILED).toBe(true) + // Child should have parent step's id + expect((childStep as any).parentStepId).toBe(parentStep.id) + } + }) + + it('should cancel all nested steps when action is cancelled', async () => { + const jobId = await client.runAction('deeplyNestedAction', {}) + + // Start fetching but cancel quickly + const fetchPromise = client.fetch({ batchSize: 10 }) + + // Cancel the job after a short delay + await new Promise((resolve) => setTimeout(resolve, 10)) + await client.cancelJob(jobId) + + await fetchPromise + + // Wait for job to settle + await new Promise((resolve) => setTimeout(resolve, 200)) + + const job = await client.getJobById(jobId) + expectToBeDefined(job) + + // Job should be cancelled or failed + expect([JOB_STATUS_COMPLETED, JOB_STATUS_FAILED, 'cancelled']).toContain(job.status) + }) + }) + }) +} + +// Run tests with both adapters +runNestedStepsTests(pgliteFactory) +runNestedStepsTests(postgresFactory) diff --git a/packages/shared-actions/index.ts b/packages/shared-actions/index.ts index 1c6a517..bea5734 100644 --- a/packages/shared-actions/index.ts +++ b/packages/shared-actions/index.ts @@ -263,3 +263,351 @@ export const getWeather = defineAction()({ } }, }) + +/** + * Example action demonstrating nested steps feature. + * This simulates an e-commerce order processing workflow with: + * - Parent steps that contain child steps + * - Deep nesting (3 levels) + * - Shared abort signal propagation + * - Parent-child step tracking in the database + * - Promise.all of parent steps, each with their own nested children + * + * The workflow: + * 1. validate-order (parent) + * ├── check-inventory (child) + * └── verify-customer (child) + * + * 2. process-payment (parent) + * ├── authorize-payment (child) + * │ └── fraud-check (grandchild - 3 levels deep!) + * └── capture-payment (child) + * + * 3. fulfill-order (parent) + * ├── reserve-inventory (child) + * └── create-shipment (child) + * + * 4. send-notifications (parent) + * ├── email-confirmation (child) ─┐ + * └── sms-notification (child) ──┴── concurrent child steps + * + * 5. post-order-processing (Promise.all of 3 parent steps with nested children) + * ├── analytics-tracking (parent) ─────┐ + * │ ├── track-purchase (child) │ + * │ └── update-recommendations │ + * ├── loyalty-update (parent) ─────────┼── all 3 run in parallel + * │ ├── calculate-points (child) │ + * │ └── update-tier (child) │ + * └── partner-sync (parent) ───────────┘ + * ├── sync-supplier (child) + * └── sync-warehouse (child) + */ +export const processOrder = defineAction()({ + name: 'processOrder', + input: z.object({ + orderId: z.string().min(1).describe('The order ID to process'), + customerId: z.string().min(1).describe('The customer ID'), + items: z + .array( + z.object({ + productId: z.string(), + quantity: z.number().min(1), + price: z.number().min(0), + }), + ) + .min(1) + .describe('Order items'), + paymentMethod: z.enum(['credit_card', 'paypal', 'bank_transfer']).default('credit_card'), + shippingAddress: z.object({ + street: z.string(), + city: z.string(), + country: z.string(), + postalCode: z.string(), + }), + }), + output: z.object({ + orderId: z.string(), + status: z.enum(['completed', 'failed']), + transactionId: z.string().nullable(), + shipmentId: z.string().nullable(), + timeline: z.array( + z.object({ + step: z.string(), + status: z.enum(['success', 'failed']), + timestamp: z.string(), + details: z.string().optional(), + }), + ), + }), + handler: async (ctx) => { + const { orderId, customerId, items, paymentMethod, shippingAddress } = ctx.input + const timeline: Array<{ step: string; status: 'success' | 'failed'; timestamp: string; details?: string }> = [] + const totalAmount = items.reduce((sum, item) => sum + item.price * item.quantity, 0) + + // Helper to add timeline entry + const addTimeline = (step: string, status: 'success' | 'failed', details?: string) => { + timeline.push({ step, status, timestamp: new Date().toISOString(), details }) + } + + // ========================================================================= + // Step 1: Validate Order (with nested child steps) + // ========================================================================= + const validation = await ctx.step( + 'validate-order', + async ({ step: nestedStep }) => { + // Child step: Check inventory for all items + const inventoryCheck = await nestedStep('check-inventory', async () => { + // Simulate inventory check + await new Promise((resolve) => setTimeout(resolve, 100)) + const allInStock = items.every((item) => item.quantity <= 10) // Mock: max 10 per item + addTimeline('check-inventory', allInStock ? 'success' : 'failed', `Checked ${items.length} items`) + return { allInStock, checkedItems: items.length } + }) + + // Child step: Verify customer + const customerVerification = await nestedStep('verify-customer', async () => { + // Simulate customer verification + await new Promise((resolve) => setTimeout(resolve, 80)) + const isValid = customerId.length > 0 + addTimeline('verify-customer', isValid ? 'success' : 'failed', `Customer: ${customerId}`) + return { isValid, customerId } + }) + + addTimeline( + 'validate-order', + inventoryCheck.allInStock && customerVerification.isValid ? 'success' : 'failed', + `Inventory: ${inventoryCheck.allInStock}, Customer: ${customerVerification.isValid}`, + ) + + return { + isValid: inventoryCheck.allInStock && customerVerification.isValid, + inventoryCheck, + customerVerification, + } + }, + { expire: 30_000 }, + ) + + if (!validation.isValid) { + return { + orderId, + status: 'failed' as const, + transactionId: null, + shipmentId: null, + timeline, + } + } + + // ========================================================================= + // Step 2: Process Payment (with deeply nested steps - 3 levels) + // ========================================================================= + const payment = await ctx.step( + 'process-payment', + async ({ step: paymentStep }) => { + // Child step: Authorize payment (contains grandchild step) + const authorization = await paymentStep('authorize-payment', async ({ step: authStep }) => { + // Grandchild step: Fraud check (3 levels deep!) + const fraudCheck = await authStep('fraud-check', async () => { + await new Promise((resolve) => setTimeout(resolve, 150)) + const isSafe = totalAmount < 10000 // Mock: flag large orders + addTimeline('fraud-check', isSafe ? 'success' : 'failed', `Amount: $${totalAmount.toFixed(2)}`) + return { isSafe, riskScore: isSafe ? 0.1 : 0.9 } + }) + + if (!fraudCheck.isSafe) { + addTimeline('authorize-payment', 'failed', 'Fraud check failed') + return { authorized: false, authCode: null, fraudCheck } + } + + await new Promise((resolve) => setTimeout(resolve, 100)) + const authCode = `AUTH-${Date.now()}` + addTimeline('authorize-payment', 'success', `Auth code: ${authCode}`) + return { authorized: true, authCode, fraudCheck } + }) + + if (!authorization.authorized) { + addTimeline('process-payment', 'failed', 'Authorization failed') + return { success: false, transactionId: null, authorization } + } + + // Child step: Capture payment + const capture = await paymentStep('capture-payment', async () => { + await new Promise((resolve) => setTimeout(resolve, 120)) + const transactionId = `TXN-${Date.now()}` + addTimeline('capture-payment', 'success', `Transaction: ${transactionId}, Method: ${paymentMethod}`) + return { captured: true, transactionId } + }) + + addTimeline('process-payment', 'success', `Transaction ID: ${capture.transactionId}`) + return { + success: true, + transactionId: capture.transactionId, + authorization, + } + }, + { expire: 60_000 }, + ) + + if (!payment.success) { + return { + orderId, + status: 'failed' as const, + transactionId: null, + shipmentId: null, + timeline, + } + } + + // ========================================================================= + // Step 3: Fulfill Order (with nested steps) + // ========================================================================= + const fulfillment = await ctx.step( + 'fulfill-order', + async ({ step: fulfillStep }) => { + // Child step: Reserve inventory + const reservation = await fulfillStep('reserve-inventory', async () => { + await new Promise((resolve) => setTimeout(resolve, 90)) + const reservationId = `RES-${Date.now()}` + addTimeline('reserve-inventory', 'success', `Reserved ${items.length} items`) + return { reserved: true, reservationId } + }) + + // Child step: Create shipment + const shipment = await fulfillStep('create-shipment', async () => { + await new Promise((resolve) => setTimeout(resolve, 110)) + const shipmentId = `SHIP-${Date.now()}` + addTimeline('create-shipment', 'success', `Shipment to ${shippingAddress.city}, ${shippingAddress.country}`) + return { shipmentId, carrier: 'FastShip', estimatedDays: 3 } + }) + + addTimeline('fulfill-order', 'success', `Shipment: ${shipment.shipmentId}`) + return { reservation, shipment } + }, + { expire: 30_000 }, + ) + + // ========================================================================= + // Step 4: Send Notifications (with concurrent nested steps) + // ========================================================================= + await ctx.step( + 'send-notifications', + async ({ step: notifyStep }) => { + // Run notification child steps concurrently + const [emailResult, smsResult] = await Promise.all([ + // Child step: Send email confirmation + notifyStep('email-confirmation', async () => { + await new Promise((resolve) => setTimeout(resolve, 70)) + addTimeline('email-confirmation', 'success', `Sent to customer ${customerId}`) + return { sent: true, type: 'email' } + }), + + // Child step: Send SMS notification + notifyStep('sms-notification', async () => { + await new Promise((resolve) => setTimeout(resolve, 50)) + addTimeline('sms-notification', 'success', 'Order confirmation SMS sent') + return { sent: true, type: 'sms' } + }), + ]) + + addTimeline('send-notifications', 'success', `Email: ${emailResult.sent}, SMS: ${smsResult.sent}`) + return { email: emailResult, sms: smsResult } + }, + { expire: 15_000 }, + ) + + // ========================================================================= + // Step 5: Post-Order Processing (Promise.all of steps with nested steps) + // This demonstrates running multiple parent steps in parallel, + // where each parent step has its own nested child steps. + // ========================================================================= + const [analytics, loyalty, partnerSync] = await Promise.all([ + // Parent step 1: Analytics Tracking (with nested steps) + ctx.step( + 'analytics-tracking', + async ({ step: analyticsStep }) => { + // Nested child: Track purchase event + const purchase = await analyticsStep('track-purchase', async () => { + await new Promise((resolve) => setTimeout(resolve, 40)) + addTimeline('track-purchase', 'success', `Tracked order ${orderId}`) + return { eventId: `EVT-${Date.now()}`, type: 'purchase' } + }) + + // Nested child: Update product recommendations + const recommendations = await analyticsStep('update-recommendations', async () => { + await new Promise((resolve) => setTimeout(resolve, 60)) + addTimeline('update-recommendations', 'success', `Updated for ${items.length} products`) + return { updated: true, productsAnalyzed: items.length } + }) + + addTimeline('analytics-tracking', 'success', 'Analytics updated') + return { purchase, recommendations } + }, + { expire: 10_000 }, + ), + + // Parent step 2: Loyalty Program Update (with nested steps) + ctx.step( + 'loyalty-update', + async ({ step: loyaltyStep }) => { + // Nested child: Calculate loyalty points + const points = await loyaltyStep('calculate-points', async () => { + await new Promise((resolve) => setTimeout(resolve, 50)) + const earnedPoints = Math.floor(totalAmount * 10) // 10 points per dollar + addTimeline('calculate-points', 'success', `Earned ${earnedPoints} points`) + return { earnedPoints, multiplier: 1.0 } + }) + + // Nested child: Update customer tier + const tier = await loyaltyStep('update-tier', async () => { + await new Promise((resolve) => setTimeout(resolve, 45)) + const newTier = totalAmount > 500 ? 'gold' : totalAmount > 100 ? 'silver' : 'bronze' + addTimeline('update-tier', 'success', `Tier: ${newTier}`) + return { tier: newTier, upgraded: totalAmount > 500 } + }) + + addTimeline('loyalty-update', 'success', `${points.earnedPoints} points, tier: ${tier.tier}`) + return { points, tier } + }, + { expire: 10_000 }, + ), + + // Parent step 3: Partner Sync (with nested steps) + ctx.step( + 'partner-sync', + async ({ step: syncStep }) => { + // Nested child: Sync with supplier + const supplier = await syncStep('sync-supplier', async () => { + await new Promise((resolve) => setTimeout(resolve, 80)) + addTimeline('sync-supplier', 'success', 'Supplier inventory updated') + return { synced: true, supplierId: 'SUP-001' } + }) + + // Nested child: Sync with warehouse + const warehouse = await syncStep('sync-warehouse', async () => { + await new Promise((resolve) => setTimeout(resolve, 70)) + addTimeline('sync-warehouse', 'success', 'Warehouse notified for picking') + return { synced: true, warehouseId: 'WH-MAIN' } + }) + + addTimeline('partner-sync', 'success', 'All partners synced') + return { supplier, warehouse } + }, + { expire: 10_000 }, + ), + ]) + + addTimeline( + 'post-order-processing', + 'success', + `Analytics: done, Loyalty: ${loyalty.points.earnedPoints}pts, Partners: ${analytics && partnerSync ? 'synced' : 'pending'}`, + ) + + return { + orderId, + status: 'completed' as const, + transactionId: payment.transactionId, + shipmentId: fulfillment.shipment.shipmentId, + timeline, + } + }, +}) From 6789dcf842a8039752f98bb0abdd62d9fe002a7d Mon Sep 17 00:00:00 2001 From: Martin Acosta Date: Sat, 17 Jan 2026 15:01:54 -0300 Subject: [PATCH 02/78] Add time travel feature to Duron job steps - Implemented a new time travel functionality allowing jobs to restart from specific steps, preserving completed branch siblings. - Updated the database schema to include a 'branch' column in job steps for independent step management. - Enhanced the StepOptions schema to support branch steps and added relevant logic in the StepManager. - Introduced new API endpoints and client methods for time traveling jobs. - Updated the dashboard to allow users to initiate time travel from the UI. - Added comprehensive tests to validate time travel behavior and edge cases. --- bun.lock | 5 + .../src/components/timeline-modal.tsx | 5 +- packages/duron-dashboard/src/lib/api.ts | 20 +- .../duron-dashboard/src/views/step-list.tsx | 94 +- .../migration.sql | 1 + .../snapshot.json | 1001 +++++++++++++++++ packages/duron/package.json | 1 + packages/duron/src/action.ts | 9 + packages/duron/src/adapters/adapter.ts | 38 + packages/duron/src/adapters/postgres/base.ts | 166 ++- .../duron/src/adapters/postgres/schema.ts | 3 +- packages/duron/src/adapters/schemas.ts | 17 +- packages/duron/src/client.ts | 15 + packages/duron/src/server.ts | 41 +- packages/duron/src/step-manager.ts | 24 +- packages/duron/test/server.test.ts | 24 +- packages/duron/test/time-travel.test.ts | 476 ++++++++ packages/shared-actions/index.ts | 162 +-- 18 files changed, 1925 insertions(+), 177 deletions(-) create mode 100644 packages/duron/migrations/postgres/20260117172938_panoramic_photon/migration.sql create mode 100644 packages/duron/migrations/postgres/20260117172938_panoramic_photon/snapshot.json create mode 100644 packages/duron/test/time-travel.test.ts diff --git a/bun.lock b/bun.lock index 3559d4a..324340e 100644 --- a/bun.lock +++ b/bun.lock @@ -49,6 +49,7 @@ "elysia": "^1.4.22", "fastq": "^1.19.1", "jose": "^6.1.2", + "p-all": "^5.0.1", "p-retry": "^7.1.0", "p-timeout": "^7.0.1", "pino": "^10.1.0", @@ -1346,6 +1347,10 @@ "openapi-types": ["openapi-types@12.1.3", "", {}, "sha512-N4YtSYJqghVu4iek2ZUvcN/0aqH1kRDuNqzcycDxhOUpg7GdvLa2F3DgS6yBNhInhv2r/6I0Flkn7CqL8+nIcw=="], + "p-all": ["p-all@5.0.1", "", { "dependencies": { "p-map": "^6.0.0" } }, "sha512-LMT7WX9ZSaq3J1zjloApkIVmtz0ZdMFSIqbuiEa3txGYPLjUPOvgOPOx3nFjo+f37ZYL+1aY666I2SG7GVwLOA=="], + + "p-map": ["p-map@6.0.0", "", {}, "sha512-T8BatKGY+k5rU+Q/GTYgrEf2r4xRMevAN5mtXc2aPc4rS1j3s+vWTaO2Wag94neXuCAUAs8cxBL9EeB5EA6diw=="], + "p-retry": ["p-retry@7.1.0", "", { "dependencies": { "is-network-error": "^1.1.0" } }, "sha512-xL4PiFRQa/f9L9ZvR4/gUCRNus4N8YX80ku8kv9Jqz+ZokkiZLM0bcvX0gm1F3PDi9SPRsww1BDsTWgE6Y1GLQ=="], "p-timeout": ["p-timeout@7.0.1", "", {}, "sha512-AxTM2wDGORHGEkPCt8yqxOTMgpfbEHqF51f/5fJCmwFC3C/zNcGT63SymH2ttOAaiIws2zVg4+izQCjrakcwHg=="], diff --git a/packages/duron-dashboard/src/components/timeline-modal.tsx b/packages/duron-dashboard/src/components/timeline-modal.tsx index f9737d6..79f9c11 100644 --- a/packages/duron-dashboard/src/components/timeline-modal.tsx +++ b/packages/duron-dashboard/src/components/timeline-modal.tsx @@ -20,10 +20,7 @@ interface TimelineModalProps { export function TimelineModal({ jobId, open, onOpenChange }: TimelineModalProps) { const [selectedStepId, setSelectedStepId] = useState(null) const { data: job, isLoading: jobLoading } = useJob(jobId) - const { data: stepsData, isLoading: stepsLoading } = useJobSteps(jobId, { - page: 1, - pageSize: 1000, // Get all steps for timeline - }) + const { data: stepsData, isLoading: stepsLoading } = useJobSteps(jobId, {}) // Enable polling for step updates useStepsPolling(jobId, open) diff --git a/packages/duron-dashboard/src/lib/api.ts b/packages/duron-dashboard/src/lib/api.ts index d2bce2d..7870f6e 100644 --- a/packages/duron-dashboard/src/lib/api.ts +++ b/packages/duron-dashboard/src/lib/api.ts @@ -344,8 +344,6 @@ export function useDeleteJobs() { export function useJobSteps(jobId: string | null, params: GetJobStepsParams = {}) { const apiRequest = useApiRequest() const queryParams = new URLSearchParams() - if (params.page) queryParams.set('page', params.page.toString()) - if (params.pageSize) queryParams.set('pageSize', params.pageSize.toString()) if (params.search) queryParams.set('search', params.search) if (params.fUpdatedAfter) { const dateObj = @@ -403,6 +401,24 @@ export function useRetryJob() { }) } +export function useTimeTravelJob() { + const apiRequest = useApiRequest() + const queryClient = useQueryClient() + return useMutation({ + mutationFn: async ({ jobId, stepId }: { jobId: string; stepId: string }) => { + return apiRequest<{ success: boolean; message: string }>(`/jobs/${jobId}/time-travel`, { + method: 'POST', + body: JSON.stringify({ stepId }), + }) + }, + onSuccess: (_, { jobId }) => { + queryClient.invalidateQueries({ queryKey: ['job', jobId] }) + queryClient.invalidateQueries({ queryKey: ['job-steps', jobId] }) + queryClient.invalidateQueries({ queryKey: ['jobs'] }) + }, + }) +} + // Actions API export function useActions() { const apiRequest = useApiRequest() diff --git a/packages/duron-dashboard/src/views/step-list.tsx b/packages/duron-dashboard/src/views/step-list.tsx index 88b97f2..f2938ab 100644 --- a/packages/duron-dashboard/src/views/step-list.tsx +++ b/packages/duron-dashboard/src/views/step-list.tsx @@ -1,6 +1,6 @@ 'use client' -import { ChevronRight, Clock, Search } from 'lucide-react' +import { ChevronRight, Clock, GitBranch, History, Search } from 'lucide-react' import { useCallback, useMemo, useState } from 'react' import { TimelineModal } from '@/components/timeline-modal' @@ -8,13 +8,14 @@ import { Accordion, AccordionContent, AccordionItem, AccordionTrigger } from '@/ import { Button } from '@/components/ui/button' import { Input } from '@/components/ui/input' import { ScrollArea } from '@/components/ui/scroll-area' +import { Tooltip, TooltipContent, TooltipTrigger } from '@/components/ui/tooltip' import { useDebouncedCallback } from '@/hooks/use-debounced-callback' import { useStepsPolling } from '@/hooks/use-steps-polling' -import { type GetJobStepsResponse, useJobSteps } from '@/lib/api' +import { type GetJobStepsResponse, useJob, useJobSteps, useTimeTravelJob } from '@/lib/api' import { BadgeStatus } from '../components/badge-status' // Step type from the API response (without output field) -type JobStepWithoutOutput = GetJobStepsResponse['steps'][number] & { parentStepId?: string | null } +type JobStepWithoutOutput = GetJobStepsResponse['steps'][number] & { parentStepId?: string | null; branch?: boolean } import { StepDetailsContent } from './step-details-content' @@ -83,14 +84,11 @@ function flattenStepTree(nodes: StepNode[]): Array<{ step: JobStepWithoutOutput; export function StepList({ jobId, selectedStepId, onStepSelect }: StepListProps) { const [inputValue, setInputValue] = useState('') const [searchTerm, setSearchTerm] = useState('') - const [page, setPage] = useState(1) const [timelineOpen, setTimelineOpen] = useState(false) - const pageSize = 20 // Debounce the search term update with 1000ms delay const debouncedSetSearchTerm = useDebouncedCallback((value: string) => { setSearchTerm(value) - setPage(1) // Reset to first page when searching }, 1000) const handleSearchChange = useCallback( @@ -101,12 +99,26 @@ export function StepList({ jobId, selectedStepId, onStepSelect }: StepListProps) [debouncedSetSearchTerm], ) + // Fetch all steps (no pagination) const { data: stepsData, isLoading: stepsLoading } = useJobSteps(jobId, { - page, - pageSize, search: searchTerm || undefined, }) + const { data: job } = useJob(jobId) + const timeTravelMutation = useTimeTravelJob() + + // Check if job is in a terminal state (can time travel) + const canTimeTravel = job?.status === 'completed' || job?.status === 'failed' || job?.status === 'cancelled' + + const handleTimeTravel = useCallback( + (stepId: string, e: React.MouseEvent) => { + e.stopPropagation() + if (!jobId || !canTimeTravel) return + timeTravelMutation.mutate({ jobId, stepId }) + }, + [jobId, canTimeTravel, timeTravelMutation], + ) + // Enable polling for step updates useStepsPolling(jobId, true) @@ -170,8 +182,9 @@ export function StepList({ jobId, selectedStepId, onStepSelect }: StepListProps) onValueChange={onStepSelect} > {orderedSteps.map(({ step, depth }, index) => { - const stepNumber = (page - 1) * pageSize + index + 1 + const stepNumber = index + 1 const isNested = depth > 0 + const isBranch = (step as any).branch === true // Calculate left padding based on depth (16px per level) const paddingLeft = depth * 16 @@ -186,10 +199,45 @@ export function StepList({ jobId, selectedStepId, onStepSelect }: StepListProps)
{isNested && } + {isBranch && ( + + + + + +

Branch step (independent from siblings)

+ + + )} #{stepNumber} {step.name}
- +
+ {canTimeTravel && ( + + + handleTimeTravel(step.id, e)} + onKeyDown={(e) => { + if (e.key === 'Enter' || e.key === ' ') { + handleTimeTravel(step.id, e as unknown as React.MouseEvent) + } + }} + aria-disabled={timeTravelMutation.isPending} + > + + + + +

Time travel: restart from this step

+ + + )} + +
@@ -203,32 +251,6 @@ export function StepList({ jobId, selectedStepId, onStepSelect }: StepListProps)
- {stepsData && stepsData.total > pageSize && ( -
-
- Showing {(page - 1) * pageSize + 1} - {Math.min(page * pageSize, stepsData.total)} of {stepsData.total}{' '} - steps -
-
- - -
-
- )}
diff --git a/packages/duron/migrations/postgres/20260117172938_panoramic_photon/migration.sql b/packages/duron/migrations/postgres/20260117172938_panoramic_photon/migration.sql new file mode 100644 index 0000000..a4ff7b2 --- /dev/null +++ b/packages/duron/migrations/postgres/20260117172938_panoramic_photon/migration.sql @@ -0,0 +1 @@ +ALTER TABLE "duron"."job_steps" ADD COLUMN "branch" boolean DEFAULT false NOT NULL; \ No newline at end of file diff --git a/packages/duron/migrations/postgres/20260117172938_panoramic_photon/snapshot.json b/packages/duron/migrations/postgres/20260117172938_panoramic_photon/snapshot.json new file mode 100644 index 0000000..4ad1558 --- /dev/null +++ b/packages/duron/migrations/postgres/20260117172938_panoramic_photon/snapshot.json @@ -0,0 +1,1001 @@ +{ + "version": "8", + "dialect": "postgres", + "id": "cc8b34a2-aa75-4928-927c-341d737fe2dc", + "prevIds": [ + "25beb926-865c-41eb-8525-bbc1e0d3a19c" + ], + "ddl": [ + { + "name": "duron", + "entityType": "schemas" + }, + { + "isRlsEnabled": false, + "name": "job_steps", + "entityType": "tables", + "schema": "duron" + }, + { + "isRlsEnabled": false, + "name": "jobs", + "entityType": "tables", + "schema": "duron" + }, + { + "type": "uuid", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "gen_random_uuid()", + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "duron", + "table": "job_steps" + }, + { + "type": "uuid", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "job_id", + "entityType": "columns", + "schema": "duron", + "table": "job_steps" + }, + { + "type": "uuid", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "parent_step_id", + "entityType": "columns", + "schema": "duron", + "table": "job_steps" + }, + { + "type": "boolean", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "false", + "generated": null, + "identity": null, + "name": "branch", + "entityType": "columns", + "schema": "duron", + "table": "job_steps" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "name", + "entityType": "columns", + "schema": "duron", + "table": "job_steps" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'active'", + "generated": null, + "identity": null, + "name": "status", + "entityType": "columns", + "schema": "duron", + "table": "job_steps" + }, + { + "type": "jsonb", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "output", + "entityType": "columns", + "schema": "duron", + "table": "job_steps" + }, + { + "type": "jsonb", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "error", + "entityType": "columns", + "schema": "duron", + "table": "job_steps" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "started_at", + "entityType": "columns", + "schema": "duron", + "table": "job_steps" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "finished_at", + "entityType": "columns", + "schema": "duron", + "table": "job_steps" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "timeout_ms", + "entityType": "columns", + "schema": "duron", + "table": "job_steps" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "expires_at", + "entityType": "columns", + "schema": "duron", + "table": "job_steps" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "0", + "generated": null, + "identity": null, + "name": "retries_limit", + "entityType": "columns", + "schema": "duron", + "table": "job_steps" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "0", + "generated": null, + "identity": null, + "name": "retries_count", + "entityType": "columns", + "schema": "duron", + "table": "job_steps" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "delayed_ms", + "entityType": "columns", + "schema": "duron", + "table": "job_steps" + }, + { + "type": "jsonb", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'{}'", + "generated": null, + "identity": null, + "name": "history_failed_attempts", + "entityType": "columns", + "schema": "duron", + "table": "job_steps" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "duron", + "table": "job_steps" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "duron", + "table": "job_steps" + }, + { + "type": "uuid", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "gen_random_uuid()", + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "duron", + "table": "jobs" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "action_name", + "entityType": "columns", + "schema": "duron", + "table": "jobs" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "group_key", + "entityType": "columns", + "schema": "duron", + "table": "jobs" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'created'", + "generated": null, + "identity": null, + "name": "status", + "entityType": "columns", + "schema": "duron", + "table": "jobs" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "checksum", + "entityType": "columns", + "schema": "duron", + "table": "jobs" + }, + { + "type": "jsonb", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'{}'", + "generated": null, + "identity": null, + "name": "input", + "entityType": "columns", + "schema": "duron", + "table": "jobs" + }, + { + "type": "jsonb", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "output", + "entityType": "columns", + "schema": "duron", + "table": "jobs" + }, + { + "type": "jsonb", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "error", + "entityType": "columns", + "schema": "duron", + "table": "jobs" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "timeout_ms", + "entityType": "columns", + "schema": "duron", + "table": "jobs" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "expires_at", + "entityType": "columns", + "schema": "duron", + "table": "jobs" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "started_at", + "entityType": "columns", + "schema": "duron", + "table": "jobs" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "finished_at", + "entityType": "columns", + "schema": "duron", + "table": "jobs" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "client_id", + "entityType": "columns", + "schema": "duron", + "table": "jobs" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "10", + "generated": null, + "identity": null, + "name": "concurrency_limit", + "entityType": "columns", + "schema": "duron", + "table": "jobs" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "duron", + "table": "jobs" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "duron", + "table": "jobs" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "job_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_job_steps_job_id", + "entityType": "indexes", + "schema": "duron", + "table": "job_steps" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "status", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_job_steps_status", + "entityType": "indexes", + "schema": "duron", + "table": "job_steps" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "name", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_job_steps_name", + "entityType": "indexes", + "schema": "duron", + "table": "job_steps" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "expires_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_job_steps_expires_at", + "entityType": "indexes", + "schema": "duron", + "table": "job_steps" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "parent_step_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_job_steps_parent_step_id", + "entityType": "indexes", + "schema": "duron", + "table": "job_steps" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "job_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "status", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_job_steps_job_status", + "entityType": "indexes", + "schema": "duron", + "table": "job_steps" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "job_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "name", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_job_steps_job_name", + "entityType": "indexes", + "schema": "duron", + "table": "job_steps" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "to_tsvector('english', \"output\"::text)", + "isExpression": true, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "gin", + "concurrently": false, + "name": "idx_job_steps_output_fts", + "entityType": "indexes", + "schema": "duron", + "table": "job_steps" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "action_name", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_jobs_action_name", + "entityType": "indexes", + "schema": "duron", + "table": "jobs" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "status", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_jobs_status", + "entityType": "indexes", + "schema": "duron", + "table": "jobs" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "group_key", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_jobs_group_key", + "entityType": "indexes", + "schema": "duron", + "table": "jobs" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "started_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_jobs_started_at", + "entityType": "indexes", + "schema": "duron", + "table": "jobs" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "finished_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_jobs_finished_at", + "entityType": "indexes", + "schema": "duron", + "table": "jobs" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "expires_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_jobs_expires_at", + "entityType": "indexes", + "schema": "duron", + "table": "jobs" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "client_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_jobs_client_id", + "entityType": "indexes", + "schema": "duron", + "table": "jobs" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "checksum", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_jobs_checksum", + "entityType": "indexes", + "schema": "duron", + "table": "jobs" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "concurrency_limit", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_jobs_concurrency_limit", + "entityType": "indexes", + "schema": "duron", + "table": "jobs" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "action_name", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "status", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_jobs_action_status", + "entityType": "indexes", + "schema": "duron", + "table": "jobs" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "action_name", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "group_key", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_jobs_action_group", + "entityType": "indexes", + "schema": "duron", + "table": "jobs" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "to_tsvector('english', \"input\"::text)", + "isExpression": true, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "gin", + "concurrently": false, + "name": "idx_jobs_input_fts", + "entityType": "indexes", + "schema": "duron", + "table": "jobs" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "to_tsvector('english', \"output\"::text)", + "isExpression": true, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "gin", + "concurrently": false, + "name": "idx_jobs_output_fts", + "entityType": "indexes", + "schema": "duron", + "table": "jobs" + }, + { + "nameExplicit": false, + "columns": [ + "job_id" + ], + "schemaTo": "duron", + "tableTo": "jobs", + "columnsTo": [ + "id" + ], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "job_steps_job_id_jobs_id_fkey", + "entityType": "fks", + "schema": "duron", + "table": "job_steps" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "job_steps_pkey", + "schema": "duron", + "table": "job_steps", + "entityType": "pks" + }, + { + "columns": [ + "id" + ], + "nameExplicit": false, + "name": "jobs_pkey", + "schema": "duron", + "table": "jobs", + "entityType": "pks" + }, + { + "nameExplicit": true, + "columns": [ + "job_id", + "name" + ], + "nullsNotDistinct": false, + "name": "unique_job_step_name", + "entityType": "uniques", + "schema": "duron", + "table": "job_steps" + }, + { + "value": "\"status\" IN ('active','completed','failed','cancelled')", + "name": "job_steps_status_check", + "entityType": "checks", + "schema": "duron", + "table": "job_steps" + }, + { + "value": "\"status\" IN ('created','active','completed','failed','cancelled')", + "name": "jobs_status_check", + "entityType": "checks", + "schema": "duron", + "table": "jobs" + } + ], + "renames": [] +} \ No newline at end of file diff --git a/packages/duron/package.json b/packages/duron/package.json index 25dcb46..ed65d7b 100644 --- a/packages/duron/package.json +++ b/packages/duron/package.json @@ -78,6 +78,7 @@ "elysia": "^1.4.22", "fastq": "^1.19.1", "jose": "^6.1.2", + "p-all": "^5.0.1", "p-retry": "^7.1.0", "p-timeout": "^7.0.1", "pino": "^10.1.0", diff --git a/packages/duron/src/action.ts b/packages/duron/src/action.ts index b79aa52..1dc2a80 100644 --- a/packages/duron/src/action.ts +++ b/packages/duron/src/action.ts @@ -132,6 +132,15 @@ export const StepOptionsSchema = z.object({ .number() .default(5 * 60 * 1000) .describe('The expire time for the step (milliseconds)'), + + /** + * Whether this step is a branch. + * Branch steps are independent from siblings during time travel. + * When time traveling to a step, completed branch siblings are preserved. + * + * @default false + */ + branch: z.boolean().default(false).describe('Whether this step is a branch (independent from siblings)'), }) /** diff --git a/packages/duron/src/adapters/adapter.ts b/packages/duron/src/adapters/adapter.ts index 5ab3e3c..993b86e 100644 --- a/packages/duron/src/adapters/adapter.ts +++ b/packages/duron/src/adapters/adapter.ts @@ -38,6 +38,7 @@ import type { JobStepStatusResult, RecoverJobsOptions, RetryJobOptions, + TimeTravelJobOptions, } from './schemas.js' import { BooleanResultSchema, @@ -68,6 +69,7 @@ import { NumberResultSchema, RecoverJobsOptionsSchema, RetryJobOptionsSchema, + TimeTravelJobOptionsSchema, } from './schemas.js' // Re-export types from schemas for backward compatibility @@ -101,6 +103,7 @@ export type { RecoverJobsOptions, RetryJobOptions, SortOrder, + TimeTravelJobOptions, } from './schemas.js' // ============================================================================ @@ -400,6 +403,30 @@ export abstract class Adapter extends EventEmitter { } } + /** + * Time travel a job to restart from a specific step. + * The job must be in completed, failed, or cancelled status. + * Resets the job and ancestor steps to active status, deletes subsequent steps, + * and preserves completed branch siblings. + * + * @returns Promise resolving to `true` if time travel succeeded, `false` otherwise + */ + async timeTravelJob(options: TimeTravelJobOptions): Promise { + try { + await this.start() + const parsedOptions = TimeTravelJobOptionsSchema.parse(options) + const result = await this._timeTravelJob(parsedOptions) + const success = BooleanResultSchema.parse(result) + if (success) { + await this._notify('job-available', { jobId: parsedOptions.jobId }) + } + return success + } catch (error) { + this.#logger?.error(error, 'Error in Adapter.timeTravelJob()') + throw error + } + } + /** * Delete a job by its ID. * Active jobs cannot be deleted. @@ -659,6 +686,17 @@ export abstract class Adapter extends EventEmitter { */ protected abstract _retryJob(options: RetryJobOptions): Promise + /** + * Internal method to time travel a job to restart from a specific step. + * The job must be in completed, failed, or cancelled status. + * Resets the job and ancestor steps to active status, deletes subsequent steps, + * and preserves completed branch siblings. + * + * @param options - Validated time travel options + * @returns Promise resolving to `true` if time travel succeeded, `false` otherwise + */ + protected abstract _timeTravelJob(options: TimeTravelJobOptions): Promise + /** * Internal method to delete a job by its ID. * Active jobs cannot be deleted. diff --git a/packages/duron/src/adapters/postgres/base.ts b/packages/duron/src/adapters/postgres/base.ts index 10e8ad5..637fbe6 100644 --- a/packages/duron/src/adapters/postgres/base.ts +++ b/packages/duron/src/adapters/postgres/base.ts @@ -39,6 +39,7 @@ import { type JobStepStatusResult, type RecoverJobsOptions, type RetryJobOptions, + type TimeTravelJobOptions, } from '../adapter.js' import createSchema from './schema.js' @@ -316,6 +317,144 @@ export class PostgresBaseAdapter e return result[0]!.id } + /** + * Internal method to time travel a job to restart from a specific step. + * The job must be in completed, failed, or cancelled status. + * Resets the job and ancestor steps to active status, deletes subsequent steps, + * and preserves completed branch siblings. + * + * Algorithm: + * 1. Validate job is in terminal state (completed/failed/cancelled) + * 2. Find the target step and all its ancestors (using parent_step_id) + * 3. Determine which steps to keep: + * - Steps completed BEFORE the target step (by created_at) + * - Branch siblings that are completed (independent) + * 4. Delete steps that should not be kept + * 5. Reset ancestor steps to active status (they need to re-run) + * 6. Reset the target step to active status + * 7. Reset job to created status + * + * @returns Promise resolving to `true` if time travel succeeded, `false` otherwise + */ + protected async _timeTravelJob({ jobId, stepId }: TimeTravelJobOptions): Promise { + const result = this._map( + await this.db.execute<{ success: boolean }>(sql` + WITH RECURSIVE + -- Lock and validate the job + locked_job AS ( + SELECT j.id + FROM ${this.tables.jobsTable} j + WHERE j.id = ${jobId} + AND j.status IN (${JOB_STATUS_COMPLETED}, ${JOB_STATUS_FAILED}, ${JOB_STATUS_CANCELLED}) + FOR UPDATE OF j + ), + -- Validate target step exists and belongs to job + target_step AS ( + SELECT s.id, s.parent_step_id, s.created_at + FROM ${this.tables.jobStepsTable} s + WHERE s.id = ${stepId} + AND s.job_id = ${jobId} + AND EXISTS (SELECT 1 FROM locked_job) + ), + -- Find all ancestor steps recursively (from target up to root) + ancestors AS ( + SELECT s.id, s.parent_step_id, 0 AS depth + FROM ${this.tables.jobStepsTable} s + WHERE s.id = (SELECT parent_step_id FROM target_step) + AND EXISTS (SELECT 1 FROM target_step) + UNION ALL + SELECT s.id, s.parent_step_id, a.depth + 1 + FROM ${this.tables.jobStepsTable} s + INNER JOIN ancestors a ON s.id = a.parent_step_id + ), + -- Steps to keep: completed steps created before target + completed branch siblings + steps_to_keep AS ( + -- Steps created before target that are completed (siblings or unrelated) + SELECT s.id + FROM ${this.tables.jobStepsTable} s + CROSS JOIN target_step ts + WHERE s.job_id = ${jobId} + AND s.created_at < ts.created_at + AND s.status = ${STEP_STATUS_COMPLETED} + AND s.id NOT IN (SELECT id FROM ancestors) + AND s.id != ts.id + UNION + -- Completed branch siblings at same level as target (same parent, branch=true, completed) + SELECT s.id + FROM ${this.tables.jobStepsTable} s + CROSS JOIN target_step ts + WHERE s.job_id = ${jobId} + AND s.id != ts.id + AND s.branch = true + AND s.status = ${STEP_STATUS_COMPLETED} + AND ( + (s.parent_step_id IS NULL AND ts.parent_step_id IS NULL) + OR s.parent_step_id = ts.parent_step_id + ) + ), + -- Delete steps that are not in the keep list and are not ancestors/target + deleted_steps AS ( + DELETE FROM ${this.tables.jobStepsTable} + WHERE job_id = ${jobId} + AND id NOT IN (SELECT id FROM steps_to_keep) + AND id NOT IN (SELECT id FROM ancestors) + AND id != (SELECT id FROM target_step) + RETURNING id + ), + -- Reset ancestor steps to active + reset_ancestors AS ( + UPDATE ${this.tables.jobStepsTable} + SET + status = ${STEP_STATUS_ACTIVE}, + output = NULL, + error = NULL, + finished_at = NULL, + started_at = now(), + expires_at = now() + (timeout_ms || ' milliseconds')::interval, + retries_count = 0, + delayed_ms = NULL, + history_failed_attempts = '{}'::jsonb + WHERE id IN (SELECT id FROM ancestors) + RETURNING id + ), + -- Reset target step to active + reset_target AS ( + UPDATE ${this.tables.jobStepsTable} + SET + status = ${STEP_STATUS_ACTIVE}, + output = NULL, + error = NULL, + finished_at = NULL, + started_at = now(), + expires_at = now() + (timeout_ms || ' milliseconds')::interval, + retries_count = 0, + delayed_ms = NULL, + history_failed_attempts = '{}'::jsonb + WHERE id = (SELECT id FROM target_step) + RETURNING id + ), + -- Reset job to created status + reset_job AS ( + UPDATE ${this.tables.jobsTable} + SET + status = ${JOB_STATUS_CREATED}, + output = NULL, + error = NULL, + started_at = NULL, + finished_at = NULL, + client_id = NULL, + expires_at = NULL + WHERE id = ${jobId} + AND EXISTS (SELECT 1 FROM target_step) + RETURNING id + ) + SELECT EXISTS(SELECT 1 FROM reset_job) AS success + `), + ) + + return result.length > 0 && result[0]!.success === true + } + /** * Internal method to delete a job by its ID. * Active jobs cannot be deleted. @@ -571,6 +710,7 @@ export class PostgresBaseAdapter e timeoutMs, retriesLimit, parentStepId, + branch = false, }: CreateOrRecoverJobStepOptions): Promise { type StepResult = CreateOrRecoverJobStepResult @@ -593,6 +733,7 @@ export class PostgresBaseAdapter e INSERT INTO ${this.tables.jobStepsTable} ( job_id, parent_step_id, + branch, name, timeout_ms, retries_limit, @@ -605,6 +746,7 @@ export class PostgresBaseAdapter e SELECT ${jobId}, ${parentStepId}, + ${branch}, ${name}, ${timeoutMs}, ${retriesLimit}, @@ -826,12 +968,12 @@ export class PostgresBaseAdapter e } /** - * Internal method to get steps for a job with pagination and fuzzy search. + * Internal method to get all steps for a job with optional fuzzy search. * Steps are always ordered by created_at ASC. * Steps do not include output data. */ protected async _getJobSteps(options: GetJobStepsOptions): Promise { - const { jobId, page = 1, pageSize = 10, search } = options + const { jobId, search } = options const jobStepsTable = this.tables.jobStepsTable @@ -850,23 +992,12 @@ export class PostgresBaseAdapter e : undefined, ) - // Get total count - const total = await this.db.$count(jobStepsTable, where) - - if (!total) { - return { - steps: [], - total: 0, - page, - pageSize, - } - } - const steps = await this.db .select({ id: jobStepsTable.id, jobId: jobStepsTable.job_id, parentStepId: jobStepsTable.parent_step_id, + branch: jobStepsTable.branch, name: jobStepsTable.name, status: jobStepsTable.status, error: jobStepsTable.error, @@ -884,14 +1015,10 @@ export class PostgresBaseAdapter e .from(jobStepsTable) .where(where) .orderBy(asc(jobStepsTable.created_at)) - .limit(pageSize) - .offset((page - 1) * pageSize) return { steps, - total, - page, - pageSize, + total: steps.length, } } @@ -1059,6 +1186,7 @@ export class PostgresBaseAdapter e id: this.tables.jobStepsTable.id, jobId: this.tables.jobStepsTable.job_id, parentStepId: this.tables.jobStepsTable.parent_step_id, + branch: this.tables.jobStepsTable.branch, name: this.tables.jobStepsTable.name, output: this.tables.jobStepsTable.output, status: this.tables.jobStepsTable.status, diff --git a/packages/duron/src/adapters/postgres/schema.ts b/packages/duron/src/adapters/postgres/schema.ts index a9b39b2..501688f 100644 --- a/packages/duron/src/adapters/postgres/schema.ts +++ b/packages/duron/src/adapters/postgres/schema.ts @@ -1,5 +1,5 @@ import { sql } from 'drizzle-orm' -import { check, index, integer, jsonb, pgSchema, text, timestamp, unique, uuid } from 'drizzle-orm/pg-core' +import { boolean, check, index, integer, jsonb, pgSchema, text, timestamp, unique, uuid } from 'drizzle-orm/pg-core' import { JOB_STATUSES, type JobStatus, STEP_STATUS_ACTIVE, STEP_STATUSES, type StepStatus } from '../../constants.js' import type { SerializableError } from '../../errors.js' @@ -67,6 +67,7 @@ export default function createSchema(schemaName: string) { .notNull() .references(() => jobsTable.id, { onDelete: 'cascade' }), parent_step_id: uuid('parent_step_id'), + branch: boolean('branch').notNull().default(false), name: text('name').notNull(), status: text('status').$type().notNull().default(STEP_STATUS_ACTIVE), output: jsonb('output'), diff --git a/packages/duron/src/adapters/schemas.ts b/packages/duron/src/adapters/schemas.ts index 74a1b9b..4ade81d 100644 --- a/packages/duron/src/adapters/schemas.ts +++ b/packages/duron/src/adapters/schemas.ts @@ -56,6 +56,7 @@ export const JobStepSchema = z.object({ id: z.string(), jobId: z.string(), parentStepId: z.string().nullable().default(null), + branch: z.boolean().default(false), name: z.string(), output: z.any().nullable().default(null), status: StepStatusSchema, @@ -114,8 +115,6 @@ export const GetJobsOptionsSchema = z.object({ export const GetJobStepsOptionsSchema = z.object({ jobId: z.string(), - page: z.number().int().positive().optional(), - pageSize: z.number().int().positive().optional(), search: z.string().optional(), updatedAfter: DateSchema.optional(), }) @@ -184,6 +183,13 @@ export const DeleteJobOptionsSchema = z.object({ export const DeleteJobsOptionsSchema = GetJobsOptionsSchema.optional() +export const TimeTravelJobOptionsSchema = z.object({ + /** The ID of the job to time travel */ + jobId: z.string(), + /** The ID of the step to restart from */ + stepId: z.string(), +}) + // ============================================================================ // Step Option Schemas // ============================================================================ @@ -193,6 +199,8 @@ export const CreateOrRecoverJobStepOptionsSchema = z.object({ jobId: z.string(), /** The ID of the parent step (null for root steps) */ parentStepId: z.string().nullable().default(null), + /** Whether this step is a branch (independent from siblings during time travel) */ + branch: z.boolean().default(false), /** The name of the step */ name: z.string(), /** Timeout in milliseconds for the step */ @@ -261,8 +269,6 @@ export const GetJobsResultSchema = z.object({ export const GetJobStepsResultSchema = z.object({ steps: z.array(JobStepWithoutOutputSchema), total: z.number().int().nonnegative(), - page: z.number().int().positive(), - pageSize: z.number().int().positive(), }) export const ActionStatsSchema = z.object({ @@ -316,9 +322,10 @@ export type CancelJobOptions = z.infer export type RetryJobOptions = z.infer export type DeleteJobOptions = z.infer export type DeleteJobsOptions = z.infer -export type CreateOrRecoverJobStepOptions = z.infer +export type CreateOrRecoverJobStepOptions = z.input export type CompleteJobStepOptions = z.infer export type FailJobStepOptions = z.infer export type DelayJobStepOptions = z.infer export type CancelJobStepOptions = z.infer export type CreateOrRecoverJobStepResult = z.infer +export type TimeTravelJobOptions = z.infer diff --git a/packages/duron/src/client.ts b/packages/duron/src/client.ts index ff6c358..f959dd7 100644 --- a/packages/duron/src/client.ts +++ b/packages/duron/src/client.ts @@ -361,6 +361,21 @@ export class Client< return this.#database.retryJob({ jobId }) } + /** + * Time travel a job to restart from a specific step. + * The job must be in completed, failed, or cancelled status. + * Resets the job and ancestor steps to active status, deletes subsequent steps, + * and preserves completed branch siblings. + * + * @param jobId - The ID of the job to time travel + * @param stepId - The ID of the step to restart from + * @returns Promise resolving to `true` if time travel succeeded, `false` otherwise + */ + async timeTravelJob(jobId: string, stepId: string): Promise { + await this.start() + return this.#database.timeTravelJob({ jobId, stepId }) + } + /** * Delete a job by its ID. * Active jobs cannot be deleted. diff --git a/packages/duron/src/server.ts b/packages/duron/src/server.ts index 89b219a..1d3ecfb 100644 --- a/packages/duron/src/server.ts +++ b/packages/duron/src/server.ts @@ -61,14 +61,10 @@ export class UnauthorizedError extends Error { export const GetJobStepsQuerySchema = z .object({ - page: z.coerce.number().int().min(1).optional(), - pageSize: z.coerce.number().int().min(1).max(1000).optional(), search: z.string().optional(), fUpdatedAfter: z.coerce.date().optional(), }) .transform((data) => ({ - page: data.page, - pageSize: data.pageSize, search: data.search, updatedAfter: data.fUpdatedAfter, })) @@ -201,6 +197,15 @@ export const RetryJobResponseSchema = z.object({ newJobId: z.string(), }) +export const TimeTravelJobBodySchema = z.object({ + stepId: z.uuid(), +}) + +export const TimeTravelJobResponseSchema = z.object({ + success: z.boolean(), + message: z.string(), +}) + // ============================================================================ // Server Factory // ============================================================================ @@ -345,8 +350,6 @@ export function createServer

({ client, prefix, login }: Create async ({ params, query }) => { const options: GetJobStepsOptions = { jobId: params.id, - page: query.page, - pageSize: query.pageSize, search: query.search, updatedAfter: query.updatedAfter, } @@ -490,6 +493,32 @@ export function createServer

({ client, prefix, login }: Create auth: true, }, ) + .post( + '/jobs/:id/time-travel', + async ({ params, body }) => { + const success = await client.timeTravelJob(params.id, body.stepId) + if (!success) { + throw new Error( + `Could not time travel job ${params.id}. The job may not be in a terminal state or the step may not exist.`, + ) + } + return { + success: true, + message: `Job ${params.id} has been time traveled to step ${body.stepId}`, + } + }, + { + params: JobIdParamsSchema, + body: TimeTravelJobBodySchema, + response: { + 200: TimeTravelJobResponseSchema, + 400: ErrorResponseSchema, + 500: ErrorResponseSchema, + 401: ErrorResponseSchema, + }, + auth: true, + }, + ) .delete( '/jobs/:id', async ({ params }) => { diff --git a/packages/duron/src/step-manager.ts b/packages/duron/src/step-manager.ts index 6b77ad3..a6495ab 100644 --- a/packages/duron/src/step-manager.ts +++ b/packages/duron/src/step-manager.ts @@ -30,6 +30,7 @@ export interface TaskStep { options: StepOptions abortSignal: AbortSignal parentStepId: string | null + branch: boolean } /** @@ -64,6 +65,7 @@ export class StepStore { * @param timeoutMs - Timeout in milliseconds for the step * @param retriesLimit - Maximum number of retries for the step * @param parentStepId - The ID of the parent step (null for root steps) + * @param branch - Whether this step is a branch (independent from siblings during time travel) * @returns Promise resolving to the created step ID * @throws Error if step creation fails */ @@ -73,6 +75,7 @@ export class StepStore { timeoutMs: number, retriesLimit: number, parentStepId: string | null = null, + branch: boolean = false, ) { try { return await this.#adapter.createOrRecoverJobStep({ @@ -81,6 +84,7 @@ export class StepStore { timeoutMs, retriesLimit, parentStepId, + branch, }) } catch (error) { throw new NonRetriableError(`Failed to get or create step "${name}" for job "${jobId}"`, { cause: error }) @@ -161,7 +165,7 @@ export class StepManager { throw new StepAlreadyExecutedError(task.name, this.#jobId, this.#actionName) } this.#historySteps.add(task.name) - return this.#executeStep(task.name, task.cb, task.options, task.abortSignal, task.parentStepId) + return this.#executeStep(task.name, task.cb, task.options, task.abortSignal, task.parentStepId, task.branch) }, options.concurrencyLimit) } @@ -229,6 +233,7 @@ export class StepManager { options: StepOptions, abortSignal: AbortSignal, parentStepId: string | null, + branch: boolean, ): Promise { const expire = options.expire const retryOptions = options.retry @@ -240,8 +245,15 @@ export class StepManager { throw new ActionCancelError(this.#actionName, this.#jobId, { cause: 'step cancelled before create step' }) } - // Create step record with parentStepId - const newStep = await this.#stepStore.getOrCreate(this.#jobId, name, expire, retryOptions.limit, parentStepId) + // Create step record with parentStepId and branch + const newStep = await this.#stepStore.getOrCreate( + this.#jobId, + name, + expire, + retryOptions.limit, + parentStepId, + branch, + ) if (!newStep) { throw new NonRetriableError( `Failed to create step "${name}" for job "${this.#jobId}" action "${this.#actionName}"`, @@ -316,8 +328,10 @@ export class StepManager { childCb: (ctx: StepHandlerContext) => Promise, childOptions: z.input = {}, ): Promise => { + // Inherit parent step options EXCEPT branch (each step's branch status is independent) + const { branch: _parentBranch, ...inheritableOptions } = options const parsedChildOptions = StepOptionsSchema.parse({ - ...options, // Inherit parent step options as defaults + ...inheritableOptions, ...childOptions, }) @@ -328,6 +342,7 @@ export class StepManager { options: parsedChildOptions, abortSignal: childSignal, // Child uses composed signal parentStepId: step!.id, // This step is the parent + branch: parsedChildOptions.branch, // Pass branch option }) // Track the child promise @@ -583,6 +598,7 @@ class ActionContext { - it('should get job steps', async () => { + it('should get all job steps', async () => { const jobId = await client.runAction('testAction', { message: 'Get steps', }) @@ -192,26 +192,8 @@ function runServerTests(adapterFactory: AdapterFactory) { expect(result.steps).toBeInstanceOf(Array) expect(result.total).toBeGreaterThan(0) - expect(result.page).toBe(1) - expect(result.pageSize).toBeGreaterThan(0) - }) - - it('should paginate job steps', async () => { - const jobId = await client.runAction('testAction', { - message: 'Paginated steps', - }) - - await client.fetch({ batchSize: 10 }) - await new Promise((resolve) => setTimeout(resolve, 500)) - - const response = await server.handle(new Request(`http://localhost/api/jobs/${jobId}/steps?page=1&pageSize=1`)) - expect(response.status).toBe(200) - - const result = GetJobStepsResponseParsedSchema.parse(await response.json()) as GetJobStepsResponse - - expect(result.steps.length).toBeLessThanOrEqual(1) - expect(result.page).toBe(1) - expect(result.pageSize).toBe(1) + // No pagination - all steps returned at once + expect(result.steps.length).toBe(result.total) }) it('should filter steps by search query', async () => { diff --git a/packages/duron/test/time-travel.test.ts b/packages/duron/test/time-travel.test.ts new file mode 100644 index 0000000..c08c8cc --- /dev/null +++ b/packages/duron/test/time-travel.test.ts @@ -0,0 +1,476 @@ +import { afterEach, beforeEach, describe, expect, it } from 'bun:test' + +import { z } from 'zod' + +import { defineAction } from '../src/action.js' +import { Client } from '../src/client.js' +import { + JOB_STATUS_ACTIVE, + JOB_STATUS_COMPLETED, + JOB_STATUS_CREATED, + JOB_STATUS_FAILED, + STEP_STATUS_ACTIVE, + STEP_STATUS_COMPLETED, +} from '../src/constants.js' +import { type AdapterFactory, type AdapterInstance, pgliteFactory, postgresFactory } from './adapters.js' +import { expectToBeDefined } from './asserts.js' + +// ============================================================================= +// Test Actions for Time Travel +// ============================================================================= + +/** + * Simple linear action with 3 steps + */ +const linearAction = defineAction()({ + name: 'linear-action', + version: '1.0.0', + input: z.object({ + failAtStep: z.number().optional(), + }), + output: z.object({ + steps: z.array(z.string()), + }), + steps: { + concurrency: 10, + retry: { limit: 0, factor: 2, minTimeout: 100, maxTimeout: 500 }, + expire: 60000, + }, + handler: async (ctx) => { + const steps: string[] = [] + + await ctx.step('step-1', async () => { + if (ctx.input.failAtStep === 1) throw new Error('Step 1 failed') + steps.push('step-1') + return { completed: 'step-1' } + }) + + await ctx.step('step-2', async () => { + if (ctx.input.failAtStep === 2) throw new Error('Step 2 failed') + steps.push('step-2') + return { completed: 'step-2' } + }) + + await ctx.step('step-3', async () => { + if (ctx.input.failAtStep === 3) throw new Error('Step 3 failed') + steps.push('step-3') + return { completed: 'step-3' } + }) + + return { steps } + }, +}) + +/** + * Action with nested steps for testing time travel from nested step + */ +const nestedAction = defineAction()({ + name: 'nested-action', + version: '1.0.0', + input: z.object({ + failAtStep: z.string().optional(), + }), + output: z.object({ + executed: z.array(z.string()), + }), + steps: { + concurrency: 10, + retry: { limit: 0, factor: 2, minTimeout: 100, maxTimeout: 500 }, + expire: 60000, + }, + handler: async (ctx) => { + const executed: string[] = [] + + await ctx.step('parent-1', async (stepCtx) => { + executed.push('parent-1-start') + + await stepCtx.step('child-1-1', async () => { + if (ctx.input.failAtStep === 'child-1-1') throw new Error('child-1-1 failed') + executed.push('child-1-1') + return { done: true } + }) + + await stepCtx.step('child-1-2', async () => { + if (ctx.input.failAtStep === 'child-1-2') throw new Error('child-1-2 failed') + executed.push('child-1-2') + return { done: true } + }) + + executed.push('parent-1-end') + return { completed: true } + }) + + await ctx.step('parent-2', async () => { + if (ctx.input.failAtStep === 'parent-2') throw new Error('parent-2 failed') + executed.push('parent-2') + return { completed: true } + }) + + return { executed } + }, +}) + +/** + * Action with branch steps for testing branch preservation during time travel + */ +const branchAction = defineAction()({ + name: 'branch-action', + version: '1.0.0', + input: z.object({ + failAtStep: z.string().optional(), + }), + output: z.object({ + executed: z.array(z.string()), + }), + steps: { + concurrency: 10, + retry: { limit: 0, factor: 2, minTimeout: 100, maxTimeout: 500 }, + expire: 60000, + }, + handler: async (ctx) => { + const executed: string[] = [] + + // Two root-level branch steps + await Promise.all([ + ctx.step( + 'branch-a', + async () => { + executed.push('branch-a') + return { result: 'a' } + }, + { branch: true }, + ), + ctx.step( + 'branch-b', + async () => { + if (ctx.input.failAtStep === 'branch-b') throw new Error('branch-b failed') + executed.push('branch-b') + return { result: 'b' } + }, + { branch: true }, + ), + ]) + + await ctx.step('final-step', async () => { + if (ctx.input.failAtStep === 'final-step') throw new Error('final-step failed') + executed.push('final-step') + return { done: true } + }) + + return { executed } + }, +}) + +// ============================================================================= +// Test Suite Runner +// ============================================================================= + +function runTests(name: string, factory: AdapterFactory) { + describe(`Time Travel Tests with ${name}`, () => { + let adapterInstance: AdapterInstance + let client: Client + + beforeEach(async () => { + adapterInstance = await factory.create() + client = new Client({ + database: adapterInstance.adapter, + actions: { linearAction, nestedAction, branchAction }, + syncPattern: false, // Manual fetching for tests + logger: 'silent', + }) + await client.start() + }) + + afterEach(async () => { + await client?.stop() + await adapterInstance?.deleteDb() + }) + + // ========================================================================= + // Basic Time Travel Tests + // ========================================================================= + + describe('Basic Time Travel', () => { + it('should time travel from a root step in completed job', async () => { + // Run a successful job + const jobId = await client.runAction('linearAction', {}) + await client.fetch({ batchSize: 1 }) + const job = await client.waitForJob(jobId, { timeout: 5000 }) + expectToBeDefined(job) + expect(job?.status).toBe(JOB_STATUS_COMPLETED) + + // Get steps + const { steps } = await client.getJobSteps({ jobId }) + expect(steps.length).toBe(3) + const step2 = steps.find((s) => s.name === 'step-2') + expectToBeDefined(step2) + + // Time travel to step-2 + const success = await client.timeTravelJob(jobId, step2!.id) + expect(success).toBe(true) + + // Job should be reset to created + const resetJob = await client.getJobById(jobId) + expectToBeDefined(resetJob) + expect(resetJob?.status).toBe(JOB_STATUS_CREATED) + + // step-1 should still be completed, step-2 and step-3 should be reset/deleted + const { steps: stepsAfter } = await client.getJobSteps({ jobId }) + const step1After = stepsAfter.find((s) => s.name === 'step-1') + const step2After = stepsAfter.find((s) => s.name === 'step-2') + const step3After = stepsAfter.find((s) => s.name === 'step-3') + + expectToBeDefined(step1After) + expect(step1After?.status).toBe(STEP_STATUS_COMPLETED) // Kept + expectToBeDefined(step2After) + expect(step2After?.status).toBe(STEP_STATUS_ACTIVE) // Reset + expect(step3After).toBeUndefined() // Deleted + }) + + it('should time travel from a step in failed job', async () => { + // Run a job that fails at step 2 + const jobId = await client.runAction('linearAction', { failAtStep: 2 }) + await client.fetch({ batchSize: 1 }) + const job = await client.waitForJob(jobId, { timeout: 5000 }) + expectToBeDefined(job) + expect(job?.status).toBe(JOB_STATUS_FAILED) + + // Get steps + const { steps } = await client.getJobSteps({ jobId }) + expect(steps.length).toBe(2) // step-1 and step-2 + + const step2 = steps.find((s) => s.name === 'step-2') + expectToBeDefined(step2) + + // Time travel to step-2 (the failed step) + const success = await client.timeTravelJob(jobId, step2!.id) + expect(success).toBe(true) + + // Job should be reset to created + const resetJob = await client.getJobById(jobId) + expectToBeDefined(resetJob) + expect(resetJob?.status).toBe(JOB_STATUS_CREATED) + }) + + it('should not time travel for active job', async () => { + // Run a job + const jobId = await client.runAction('linearAction', {}) + + // Don't wait for completion - job is still created/active + const job = await client.getJobById(jobId) + expectToBeDefined(job) + expect(job.status === JOB_STATUS_CREATED || job.status === JOB_STATUS_ACTIVE).toBe(true) + + // There are no steps yet, so we can't time travel + const { steps } = await client.getJobSteps({ jobId }) + if (steps.length > 0) { + const success = await client.timeTravelJob(jobId, steps[0]!.id) + expect(success).toBe(false) + } + }) + }) + + // ========================================================================= + // Nested Steps Time Travel Tests + // ========================================================================= + + describe('Nested Steps Time Travel', () => { + it('should time travel from nested child step (ancestors re-run)', async () => { + // Run a successful job with nested steps + const jobId = await client.runAction('nestedAction', {}) + await client.fetch({ batchSize: 1 }) + const job = await client.waitForJob(jobId, { timeout: 5000 }) + expectToBeDefined(job) + expect(job?.status).toBe(JOB_STATUS_COMPLETED) + + // Get steps + const { steps } = await client.getJobSteps({ jobId }) + expect(steps.length).toBe(4) // parent-1, child-1-1, child-1-2, parent-2 + + const child11 = steps.find((s) => s.name === 'child-1-1') + const parent1 = steps.find((s) => s.name === 'parent-1') + expectToBeDefined(child11) + expectToBeDefined(parent1) + + // Time travel to child-1-1 + const success = await client.timeTravelJob(jobId, child11!.id) + expect(success).toBe(true) + + // Job should be reset to created + const resetJob = await client.getJobById(jobId) + expectToBeDefined(resetJob) + expect(resetJob?.status).toBe(JOB_STATUS_CREATED) + + // parent-1 should be reset (ancestor) + // child-1-1 should be reset (target) + // child-1-2 and parent-2 should be deleted + const { steps: stepsAfter } = await client.getJobSteps({ jobId }) + + const parent1After = stepsAfter.find((s) => s.name === 'parent-1') + const child11After = stepsAfter.find((s) => s.name === 'child-1-1') + const child12After = stepsAfter.find((s) => s.name === 'child-1-2') + const parent2After = stepsAfter.find((s) => s.name === 'parent-2') + + expectToBeDefined(parent1After) + expect(parent1After?.status).toBe(STEP_STATUS_ACTIVE) // Ancestor - reset + expectToBeDefined(child11After) + expect(child11After?.status).toBe(STEP_STATUS_ACTIVE) // Target - reset + expect(child12After).toBeUndefined() // Deleted (came after) + expect(parent2After).toBeUndefined() // Deleted (came after) + }) + + it('should time travel to second child step', async () => { + // Run a successful job with nested steps + const jobId = await client.runAction('nestedAction', {}) + await client.fetch({ batchSize: 1 }) + const job = await client.waitForJob(jobId, { timeout: 5000 }) + expectToBeDefined(job) + expect(job?.status).toBe(JOB_STATUS_COMPLETED) + + // Get steps + const { steps } = await client.getJobSteps({ jobId }) + + const child12 = steps.find((s) => s.name === 'child-1-2') + expectToBeDefined(child12) + + // Time travel to child-1-2 + const success = await client.timeTravelJob(jobId, child12!.id) + expect(success).toBe(true) + + // Check steps after time travel + const { steps: stepsAfter } = await client.getJobSteps({ jobId }) + + const parent1After = stepsAfter.find((s) => s.name === 'parent-1') + const child11After = stepsAfter.find((s) => s.name === 'child-1-1') + const child12After = stepsAfter.find((s) => s.name === 'child-1-2') + const parent2After = stepsAfter.find((s) => s.name === 'parent-2') + + expectToBeDefined(parent1After) + expect(parent1After?.status).toBe(STEP_STATUS_ACTIVE) // Ancestor - reset + expectToBeDefined(child11After) + expect(child11After?.status).toBe(STEP_STATUS_COMPLETED) // Completed before target - kept + expectToBeDefined(child12After) + expect(child12After?.status).toBe(STEP_STATUS_ACTIVE) // Target - reset + expect(parent2After).toBeUndefined() // Deleted (came after parent-1) + }) + }) + + // ========================================================================= + // Branch Steps Time Travel Tests + // ========================================================================= + + describe('Branch Steps Time Travel', () => { + it('should preserve completed branch siblings during time travel', async () => { + // Run a job that fails at final-step + const jobId = await client.runAction('branchAction', { failAtStep: 'final-step' }) + await client.fetch({ batchSize: 1 }) + const job = await client.waitForJob(jobId, { timeout: 5000 }) + expectToBeDefined(job) + expect(job?.status).toBe(JOB_STATUS_FAILED) + + // Get steps + const { steps } = await client.getJobSteps({ jobId }) + + // Find the final-step + const finalStep = steps.find((s) => s.name === 'final-step') + expectToBeDefined(finalStep) + + // Time travel to final-step + const success = await client.timeTravelJob(jobId, finalStep!.id) + expect(success).toBe(true) + + // Check steps after time travel + const { steps: stepsAfter } = await client.getJobSteps({ jobId }) + + const branchAAfter = stepsAfter.find((s) => s.name === 'branch-a') + const branchBAfter = stepsAfter.find((s) => s.name === 'branch-b') + const finalStepAfter = stepsAfter.find((s) => s.name === 'final-step') + + // Both branch steps should be preserved (completed before final-step) + expectToBeDefined(branchAAfter) + expect(branchAAfter?.status).toBe(STEP_STATUS_COMPLETED) + expectToBeDefined(branchBAfter) + expect(branchBAfter?.status).toBe(STEP_STATUS_COMPLETED) + + // final-step should be reset + expectToBeDefined(finalStepAfter) + expect(finalStepAfter?.status).toBe(STEP_STATUS_ACTIVE) + }) + + it('should keep branch sibling when time traveling to another branch', async () => { + // Run a job that fails at branch-b + const jobId = await client.runAction('branchAction', { failAtStep: 'branch-b' }) + await client.fetch({ batchSize: 1 }) + const job = await client.waitForJob(jobId, { timeout: 5000 }) + expectToBeDefined(job) + expect(job?.status).toBe(JOB_STATUS_FAILED) + + // Get steps - branch-a should be completed, branch-b should be failed + const { steps } = await client.getJobSteps({ jobId }) + + const branchA = steps.find((s) => s.name === 'branch-a') + const branchB = steps.find((s) => s.name === 'branch-b') + expectToBeDefined(branchA) + expectToBeDefined(branchB) + + // Time travel to branch-b + const success = await client.timeTravelJob(jobId, branchB!.id) + expect(success).toBe(true) + + // Check steps after time travel + const { steps: stepsAfter } = await client.getJobSteps({ jobId }) + + const branchAAfter = stepsAfter.find((s) => s.name === 'branch-a') + const branchBAfter = stepsAfter.find((s) => s.name === 'branch-b') + + // branch-a should be preserved (completed branch sibling) + expectToBeDefined(branchAAfter) + expect(branchAAfter?.status).toBe(STEP_STATUS_COMPLETED) + + // branch-b should be reset (target) + expectToBeDefined(branchBAfter) + expect(branchBAfter?.status).toBe(STEP_STATUS_ACTIVE) + }) + }) + + // ========================================================================= + // Re-execution After Time Travel Tests + // ========================================================================= + + describe('Re-execution After Time Travel', () => { + it('should complete job after time travel and re-execution', async () => { + // Run a job that fails at step 2 + const jobId = await client.runAction('linearAction', { failAtStep: 2 }) + await client.fetch({ batchSize: 1 }) + const job = await client.waitForJob(jobId, { timeout: 5000 }) + expectToBeDefined(job) + expect(job?.status).toBe(JOB_STATUS_FAILED) + + // Get the failed step + const { steps } = await client.getJobSteps({ jobId }) + const step2 = steps.find((s) => s.name === 'step-2') + expectToBeDefined(step2) + + // Time travel to step-2 + const success = await client.timeTravelJob(jobId, step2!.id) + expect(success).toBe(true) + + // Need to update the input to not fail (simulate fix) + // Since we can't update input, we just verify the job is re-runnable + const resetJob = await client.getJobById(jobId) + expectToBeDefined(resetJob) + expect(resetJob?.status).toBe(JOB_STATUS_CREATED) + + // The job is ready to be fetched and executed again + // In a real scenario, you'd fix the input or the code before re-running + }) + }) + }) +} + +// ============================================================================= +// Run Tests for Each Adapter +// ============================================================================= + +runTests('pglite', pgliteFactory) +runTests('postgres', postgresFactory) diff --git a/packages/shared-actions/index.ts b/packages/shared-actions/index.ts index bea5734..72a7b40 100644 --- a/packages/shared-actions/index.ts +++ b/packages/shared-actions/index.ts @@ -515,86 +515,90 @@ export const processOrder = defineAction()({ { expire: 15_000 }, ) - // ========================================================================= - // Step 5: Post-Order Processing (Promise.all of steps with nested steps) - // This demonstrates running multiple parent steps in parallel, - // where each parent step has its own nested child steps. - // ========================================================================= - const [analytics, loyalty, partnerSync] = await Promise.all([ - // Parent step 1: Analytics Tracking (with nested steps) - ctx.step( - 'analytics-tracking', - async ({ step: analyticsStep }) => { - // Nested child: Track purchase event - const purchase = await analyticsStep('track-purchase', async () => { - await new Promise((resolve) => setTimeout(resolve, 40)) - addTimeline('track-purchase', 'success', `Tracked order ${orderId}`) - return { eventId: `EVT-${Date.now()}`, type: 'purchase' } - }) - - // Nested child: Update product recommendations - const recommendations = await analyticsStep('update-recommendations', async () => { - await new Promise((resolve) => setTimeout(resolve, 60)) - addTimeline('update-recommendations', 'success', `Updated for ${items.length} products`) - return { updated: true, productsAnalyzed: items.length } - }) - - addTimeline('analytics-tracking', 'success', 'Analytics updated') - return { purchase, recommendations } - }, - { expire: 10_000 }, - ), - - // Parent step 2: Loyalty Program Update (with nested steps) - ctx.step( - 'loyalty-update', - async ({ step: loyaltyStep }) => { - // Nested child: Calculate loyalty points - const points = await loyaltyStep('calculate-points', async () => { - await new Promise((resolve) => setTimeout(resolve, 50)) - const earnedPoints = Math.floor(totalAmount * 10) // 10 points per dollar - addTimeline('calculate-points', 'success', `Earned ${earnedPoints} points`) - return { earnedPoints, multiplier: 1.0 } - }) - - // Nested child: Update customer tier - const tier = await loyaltyStep('update-tier', async () => { - await new Promise((resolve) => setTimeout(resolve, 45)) - const newTier = totalAmount > 500 ? 'gold' : totalAmount > 100 ? 'silver' : 'bronze' - addTimeline('update-tier', 'success', `Tier: ${newTier}`) - return { tier: newTier, upgraded: totalAmount > 500 } - }) - - addTimeline('loyalty-update', 'success', `${points.earnedPoints} points, tier: ${tier.tier}`) - return { points, tier } - }, - { expire: 10_000 }, - ), - - // Parent step 3: Partner Sync (with nested steps) - ctx.step( - 'partner-sync', - async ({ step: syncStep }) => { - // Nested child: Sync with supplier - const supplier = await syncStep('sync-supplier', async () => { - await new Promise((resolve) => setTimeout(resolve, 80)) - addTimeline('sync-supplier', 'success', 'Supplier inventory updated') - return { synced: true, supplierId: 'SUP-001' } - }) - - // Nested child: Sync with warehouse - const warehouse = await syncStep('sync-warehouse', async () => { - await new Promise((resolve) => setTimeout(resolve, 70)) - addTimeline('sync-warehouse', 'success', 'Warehouse notified for picking') - return { synced: true, warehouseId: 'WH-MAIN' } - }) + const { analytics, loyalty, partnerSync } = await ctx.step('post-order-processing', async (ctx) => { + // ========================================================================= + // Step 5: Post-Order Processing (Promise.all of steps with nested steps) + // This demonstrates running multiple parent steps in parallel, + // where each parent step has its own nested child steps. + // ========================================================================= + const [analytics, loyalty, partnerSync] = await Promise.all([ + // Parent step 1: Analytics Tracking (with nested steps) + ctx.step( + 'analytics-tracking', + async ({ step: analyticsStep }) => { + // Nested child: Track purchase event + const purchase = await analyticsStep('track-purchase', async () => { + await new Promise((resolve) => setTimeout(resolve, 40)) + addTimeline('track-purchase', 'success', `Tracked order ${orderId}`) + return { eventId: `EVT-${Date.now()}`, type: 'purchase' } + }) + + // Nested child: Update product recommendations + const recommendations = await analyticsStep('update-recommendations', async () => { + await new Promise((resolve) => setTimeout(resolve, 60)) + addTimeline('update-recommendations', 'success', `Updated for ${items.length} products`) + return { updated: true, productsAnalyzed: items.length } + }) + + addTimeline('analytics-tracking', 'success', 'Analytics updated') + return { purchase, recommendations } + }, + { expire: 10_000, branch: true }, + ), + + // Parent step 2: Loyalty Program Update (with nested steps) + ctx.step( + 'loyalty-update', + async ({ step: loyaltyStep }) => { + // Nested child: Calculate loyalty points + const points = await loyaltyStep('calculate-points', async () => { + await new Promise((resolve) => setTimeout(resolve, 50)) + const earnedPoints = Math.floor(totalAmount * 10) // 10 points per dollar + addTimeline('calculate-points', 'success', `Earned ${earnedPoints} points`) + return { earnedPoints, multiplier: 1.0 } + }) + + // Nested child: Update customer tier + const tier = await loyaltyStep('update-tier', async () => { + await new Promise((resolve) => setTimeout(resolve, 45)) + const newTier = totalAmount > 500 ? 'gold' : totalAmount > 100 ? 'silver' : 'bronze' + addTimeline('update-tier', 'success', `Tier: ${newTier}`) + return { tier: newTier, upgraded: totalAmount > 500 } + }) + + addTimeline('loyalty-update', 'success', `${points.earnedPoints} points, tier: ${tier.tier}`) + return { points, tier } + }, + { expire: 10_000, branch: true }, + ), + + // Parent step 3: Partner Sync (with nested steps) + ctx.step( + 'partner-sync', + async ({ step: syncStep }) => { + // Nested child: Sync with supplier + const supplier = await syncStep('sync-supplier', async () => { + await new Promise((resolve) => setTimeout(resolve, 80)) + addTimeline('sync-supplier', 'success', 'Supplier inventory updated') + return { synced: true, supplierId: 'SUP-001' } + }) + + // Nested child: Sync with warehouse + const warehouse = await syncStep('sync-warehouse', async () => { + await new Promise((resolve) => setTimeout(resolve, 70)) + addTimeline('sync-warehouse', 'success', 'Warehouse notified for picking') + return { synced: true, warehouseId: 'WH-MAIN' } + }) + + addTimeline('partner-sync', 'success', 'All partners synced') + return { supplier, warehouse } + }, + { expire: 10_000, branch: true }, + ), + ]) - addTimeline('partner-sync', 'success', 'All partners synced') - return { supplier, warehouse } - }, - { expire: 10_000 }, - ), - ]) + return { analytics, loyalty, partnerSync } + }) addTimeline( 'post-order-processing', From 817ce795196c2997d4c7233f2c5415d823355e44 Mon Sep 17 00:00:00 2001 From: Martin Acosta Date: Sat, 17 Jan 2026 15:09:51 -0300 Subject: [PATCH 03/78] Refactor time travel logic to preserve nested branches and their children - Updated SQL queries to enhance the logic for keeping completed steps, including branch siblings and their descendants. - Introduced a new action for testing nested branches, ensuring that time travel correctly preserves sibling branches and their nested steps. - Added comprehensive tests to validate the behavior of time travel when targeting steps within nested branches. --- packages/duron/src/adapters/postgres/base.ts | 53 ++++-- packages/duron/test/time-travel.test.ts | 169 ++++++++++++++++++- 2 files changed, 208 insertions(+), 14 deletions(-) diff --git a/packages/duron/src/adapters/postgres/base.ts b/packages/duron/src/adapters/postgres/base.ts index 637fbe6..3719a04 100644 --- a/packages/duron/src/adapters/postgres/base.ts +++ b/packages/duron/src/adapters/postgres/base.ts @@ -367,19 +367,9 @@ export class PostgresBaseAdapter e FROM ${this.tables.jobStepsTable} s INNER JOIN ancestors a ON s.id = a.parent_step_id ), - -- Steps to keep: completed steps created before target + completed branch siblings - steps_to_keep AS ( - -- Steps created before target that are completed (siblings or unrelated) - SELECT s.id - FROM ${this.tables.jobStepsTable} s - CROSS JOIN target_step ts - WHERE s.job_id = ${jobId} - AND s.created_at < ts.created_at - AND s.status = ${STEP_STATUS_COMPLETED} - AND s.id NOT IN (SELECT id FROM ancestors) - AND s.id != ts.id - UNION - -- Completed branch siblings at same level as target (same parent, branch=true, completed) + -- Steps to keep: completed steps created before target + completed branch siblings of target and ancestors + their descendants + branch_siblings AS ( + -- Completed branch siblings of target step SELECT s.id FROM ${this.tables.jobStepsTable} s CROSS JOIN target_step ts @@ -391,6 +381,43 @@ export class PostgresBaseAdapter e (s.parent_step_id IS NULL AND ts.parent_step_id IS NULL) OR s.parent_step_id = ts.parent_step_id ) + UNION + -- Completed branch siblings of each ancestor + SELECT s.id + FROM ${this.tables.jobStepsTable} s + INNER JOIN ancestors a ON ( + (s.parent_step_id IS NULL AND a.parent_step_id IS NULL) + OR s.parent_step_id = a.parent_step_id + ) + WHERE s.job_id = ${jobId} + AND s.id NOT IN (SELECT id FROM ancestors) + AND s.branch = true + AND s.status = ${STEP_STATUS_COMPLETED} + ), + -- Find all descendants of branch siblings (to keep their children too) + branch_descendants AS ( + SELECT s.id + FROM ${this.tables.jobStepsTable} s + WHERE s.id IN (SELECT id FROM branch_siblings) + UNION ALL + SELECT s.id + FROM ${this.tables.jobStepsTable} s + INNER JOIN branch_descendants bd ON s.parent_step_id = bd.id + WHERE s.job_id = ${jobId} + ), + steps_to_keep AS ( + -- Steps created before target that are completed (non-ancestor, non-target) + SELECT s.id + FROM ${this.tables.jobStepsTable} s + CROSS JOIN target_step ts + WHERE s.job_id = ${jobId} + AND s.created_at < ts.created_at + AND s.status = ${STEP_STATUS_COMPLETED} + AND s.id NOT IN (SELECT id FROM ancestors) + AND s.id != ts.id + UNION + -- All branch siblings and their descendants + SELECT id FROM branch_descendants ), -- Delete steps that are not in the keep list and are not ancestors/target deleted_steps AS ( diff --git a/packages/duron/test/time-travel.test.ts b/packages/duron/test/time-travel.test.ts index c08c8cc..8c67033 100644 --- a/packages/duron/test/time-travel.test.ts +++ b/packages/duron/test/time-travel.test.ts @@ -161,6 +161,95 @@ const branchAction = defineAction()({ }, }) +/** + * Action with nested branches - branches that contain nested steps + * This tests the edge case where time travel targets a step inside a branch + * and sibling branches with their own children should be preserved + */ +const nestedBranchAction = defineAction()({ + name: 'nested-branch-action', + version: '1.0.0', + input: z.object({ + failAtStep: z.string().optional(), + }), + output: z.object({ + executed: z.array(z.string()), + }), + steps: { + concurrency: 10, + retry: { limit: 0, factor: 2, minTimeout: 100, maxTimeout: 500 }, + expire: 60000, + }, + handler: async (ctx) => { + const executed: string[] = [] + + // Parent step with multiple branch children, each with nested steps + await ctx.step('parent', async (parentCtx) => { + executed.push('parent-start') + + await Promise.all([ + // Branch A with nested steps + parentCtx.step( + 'branch-a', + async (branchACtx) => { + executed.push('branch-a-start') + await branchACtx.step('child-a-1', async () => { + executed.push('child-a-1') + return { done: true } + }) + await branchACtx.step('child-a-2', async () => { + executed.push('child-a-2') + return { done: true } + }) + executed.push('branch-a-end') + return { result: 'a' } + }, + { branch: true }, + ), + // Branch B with nested steps (target for time travel) + parentCtx.step( + 'branch-b', + async (branchBCtx) => { + executed.push('branch-b-start') + await branchBCtx.step('child-b-1', async () => { + if (ctx.input.failAtStep === 'child-b-1') throw new Error('child-b-1 failed') + executed.push('child-b-1') + return { done: true } + }) + await branchBCtx.step('child-b-2', async () => { + if (ctx.input.failAtStep === 'child-b-2') throw new Error('child-b-2 failed') + executed.push('child-b-2') + return { done: true } + }) + executed.push('branch-b-end') + return { result: 'b' } + }, + { branch: true }, + ), + // Branch C with nested steps + parentCtx.step( + 'branch-c', + async (branchCCtx) => { + executed.push('branch-c-start') + await branchCCtx.step('child-c-1', async () => { + executed.push('child-c-1') + return { done: true } + }) + executed.push('branch-c-end') + return { result: 'c' } + }, + { branch: true }, + ), + ]) + + executed.push('parent-end') + return { completed: true } + }) + + return { executed } + }, +}) + // ============================================================================= // Test Suite Runner // ============================================================================= @@ -174,7 +263,7 @@ function runTests(name: string, factory: AdapterFactory) { adapterInstance = await factory.create() client = new Client({ database: adapterInstance.adapter, - actions: { linearAction, nestedAction, branchAction }, + actions: { linearAction, nestedAction, branchAction, nestedBranchAction }, syncPattern: false, // Manual fetching for tests logger: 'silent', }) @@ -431,6 +520,84 @@ function runTests(name: string, factory: AdapterFactory) { expectToBeDefined(branchBAfter) expect(branchBAfter?.status).toBe(STEP_STATUS_ACTIVE) }) + + it('should preserve sibling branches AND their children when time traveling to nested step in a branch', async () => { + // This tests the edge case from processOrder: time traveling to a step inside + // a branch should preserve sibling branches AND all their nested children + const jobId = await client.runAction('nestedBranchAction', { failAtStep: 'child-b-2' }) + await client.fetch({ batchSize: 1 }) + const job = await client.waitForJob(jobId, { timeout: 10000 }) + expectToBeDefined(job) + expect(job?.status).toBe(JOB_STATUS_FAILED) + + // Get steps before time travel + const { steps } = await client.getJobSteps({ jobId }) + + // Verify initial state: branch-a and branch-c completed with children, branch-b failed + const branchA = steps.find((s) => s.name === 'branch-a') + const childA1 = steps.find((s) => s.name === 'child-a-1') + const childA2 = steps.find((s) => s.name === 'child-a-2') + const branchC = steps.find((s) => s.name === 'branch-c') + const childC1 = steps.find((s) => s.name === 'child-c-1') + const childB1 = steps.find((s) => s.name === 'child-b-1') + const childB2 = steps.find((s) => s.name === 'child-b-2') + + expectToBeDefined(branchA) + expectToBeDefined(childA1) + expectToBeDefined(childA2) + expectToBeDefined(branchC) + expectToBeDefined(childC1) + expectToBeDefined(childB1) + expectToBeDefined(childB2) + + // Time travel to child-b-2 (nested inside branch-b) + const success = await client.timeTravelJob(jobId, childB2!.id) + expect(success).toBe(true) + + // Check steps after time travel + const { steps: stepsAfter } = await client.getJobSteps({ jobId }) + + // branch-a should be preserved WITH its children + const branchAAfter = stepsAfter.find((s) => s.name === 'branch-a') + const childA1After = stepsAfter.find((s) => s.name === 'child-a-1') + const childA2After = stepsAfter.find((s) => s.name === 'child-a-2') + + expectToBeDefined(branchAAfter) + expect(branchAAfter?.status).toBe(STEP_STATUS_COMPLETED) + expectToBeDefined(childA1After) + expect(childA1After?.status).toBe(STEP_STATUS_COMPLETED) + expectToBeDefined(childA2After) + expect(childA2After?.status).toBe(STEP_STATUS_COMPLETED) + + // branch-c should be preserved WITH its children + const branchCAfter = stepsAfter.find((s) => s.name === 'branch-c') + const childC1After = stepsAfter.find((s) => s.name === 'child-c-1') + + expectToBeDefined(branchCAfter) + expect(branchCAfter?.status).toBe(STEP_STATUS_COMPLETED) + expectToBeDefined(childC1After) + expect(childC1After?.status).toBe(STEP_STATUS_COMPLETED) + + // branch-b should be reset to active (ancestor of target) + const branchBAfter = stepsAfter.find((s) => s.name === 'branch-b') + expectToBeDefined(branchBAfter) + expect(branchBAfter?.status).toBe(STEP_STATUS_ACTIVE) + + // child-b-1 should be preserved (sibling before target in same branch, completed) + const childB1After = stepsAfter.find((s) => s.name === 'child-b-1') + expectToBeDefined(childB1After) + expect(childB1After?.status).toBe(STEP_STATUS_COMPLETED) + + // child-b-2 should be reset (target) + const childB2After = stepsAfter.find((s) => s.name === 'child-b-2') + expectToBeDefined(childB2After) + expect(childB2After?.status).toBe(STEP_STATUS_ACTIVE) + + // parent should be reset (ancestor of target) + const parentAfter = stepsAfter.find((s) => s.name === 'parent') + expectToBeDefined(parentAfter) + expect(parentAfter?.status).toBe(STEP_STATUS_ACTIVE) + }) }) // ========================================================================= From 79f24cdfeec876f569a89bd33e9db6b0688cb612 Mon Sep 17 00:00:00 2001 From: Martin Acosta Date: Sat, 17 Jan 2026 15:17:19 -0300 Subject: [PATCH 04/78] Implement time offset adjustment for preserved job steps - Added SQL logic to calculate and apply a time offset for preserved job steps, aligning their start and finish times with the current time. - Enhanced the update process for job steps to ensure accurate time tracking while maintaining the integrity of completed steps. --- packages/duron/src/adapters/postgres/base.ts | 22 ++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/packages/duron/src/adapters/postgres/base.ts b/packages/duron/src/adapters/postgres/base.ts index 3719a04..bcae474 100644 --- a/packages/duron/src/adapters/postgres/base.ts +++ b/packages/duron/src/adapters/postgres/base.ts @@ -419,6 +419,28 @@ export class PostgresBaseAdapter e -- All branch siblings and their descendants SELECT id FROM branch_descendants ), + -- Calculate time offset: shift preserved steps to start from "now" + time_offset AS ( + SELECT + now() - MIN(s.started_at) AS offset_interval + FROM ${this.tables.jobStepsTable} s + WHERE s.id IN (SELECT id FROM steps_to_keep) + ), + -- Shift times of preserved steps to align with current time (only started_at/finished_at, NOT created_at to preserve ordering) + shift_preserved_times AS ( + UPDATE ${this.tables.jobStepsTable} + SET + started_at = started_at + (SELECT offset_interval FROM time_offset), + finished_at = CASE + WHEN finished_at IS NOT NULL + THEN finished_at + (SELECT offset_interval FROM time_offset) + ELSE NULL + END, + updated_at = now() + WHERE id IN (SELECT id FROM steps_to_keep) + AND (SELECT offset_interval FROM time_offset) IS NOT NULL + RETURNING id + ), -- Delete steps that are not in the keep list and are not ancestors/target deleted_steps AS ( DELETE FROM ${this.tables.jobStepsTable} From 97b06072c1c8529d5b9a54c432ab7ef01cde750f Mon Sep 17 00:00:00 2001 From: Martin Acosta Date: Sat, 17 Jan 2026 17:54:22 -0300 Subject: [PATCH 05/78] Refactor CSS variables and enhance badge status component - Updated CSS variables for light and dark themes to improve color consistency and accessibility. - Refactored the BadgeStatus component to utilize a configuration object for status management, enhancing maintainability and readability. - Removed hardcoded color classes in favor of a structured status configuration, allowing for easier updates and customization. --- packages/docs/src/styles/app.css | 114 ++++++++++-------- .../src/components/badge-status.tsx | 63 ++++++++-- packages/duron/test/adapter.test.ts | 7 +- packages/duron/test/nested-steps.test.ts | 8 +- 4 files changed, 120 insertions(+), 72 deletions(-) diff --git a/packages/docs/src/styles/app.css b/packages/docs/src/styles/app.css index 21f0a2e..4ae0adf 100644 --- a/packages/docs/src/styles/app.css +++ b/packages/docs/src/styles/app.css @@ -3,59 +3,73 @@ @import 'fumadocs-ui/css/preset.css'; :root { - --background: oklch(95.78% 0.006 264.5); - --foreground: oklch(43.18% 0.043 279.3); - --muted: oklch(91.88% 0.007 268.5); - --muted-foreground: oklch(40.50% 0.023 265.7); - --popover: oklch(93.35% 0.009 264.5); - --popover-foreground: oklch(34.64% 0.031 279.7); - --card: oklch(94.22% 0.007 260.7); - --card-foreground: oklch(38.93% 0.038 278.5); - --border: oklch(91.88% 0.007 268.5); - --input: oklch(89.47% 0.008 271.3); - --primary: oklch(55.47% 0.250 297.0); - --primary-foreground: oklch(100.00% 0.000 89.9); - --secondary: oklch(77.13% 0.056 305.4); - --secondary-foreground: oklch(24.58% 0.045 303.3); - --accent: oklch(83.15% 0.024 264.4); - --accent-foreground: oklch(30.49% 0.031 263.9); - --destructive: oklch(48.29% 0.188 29.3); - --destructive-foreground: oklch(96.83% 0.014 17.4); - --ring: oklch(55.47% 0.250 297.0); - --chart-1: oklch(55.47% 0.250 297.0); - --chart-2: oklch(77.13% 0.056 305.4); - --chart-3: oklch(83.15% 0.024 264.4); - --chart-4: oklch(79.96% 0.050 305.2); - --chart-5: oklch(55.33% 0.256 296.4); - --radius: 0.5rem; + --radius: 0.65rem; + --background: oklch(1 0 0); + --foreground: oklch(0.145 0 0); + --card: oklch(1 0 0); + --card-foreground: oklch(0.145 0 0); + --popover: oklch(1 0 0); + --popover-foreground: oklch(0.145 0 0); + --primary: oklch(0.205 0 0); + --primary-foreground: oklch(0.985 0 0); + --secondary: oklch(0.97 0 0); + --secondary-foreground: oklch(0.205 0 0); + --muted: oklch(0.97 0 0); + --muted-foreground: oklch(0.556 0 0); + --accent: oklch(0.97 0 0); + --accent-foreground: oklch(0.205 0 0); + --destructive: oklch(0.577 0.245 27.325); + --border: oklch(0.922 0 0); + --input: oklch(0.922 0 0); + --ring: oklch(0.708 0 0); + --chart-1: oklch(0.646 0.222 41.116); + --chart-2: oklch(0.6 0.118 184.704); + --chart-3: oklch(0.398 0.07 227.392); + --chart-4: oklch(0.828 0.189 84.429); + --chart-5: oklch(0.769 0.188 70.08); + --radius: 0.625rem; + --sidebar: oklch(0.985 0 0); + --sidebar-foreground: oklch(0.145 0 0); + --sidebar-primary: oklch(0.205 0 0); + --sidebar-primary-foreground: oklch(0.985 0 0); + --sidebar-accent: oklch(0.97 0 0); + --sidebar-accent-foreground: oklch(0.205 0 0); + --sidebar-border: oklch(0.922 0 0); + --sidebar-ring: oklch(0.708 0 0); } .dark { - --background: oklch(32.91% 0.032 274.8); - --foreground: oklch(86.46% 0.051 273.1); - --muted: oklch(37.38% 0.023 275.0); - --muted-foreground: oklch(80.88% 0.016 273.8); - --popover: oklch(30.04% 0.030 273.6); - --popover-foreground: oklch(96.82% 0.012 276.1); - --card: oklch(30.89% 0.031 274.2); - --card-foreground: oklch(91.72% 0.031 271.7); - --border: oklch(38.52% 0.019 277.3); - --input: oklch(41.42% 0.019 273.5); - --primary: oklch(76.48% 0.111 311.7); - --primary-foreground: oklch(24.97% 0.089 309.1); - --secondary: oklch(34.18% 0.070 311.0); - --secondary-foreground: oklch(86.66% 0.035 312.4); - --accent: oklch(45.57% 0.050 274.1); - --accent-foreground: oklch(98.27% 0.003 286.4); - --destructive: oklch(64.41% 0.232 28.6); - --destructive-foreground: oklch(100.00% 0.000 89.9); - --ring: oklch(76.48% 0.111 311.7); - --chart-1: oklch(76.48% 0.111 311.7); - --chart-2: oklch(34.18% 0.070 311.0); - --chart-3: oklch(45.57% 0.050 274.1); - --chart-4: oklch(36.70% 0.078 310.9); - --chart-5: oklch(76.30% 0.117 312.0); - --radius: 0.5rem; + --background: oklch(0.145 0 0); + --foreground: oklch(0.985 0 0); + --card: oklch(0.205 0 0); + --card-foreground: oklch(0.985 0 0); + --popover: oklch(0.205 0 0); + --popover-foreground: oklch(0.985 0 0); + --primary: oklch(0.922 0 0); + --primary-foreground: oklch(0.205 0 0); + --secondary: oklch(0.269 0 0); + --secondary-foreground: oklch(0.985 0 0); + --muted: oklch(0.269 0 0); + --muted-foreground: oklch(0.708 0 0); + --accent: oklch(0.269 0 0); + --accent-foreground: oklch(0.985 0 0); + --destructive: oklch(0.704 0.191 22.216); + --border: oklch(1 0 0 / 10%); + --input: oklch(1 0 0 / 15%); + --ring: oklch(0.556 0 0); + --chart-1: oklch(0.488 0.243 264.376); + --chart-2: oklch(0.696 0.17 162.48); + --chart-3: oklch(0.769 0.188 70.08); + --chart-4: oklch(0.627 0.265 303.9); + --chart-5: oklch(0.645 0.246 16.439); + --sidebar: oklch(0.205 0 0); + --sidebar-foreground: oklch(0.985 0 0); + --sidebar-primary: oklch(0.488 0.243 264.376); + --sidebar-primary-foreground: oklch(0.985 0 0); + --sidebar-accent: oklch(0.269 0 0); + --sidebar-accent-foreground: oklch(0.985 0 0); + --sidebar-border: oklch(1 0 0 / 10%); + --sidebar-ring: oklch(0.556 0 0); } .logo-word { diff --git a/packages/duron-dashboard/src/components/badge-status.tsx b/packages/duron-dashboard/src/components/badge-status.tsx index fdc23cc..572520f 100644 --- a/packages/duron-dashboard/src/components/badge-status.tsx +++ b/packages/duron-dashboard/src/components/badge-status.tsx @@ -11,27 +11,64 @@ const icons = { cancelled: Ban, } -const colors = { - created: 'bg-gray-100 text-gray-800 border-gray-800', - active: 'bg-blue-100 text-blue-800 border-blue-800', - completed: 'bg-green-100 text-green-800 border-green-800', - failed: 'bg-red-100 text-red-800 border-red-800', - cancelled: 'bg-yellow-100 text-yellow-800 border-yellow-800', -} +export const statusConfig = { + created: { + label: 'Created', + variant: 'outline', + badgeClassName: '', + iconClassName: '', + }, + active: { + label: 'Active', + variant: 'outline', + badgeClassName: + 'border-none bg-amber-600/10 text-amber-600 focus-visible:ring-amber-600/20 focus-visible:outline-none dark:bg-amber-400/10 dark:text-amber-400 dark:focus-visible:ring-amber-400/40 [a&]:hover:bg-amber-600/5 dark:[a&]:hover:bg-amber-400/5', + iconClassName: 'rounded-full bg-amber-600 dark:bg-amber-400', + }, + completed: { + label: 'Completed', + variant: 'outline', + badgeClassName: + 'border-none bg-green-600/10 text-green-600 focus-visible:ring-green-600/20 focus-visible:outline-none dark:bg-green-400/10 dark:text-green-400 dark:focus-visible:ring-green-400/40 [a&]:hover:bg-green-600/5 dark:[a&]:hover:bg-green-400/5', + iconClassName: 'rounded-full bg-green-600 dark:bg-green-400', + }, + failed: { + label: 'Failed', + variant: 'outline', + badgeClassName: + 'bg-destructive/10 [a&]:hover:bg-destructive/5 focus-visible:ring-destructive/20 dark:focus-visible:ring-destructive/40 text-destructive border-none focus-visible:outline-none', + iconClassName: 'bg-destructive rounded-full', + }, + cancelled: { + label: 'Cancelled', + variant: 'secondary', + badgeClassName: '', + iconClassName: '', + }, +} as const + +export type Status = keyof typeof statusConfig export function BadgeStatus({ status, justIcon = false }: { status: string; justIcon?: boolean }) { - const Icon = icons[status as keyof typeof icons] - const color = colors[status as keyof typeof colors] + const Icon = icons[status as Status] + const config = statusConfig[status as Status] + + if (!config) { + return {status.charAt(0).toUpperCase() + status.slice(1)} + } + if (justIcon) { return ( - {Icon && } + + {Icon && } + ) } return ( - - {Icon && } - {status.charAt(0).toUpperCase() + status.slice(1)} + + {Icon && } + {config.label} ) } diff --git a/packages/duron/test/adapter.test.ts b/packages/duron/test/adapter.test.ts index 5bce634..ae82eee 100644 --- a/packages/duron/test/adapter.test.ts +++ b/packages/duron/test/adapter.test.ts @@ -726,14 +726,11 @@ function runAdapterTests(adapterFactory: AdapterFactory) { const result = await adapter.getJobSteps({ jobId: jobId!, - page: 1, - pageSize: 2, }) - expect(result.steps.length).toBe(2) + // All steps returned (no pagination) + expect(result.steps.length).toBe(5) expect(result.total).toBe(5) - expect(result.page).toBe(1) - expect(result.pageSize).toBe(2) }) it('should get action statistics', async () => { diff --git a/packages/duron/test/nested-steps.test.ts b/packages/duron/test/nested-steps.test.ts index 39de38e..13e7620 100644 --- a/packages/duron/test/nested-steps.test.ts +++ b/packages/duron/test/nested-steps.test.ts @@ -357,7 +357,7 @@ function runNestedStepsTests(adapterFactory: AdapterFactory) { expectToBeDefined(job) // Get all steps for the job - const stepsResult = await client.getJobSteps({ jobId, pageSize: 100 }) + const stepsResult = await client.getJobSteps({ jobId }) const steps = stepsResult.steps expect(steps.length).toBe(2) // parent + child @@ -432,7 +432,7 @@ function runNestedStepsTests(adapterFactory: AdapterFactory) { expect(output.childResults.sort()).toEqual([0, 1, 2, 3, 4]) // Verify all steps exist in database - const stepsResult = await client.getJobSteps({ jobId, pageSize: 100 }) + const stepsResult = await client.getJobSteps({ jobId }) expect(stepsResult.steps.length).toBe(6) // 1 parent + 5 children const parentStep = stepsResult.steps.find((s) => s.name === 'parent-step') @@ -461,7 +461,7 @@ function runNestedStepsTests(adapterFactory: AdapterFactory) { expect(job.error.name).toBe('UnhandledChildStepsError') // The orphaned child step should be cancelled - const stepsResult = await client.getJobSteps({ jobId, pageSize: 100 }) + const stepsResult = await client.getJobSteps({ jobId }) const orphanedChild = stepsResult.steps.find((s) => s.name === 'orphaned-child') expectToBeDefined(orphanedChild) expect(orphanedChild.status).toBe(STEP_STATUS_CANCELLED) @@ -482,7 +482,7 @@ function runNestedStepsTests(adapterFactory: AdapterFactory) { expect(job.error.name).toBe('StepTimeoutError') // Get steps - const stepsResult = await client.getJobSteps({ jobId, pageSize: 100 }) + const stepsResult = await client.getJobSteps({ jobId }) // Parent step should be present and failed const parentStep = stepsResult.steps.find((s) => s.name === 'parent-step') From 4e06f36f548a9b15c18b3bd17bea7104c12542db Mon Sep 17 00:00:00 2001 From: Martin Acosta Date: Sat, 17 Jan 2026 18:06:04 -0300 Subject: [PATCH 06/78] Refactor DuronDashboard and enhance StepList functionality - Wrapped the ApiProvider and AuthProvider in a StepViewProvider to manage step view context. - Removed the TimelineModal component and integrated timeline functionality directly into the StepList component. - Updated Dashboard and JobDetails components to streamline visibility logic and improve layout for mobile and desktop views. - Enhanced StepList to toggle between list and timeline views, improving user experience and accessibility. --- .../duron-dashboard/src/DuronDashboard.tsx | 13 +- .../src/components/timeline-modal.tsx | 85 --------- .../src/contexts/step-view-context.tsx | 41 +++++ .../duron-dashboard/src/views/dashboard.tsx | 161 +++++++----------- .../duron-dashboard/src/views/job-details.tsx | 12 +- .../duron-dashboard/src/views/step-list.tsx | 78 +++++---- 6 files changed, 161 insertions(+), 229 deletions(-) delete mode 100644 packages/duron-dashboard/src/components/timeline-modal.tsx create mode 100644 packages/duron-dashboard/src/contexts/step-view-context.tsx diff --git a/packages/duron-dashboard/src/DuronDashboard.tsx b/packages/duron-dashboard/src/DuronDashboard.tsx index 705ed23..c56f037 100644 --- a/packages/duron-dashboard/src/DuronDashboard.tsx +++ b/packages/duron-dashboard/src/DuronDashboard.tsx @@ -3,6 +3,7 @@ import { NuqsAdapter } from 'nuqs/adapters/react' import { ApiProvider } from './contexts/api-context' import { AuthProvider, useAuth } from './contexts/auth-context' +import { StepViewProvider } from './contexts/step-view-context' import { ThemeProvider } from './contexts/theme-context' import { Dashboard } from './views/dashboard' import Login from './views/login' @@ -54,11 +55,13 @@ export function DuronDashboard({ url, enableLogin = false, showLogo = true }: Du return ( - - - - - + + + + + + + ) diff --git a/packages/duron-dashboard/src/components/timeline-modal.tsx b/packages/duron-dashboard/src/components/timeline-modal.tsx deleted file mode 100644 index 79f9c11..0000000 --- a/packages/duron-dashboard/src/components/timeline-modal.tsx +++ /dev/null @@ -1,85 +0,0 @@ -'use client' - -import { Menu } from 'lucide-react' -import { useState } from 'react' - -import { Timeline } from '@/components/timeline' -import { Dialog, DialogContent, DialogHeader, DialogTitle } from '@/components/ui/dialog' -import { ScrollArea, ScrollBar } from '@/components/ui/scroll-area' -import { Separator } from '@/components/ui/separator' -import { useStepsPolling } from '@/hooks/use-steps-polling' -import { useJob, useJobSteps } from '@/lib/api' -import { StepDetailsContent } from '@/views/step-details-content' - -interface TimelineModalProps { - jobId: string | null - open: boolean - onOpenChange: (open: boolean) => void -} - -export function TimelineModal({ jobId, open, onOpenChange }: TimelineModalProps) { - const [selectedStepId, setSelectedStepId] = useState(null) - const { data: job, isLoading: jobLoading } = useJob(jobId) - const { data: stepsData, isLoading: stepsLoading } = useJobSteps(jobId, {}) - - // Enable polling for step updates - useStepsPolling(jobId, open) - - const steps = stepsData?.steps ?? [] - const isLoading = jobLoading || stepsLoading - - // Reset selected step when modal closes - const handleOpenChange = (newOpen: boolean) => { - if (!newOpen) { - setSelectedStepId(null) - } - onOpenChange(newOpen) - } - - // Toggle step selection - if clicking the same step, deselect it - const handleStepSelect = (stepId: string) => { - setSelectedStepId((current) => (current === stepId ? null : stepId)) - } - - return ( -

- - -
- - Timeline -
- -
- {/* Timeline Section */} - - {isLoading ? ( -
Loading timeline...
- ) : ( - - )} - - - - {/* Step Details Section */} - {selectedStepId && ( - <> - - -
- -
- - - - )} -
- - - ) -} diff --git a/packages/duron-dashboard/src/contexts/step-view-context.tsx b/packages/duron-dashboard/src/contexts/step-view-context.tsx new file mode 100644 index 0000000..35e5a19 --- /dev/null +++ b/packages/duron-dashboard/src/contexts/step-view-context.tsx @@ -0,0 +1,41 @@ +'use client' + +import { createContext, useCallback, useContext, useEffect, useState, type ReactNode } from 'react' + +type StepViewType = 'list' | 'timeline' + +interface StepViewContextValue { + viewType: StepViewType + setViewType: (type: StepViewType) => void +} + +const STORAGE_KEY = 'duron-step-view-type' + +const StepViewContext = createContext(null) + +export function StepViewProvider({ children }: { children: ReactNode }) { + const [viewType, setViewTypeState] = useState('list') + + // Load from localStorage on mount + useEffect(() => { + const stored = localStorage.getItem(STORAGE_KEY) + if (stored === 'list' || stored === 'timeline') { + setViewTypeState(stored) + } + }, []) + + const setViewType = useCallback((type: StepViewType) => { + setViewTypeState(type) + localStorage.setItem(STORAGE_KEY, type) + }, []) + + return {children} +} + +export function useStepView() { + const context = useContext(StepViewContext) + if (!context) { + throw new Error('useStepView must be used within a StepViewProvider') + } + return context +} diff --git a/packages/duron-dashboard/src/views/dashboard.tsx b/packages/duron-dashboard/src/views/dashboard.tsx index 0b58c06..49f5653 100644 --- a/packages/duron-dashboard/src/views/dashboard.tsx +++ b/packages/duron-dashboard/src/views/dashboard.tsx @@ -1,6 +1,6 @@ 'use client' -import { ChevronLeft, LogOut, MoreVertical, Plus, Trash2, X } from 'lucide-react' +import { LogOut, MoreVertical, Plus, Trash2, X } from 'lucide-react' import { useCallback, useEffect, useState } from 'react' import { CreateJobDialog } from '@/components/create-job-dialog' @@ -31,10 +31,8 @@ interface DashboardProps { export function Dashboard({ showLogo = true, enableLogin = true }: DashboardProps) { const [selectedJobId, setSelectedJobId] = useState(null) const [selectedStepId, setSelectedStepId] = useState(null) - const [stepListFullScreenVisible, setStepListFullScreenVisible] = useState(false) const [createJobDialogOpen, setCreateJobDialogOpen] = useState(false) const [jobDetailsVisible, setJobDetailsVisible] = useState(false) - const [stepListVisible, setStepListVisible] = useState(false) const isMobile = useIsMobile() const { logout } = useAuth() @@ -43,23 +41,17 @@ export function Dashboard({ showLogo = true, enableLogin = true }: DashboardProp }, []) useEffect(() => { - if (!jobDetailsVisible && !stepListVisible) { + if (!jobDetailsVisible) { handleJobSelect(null) } - }, [jobDetailsVisible, stepListVisible, handleJobSelect]) + }, [jobDetailsVisible, handleJobSelect]) useEffect(() => { if (!selectedJobId) { setSelectedStepId(null) } - - if (isMobile) { - setJobDetailsVisible(!!selectedJobId) - } else { - setJobDetailsVisible(!!selectedJobId) - setStepListVisible(!!selectedJobId) - } - }, [selectedJobId, isMobile]) + setJobDetailsVisible(!!selectedJobId) + }, [selectedJobId]) const handleJobCreated = useCallback((jobId: string) => { setSelectedJobId(jobId) @@ -161,71 +153,55 @@ export function Dashboard({ showLogo = true, enableLogin = true }: DashboardProp -
- {/* Desktop: Three Horizontal Views with Collapse */} +
+ {/* Desktop Layout: Top row (Jobs | Details), Bottom row (Steps) */} {!isMobile && ( -
- {/* Jobs Section */} +
+ {/* Top Row: Jobs and Job Details */}
-
-

Jobs

-
-
- -
-
- - {/* Job Details Section */} - {jobDetailsVisible && ( + {/* Jobs Section */}
-

Job Details

- +

Jobs

- setStepListFullScreenVisible(true)} /> +
- )} - {/* Step List Section */} - {stepListVisible && ( -
+ {/* Job Details Section */} + {jobDetailsVisible && ( +
+
+

Job Details

+ +
+
+ +
+
+ )} +
+ + {/* Bottom Row: Steps (full width) */} + {selectedJobId && ( +

Steps

-
@@ -235,56 +211,43 @@ export function Dashboard({ showLogo = true, enableLogin = true }: DashboardProp
)} - {/* Mobile: Vertical Layout - Jobs and Job Details 50% each */} + {/* Mobile: Vertical Layout - Jobs, Job Details, and Steps */} {isMobile && ( - <> -
- {/* Jobs Section */} -
+
+ {/* Jobs Section */} +
+
+

Jobs

+
+
+ +
+
+ + {/* Job Details Section */} + {selectedJobId && ( +
-

Jobs

+

Job Details

- +
+ )} - {/* Job Details Section */} - {selectedJobId && ( -
-
-

Job Details

-
-
- setStepListFullScreenVisible(true)} /> -
-
- )} -
- - {/* Step List Overlay - Mobile */} - {stepListFullScreenVisible && ( -
-
+ {/* Steps Section */} + {selectedJobId && ( +
+

Steps

-
)} - +
)}
void } -export function JobDetails({ jobId, onOpenStepList }: JobDetailsProps) { +export function JobDetails({ jobId }: JobDetailsProps) { const { data: job, isLoading: jobLoading } = useJob(jobId) // Enable polling for job status updates - refetches entire job detail when status changes @@ -117,13 +116,8 @@ export function JobDetails({ jobId, onOpenStepList }: JobDetailsProps) {
- {/* Mobile: Dropdown menu and Step List button */} + {/* Mobile: Dropdown menu */}
- {onOpenStepList && ( - - )} +
+
+
+
+ + handleSearchChange(e.target.value)} + className="pl-8" + />
+ + + + + +

Switch to {viewType === 'list' ? 'timeline' : 'list'} view

+ +
+
-
+
+ {viewType === 'timeline' ? ( + + ) : (
{stepsLoading ? ( @@ -250,9 +267,8 @@ export function StepList({ jobId, selectedStepId, onStepSelect }: StepListProps) )}
-
+ )}
- - +
) } From ffb995c6d1914a04c29fa12cc4b7805c8aa342ad Mon Sep 17 00:00:00 2001 From: Martin Acosta Date: Sat, 17 Jan 2026 18:23:34 -0300 Subject: [PATCH 07/78] Enhance DataTable component and streamline JobsTable layout - Added a new `fillHeight` prop to the DataTable component, allowing it to expand and fill available height with a fixed header, scrollable body, and fixed pagination footer. - Refactored the JobsTable component to simplify its layout by removing unnecessary wrapper elements, improving readability and performance. - Updated the Dashboard view to utilize the new JobsTable layout, enhancing the overall user experience. --- .../src/components/data-table/data-table.tsx | 85 ++++++++++++++++++- .../duron-dashboard/src/views/dashboard.tsx | 14 +-- .../duron-dashboard/src/views/jobs-table.tsx | 22 ++--- 3 files changed, 98 insertions(+), 23 deletions(-) diff --git a/packages/duron-dashboard/src/components/data-table/data-table.tsx b/packages/duron-dashboard/src/components/data-table/data-table.tsx index 367ea3a..ff33abc 100644 --- a/packages/duron-dashboard/src/components/data-table/data-table.tsx +++ b/packages/duron-dashboard/src/components/data-table/data-table.tsx @@ -10,9 +10,92 @@ import { cn } from '@/lib/utils' interface DataTableProps extends React.ComponentProps<'div'> { table: TanstackTable actionBar?: React.ReactNode + /** + * When true, the table will expand to fill available height with: + * - Fixed header + * - Scrollable body + * - Fixed pagination footer + */ + fillHeight?: boolean } -export function DataTable({ table, actionBar, children, className, ...props }: DataTableProps) { +export function DataTable({ + table, + actionBar, + children, + className, + fillHeight = false, + ...props +}: DataTableProps) { + if (fillHeight) { + return ( +
+ {/* Toolbar passed as children */} + {children &&
{children}
} + + {/* Scrollable table container */} +
+ + + + {table.getHeaderGroups().map((headerGroup) => ( + + {headerGroup.headers.map((header) => ( + + {header.isPlaceholder ? null : flexRender(header.column.columnDef.header, header.getContext())} + + ))} + + ))} + + + {table.getRowModel().rows?.length ? ( + table.getRowModel().rows.map((row) => { + const isSelected = row.getIsSelected() + return ( + + {row.getVisibleCells().map((cell) => ( + + {flexRender(cell.column.columnDef.cell, cell.getContext())} + + ))} + + ) + }) + ) : ( + + + No results. + + + )} + +
+ + +
+ + {/* Fixed pagination footer */} +
+ + {actionBar && table.getFilteredSelectedRowModel().rows.length > 0 && actionBar} +
+
+ ) + } + + // Original layout (non-fillHeight) return (
{children} diff --git a/packages/duron-dashboard/src/views/dashboard.tsx b/packages/duron-dashboard/src/views/dashboard.tsx index 49f5653..5fb499e 100644 --- a/packages/duron-dashboard/src/views/dashboard.tsx +++ b/packages/duron-dashboard/src/views/dashboard.tsx @@ -167,12 +167,7 @@ export function Dashboard({ showLogo = true, enableLogin = true }: DashboardProp jobDetailsVisible ? 'w-1/2' : 'w-full' } border-r flex flex-col overflow-hidden transition-all duration-200 min-w-0`} > -
-

Jobs

-
-
- -
+
{/* Job Details Section */} @@ -216,12 +211,7 @@ export function Dashboard({ showLogo = true, enableLogin = true }: DashboardProp
{/* Jobs Section */}
-
-

Jobs

-
-
- -
+
{/* Job Details Section */} diff --git a/packages/duron-dashboard/src/views/jobs-table.tsx b/packages/duron-dashboard/src/views/jobs-table.tsx index 8e13cae..41a21e4 100644 --- a/packages/duron-dashboard/src/views/jobs-table.tsx +++ b/packages/duron-dashboard/src/views/jobs-table.tsx @@ -9,7 +9,6 @@ import { DataTableColumnHeader } from '@/components/data-table/data-table-column import { DataTableSortList } from '@/components/data-table/data-table-sort-list' import { DataTableToolbar } from '@/components/data-table/data-table-toolbar' import { Button } from '@/components/ui/button' -import { ScrollArea } from '@/components/ui/scroll-area' import { Tooltip, TooltipContent, TooltipTrigger } from '@/components/ui/tooltip' import { useDataTable } from '@/hooks/use-data-table' import { useJobParams } from '@/hooks/use-job-params' @@ -250,15 +249,18 @@ export function JobsTable({ onJobSelect, selectedJobId }: JobsTableProps) { return (
- -
- - - - - -
- + {/* Header with title and toolbar */} +
+

Jobs

+ + + +
+ + {/* Table content */} +
+ +
) } From 580ebee98288b83c8ffbb50dbe6bb960e95baa60 Mon Sep 17 00:00:00 2001 From: Martin Acosta Date: Sat, 17 Jan 2026 18:25:57 -0300 Subject: [PATCH 08/78] Refactor DataTable and JobsTable components for improved layout - Updated the DataTable component to adjust the border styling for better visual separation. - Simplified the JobsTable layout by removing unnecessary padding, enhancing the overall design and usability. --- .../duron-dashboard/src/components/data-table/data-table.tsx | 2 +- packages/duron-dashboard/src/views/jobs-table.tsx | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/duron-dashboard/src/components/data-table/data-table.tsx b/packages/duron-dashboard/src/components/data-table/data-table.tsx index ff33abc..79d7ed7 100644 --- a/packages/duron-dashboard/src/components/data-table/data-table.tsx +++ b/packages/duron-dashboard/src/components/data-table/data-table.tsx @@ -34,7 +34,7 @@ export function DataTable({ {children &&
{children}
} {/* Scrollable table container */} -
+
diff --git a/packages/duron-dashboard/src/views/jobs-table.tsx b/packages/duron-dashboard/src/views/jobs-table.tsx index 41a21e4..0ea3c57 100644 --- a/packages/duron-dashboard/src/views/jobs-table.tsx +++ b/packages/duron-dashboard/src/views/jobs-table.tsx @@ -258,7 +258,7 @@ export function JobsTable({ onJobSelect, selectedJobId }: JobsTableProps) { {/* Table content */} -
+
From 02349c58044552f11d61a8ae6f8f2e9aea77f0aa Mon Sep 17 00:00:00 2001 From: Martin Acosta Date: Sat, 17 Jan 2026 18:33:33 -0300 Subject: [PATCH 09/78] Refactor JobDetails and Dashboard components for improved layout and functionality - Updated the JobDetails component to include an optional onClose prop for better control over visibility. - Simplified the Dashboard layout by integrating JobDetails directly, enhancing the user experience. - Adjusted padding and border styles across various components for a more consistent design. --- .../duron-dashboard/src/views/dashboard.tsx | 27 +--- .../duron-dashboard/src/views/job-details.tsx | 153 +++++++----------- .../duron-dashboard/src/views/jobs-table.tsx | 2 +- .../duron-dashboard/src/views/step-list.tsx | 2 +- 4 files changed, 68 insertions(+), 116 deletions(-) diff --git a/packages/duron-dashboard/src/views/dashboard.tsx b/packages/duron-dashboard/src/views/dashboard.tsx index 5fb499e..288a946 100644 --- a/packages/duron-dashboard/src/views/dashboard.tsx +++ b/packages/duron-dashboard/src/views/dashboard.tsx @@ -173,21 +173,7 @@ export function Dashboard({ showLogo = true, enableLogin = true }: DashboardProp {/* Job Details Section */} {jobDetailsVisible && (
-
-

Job Details

- -
-
- -
+ setJobDetailsVisible(false)} />
)} @@ -195,7 +181,7 @@ export function Dashboard({ showLogo = true, enableLogin = true }: DashboardProp {/* Bottom Row: Steps (full width) */} {selectedJobId && (
-
+

Steps

@@ -217,19 +203,14 @@ export function Dashboard({ showLogo = true, enableLogin = true }: DashboardProp {/* Job Details Section */} {selectedJobId && (
-
-

Job Details

-
-
- -
+
)} {/* Steps Section */} {selectedJobId && (
-
+

Steps

diff --git a/packages/duron-dashboard/src/views/job-details.tsx b/packages/duron-dashboard/src/views/job-details.tsx index b935c29..4928980 100644 --- a/packages/duron-dashboard/src/views/job-details.tsx +++ b/packages/duron-dashboard/src/views/job-details.tsx @@ -15,9 +15,10 @@ import { isExpiring } from '../lib/is-expiring' interface JobDetailsProps { jobId: string | null + onClose?: () => void } -export function JobDetails({ jobId }: JobDetailsProps) { +export function JobDetails({ jobId, onClose }: JobDetailsProps) { const { data: job, isLoading: jobLoading } = useJob(jobId) // Enable polling for job status updates - refetches entire job detail when status changes @@ -113,96 +114,65 @@ export function JobDetails({ jobId }: JobDetailsProps) { } return ( - -
-
- {/* Mobile: Dropdown menu */} -
- - - - - - - - Retry - - - - Cancel - - - - Delete - - - -
- {/* Desktop: Individual buttons */} -
- - - + + + + + Retry + + + + Cancel + + + + Delete + + + + {/* Close button */} + {onClose && ( + -
+ )}
+
-
+ {/* Scrollable content */} + +
+
ID: {job.id}
@@ -302,8 +272,9 @@ export function JobDetails({ jobId }: JobDetailsProps) { {!job.output &&
No output available
}
-
- - +
+ + +
) } diff --git a/packages/duron-dashboard/src/views/jobs-table.tsx b/packages/duron-dashboard/src/views/jobs-table.tsx index 0ea3c57..9231708 100644 --- a/packages/duron-dashboard/src/views/jobs-table.tsx +++ b/packages/duron-dashboard/src/views/jobs-table.tsx @@ -250,7 +250,7 @@ export function JobsTable({ onJobSelect, selectedJobId }: JobsTableProps) { return (
{/* Header with title and toolbar */} -
+

Jobs

diff --git a/packages/duron-dashboard/src/views/step-list.tsx b/packages/duron-dashboard/src/views/step-list.tsx index ddf567e..e89a035 100644 --- a/packages/duron-dashboard/src/views/step-list.tsx +++ b/packages/duron-dashboard/src/views/step-list.tsx @@ -144,7 +144,7 @@ export function StepList({ jobId, selectedStepId, onStepSelect }: StepListProps) return (
-
+
From fe3f2b5710d0bdb8f758c1da216f3bc5021b8e92 Mon Sep 17 00:00:00 2001 From: Martin Acosta Date: Sat, 17 Jan 2026 20:33:24 -0300 Subject: [PATCH 10/78] Refactor job step handling to support parallel execution - Replaced the 'branch' concept with 'parallel' in job step options, allowing steps to run independently from siblings during time travel. - Updated related components, schemas, and tests to reflect the new parallel execution model. - Enhanced documentation and comments to clarify the changes in step behavior and time travel logic. --- .../src/components/ui/scroll-area.tsx | 4 +- .../duron-dashboard/src/views/step-list.tsx | 8 +- .../snapshot.json | 988 ------------------ .../migration.sql | 1 - .../migration.sql | 1 + .../snapshot.json | 4 +- packages/duron/src/action.ts | 8 +- packages/duron/src/adapters/adapter.ts | 4 +- packages/duron/src/adapters/postgres/base.ts | 30 +- .../duron/src/adapters/postgres/schema.ts | 2 +- packages/duron/src/adapters/schemas.ts | 6 +- packages/duron/src/client.ts | 2 +- packages/duron/src/step-manager.ts | 24 +- packages/duron/test/client.test.ts | 4 - packages/duron/test/time-travel.test.ts | 40 +- packages/shared-actions/index.ts | 6 +- 16 files changed, 70 insertions(+), 1062 deletions(-) delete mode 100644 packages/duron/migrations/postgres/20260117142756_pretty_vulcan/snapshot.json delete mode 100644 packages/duron/migrations/postgres/20260117172938_panoramic_photon/migration.sql rename packages/duron/migrations/postgres/{20260117142756_pretty_vulcan => 20260117231749_clumsy_penance}/migration.sql (61%) rename packages/duron/migrations/postgres/{20260117172938_panoramic_photon => 20260117231749_clumsy_penance}/snapshot.json (99%) diff --git a/packages/duron-dashboard/src/components/ui/scroll-area.tsx b/packages/duron-dashboard/src/components/ui/scroll-area.tsx index faef452..a59a640 100644 --- a/packages/duron-dashboard/src/components/ui/scroll-area.tsx +++ b/packages/duron-dashboard/src/components/ui/scroll-area.tsx @@ -5,10 +5,10 @@ import { cn } from '@/lib/utils' function ScrollArea({ className, children, ...props }: React.ComponentProps) { return ( - + {children} diff --git a/packages/duron-dashboard/src/views/step-list.tsx b/packages/duron-dashboard/src/views/step-list.tsx index e89a035..7873e0b 100644 --- a/packages/duron-dashboard/src/views/step-list.tsx +++ b/packages/duron-dashboard/src/views/step-list.tsx @@ -16,7 +16,7 @@ import { type GetJobStepsResponse, useJob, useJobSteps, useTimeTravelJob } from import { BadgeStatus } from '../components/badge-status' // Step type from the API response (without output field) -type JobStepWithoutOutput = GetJobStepsResponse['steps'][number] & { parentStepId?: string | null; branch?: boolean } +type JobStepWithoutOutput = GetJobStepsResponse['steps'][number] & { parentStepId?: string | null; parallel?: boolean } import { StepDetailsContent } from './step-details-content' @@ -201,7 +201,7 @@ export function StepList({ jobId, selectedStepId, onStepSelect }: StepListProps) {orderedSteps.map(({ step, depth }, index) => { const stepNumber = index + 1 const isNested = depth > 0 - const isBranch = (step as any).branch === true + const isParallel = (step as any).parallel === true // Calculate left padding based on depth (16px per level) const paddingLeft = depth * 16 @@ -216,13 +216,13 @@ export function StepList({ jobId, selectedStepId, onStepSelect }: StepListProps)
{isNested && } - {isBranch && ( + {isParallel && ( -

Branch step (independent from siblings)

+

Parallel step (independent from siblings)

 )} diff --git a/packages/duron/migrations/postgres/20260117142756_pretty_vulcan/snapshot.json b/packages/duron/migrations/postgres/20260117142756_pretty_vulcan/snapshot.json deleted file mode 100644 index 4db21a3..0000000 --- a/packages/duron/migrations/postgres/20260117142756_pretty_vulcan/snapshot.json +++ /dev/null @@ -1,988 +0,0 @@ -{ - "version": "8", - "dialect": "postgres", - "id": "25beb926-865c-41eb-8525-bbc1e0d3a19c", - "prevIds": [ - "e32fddc5-6d55-4c79-87b4-13d3a01b7d09" - ], - "ddl": [ - { - "name": "duron", - "entityType": "schemas" - }, - { - "isRlsEnabled": false, - "name": "job_steps", - "entityType": "tables", - "schema": "duron" - }, - { - "isRlsEnabled": false, - "name": "jobs", - "entityType": "tables", - "schema": "duron" - }, - { - "type": "uuid", - "typeSchema": null, - "notNull": true, - "dimensions": 0, - "default": "gen_random_uuid()", - "generated": null, - "identity": null, - "name": "id", - "entityType": "columns", - "schema": "duron", - "table": "job_steps" - }, - { - "type": "uuid", - "typeSchema": null, - "notNull": true, - "dimensions": 0, - "default": null, - "generated": null, - "identity": null, - "name": "job_id", - "entityType": "columns", - "schema": "duron", - "table": "job_steps" - }, - { - "type": "uuid", - "typeSchema": null, - "notNull": false, - "dimensions": 0, - "default": null, - "generated": null, - "identity": null, - "name": "parent_step_id", - "entityType": "columns", - "schema": "duron", - "table": "job_steps" - }, - { - "type": "text", - "typeSchema": null, - "notNull": true, - "dimensions": 0, - "default": null, - "generated": null, - "identity": null, - "name": "name", - "entityType": "columns", - "schema": "duron", - "table": "job_steps" - }, - { - "type": "text", - "typeSchema": null, - "notNull": true, - "dimensions": 0, - "default": "'active'", - "generated": null, - "identity": null, - "name": "status", - "entityType": "columns", - "schema": "duron", - "table": "job_steps" - }, - { - "type": "jsonb", - "typeSchema": null, - "notNull": false, - "dimensions": 0, - "default": null, - "generated": null, - "identity": null, - "name": "output", - "entityType": "columns", - "schema": "duron", - "table": "job_steps" - }, - { - "type": "jsonb", - "typeSchema": null, - "notNull": false, - "dimensions": 0, - "default": null, - "generated": null, - "identity": null, - "name": "error", - "entityType": "columns", - "schema": "duron", - "table": "job_steps" - }, - { - "type": "timestamp with time zone", - "typeSchema": null, - "notNull": true, - "dimensions": 0, - "default": "now()", - "generated": null, - "identity": null, - "name": "started_at", - "entityType": "columns", - "schema": "duron", - "table": "job_steps" - }, - { - "type": "timestamp with time zone", - "typeSchema": null, - "notNull": false, - "dimensions": 0, - "default": null, - "generated": null, - "identity": null, - "name": "finished_at", - "entityType": "columns", - "schema": "duron", - "table": "job_steps" - }, - { - "type": "integer", - "typeSchema": null, - "notNull": true, - "dimensions": 0, - "default": null, - "generated": null, - "identity": null, - "name": "timeout_ms", - "entityType": "columns", - "schema": "duron", - "table": "job_steps" - }, - { - "type": "timestamp with time zone", - "typeSchema": null, - "notNull": false, - "dimensions": 0, - "default": null, - "generated": null, - "identity": null, - "name": "expires_at", - "entityType": "columns", - "schema": "duron", - "table": "job_steps" - }, - { - "type": "integer", - "typeSchema": null, - "notNull": true, - "dimensions": 0, - "default": "0", - "generated": null, - "identity": null, - "name": "retries_limit", - "entityType": "columns", - "schema": "duron", - "table": "job_steps" - }, - { - "type": "integer", - "typeSchema": null, - "notNull": true, - "dimensions": 0, - "default": "0", - "generated": null, - "identity": null, - "name": "retries_count", - "entityType": "columns", - "schema": "duron", - "table": "job_steps" - }, - { - "type": "integer", - "typeSchema": null, - "notNull": false, - "dimensions": 0, - "default": null, - "generated": null, - "identity": null, - "name": "delayed_ms", - "entityType": "columns", - "schema": "duron", - "table": "job_steps" - }, - { - "type": "jsonb", - "typeSchema": null, - "notNull": true, - "dimensions": 0, - "default": "'{}'", - "generated": null, - "identity": null, - "name": "history_failed_attempts", - "entityType": "columns", - "schema": "duron", - "table": "job_steps" - }, - { - "type": "timestamp with time zone", - "typeSchema": null, - "notNull": true, - "dimensions": 0, - "default": "now()", - "generated": null, - "identity": null, - "name": "created_at", - "entityType": "columns", - "schema": "duron", - "table": "job_steps" - }, - { - "type": "timestamp with time zone", - "typeSchema": null, - "notNull": true, - "dimensions": 0, - "default": "now()", - "generated": null, - "identity": null, - "name": "updated_at", - "entityType": "columns", - "schema": "duron", - "table": "job_steps" - }, - { - "type": "uuid", - "typeSchema": null, - "notNull": true, - "dimensions": 0, - "default": "gen_random_uuid()", - "generated": null, - "identity": null, - "name": "id", - "entityType": "columns", - "schema": "duron", - "table": "jobs" - }, - { - "type": "text", - "typeSchema": null, - "notNull": true, - "dimensions": 0, - "default": null, - "generated": null, - "identity": null, - "name": "action_name", - "entityType": "columns", - "schema": "duron", - "table": "jobs" - }, - { - "type": "text", - "typeSchema": null, - "notNull": true, - "dimensions": 0, - "default": null, - "generated": null, - "identity": null, - "name": "group_key", - "entityType": "columns", - "schema": "duron", - "table": "jobs" - }, - { - "type": "text", - "typeSchema": null, - "notNull": true, - "dimensions": 0, - "default": "'created'", - "generated": null, - "identity": null, - "name": "status", - "entityType": "columns", - "schema": "duron", - "table": "jobs" - }, - { - "type": "text", - "typeSchema": null, - "notNull": true, - "dimensions": 0, - "default": null, - "generated": null, - "identity": null, - "name": "checksum", - "entityType": "columns", - "schema": "duron", - "table": "jobs" - }, - { - "type": "jsonb", - "typeSchema": null, - "notNull": true, - "dimensions": 0, - "default": "'{}'", - "generated": null, - "identity": null, - "name": "input", - "entityType": "columns", - "schema": "duron", - "table": "jobs" - }, - { - "type": "jsonb", - "typeSchema": null, - "notNull": false, - "dimensions": 0, - "default": null, - "generated": null, - "identity": null, - "name": "output", - "entityType": "columns", - "schema": "duron", - "table": "jobs" - }, - { - "type": "jsonb", - "typeSchema": null, - "notNull": false, - "dimensions": 0, - "default": null, - "generated": null, - "identity": null, - "name": "error", - "entityType": "columns", - "schema": "duron", - "table": "jobs" - }, - { - "type": "integer", - "typeSchema": null, - "notNull": true, - "dimensions": 0, - "default": null, - "generated": null, - "identity": null, - "name": "timeout_ms", - "entityType": "columns", - "schema": "duron", - "table": "jobs" - }, - { - "type": "timestamp with time zone", - "typeSchema": null, - "notNull": false, - "dimensions": 0, - "default": null, - "generated": null, - "identity": null, - "name": "expires_at", - "entityType": "columns", - "schema": "duron", - "table": "jobs" - }, - { - "type": "timestamp with time zone", - "typeSchema": null, - "notNull": false, - "dimensions": 0, - "default": null, - "generated": null, - "identity": null, - "name": "started_at", - "entityType": "columns", - "schema": "duron", - "table": "jobs" - }, - { - "type": "timestamp with time zone", - "typeSchema": null, - "notNull": false, - "dimensions": 0, - "default": null, - "generated": null, - "identity": null, - "name": "finished_at", - "entityType": "columns", - "schema": "duron", - "table": "jobs" - }, - { - "type": "text", - "typeSchema": null, - "notNull": false, - "dimensions": 0, - "default": null, - "generated": null, - "identity": null, - "name": "client_id", - "entityType": "columns", - "schema": "duron", - "table": "jobs" - }, - { - "type": "integer", - "typeSchema": null, - "notNull": true, - "dimensions": 0, - "default": "10", - "generated": null, - "identity": null, - "name": "concurrency_limit", - "entityType": "columns", - "schema": "duron", - "table": "jobs" - }, - { - "type": "timestamp with time zone", - "typeSchema": null, - "notNull": true, - "dimensions": 0, - "default": "now()", - "generated": null, - "identity": null, - "name": "created_at", - "entityType": "columns", - "schema": "duron", - "table": "jobs" - }, - { - "type": "timestamp with time zone", - "typeSchema": null, - "notNull": true, - "dimensions": 0, - "default": "now()", - "generated": null, - "identity": null, - "name": "updated_at", - "entityType": "columns", - "schema": "duron", - "table": "jobs" - }, - { - "nameExplicit": true, - "columns": [ - { - "value": "job_id", - "isExpression": false, - "asc": true, - "nullsFirst": false, - "opclass": null - } - ], - "isUnique": false, - "where": null, - "with": "", - "method": "btree", - "concurrently": false, - "name": "idx_job_steps_job_id", - "entityType": "indexes", - "schema": "duron", - "table": "job_steps" - }, - { - "nameExplicit": true, - "columns": [ - { - "value": "status", - "isExpression": false, - "asc": true, - "nullsFirst": false, - "opclass": null - } - ], - "isUnique": false, - "where": null, - "with": "", - "method": "btree", - "concurrently": false, - "name": "idx_job_steps_status", - "entityType": "indexes", - "schema": "duron", - "table": "job_steps" - }, - { - "nameExplicit": true, - "columns": [ - { - "value": "name", - "isExpression": false, - "asc": true, - "nullsFirst": false, - "opclass": null - } - ], - "isUnique": false, - "where": null, - "with": "", - "method": "btree", - "concurrently": false, - "name": "idx_job_steps_name", - "entityType": "indexes", - "schema": "duron", - "table": "job_steps" - }, - { - "nameExplicit": true, - "columns": [ - { - "value": "expires_at", - "isExpression": false, - "asc": true, - "nullsFirst": false, - "opclass": null - } - ], - "isUnique": false, - "where": null, - "with": "", - "method": "btree", - "concurrently": false, - "name": "idx_job_steps_expires_at", - "entityType": "indexes", - "schema": "duron", - "table": "job_steps" - }, - { - "nameExplicit": true, - "columns": [ - { - "value": "parent_step_id", - "isExpression": false, - "asc": true, - "nullsFirst": false, - "opclass": null - } - ], - "isUnique": false, - "where": null, - "with": "", - "method": "btree", - "concurrently": false, - "name": "idx_job_steps_parent_step_id", - "entityType": "indexes", - "schema": "duron", - "table": "job_steps" - }, - { - "nameExplicit": true, - "columns": [ - { - "value": "job_id", - "isExpression": false, - "asc": true, - "nullsFirst": false, - "opclass": null - }, - { - "value": "status", - "isExpression": false, - "asc": true, - "nullsFirst": false, - "opclass": null - } - ], - "isUnique": false, - "where": null, - "with": "", - "method": "btree", - "concurrently": false, - "name": "idx_job_steps_job_status", - "entityType": "indexes", - "schema": "duron", - "table": "job_steps" - }, - { - "nameExplicit": true, - "columns": [ - { - "value": "job_id", - "isExpression": false, - "asc": true, - "nullsFirst": false, - "opclass": null - }, - { - "value": "name", - "isExpression": false, - "asc": true, - "nullsFirst": false, - "opclass": null - } - ], - "isUnique": false, - "where": null, - "with": "", - "method": "btree", - "concurrently": false, - "name": "idx_job_steps_job_name", - "entityType": "indexes", - "schema": "duron", - "table": "job_steps" - }, - { - "nameExplicit": true, - "columns": [ - { - "value": "to_tsvector('english', \"output\"::text)", - "isExpression": true, - "asc": true, - "nullsFirst": false, - "opclass": null - } - ], - "isUnique": false, - "where": null, - "with": "", - "method": "gin", - "concurrently": false, - "name": "idx_job_steps_output_fts", - "entityType": "indexes", - "schema": "duron", - "table": "job_steps" - }, - { - "nameExplicit": true, - "columns": [ - { - "value": "action_name", - "isExpression": false, - "asc": true, - "nullsFirst": false, - "opclass": null - } - ], - "isUnique": false, - "where": null, - "with": "", - "method": "btree", - "concurrently": false, - "name": "idx_jobs_action_name", - "entityType": "indexes", - "schema": "duron", - "table": "jobs" - }, - { - "nameExplicit": true, - "columns": [ - { - "value": "status", - "isExpression": false, - "asc": true, - "nullsFirst": false, - "opclass": null - } - ], - "isUnique": false, - "where": null, - "with": "", - "method": "btree", - "concurrently": false, - "name": "idx_jobs_status", - "entityType": "indexes", - "schema": "duron", - "table": "jobs" - }, - { - "nameExplicit": true, - "columns": [ - { - "value": "group_key", - "isExpression": false, - "asc": true, - "nullsFirst": false, - "opclass": null - } - ], - "isUnique": false, - "where": null, - "with": "", - "method": "btree", - "concurrently": false, - "name": "idx_jobs_group_key", - "entityType": "indexes", - "schema": "duron", - "table": "jobs" - }, - { - "nameExplicit": true, - "columns": [ - { - "value": "started_at", - "isExpression": false, - "asc": true, - "nullsFirst": false, - "opclass": null - } - ], - "isUnique": false, - "where": null, - "with": "", - "method": "btree", - "concurrently": false, - "name": "idx_jobs_started_at", - "entityType": "indexes", - "schema": "duron", - "table": "jobs" - }, - { - "nameExplicit": true, - "columns": [ - { - "value": "finished_at", - "isExpression": false, - "asc": true, - "nullsFirst": false, - "opclass": null - } - ], - "isUnique": false, - "where": null, - "with": "", - "method": "btree", - "concurrently": false, - "name": "idx_jobs_finished_at", - "entityType": "indexes", - "schema": "duron", - "table": "jobs" - }, - { - "nameExplicit": true, - "columns": [ - { - "value": "expires_at", - "isExpression": false, - "asc": true, - "nullsFirst": false, - "opclass": null - } - ], - "isUnique": false, - "where": null, - "with": "", - "method": "btree", - "concurrently": false, - "name": "idx_jobs_expires_at", - "entityType": "indexes", - "schema": "duron", - "table": "jobs" - }, - { - "nameExplicit": true, - "columns": [ - { - "value": "client_id", - "isExpression": false, - "asc": true, - "nullsFirst": false, - "opclass": null - } - ], - "isUnique": false, - "where": null, - "with": "", - "method": "btree", - "concurrently": false, - "name": "idx_jobs_client_id", - "entityType": "indexes", - "schema": "duron", - "table": "jobs" - }, - { - "nameExplicit": true, - "columns": [ - { - "value": "checksum", - "isExpression": false, - "asc": true, - "nullsFirst": false, - "opclass": null - } - ], - "isUnique": false, - "where": null, - "with": "", - "method": "btree", - "concurrently": false, - "name": "idx_jobs_checksum", - "entityType": "indexes", - "schema": "duron", - "table": "jobs" - }, - { - "nameExplicit": true, - "columns": [ - { - "value": "concurrency_limit", - "isExpression": false, - "asc": true, - "nullsFirst": false, - "opclass": null - } - ], - "isUnique": false, - "where": null, - "with": "", - "method": "btree", - "concurrently": false, - "name": "idx_jobs_concurrency_limit", - "entityType": "indexes", - "schema": "duron", - "table": "jobs" - }, - { - "nameExplicit": true, - "columns": [ - { - "value": "action_name", - "isExpression": false, - "asc": true, - "nullsFirst": false, - "opclass": null - }, - { - "value": "status", - "isExpression": false, - "asc": true, - "nullsFirst": false, - "opclass": null - } - ], - "isUnique": false, - "where": null, - "with": "", - "method": "btree", - "concurrently": false, - "name": "idx_jobs_action_status", - "entityType": "indexes", - "schema": "duron", - "table": "jobs" - }, - { - "nameExplicit": true, - "columns": [ - { - "value": "action_name", - "isExpression": false, - "asc": true, - "nullsFirst": false, - "opclass": null - }, - { - "value": "group_key", - "isExpression": false, - "asc": true, - "nullsFirst": false, - "opclass": null - } - ], - "isUnique": false, - "where": null, - "with": "", - "method": "btree", - "concurrently": false, - "name": "idx_jobs_action_group", - "entityType": "indexes", - "schema": "duron", - "table": "jobs" - }, - { - "nameExplicit": true, - "columns": [ - { - "value": "to_tsvector('english', \"input\"::text)", - "isExpression": true, - "asc": true, - "nullsFirst": false, - "opclass": null - } - ], - "isUnique": false, - "where": null, - "with": "", - "method": "gin", - "concurrently": false, - "name": "idx_jobs_input_fts", - "entityType": "indexes", - "schema": "duron", - "table": "jobs" - }, - { - "nameExplicit": true, - "columns": [ - { - "value": "to_tsvector('english', \"output\"::text)", - "isExpression": true, - "asc": true, - "nullsFirst": false, - "opclass": null - } - ], - "isUnique": false, - "where": null, - "with": "", - "method": "gin", - "concurrently": false, - "name": "idx_jobs_output_fts", - "entityType": "indexes", - "schema": "duron", - "table": "jobs" - }, - { - "nameExplicit": false, - "columns": [ - "job_id" - ], - "schemaTo": "duron", - "tableTo": "jobs", - "columnsTo": [ - "id" - ], - "onUpdate": "NO ACTION", - "onDelete": "CASCADE", - "name": "job_steps_job_id_jobs_id_fkey", - "entityType": "fks", - "schema": "duron", - "table": "job_steps" - }, - { - "columns": [ - "id" - ], - "nameExplicit": false, - "name": "job_steps_pkey", - "schema": "duron", - "table": "job_steps", - "entityType": "pks" - }, - { - "columns": [ - "id" - ], - "nameExplicit": false, - "name": "jobs_pkey", - "schema": "duron", - "table": "jobs", - "entityType": "pks" - }, - { - "nameExplicit": true, - "columns": [ - "job_id", - "name" - ], - "nullsNotDistinct": false, - "name": "unique_job_step_name", - "entityType": "uniques", - "schema": "duron", - "table": "job_steps" - }, - { - "value": "\"status\" IN ('active','completed','failed','cancelled')", - "name": "job_steps_status_check", - "entityType": "checks", - "schema": "duron", - "table": "job_steps" - }, - { - "value": "\"status\" IN ('created','active','completed','failed','cancelled')", - "name": "jobs_status_check", - "entityType": "checks", - "schema": "duron", - "table": "jobs" - } - ], - "renames": [] -} \ No newline at end of file diff --git a/packages/duron/migrations/postgres/20260117172938_panoramic_photon/migration.sql b/packages/duron/migrations/postgres/20260117172938_panoramic_photon/migration.sql deleted file mode 100644 index a4ff7b2..0000000 --- a/packages/duron/migrations/postgres/20260117172938_panoramic_photon/migration.sql +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE "duron"."job_steps" ADD COLUMN "branch" boolean DEFAULT false NOT NULL; \ No newline at end of file diff --git a/packages/duron/migrations/postgres/20260117142756_pretty_vulcan/migration.sql b/packages/duron/migrations/postgres/20260117231749_clumsy_penance/migration.sql similarity index 61% rename from packages/duron/migrations/postgres/20260117142756_pretty_vulcan/migration.sql rename to packages/duron/migrations/postgres/20260117231749_clumsy_penance/migration.sql index 32f82a7..05fc771 100644 --- a/packages/duron/migrations/postgres/20260117142756_pretty_vulcan/migration.sql +++ b/packages/duron/migrations/postgres/20260117231749_clumsy_penance/migration.sql @@ -1,2 +1,3 @@ ALTER TABLE "duron"."job_steps" ADD COLUMN "parent_step_id" uuid;--> statement-breakpoint +ALTER TABLE "duron"."job_steps" ADD COLUMN "branch" boolean DEFAULT false NOT NULL;--> statement-breakpoint CREATE INDEX "idx_job_steps_parent_step_id" ON "duron"."job_steps" ("parent_step_id"); \ No newline at end of file diff --git a/packages/duron/migrations/postgres/20260117172938_panoramic_photon/snapshot.json b/packages/duron/migrations/postgres/20260117231749_clumsy_penance/snapshot.json similarity index 99% rename from packages/duron/migrations/postgres/20260117172938_panoramic_photon/snapshot.json rename to packages/duron/migrations/postgres/20260117231749_clumsy_penance/snapshot.json index 4ad1558..5a536ee 100644 --- a/packages/duron/migrations/postgres/20260117172938_panoramic_photon/snapshot.json +++ b/packages/duron/migrations/postgres/20260117231749_clumsy_penance/snapshot.json @@ -1,9 +1,9 @@ { "version": "8", "dialect": "postgres", - "id": "cc8b34a2-aa75-4928-927c-341d737fe2dc", + "id": "4dab3234-6e6b-4a65-a5c5-c898bf2f3911", "prevIds": [ - "25beb926-865c-41eb-8525-bbc1e0d3a19c" + "e32fddc5-6d55-4c79-87b4-13d3a01b7d09" ], "ddl": [ { diff --git a/packages/duron/src/action.ts b/packages/duron/src/action.ts index 1dc2a80..d4263fd 100644 --- a/packages/duron/src/action.ts +++ b/packages/duron/src/action.ts @@ -134,13 +134,13 @@ export const StepOptionsSchema = z.object({ .describe('The expire time for the step (milliseconds)'), /** - * Whether this step is a branch. - * Branch steps are independent from siblings during time travel. - * When time traveling to a step, completed branch siblings are preserved. + * Whether this step runs in parallel with siblings. + * Parallel steps are independent from siblings during time travel. + * When time traveling to a step, completed parallel siblings are preserved. * * @default false */ - branch: z.boolean().default(false).describe('Whether this step is a branch (independent from siblings)'), + parallel: z.boolean().default(false).describe('Whether this step runs in parallel (independent from siblings)'), }) /** diff --git a/packages/duron/src/adapters/adapter.ts b/packages/duron/src/adapters/adapter.ts index 993b86e..d587abb 100644 --- a/packages/duron/src/adapters/adapter.ts +++ b/packages/duron/src/adapters/adapter.ts @@ -407,7 +407,7 @@ export abstract class Adapter extends EventEmitter { * Time travel a job to restart from a specific step. * The job must be in completed, failed, or cancelled status. * Resets the job and ancestor steps to active status, deletes subsequent steps, - * and preserves completed branch siblings. + * and preserves completed parallel siblings. * * @returns Promise resolving to `true` if time travel succeeded, `false` otherwise */ @@ -690,7 +690,7 @@ export abstract class Adapter extends EventEmitter { * Internal method to time travel a job to restart from a specific step. * The job must be in completed, failed, or cancelled status. * Resets the job and ancestor steps to active status, deletes subsequent steps, - * and preserves completed branch siblings. + * and preserves completed parallel siblings. * * @param options - Validated time travel options * @returns Promise resolving to `true` if time travel succeeded, `false` otherwise diff --git a/packages/duron/src/adapters/postgres/base.ts b/packages/duron/src/adapters/postgres/base.ts index bcae474..a7a5176 100644 --- a/packages/duron/src/adapters/postgres/base.ts +++ b/packages/duron/src/adapters/postgres/base.ts @@ -321,7 +321,7 @@ export class PostgresBaseAdapter e * Internal method to time travel a job to restart from a specific step. * The job must be in completed, failed, or cancelled status. * Resets the job and ancestor steps to active status, deletes subsequent steps, - * and preserves completed branch siblings. + * and preserves completed parallel siblings. * * Algorithm: * 1. Validate job is in terminal state (completed/failed/cancelled) @@ -367,9 +367,9 @@ export class PostgresBaseAdapter e FROM ${this.tables.jobStepsTable} s INNER JOIN ancestors a ON s.id = a.parent_step_id ), - -- Steps to keep: completed steps created before target + completed branch siblings of target and ancestors + their descendants - branch_siblings AS ( - -- Completed branch siblings of target step + -- Steps to keep: completed steps created before target + completed parallel siblings of target and ancestors + their descendants + parallel_siblings AS ( + -- Completed parallel siblings of target step SELECT s.id FROM ${this.tables.jobStepsTable} s CROSS JOIN target_step ts @@ -382,7 +382,7 @@ export class PostgresBaseAdapter e OR s.parent_step_id = ts.parent_step_id ) UNION - -- Completed branch siblings of each ancestor + -- Completed parallel siblings of each ancestor SELECT s.id FROM ${this.tables.jobStepsTable} s INNER JOIN ancestors a ON ( @@ -394,15 +394,15 @@ export class PostgresBaseAdapter e AND s.branch = true AND s.status = ${STEP_STATUS_COMPLETED} ), - -- Find all descendants of branch siblings (to keep their children too) - branch_descendants AS ( + -- Find all descendants of parallel siblings (to keep their children too) + parallel_descendants AS ( SELECT s.id FROM ${this.tables.jobStepsTable} s - WHERE s.id IN (SELECT id FROM branch_siblings) + WHERE s.id IN (SELECT id FROM parallel_siblings) UNION ALL SELECT s.id FROM ${this.tables.jobStepsTable} s - INNER JOIN branch_descendants bd ON s.parent_step_id = bd.id + INNER JOIN parallel_descendants pd ON s.parent_step_id = pd.id WHERE s.job_id = ${jobId} ), steps_to_keep AS ( @@ -416,8 +416,8 @@ export class PostgresBaseAdapter e AND s.id NOT IN (SELECT id FROM ancestors) AND s.id != ts.id UNION - -- All branch siblings and their descendants - SELECT id FROM branch_descendants + -- All parallel siblings and their descendants + SELECT id FROM parallel_descendants ), -- Calculate time offset: shift preserved steps to start from "now" time_offset AS ( @@ -759,7 +759,7 @@ export class PostgresBaseAdapter e timeoutMs, retriesLimit, parentStepId, - branch = false, + parallel = false, }: CreateOrRecoverJobStepOptions): Promise { type StepResult = CreateOrRecoverJobStepResult @@ -795,7 +795,7 @@ export class PostgresBaseAdapter e SELECT ${jobId}, ${parentStepId}, - ${branch}, + ${parallel}, ${name}, ${timeoutMs}, ${retriesLimit}, @@ -1046,7 +1046,7 @@ export class PostgresBaseAdapter e id: jobStepsTable.id, jobId: jobStepsTable.job_id, parentStepId: jobStepsTable.parent_step_id, - branch: jobStepsTable.branch, + parallel: jobStepsTable.parallel, name: jobStepsTable.name, status: jobStepsTable.status, error: jobStepsTable.error, @@ -1235,7 +1235,7 @@ export class PostgresBaseAdapter e id: this.tables.jobStepsTable.id, jobId: this.tables.jobStepsTable.job_id, parentStepId: this.tables.jobStepsTable.parent_step_id, - branch: this.tables.jobStepsTable.branch, + parallel: this.tables.jobStepsTable.parallel, name: this.tables.jobStepsTable.name, output: this.tables.jobStepsTable.output, status: this.tables.jobStepsTable.status, diff --git a/packages/duron/src/adapters/postgres/schema.ts b/packages/duron/src/adapters/postgres/schema.ts index 501688f..fe0f060 100644 --- a/packages/duron/src/adapters/postgres/schema.ts +++ b/packages/duron/src/adapters/postgres/schema.ts @@ -67,7 +67,7 @@ export default function createSchema(schemaName: string) { .notNull() .references(() => jobsTable.id, { onDelete: 'cascade' }), parent_step_id: uuid('parent_step_id'), - branch: boolean('branch').notNull().default(false), + parallel: boolean('branch').notNull().default(false), // DB column is 'branch', TypeScript uses 'parallel' name: text('name').notNull(), status: text('status').$type().notNull().default(STEP_STATUS_ACTIVE), output: jsonb('output'), diff --git a/packages/duron/src/adapters/schemas.ts b/packages/duron/src/adapters/schemas.ts index 4ade81d..717043b 100644 --- a/packages/duron/src/adapters/schemas.ts +++ b/packages/duron/src/adapters/schemas.ts @@ -56,7 +56,7 @@ export const JobStepSchema = z.object({ id: z.string(), jobId: z.string(), parentStepId: z.string().nullable().default(null), - branch: z.boolean().default(false), + parallel: z.boolean().default(false), name: z.string(), output: z.any().nullable().default(null), status: StepStatusSchema, @@ -199,8 +199,8 @@ export const CreateOrRecoverJobStepOptionsSchema = z.object({ jobId: z.string(), /** The ID of the parent step (null for root steps) */ parentStepId: z.string().nullable().default(null), - /** Whether this step is a branch (independent from siblings during time travel) */ - branch: z.boolean().default(false), + /** Whether this step runs in parallel (independent from siblings during time travel) */ + parallel: z.boolean().default(false), /** The name of the step */ name: z.string(), /** Timeout in milliseconds for the step */ diff --git a/packages/duron/src/client.ts b/packages/duron/src/client.ts index f959dd7..84e092b 100644 --- a/packages/duron/src/client.ts +++ b/packages/duron/src/client.ts @@ -365,7 +365,7 @@ export class Client< * Time travel a job to restart from a specific step. * The job must be in completed, failed, or cancelled status. * Resets the job and ancestor steps to active status, deletes subsequent steps, - * and preserves completed branch siblings. + * and preserves completed parallel siblings. * * @param jobId - The ID of the job to time travel * @param stepId - The ID of the step to restart from diff --git a/packages/duron/src/step-manager.ts b/packages/duron/src/step-manager.ts index a6495ab..4b86c19 100644 --- a/packages/duron/src/step-manager.ts +++ b/packages/duron/src/step-manager.ts @@ -30,7 +30,7 @@ export interface TaskStep { options: StepOptions abortSignal: AbortSignal parentStepId: string | null - branch: boolean + parallel: boolean } /** @@ -65,7 +65,7 @@ export class StepStore { * @param timeoutMs - Timeout in milliseconds for the step * @param retriesLimit - Maximum number of retries for the step * @param parentStepId - The ID of the parent step (null for root steps) - * @param branch - Whether this step is a branch (independent from siblings during time travel) + * @param parallel - Whether this step runs in parallel (independent from siblings during time travel) * @returns Promise resolving to the created step ID * @throws Error if step creation fails */ @@ -75,7 +75,7 @@ export class StepStore { timeoutMs: number, retriesLimit: number, parentStepId: string | null = null, - branch: boolean = false, + parallel: boolean = false, ) { try { return await this.#adapter.createOrRecoverJobStep({ @@ -84,7 +84,7 @@ export class StepStore { timeoutMs, retriesLimit, parentStepId, - branch, + parallel, }) } catch (error) { throw new NonRetriableError(`Failed to get or create step "${name}" for job "${jobId}"`, { cause: error }) @@ -165,7 +165,7 @@ export class StepManager { throw new StepAlreadyExecutedError(task.name, this.#jobId, this.#actionName) } this.#historySteps.add(task.name) - return this.#executeStep(task.name, task.cb, task.options, task.abortSignal, task.parentStepId, task.branch) + return this.#executeStep(task.name, task.cb, task.options, task.abortSignal, task.parentStepId, task.parallel) }, options.concurrencyLimit) } @@ -233,7 +233,7 @@ export class StepManager { options: StepOptions, abortSignal: AbortSignal, parentStepId: string | null, - branch: boolean, + parallel: boolean, ): Promise { const expire = options.expire const retryOptions = options.retry @@ -245,14 +245,14 @@ export class StepManager { throw new ActionCancelError(this.#actionName, this.#jobId, { cause: 'step cancelled before create step' }) } - // Create step record with parentStepId and branch + // Create step record with parentStepId and parallel const newStep = await this.#stepStore.getOrCreate( this.#jobId, name, expire, retryOptions.limit, parentStepId, - branch, + parallel, ) if (!newStep) { throw new NonRetriableError( @@ -328,8 +328,8 @@ export class StepManager { childCb: (ctx: StepHandlerContext) => Promise, childOptions: z.input = {}, ): Promise => { - // Inherit parent step options EXCEPT branch (each step's branch status is independent) - const { branch: _parentBranch, ...inheritableOptions } = options + // Inherit parent step options EXCEPT parallel (each step's parallel status is independent) + const { parallel: _parentParallel, ...inheritableOptions } = options const parsedChildOptions = StepOptionsSchema.parse({ ...inheritableOptions, ...childOptions, @@ -342,7 +342,7 @@ export class StepManager { options: parsedChildOptions, abortSignal: childSignal, // Child uses composed signal parentStepId: step!.id, // This step is the parent - branch: parsedChildOptions.branch, // Pass branch option + parallel: parsedChildOptions.parallel, // Pass parallel option }) // Track the child promise @@ -598,7 +598,7 @@ class ActionContext { const executed: string[] = [] - // Two root-level branch steps + // Two root-level parallel steps await Promise.all([ ctx.step( 'branch-a', @@ -138,7 +138,7 @@ const branchAction = defineAction()({ executed.push('branch-a') return { result: 'a' } }, - { branch: true }, + { parallel: true }, ), ctx.step( 'branch-b', @@ -147,7 +147,7 @@ const branchAction = defineAction()({ executed.push('branch-b') return { result: 'b' } }, - { branch: true }, + { parallel: true }, ), ]) @@ -162,9 +162,9 @@ const branchAction = defineAction()({ }) /** - * Action with nested branches - branches that contain nested steps - * This tests the edge case where time travel targets a step inside a branch - * and sibling branches with their own children should be preserved + * Action with nested parallel steps - parallel steps that contain nested steps + * This tests the edge case where time travel targets a step inside a parallel step + * and sibling parallel steps with their own children should be preserved */ const nestedBranchAction = defineAction()({ name: 'nested-branch-action', @@ -183,7 +183,7 @@ const nestedBranchAction = defineAction()({ handler: async (ctx) => { const executed: string[] = [] - // Parent step with multiple branch children, each with nested steps + // Parent step with multiple parallel children, each with nested steps await ctx.step('parent', async (parentCtx) => { executed.push('parent-start') @@ -204,7 +204,7 @@ const nestedBranchAction = defineAction()({ executed.push('branch-a-end') return { result: 'a' } }, - { branch: true }, + { parallel: true }, ), // Branch B with nested steps (target for time travel) parentCtx.step( @@ -224,7 +224,7 @@ const nestedBranchAction = defineAction()({ executed.push('branch-b-end') return { result: 'b' } }, - { branch: true }, + { parallel: true }, ), // Branch C with nested steps parentCtx.step( @@ -238,7 +238,7 @@ const nestedBranchAction = defineAction()({ executed.push('branch-c-end') return { result: 'c' } }, - { branch: true }, + { parallel: true }, ), ]) @@ -449,7 +449,7 @@ function runTests(name: string, factory: AdapterFactory) { // ========================================================================= describe('Branch Steps Time Travel', () => { - it('should preserve completed branch siblings during time travel', async () => { + it('should preserve completed parallel siblings during time travel', async () => { // Run a job that fails at final-step const jobId = await client.runAction('branchAction', { failAtStep: 'final-step' }) await client.fetch({ batchSize: 1 }) @@ -475,7 +475,7 @@ function runTests(name: string, factory: AdapterFactory) { const branchBAfter = stepsAfter.find((s) => s.name === 'branch-b') const finalStepAfter = stepsAfter.find((s) => s.name === 'final-step') - // Both branch steps should be preserved (completed before final-step) + // Both parallel steps should be preserved (completed before final-step) expectToBeDefined(branchAAfter) expect(branchAAfter?.status).toBe(STEP_STATUS_COMPLETED) expectToBeDefined(branchBAfter) @@ -486,7 +486,7 @@ function runTests(name: string, factory: AdapterFactory) { expect(finalStepAfter?.status).toBe(STEP_STATUS_ACTIVE) }) - it('should keep branch sibling when time traveling to another branch', async () => { + it('should keep parallel sibling when time traveling to another parallel step', async () => { // Run a job that fails at branch-b const jobId = await client.runAction('branchAction', { failAtStep: 'branch-b' }) await client.fetch({ batchSize: 1 }) @@ -512,7 +512,7 @@ function runTests(name: string, factory: AdapterFactory) { const branchAAfter = stepsAfter.find((s) => s.name === 'branch-a') const branchBAfter = stepsAfter.find((s) => s.name === 'branch-b') - // branch-a should be preserved (completed branch sibling) + // branch-a should be preserved (completed parallel sibling) expectToBeDefined(branchAAfter) expect(branchAAfter?.status).toBe(STEP_STATUS_COMPLETED) @@ -521,9 +521,9 @@ function runTests(name: string, factory: AdapterFactory) { expect(branchBAfter?.status).toBe(STEP_STATUS_ACTIVE) }) - it('should preserve sibling branches AND their children when time traveling to nested step in a branch', async () => { + it('should preserve sibling parallel steps AND their children when time traveling to nested step in a parallel step', async () => { // This tests the edge case from processOrder: time traveling to a step inside - // a branch should preserve sibling branches AND all their nested children + // a parallel step should preserve sibling parallel steps AND all their nested children const jobId = await client.runAction('nestedBranchAction', { failAtStep: 'child-b-2' }) await client.fetch({ batchSize: 1 }) const job = await client.waitForJob(jobId, { timeout: 10000 }) @@ -533,7 +533,7 @@ function runTests(name: string, factory: AdapterFactory) { // Get steps before time travel const { steps } = await client.getJobSteps({ jobId }) - // Verify initial state: branch-a and branch-c completed with children, branch-b failed + // Verify initial state: parallel step a and c completed with children, parallel step b failed const branchA = steps.find((s) => s.name === 'branch-a') const childA1 = steps.find((s) => s.name === 'child-a-1') const childA2 = steps.find((s) => s.name === 'child-a-2') @@ -550,7 +550,7 @@ function runTests(name: string, factory: AdapterFactory) { expectToBeDefined(childB1) expectToBeDefined(childB2) - // Time travel to child-b-2 (nested inside branch-b) + // Time travel to child-b-2 (nested inside parallel step b) const success = await client.timeTravelJob(jobId, childB2!.id) expect(success).toBe(true) @@ -583,7 +583,7 @@ function runTests(name: string, factory: AdapterFactory) { expectToBeDefined(branchBAfter) expect(branchBAfter?.status).toBe(STEP_STATUS_ACTIVE) - // child-b-1 should be preserved (sibling before target in same branch, completed) + // child-b-1 should be preserved (sibling before target in same parallel step, completed) const childB1After = stepsAfter.find((s) => s.name === 'child-b-1') expectToBeDefined(childB1After) expect(childB1After?.status).toBe(STEP_STATUS_COMPLETED) diff --git a/packages/shared-actions/index.ts b/packages/shared-actions/index.ts index 72a7b40..63abeb6 100644 --- a/packages/shared-actions/index.ts +++ b/packages/shared-actions/index.ts @@ -543,7 +543,7 @@ export const processOrder = defineAction()({ addTimeline('analytics-tracking', 'success', 'Analytics updated') return { purchase, recommendations } }, - { expire: 10_000, branch: true }, + { expire: 10_000, parallel: true }, ), // Parent step 2: Loyalty Program Update (with nested steps) @@ -569,7 +569,7 @@ export const processOrder = defineAction()({ addTimeline('loyalty-update', 'success', `${points.earnedPoints} points, tier: ${tier.tier}`) return { points, tier } }, - { expire: 10_000, branch: true }, + { expire: 10_000, parallel: true }, ), // Parent step 3: Partner Sync (with nested steps) @@ -593,7 +593,7 @@ export const processOrder = defineAction()({ addTimeline('partner-sync', 'success', 'All partners synced') return { supplier, warehouse } }, - { expire: 10_000, branch: true }, + { expire: 10_000, parallel: true }, ), ]) From 85f58c1edd0bffef77de09bb90f55b65857c2749 Mon Sep 17 00:00:00 2001 From: Martin Acosta Date: Sun, 18 Jan 2026 10:20:59 -0300 Subject: [PATCH 11/78] Add resizable panels to Dashboard layout and integrate new Resizable components - Introduced the `react-resizable-panels` library to enable resizable panel functionality in the Dashboard view. - Refactored the layout to support vertical and horizontal resizing of Jobs, Job Details, and Steps sections for improved user experience. - Created new `ResizablePanelGroup`, `ResizablePanel`, and `ResizableHandle` components for better modularity and reusability. - Updated JobDetails component to enhance the display of job information and maintain consistent styling across the application. --- packages/duron-dashboard/package.json | 1 + .../src/components/ui/resizable.tsx | 52 ++++++ .../src/contexts/step-view-context.tsx | 2 +- .../duron-dashboard/src/views/dashboard.tsx | 125 ++++++++------ .../duron-dashboard/src/views/job-details.tsx | 163 +++++++++--------- 5 files changed, 213 insertions(+), 130 deletions(-) create mode 100644 packages/duron-dashboard/src/components/ui/resizable.tsx diff --git a/packages/duron-dashboard/package.json b/packages/duron-dashboard/package.json index d6d3e48..b21da03 100644 --- a/packages/duron-dashboard/package.json +++ b/packages/duron-dashboard/package.json @@ -70,6 +70,7 @@ "react": "^19", "react-day-picker": "^9.11.2", "react-dom": "^19", + "react-resizable-panels": "^4.4.1", "tailwind-merge": "^3.4.0", "zod": "^4.1.12" }, diff --git a/packages/duron-dashboard/src/components/ui/resizable.tsx b/packages/duron-dashboard/src/components/ui/resizable.tsx new file mode 100644 index 0000000..db7350a --- /dev/null +++ b/packages/duron-dashboard/src/components/ui/resizable.tsx @@ -0,0 +1,52 @@ +import { GripVerticalIcon } from 'lucide-react' +import type * as React from 'react' +import { Group, Panel, Separator } from 'react-resizable-panels' + +import { cn } from '@/lib/utils' + +function ResizablePanelGroup({ + className, + direction = 'horizontal', + ...props +}: React.ComponentProps & { direction?: 'horizontal' | 'vertical' }) { + return ( + + ) +} + +function ResizablePanel({ ...props }: React.ComponentProps) { + return +} + +function ResizableHandle({ + withHandle, + className, + ...props +}: React.ComponentProps & { + withHandle?: boolean +}) { + return ( + div]:rotate-90', + className, + )} + {...props} + > + {withHandle && ( +
+ +
+ )} + + ) +} + +export { ResizablePanelGroup, ResizablePanel, ResizableHandle } diff --git a/packages/duron-dashboard/src/contexts/step-view-context.tsx b/packages/duron-dashboard/src/contexts/step-view-context.tsx index 35e5a19..e65fa65 100644 --- a/packages/duron-dashboard/src/contexts/step-view-context.tsx +++ b/packages/duron-dashboard/src/contexts/step-view-context.tsx @@ -1,6 +1,6 @@ 'use client' -import { createContext, useCallback, useContext, useEffect, useState, type ReactNode } from 'react' +import { createContext, type ReactNode, useCallback, useContext, useEffect, useState } from 'react' type StepViewType = 'list' | 'timeline' diff --git a/packages/duron-dashboard/src/views/dashboard.tsx b/packages/duron-dashboard/src/views/dashboard.tsx index 288a946..06fd44f 100644 --- a/packages/duron-dashboard/src/views/dashboard.tsx +++ b/packages/duron-dashboard/src/views/dashboard.tsx @@ -1,6 +1,6 @@ 'use client' -import { LogOut, MoreVertical, Plus, Trash2, X } from 'lucide-react' +import { LogOut, MoreVertical, Plus, Trash2 } from 'lucide-react' import { useCallback, useEffect, useState } from 'react' import { CreateJobDialog } from '@/components/create-job-dialog' @@ -15,6 +15,7 @@ import { DropdownMenuSeparator, DropdownMenuTrigger, } from '@/components/ui/dropdown-menu' +import { ResizableHandle, ResizablePanel, ResizablePanelGroup } from '@/components/ui/resizable' import { useAuth } from '@/contexts/auth-context' import { useIsMobile } from '@/hooks/use-is-mobile' import { useJobParams } from '@/hooks/use-job-params' @@ -153,72 +154,100 @@ export function Dashboard({ showLogo = true, enableLogin = true }: DashboardProp
-
- {/* Desktop Layout: Top row (Jobs | Details), Bottom row (Steps) */} +
+ {/* Desktop Layout with Resizable Panels */} {!isMobile && ( -
+ {/* Top Row: Jobs and Job Details */} -
- {/* Jobs Section */} -
- -
+ + + {/* Jobs Section */} + +
+ +
+ - {/* Job Details Section */} - {jobDetailsVisible && ( -
- setJobDetailsVisible(false)} /> -
- )} -
+ {/* Job Details Section */} + {jobDetailsVisible && ( + <> + + +
+ setJobDetailsVisible(false)} /> +
+ + + )} + + {/* Bottom Row: Steps (full width) */} {selectedJobId && ( -
-
-

Steps

-
-
- -
-
+ <> + + +
+
+

Steps

+
+
+ +
+
+ + )} -
+ )} - {/* Mobile: Vertical Layout - Jobs, Job Details, and Steps */} + {/* Mobile: Vertical Resizable Layout */} {isMobile && ( -
+ {/* Jobs Section */} -
- -
+ +
+ +
+ {/* Job Details Section */} {selectedJobId && ( -
- -
+ <> + + +
+ +
+ + )} {/* Steps Section */} {selectedJobId && ( -
-
-

Steps

-
-
- -
-
+ <> + + +
+
+

Steps

+
+
+ +
+
+ + )} -
+ )}
-
- ID: {job.id} -
-
- Action: {job.actionName} -
-
- Group Key: {job.groupKey} -
- {job.clientId && ( -
- Client ID: {job.clientId} -
- )} -
- Status: -
-
- Created: {formatDate(job.createdAt)} -
- {job.startedAt && (
- Started: {formatDate(job.startedAt)} + ID: {job.id}
- )} - {job.finishedAt && (
- Completed: {formatDate(job.finishedAt)} + Action: {job.actionName}
- )} - {job.startedAt && (
- Duration: {jobDuration} + Group Key: {job.groupKey}
- )} - {job.concurrencyLimit && ( + {job.clientId && ( +
+ Client ID:{' '} + {job.clientId} +
+ )}
- Concurrency Limit: {job.concurrencyLimit} + Status:
- )} - {job.timeoutMs && (
- Timeout: {job.timeoutMs}ms + Created: {formatDate(job.createdAt)}
- )} - {job.expiresAt && ( -
- Expires:{' '} - - {formatDate(job.expiresAt)} - -
- )} -
+ {job.startedAt && ( +
+ Started: {formatDate(job.startedAt)} +
+ )} + {job.finishedAt && ( +
+ Completed: {formatDate(job.finishedAt)} +
+ )} + {job.startedAt && ( +
+ Duration: {jobDuration} +
+ )} + {job.concurrencyLimit && ( +
+ Concurrency Limit: {job.concurrencyLimit} +
+ )} + {job.timeoutMs && ( +
+ Timeout: {job.timeoutMs}ms +
+ )} + {job.expiresAt && ( +
+ Expires:{' '} + + {formatDate(job.expiresAt)} + +
+ )} +
- {/* Job Input/Output */} -
- {job.input && ( -
-
Input
-
- + {/* Job Input/Output */} +
+ {job.input && ( +
+
Input
+
+ +
-
- )} + )} - {!job.input &&
No input available
} + {!job.input &&
No input available
} - {job.error && ( -
-
Error
-
- + {job.error && ( +
+
Error
+
+ +
-
- )} + )} - {job.output && ( -
-
Output
-
- + {job.output && ( +
+
Output
+
+ +
-
- )} + )} - {!job.output &&
No output available
} -
+ {!job.output &&
No output available
} +
From 3166ea8d4fd2f8842ae669d67f34b0ed2fabdacc Mon Sep 17 00:00:00 2001 From: Martin Acosta Date: Sun, 18 Jan 2026 10:59:57 -0300 Subject: [PATCH 12/78] Refactor DuronDashboard layout and introduce LayoutProvider for improved configuration management - Replaced the StepViewProvider with a new LayoutProvider to manage layout configurations for both desktop and mobile views. - Added a LayoutContext to handle layout settings, including step view types and panel sizes, enhancing user customization. - Updated the Dashboard component to utilize the new layout context, allowing for dynamic resizing of panels based on user preferences. - Removed the deprecated StepViewContext to streamline the codebase and improve maintainability. --- .../duron-dashboard/src/DuronDashboard.tsx | 6 +- .../src/contexts/layout-context.tsx | 189 ++++++++++++++++++ .../src/contexts/step-view-context.tsx | 41 ---- .../duron-dashboard/src/views/dashboard.tsx | 183 +++++++++++------ .../duron-dashboard/src/views/step-list.tsx | 2 +- 5 files changed, 316 insertions(+), 105 deletions(-) create mode 100644 packages/duron-dashboard/src/contexts/layout-context.tsx delete mode 100644 packages/duron-dashboard/src/contexts/step-view-context.tsx diff --git a/packages/duron-dashboard/src/DuronDashboard.tsx b/packages/duron-dashboard/src/DuronDashboard.tsx index c56f037..828a2a6 100644 --- a/packages/duron-dashboard/src/DuronDashboard.tsx +++ b/packages/duron-dashboard/src/DuronDashboard.tsx @@ -3,7 +3,7 @@ import { NuqsAdapter } from 'nuqs/adapters/react' import { ApiProvider } from './contexts/api-context' import { AuthProvider, useAuth } from './contexts/auth-context' -import { StepViewProvider } from './contexts/step-view-context' +import { LayoutProvider } from './contexts/layout-context' import { ThemeProvider } from './contexts/theme-context' import { Dashboard } from './views/dashboard' import Login from './views/login' @@ -55,13 +55,13 @@ export function DuronDashboard({ url, enableLogin = false, showLogo = true }: Du return ( - + - + ) diff --git a/packages/duron-dashboard/src/contexts/layout-context.tsx b/packages/duron-dashboard/src/contexts/layout-context.tsx new file mode 100644 index 0000000..cdb825d --- /dev/null +++ b/packages/duron-dashboard/src/contexts/layout-context.tsx @@ -0,0 +1,189 @@ +'use client' + +import { createContext, type ReactNode, useCallback, useContext, useEffect, useMemo, useState } from 'react' + +type StepViewType = 'list' | 'timeline' + +interface DesktopLayout { + /** Horizontal panel sizes: [jobs, details] as percentages */ + horizontalSizes: number[] + /** Vertical panel sizes: [top, bottom] as percentages */ + verticalSizes: number[] +} + +interface MobileLayout { + /** Vertical panel sizes: [jobs, details, steps] as percentages */ + verticalSizes: number[] +} + +interface LayoutConfig { + /** Step view type: 'list' or 'timeline' */ + stepViewType: StepViewType + /** Desktop layout configuration */ + desktop: DesktopLayout + /** Mobile layout configuration */ + mobile: MobileLayout +} + +interface LayoutContextValue { + config: LayoutConfig + setStepViewType: (type: StepViewType) => void + setDesktopHorizontalSizes: (sizes: number[]) => void + setDesktopVerticalSizes: (sizes: number[]) => void + setMobileVerticalSizes: (sizes: number[]) => void +} + +const STORAGE_KEY = 'duron-layout-config' + +const DEFAULT_CONFIG: LayoutConfig = { + stepViewType: 'list', + desktop: { + horizontalSizes: [50, 50], + verticalSizes: [50, 50], + }, + mobile: { + verticalSizes: [33, 33, 34], + }, +} + +const LayoutContext = createContext(null) + +function loadConfig(): LayoutConfig { + if (typeof window === 'undefined') { + return DEFAULT_CONFIG + } + + try { + const stored = localStorage.getItem(STORAGE_KEY) + if (stored) { + const parsed = JSON.parse(stored) + + // Handle migration from old format (horizontalSizes/verticalSizes at root) + const desktop: DesktopLayout = parsed.desktop ?? { + horizontalSizes: parsed.horizontalSizes ?? DEFAULT_CONFIG.desktop.horizontalSizes, + verticalSizes: parsed.verticalSizes ?? DEFAULT_CONFIG.desktop.verticalSizes, + } + + const mobile: MobileLayout = parsed.mobile ?? { + verticalSizes: DEFAULT_CONFIG.mobile.verticalSizes, + } + + return { + stepViewType: parsed.stepViewType === 'timeline' ? 'timeline' : 'list', + desktop: { + horizontalSizes: Array.isArray(desktop.horizontalSizes) + ? desktop.horizontalSizes + : DEFAULT_CONFIG.desktop.horizontalSizes, + verticalSizes: Array.isArray(desktop.verticalSizes) + ? desktop.verticalSizes + : DEFAULT_CONFIG.desktop.verticalSizes, + }, + mobile: { + verticalSizes: Array.isArray(mobile.verticalSizes) + ? mobile.verticalSizes + : DEFAULT_CONFIG.mobile.verticalSizes, + }, + } + } + } catch { + // Ignore parsing errors + } + + // Migrate from old step view type storage + try { + const oldStepViewType = localStorage.getItem('duron-step-view-type') + if (oldStepViewType === 'timeline' || oldStepViewType === 'list') { + return { + ...DEFAULT_CONFIG, + stepViewType: oldStepViewType, + } + } + } catch { + // Ignore errors + } + + return DEFAULT_CONFIG +} + +function saveConfig(config: LayoutConfig): void { + if (typeof window === 'undefined') { + return + } + + try { + localStorage.setItem(STORAGE_KEY, JSON.stringify(config)) + } catch { + // Ignore storage errors + } +} + +export function LayoutProvider({ children }: { children: ReactNode }) { + const [config, setConfig] = useState(DEFAULT_CONFIG) + + // Load from localStorage on mount + useEffect(() => { + setConfig(loadConfig()) + }, []) + + const setStepViewType = useCallback((type: StepViewType) => { + setConfig((prev) => { + const next = { ...prev, stepViewType: type } + saveConfig(next) + return next + }) + }, []) + + const setDesktopHorizontalSizes = useCallback((sizes: number[]) => { + setConfig((prev) => { + const next = { ...prev, desktop: { ...prev.desktop, horizontalSizes: sizes } } + saveConfig(next) + return next + }) + }, []) + + const setDesktopVerticalSizes = useCallback((sizes: number[]) => { + setConfig((prev) => { + const next = { ...prev, desktop: { ...prev.desktop, verticalSizes: sizes } } + saveConfig(next) + return next + }) + }, []) + + const setMobileVerticalSizes = useCallback((sizes: number[]) => { + setConfig((prev) => { + const next = { ...prev, mobile: { ...prev.mobile, verticalSizes: sizes } } + saveConfig(next) + return next + }) + }, []) + + const value = useMemo( + () => ({ + config, + setStepViewType, + setDesktopHorizontalSizes, + setDesktopVerticalSizes, + setMobileVerticalSizes, + }), + [config, setStepViewType, setDesktopHorizontalSizes, setDesktopVerticalSizes, setMobileVerticalSizes], + ) + + return {children} +} + +export function useLayout() { + const context = useContext(LayoutContext) + if (!context) { + throw new Error('useLayout must be used within a LayoutProvider') + } + return context +} + +// Convenience hook for step view type (backwards compatible) +export function useStepView() { + const { config, setStepViewType } = useLayout() + return { + viewType: config.stepViewType, + setViewType: setStepViewType, + } +} diff --git a/packages/duron-dashboard/src/contexts/step-view-context.tsx b/packages/duron-dashboard/src/contexts/step-view-context.tsx deleted file mode 100644 index e65fa65..0000000 --- a/packages/duron-dashboard/src/contexts/step-view-context.tsx +++ /dev/null @@ -1,41 +0,0 @@ -'use client' - -import { createContext, type ReactNode, useCallback, useContext, useEffect, useState } from 'react' - -type StepViewType = 'list' | 'timeline' - -interface StepViewContextValue { - viewType: StepViewType - setViewType: (type: StepViewType) => void -} - -const STORAGE_KEY = 'duron-step-view-type' - -const StepViewContext = createContext(null) - -export function StepViewProvider({ children }: { children: ReactNode }) { - const [viewType, setViewTypeState] = useState('list') - - // Load from localStorage on mount - useEffect(() => { - const stored = localStorage.getItem(STORAGE_KEY) - if (stored === 'list' || stored === 'timeline') { - setViewTypeState(stored) - } - }, []) - - const setViewType = useCallback((type: StepViewType) => { - setViewTypeState(type) - localStorage.setItem(STORAGE_KEY, type) - }, []) - - return {children} -} - -export function useStepView() { - const context = useContext(StepViewContext) - if (!context) { - throw new Error('useStepView must be used within a StepViewProvider') - } - return context -} diff --git a/packages/duron-dashboard/src/views/dashboard.tsx b/packages/duron-dashboard/src/views/dashboard.tsx index 06fd44f..f49a8c5 100644 --- a/packages/duron-dashboard/src/views/dashboard.tsx +++ b/packages/duron-dashboard/src/views/dashboard.tsx @@ -1,7 +1,7 @@ 'use client' import { LogOut, MoreVertical, Plus, Trash2 } from 'lucide-react' -import { useCallback, useEffect, useState } from 'react' +import { useCallback, useEffect, useMemo, useState } from 'react' import { CreateJobDialog } from '@/components/create-job-dialog' import { JobSearch } from '@/components/job-search' @@ -15,8 +15,9 @@ import { DropdownMenuSeparator, DropdownMenuTrigger, } from '@/components/ui/dropdown-menu' -import { ResizableHandle, ResizablePanel, ResizablePanelGroup } from '@/components/ui/resizable' +import { ResizablePanel, ResizablePanelGroup } from '@/components/ui/resizable' import { useAuth } from '@/contexts/auth-context' +import { useLayout } from '@/contexts/layout-context' import { useIsMobile } from '@/hooks/use-is-mobile' import { useJobParams } from '@/hooks/use-job-params' import { useDeleteJobs } from '@/lib/api' @@ -36,11 +37,64 @@ export function Dashboard({ showLogo = true, enableLogin = true }: DashboardProp const [jobDetailsVisible, setJobDetailsVisible] = useState(false) const isMobile = useIsMobile() const { logout } = useAuth() + const { config, setDesktopHorizontalSizes, setDesktopVerticalSizes, setMobileVerticalSizes } = useLayout() const handleJobSelect = useCallback((jobId: string | null) => { setSelectedJobId(jobId) }, []) + // Desktop layout config + const desktopHorizontalLayout = useMemo(() => { + const jobs = config.desktop?.horizontalSizes?.[0] ?? 50 + const details = config.desktop?.horizontalSizes?.[1] ?? 50 + return { jobs, details } + }, [config.desktop?.horizontalSizes]) + + const desktopVerticalLayout = useMemo(() => { + const top = config.desktop?.verticalSizes?.[0] ?? 50 + const bottom = config.desktop?.verticalSizes?.[1] ?? 50 + return { top, bottom } + }, [config.desktop?.verticalSizes]) + + // Mobile layout config + const mobileLayout = useMemo(() => { + const jobs = config.mobile?.verticalSizes?.[0] ?? 33 + const details = config.mobile?.verticalSizes?.[1] ?? 33 + const steps = config.mobile?.verticalSizes?.[2] ?? 34 + return { jobs, details, steps } + }, [config.mobile?.verticalSizes]) + + // Handle desktop horizontal panel resize (jobs/details) + const handleDesktopHorizontalLayoutChange = useCallback( + (layout: { [panelId: string]: number }) => { + const jobs = layout['jobs-panel'] ?? 50 + const details = layout['details-panel'] ?? 50 + setDesktopHorizontalSizes([jobs, details]) + }, + [setDesktopHorizontalSizes], + ) + + // Handle desktop vertical panel resize (top/bottom) + const handleDesktopVerticalLayoutChange = useCallback( + (layout: { [panelId: string]: number }) => { + const top = layout['top-panel'] ?? 50 + const bottom = layout['bottom-panel'] ?? 50 + setDesktopVerticalSizes([top, bottom]) + }, + [setDesktopVerticalSizes], + ) + + // Handle mobile vertical panel resize (jobs/details/steps) + const handleMobileVerticalLayoutChange = useCallback( + (layout: { [panelId: string]: number }) => { + const jobs = layout['mobile-jobs-panel'] ?? 33 + const details = layout['mobile-details-panel'] ?? 33 + const steps = layout['mobile-steps-panel'] ?? 34 + setMobileVerticalSizes([jobs, details, steps]) + }, + [setMobileVerticalSizes], + ) + useEffect(() => { if (!jobDetailsVisible) { handleJobSelect(null) @@ -157,95 +211,104 @@ export function Dashboard({ showLogo = true, enableLogin = true }: DashboardProp
{/* Desktop Layout with Resizable Panels */} {!isMobile && ( - + {/* Top Row: Jobs and Job Details */} - - + + {/* Jobs Section */} - -
+ +
{/* Job Details Section */} {jobDetailsVisible && ( - <> - - -
- setJobDetailsVisible(false)} /> -
- - + + setJobDetailsVisible(false)} /> + )} {/* Bottom Row: Steps (full width) */} {selectedJobId && ( - <> - - -
-
-

Steps

-
-
- -
+ +
+
+

Steps

- - +
+ +
+
+ )} )} {/* Mobile: Vertical Resizable Layout */} {isMobile && ( - + {/* Jobs Section */} - -
- -
+ + {/* Job Details Section */} {selectedJobId && ( - <> - - -
- -
- - + + + )} {/* Steps Section */} {selectedJobId && ( - <> - - -
-
-

Steps

-
-
- -
+ +
+
+

Steps

- - +
+ +
+
+ )} )} diff --git a/packages/duron-dashboard/src/views/step-list.tsx b/packages/duron-dashboard/src/views/step-list.tsx index 7873e0b..b0a3602 100644 --- a/packages/duron-dashboard/src/views/step-list.tsx +++ b/packages/duron-dashboard/src/views/step-list.tsx @@ -9,7 +9,7 @@ import { Button } from '@/components/ui/button' import { Input } from '@/components/ui/input' import { ScrollArea } from '@/components/ui/scroll-area' import { Tooltip, TooltipContent, TooltipTrigger } from '@/components/ui/tooltip' -import { useStepView } from '@/contexts/step-view-context' +import { useStepView } from '@/contexts/layout-context' import { useDebouncedCallback } from '@/hooks/use-debounced-callback' import { useStepsPolling } from '@/hooks/use-steps-polling' import { type GetJobStepsResponse, useJob, useJobSteps, useTimeTravelJob } from '@/lib/api' From 7cea671e889fac6033c6ea23b2e2ffa90c29e862 Mon Sep 17 00:00:00 2001 From: Martin Acosta Date: Sun, 18 Jan 2026 11:14:46 -0300 Subject: [PATCH 13/78] Refactor layout configuration loading and enhance Dashboard panel resizing logic - Removed deprecated migration logic for old step view types from the layout configuration loading process. - Updated the LayoutProvider to load configuration synchronously on the first render, preventing a flash of default layout. - Improved panel resizing logic in the Dashboard component by adding checks to ensure required panels are present before resizing, enhancing stability and user experience. --- .../src/contexts/layout-context.tsx | 21 ++----------------- .../duron-dashboard/src/views/dashboard.tsx | 13 ++++++++++++ 2 files changed, 15 insertions(+), 19 deletions(-) diff --git a/packages/duron-dashboard/src/contexts/layout-context.tsx b/packages/duron-dashboard/src/contexts/layout-context.tsx index cdb825d..b89d64d 100644 --- a/packages/duron-dashboard/src/contexts/layout-context.tsx +++ b/packages/duron-dashboard/src/contexts/layout-context.tsx @@ -89,19 +89,6 @@ function loadConfig(): LayoutConfig { // Ignore parsing errors } - // Migrate from old step view type storage - try { - const oldStepViewType = localStorage.getItem('duron-step-view-type') - if (oldStepViewType === 'timeline' || oldStepViewType === 'list') { - return { - ...DEFAULT_CONFIG, - stepViewType: oldStepViewType, - } - } - } catch { - // Ignore errors - } - return DEFAULT_CONFIG } @@ -118,12 +105,8 @@ function saveConfig(config: LayoutConfig): void { } export function LayoutProvider({ children }: { children: ReactNode }) { - const [config, setConfig] = useState(DEFAULT_CONFIG) - - // Load from localStorage on mount - useEffect(() => { - setConfig(loadConfig()) - }, []) + // Load config synchronously on first render to avoid flash of default layout + const [config, setConfig] = useState(() => loadConfig()) const setStepViewType = useCallback((type: StepViewType) => { setConfig((prev) => { diff --git a/packages/duron-dashboard/src/views/dashboard.tsx b/packages/duron-dashboard/src/views/dashboard.tsx index f49a8c5..2bd47ff 100644 --- a/packages/duron-dashboard/src/views/dashboard.tsx +++ b/packages/duron-dashboard/src/views/dashboard.tsx @@ -67,6 +67,9 @@ export function Dashboard({ showLogo = true, enableLogin = true }: DashboardProp // Handle desktop horizontal panel resize (jobs/details) const handleDesktopHorizontalLayoutChange = useCallback( (layout: { [panelId: string]: number }) => { + if (!('jobs-panel' in layout) || !('details-panel' in layout)) { + return + } const jobs = layout['jobs-panel'] ?? 50 const details = layout['details-panel'] ?? 50 setDesktopHorizontalSizes([jobs, details]) @@ -77,6 +80,9 @@ export function Dashboard({ showLogo = true, enableLogin = true }: DashboardProp // Handle desktop vertical panel resize (top/bottom) const handleDesktopVerticalLayoutChange = useCallback( (layout: { [panelId: string]: number }) => { + if (!('top-panel' in layout) || !('bottom-panel' in layout)) { + return + } const top = layout['top-panel'] ?? 50 const bottom = layout['bottom-panel'] ?? 50 setDesktopVerticalSizes([top, bottom]) @@ -87,6 +93,13 @@ export function Dashboard({ showLogo = true, enableLogin = true }: DashboardProp // Handle mobile vertical panel resize (jobs/details/steps) const handleMobileVerticalLayoutChange = useCallback( (layout: { [panelId: string]: number }) => { + if ( + !('mobile-jobs-panel' in layout) || + !('mobile-details-panel' in layout) || + !('mobile-steps-panel' in layout) + ) { + return + } const jobs = layout['mobile-jobs-panel'] ?? 33 const details = layout['mobile-details-panel'] ?? 33 const steps = layout['mobile-steps-panel'] ?? 34 From 43553e9cbcc22db271964c3d58d6ddf34f5842c6 Mon Sep 17 00:00:00 2001 From: Martin Acosta Date: Sun, 18 Jan 2026 11:16:01 -0300 Subject: [PATCH 14/78] Add react-resizable-panels dependency and update CLAUDE.md documentation - Added the `react-resizable-panels` library to enable resizable panel functionality in the project. - Updated CLAUDE.md to include a note on using Context7 MCP for library/API documentation and configuration steps. - Refactored JSON snapshot for PostgreSQL migrations to improve formatting and consistency. --- CLAUDE.md | 2 ++ bun.lock | 11 +++----- .../snapshot.json | 27 +++++-------------- packages/shared-actions/index.ts | 2 +- 4 files changed, 13 insertions(+), 29 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index bf032b1..188fe3e 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -447,3 +447,5 @@ if (!apiKey) { 4. Follow existing code patterns 5. Use TypeScript strict mode 6. Document public APIs with JSDoc + +Always use Context7 MCP when I need library/API documentation, code generation, setup or configuration steps without me having to explicitly ask. diff --git a/bun.lock b/bun.lock index 324340e..eb71665 100644 --- a/bun.lock +++ b/bun.lock @@ -106,6 +106,7 @@ "react": "^19", "react-day-picker": "^9.11.2", "react-dom": "^19", + "react-resizable-panels": "^4.4.1", "tailwind-merge": "^3.4.0", "zod": "^4.1.12", }, @@ -1421,6 +1422,8 @@ "react-remove-scroll-bar": ["react-remove-scroll-bar@2.3.8", "", { "dependencies": { "react-style-singleton": "^2.2.2", "tslib": "^2.0.0" }, "peerDependencies": { "@types/react": "*", "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" }, "optionalPeers": ["@types/react"] }, "sha512-9r+yi9+mgU33AKcj6IbT9oRCO78WriSj6t/cF8DWBZJ9aOGPOTEDvdUDz1FwKim7QXWwmHqtdHnRJfhAxEG46Q=="], + "react-resizable-panels": ["react-resizable-panels@4.4.1", "", { "peerDependencies": { "react": "^18.0.0 || ^19.0.0", "react-dom": "^18.0.0 || ^19.0.0" } }, "sha512-dpM9oI6rGlAq7VYDeafSRA1JmkJv8aNuKySR+tZLQQLfaeqTnQLSM52EcoI/QdowzsjVUCk6jViKS0xHWITVRQ=="], + "react-style-singleton": ["react-style-singleton@2.2.3", "", { "dependencies": { "get-nonce": "^1.0.0", "tslib": "^2.0.0" }, "peerDependencies": { "@types/react": "*", "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react"] }, "sha512-b6jSvxvVnyptAiLjbkWLE/lOnR4lfTtDAl+eUC7RZy+QQWc6wRzIV2CE6xBuMmDxc2qIihtDCZD5NPOFl7fRBQ=="], "readable-stream": ["readable-stream@4.7.0", "", { "dependencies": { "abort-controller": "^3.0.0", "buffer": "^6.0.3", "events": "^3.3.0", "process": "^0.11.10", "string_decoder": "^1.3.0" } }, "sha512-oIGGmcpTLwPga8Bn6/Z75SVaH1z5dUut2ibSyAMVhmUggWpmDn2dapB0n7f8nwaSiRtepAsfJyfXIO5DCVAODg=="], @@ -1725,10 +1728,6 @@ "drizzle-kit/esbuild": ["esbuild@0.25.12", "", { "optionalDependencies": { "@esbuild/aix-ppc64": "0.25.12", "@esbuild/android-arm": "0.25.12", "@esbuild/android-arm64": "0.25.12", "@esbuild/android-x64": "0.25.12", "@esbuild/darwin-arm64": "0.25.12", "@esbuild/darwin-x64": "0.25.12", "@esbuild/freebsd-arm64": "0.25.12", "@esbuild/freebsd-x64": "0.25.12", "@esbuild/linux-arm": "0.25.12", "@esbuild/linux-arm64": "0.25.12", "@esbuild/linux-ia32": "0.25.12", "@esbuild/linux-loong64": "0.25.12", "@esbuild/linux-mips64el": "0.25.12", "@esbuild/linux-ppc64": "0.25.12", "@esbuild/linux-riscv64": "0.25.12", "@esbuild/linux-s390x": "0.25.12", "@esbuild/linux-x64": "0.25.12", "@esbuild/netbsd-arm64": "0.25.12", "@esbuild/netbsd-x64": "0.25.12", "@esbuild/openbsd-arm64": "0.25.12", "@esbuild/openbsd-x64": "0.25.12", "@esbuild/openharmony-arm64": "0.25.12", "@esbuild/sunos-x64": "0.25.12", "@esbuild/win32-arm64": "0.25.12", "@esbuild/win32-ia32": "0.25.12", "@esbuild/win32-x64": "0.25.12" }, "bin": { "esbuild": "bin/esbuild" } }, "sha512-bbPBYYrtZbkt6Os6FiTLCTFxvq4tt3JKall1vRwshA3fdVztsLAatFaZobhkBC8/BrPetoa0oksYoKXoG4ryJg=="], - "duron-dashboard/@types/bun": ["@types/bun@1.3.3", "", { "dependencies": { "bun-types": "1.3.3" } }, "sha512-ogrKbJ2X5N0kWLLFKeytG0eHDleBYtngtlbu9cyBKFtNL3cnpDZkNdQj8flVf6WTZUX5ulI9AY1oa7ljhSrp+g=="], - - "duron-dashboard/elysia": ["elysia@1.4.16", "", { "dependencies": { "cookie": "^1.0.2", "exact-mirror": "0.2.3", "fast-decode-uri-component": "^1.0.1", "memoirist": "^0.4.0" }, "peerDependencies": { "@sinclair/typebox": ">= 0.34.0 < 1", "@types/bun": ">= 1.2.0", "file-type": ">= 20.0.0", "openapi-types": ">= 12.0.0", "typescript": ">= 5.0.0" }, "optionalPeers": ["@types/bun", "typescript"] }, "sha512-KZtKN160/bdWVKg2hEgyoNXY8jRRquc+m6PboyisaLZL891I+Ufb7Ja6lDAD7vMQur8sLEWIcidZOzj5lWw9UA=="], - "examples/@types/bun": ["@types/bun@1.3.3", "", { "dependencies": { "bun-types": "1.3.3" } }, "sha512-ogrKbJ2X5N0kWLLFKeytG0eHDleBYtngtlbu9cyBKFtNL3cnpDZkNdQj8flVf6WTZUX5ulI9AY1oa7ljhSrp+g=="], "fumadocs-ui/fumadocs-core": ["fumadocs-core@16.1.0", "", { "dependencies": { "@formatjs/intl-localematcher": "^0.6.2", "@orama/orama": "^3.1.16", "@shikijs/rehype": "^3.15.0", "@shikijs/transformers": "^3.15.0", "estree-util-value-to-estree": "^3.5.0", "github-slugger": "^2.0.0", "hast-util-to-estree": "^3.1.3", "hast-util-to-jsx-runtime": "^2.3.6", "image-size": "^2.0.2", "negotiator": "^1.0.0", "npm-to-yarn": "^3.0.1", "path-to-regexp": "^8.3.0", "remark": "^15.0.1", "remark-gfm": "^4.0.1", "remark-rehype": "^11.1.2", "scroll-into-view-if-needed": "^3.1.0", "shiki": "^3.15.0", "unist-util-visit": "^5.0.0" }, "peerDependencies": { "@mixedbread/sdk": "^0.19.0", "@orama/core": "1.x.x", "@tanstack/react-router": "1.x.x", "@types/react": "*", "algoliasearch": "5.x.x", "lucide-react": "*", "next": "16.x.x", "react": "^19.2.0", "react-dom": "^19.2.0", "react-router": "7.x.x", "waku": "^0.26.0 || ^0.27.0" }, "optionalPeers": ["@mixedbread/sdk", "@orama/core", "@tanstack/react-router", "@types/react", "algoliasearch", "lucide-react", "next", "react", "react-dom", "react-router", "waku"] }, "sha512-5pbO2bOGc/xlb2yLQSy6Oag8mvD5CNf5HzQIG80HjZzLXYWEOHW8yovRKnWKRF9gAibn6WHnbssj3YPAlitV/A=="], @@ -1831,10 +1830,6 @@ "drizzle-kit/esbuild/@esbuild/win32-x64": ["@esbuild/win32-x64@0.25.12", "", { "os": "win32", "cpu": "x64" }, "sha512-alJC0uCZpTFrSL0CCDjcgleBXPnCrEAhTBILpeAp7M/OFgoqtAetfBzX0xM00MUsVVPpVjlPuMbREqnZCXaTnA=="], - "duron-dashboard/@types/bun/bun-types": ["bun-types@1.3.3", "", { "dependencies": { "@types/node": "*" } }, "sha512-z3Xwlg7j2l9JY27x5Qn3Wlyos8YAp0kKRlrePAOjgjMGS5IG6E7Jnlx736vH9UVI4wUICwwhC9anYL++XeOgTQ=="], - - "duron-dashboard/elysia/exact-mirror": ["exact-mirror@0.2.3", "", { "peerDependencies": { "@sinclair/typebox": "^0.34.15" }, "optionalPeers": ["@sinclair/typebox"] }, "sha512-aLdARfO0W0ntufjDyytUJQMbNXoB9g+BbA8KcgIq4XOOTYRw48yUGON/Pr64iDrYNZKcKvKbqE0MPW56FF2BXA=="], - "examples/@types/bun/bun-types": ["bun-types@1.3.3", "", { "dependencies": { "@types/node": "*" } }, "sha512-z3Xwlg7j2l9JY27x5Qn3Wlyos8YAp0kKRlrePAOjgjMGS5IG6E7Jnlx736vH9UVI4wUICwwhC9anYL++XeOgTQ=="], "shared-actions/@types/bun/bun-types": ["bun-types@1.3.3", "", { "dependencies": { "@types/node": "*" } }, "sha512-z3Xwlg7j2l9JY27x5Qn3Wlyos8YAp0kKRlrePAOjgjMGS5IG6E7Jnlx736vH9UVI4wUICwwhC9anYL++XeOgTQ=="], diff --git a/packages/duron/migrations/postgres/20260117231749_clumsy_penance/snapshot.json b/packages/duron/migrations/postgres/20260117231749_clumsy_penance/snapshot.json index 5a536ee..4f989a3 100644 --- a/packages/duron/migrations/postgres/20260117231749_clumsy_penance/snapshot.json +++ b/packages/duron/migrations/postgres/20260117231749_clumsy_penance/snapshot.json @@ -2,9 +2,7 @@ "version": "8", "dialect": "postgres", "id": "4dab3234-6e6b-4a65-a5c5-c898bf2f3911", - "prevIds": [ - "e32fddc5-6d55-4c79-87b4-13d3a01b7d09" - ], + "prevIds": ["e32fddc5-6d55-4c79-87b4-13d3a01b7d09"], "ddl": [ { "name": "duron", @@ -935,14 +933,10 @@ }, { "nameExplicit": false, - "columns": [ - "job_id" - ], + "columns": ["job_id"], "schemaTo": "duron", "tableTo": "jobs", - "columnsTo": [ - "id" - ], + "columnsTo": ["id"], "onUpdate": "NO ACTION", "onDelete": "CASCADE", "name": "job_steps_job_id_jobs_id_fkey", @@ -951,9 +945,7 @@ "table": "job_steps" }, { - "columns": [ - "id" - ], + "columns": ["id"], "nameExplicit": false, "name": "job_steps_pkey", "schema": "duron", @@ -961,9 +953,7 @@ "entityType": "pks" }, { - "columns": [ - "id" - ], + "columns": ["id"], "nameExplicit": false, "name": "jobs_pkey", "schema": "duron", @@ -972,10 +962,7 @@ }, { "nameExplicit": true, - "columns": [ - "job_id", - "name" - ], + "columns": ["job_id", "name"], "nullsNotDistinct": false, "name": "unique_job_step_name", "entityType": "uniques", @@ -998,4 +985,4 @@ } ], "renames": [] -} \ No newline at end of file +} diff --git a/packages/shared-actions/index.ts b/packages/shared-actions/index.ts index 63abeb6..d221c85 100644 --- a/packages/shared-actions/index.ts +++ b/packages/shared-actions/index.ts @@ -408,7 +408,7 @@ export const processOrder = defineAction()({ const authorization = await paymentStep('authorize-payment', async ({ step: authStep }) => { // Grandchild step: Fraud check (3 levels deep!) const fraudCheck = await authStep('fraud-check', async () => { - await new Promise((resolve) => setTimeout(resolve, 150)) + await new Promise((resolve) => setTimeout(resolve, 5000)) const isSafe = totalAmount < 10000 // Mock: flag large orders addTimeline('fraud-check', isSafe ? 'success' : 'failed', `Amount: $${totalAmount.toFixed(2)}`) return { isSafe, riskScore: isSafe ? 0.1 : 0.9 } From adc2bfeea5b25feaff453d91ea4b567550b2049c Mon Sep 17 00:00:00 2001 From: Martin Acosta Date: Sun, 18 Jan 2026 11:34:37 -0300 Subject: [PATCH 15/78] Update layout configuration and Dashboard component for improved panel resizing - Adjusted the layout configuration to change default panel sizes for desktop views, setting horizontal sizes to [30, 70] for details and steps, and maintaining vertical sizes at [50, 50] for jobs and bottom. - Refactored the Dashboard component to reflect the new layout configuration, updating the logic for handling panel resizing and ensuring correct panel identifiers are used. - Enhanced comments for clarity regarding the layout structure and resizing behavior, improving maintainability and user understanding. --- .../src/contexts/layout-context.tsx | 10 +- .../duron-dashboard/src/views/dashboard.tsx | 112 ++++++++++-------- 2 files changed, 67 insertions(+), 55 deletions(-) diff --git a/packages/duron-dashboard/src/contexts/layout-context.tsx b/packages/duron-dashboard/src/contexts/layout-context.tsx index b89d64d..1f46536 100644 --- a/packages/duron-dashboard/src/contexts/layout-context.tsx +++ b/packages/duron-dashboard/src/contexts/layout-context.tsx @@ -1,13 +1,13 @@ 'use client' -import { createContext, type ReactNode, useCallback, useContext, useEffect, useMemo, useState } from 'react' +import { createContext, type ReactNode, useCallback, useContext, useMemo, useState } from 'react' type StepViewType = 'list' | 'timeline' interface DesktopLayout { - /** Horizontal panel sizes: [jobs, details] as percentages */ + /** Horizontal panel sizes: [details, steps] as percentages (bottom row) */ horizontalSizes: number[] - /** Vertical panel sizes: [top, bottom] as percentages */ + /** Vertical panel sizes: [jobs, bottom] as percentages */ verticalSizes: number[] } @@ -38,8 +38,8 @@ const STORAGE_KEY = 'duron-layout-config' const DEFAULT_CONFIG: LayoutConfig = { stepViewType: 'list', desktop: { - horizontalSizes: [50, 50], - verticalSizes: [50, 50], + horizontalSizes: [30, 70], // [details, steps] - details takes 30% by default + verticalSizes: [50, 50], // [jobs, bottom] }, mobile: { verticalSizes: [33, 33, 34], diff --git a/packages/duron-dashboard/src/views/dashboard.tsx b/packages/duron-dashboard/src/views/dashboard.tsx index 2bd47ff..6949393 100644 --- a/packages/duron-dashboard/src/views/dashboard.tsx +++ b/packages/duron-dashboard/src/views/dashboard.tsx @@ -43,17 +43,18 @@ export function Dashboard({ showLogo = true, enableLogin = true }: DashboardProp setSelectedJobId(jobId) }, []) - // Desktop layout config + // Desktop layout config (horizontal: [details, steps] in bottom row) const desktopHorizontalLayout = useMemo(() => { - const jobs = config.desktop?.horizontalSizes?.[0] ?? 50 - const details = config.desktop?.horizontalSizes?.[1] ?? 50 - return { jobs, details } + const details = config.desktop?.horizontalSizes?.[0] ?? 30 + const steps = config.desktop?.horizontalSizes?.[1] ?? 70 + return { details, steps } }, [config.desktop?.horizontalSizes]) + // Desktop layout config (vertical: [jobs, bottom] where bottom has details|steps) const desktopVerticalLayout = useMemo(() => { - const top = config.desktop?.verticalSizes?.[0] ?? 50 + const jobs = config.desktop?.verticalSizes?.[0] ?? 50 const bottom = config.desktop?.verticalSizes?.[1] ?? 50 - return { top, bottom } + return { jobs, bottom } }, [config.desktop?.verticalSizes]) // Mobile layout config @@ -64,28 +65,28 @@ export function Dashboard({ showLogo = true, enableLogin = true }: DashboardProp return { jobs, details, steps } }, [config.mobile?.verticalSizes]) - // Handle desktop horizontal panel resize (jobs/details) + // Handle desktop horizontal panel resize (details/steps in bottom row) const handleDesktopHorizontalLayoutChange = useCallback( (layout: { [panelId: string]: number }) => { - if (!('jobs-panel' in layout) || !('details-panel' in layout)) { + if (!('details-panel' in layout) || !('steps-panel' in layout)) { return } - const jobs = layout['jobs-panel'] ?? 50 - const details = layout['details-panel'] ?? 50 - setDesktopHorizontalSizes([jobs, details]) + const details = layout['details-panel'] ?? 30 + const steps = layout['steps-panel'] ?? 70 + setDesktopHorizontalSizes([details, steps]) }, [setDesktopHorizontalSizes], ) - // Handle desktop vertical panel resize (top/bottom) + // Handle desktop vertical panel resize (jobs/bottom) const handleDesktopVerticalLayoutChange = useCallback( (layout: { [panelId: string]: number }) => { - if (!('top-panel' in layout) || !('bottom-panel' in layout)) { + if (!('jobs-panel' in layout) || !('bottom-panel' in layout)) { return } - const top = layout['top-panel'] ?? 50 + const jobs = layout['jobs-panel'] ?? 50 const bottom = layout['bottom-panel'] ?? 50 - setDesktopVerticalSizes([top, bottom]) + setDesktopVerticalSizes([jobs, bottom]) }, [setDesktopVerticalSizes], ) @@ -223,54 +224,65 @@ export function Dashboard({ showLogo = true, enableLogin = true }: DashboardProp
{/* Desktop Layout with Resizable Panels */} + {/* Layout: [Jobs Table (top)] / [Job Details | Steps (bottom)] */} {!isMobile && ( - {/* Top Row: Jobs and Job Details */} - - - {/* Jobs Section */} - -
- -
- - - {/* Job Details Section */} - {jobDetailsVisible && ( - - setJobDetailsVisible(false)} /> - - )} - + {/* Top Row: Jobs Table (full width) */} + +
+ +
- {/* Bottom Row: Steps (full width) */} + {/* Bottom Row: Job Details | Steps */} {selectedJobId && ( -
-
-

Steps

-
-
- -
-
+ + {/* Job Details Section */} + {jobDetailsVisible && ( + +
+ setJobDetailsVisible(false)} /> +
+ + )} + + {/* Steps Section */} + +
+
+

Steps

+
+
+ +
+
+ + )} From 9fbfb22367761c13868bf27073d904e6541256b4 Mon Sep 17 00:00:00 2001 From: Martin Acosta Date: Sun, 18 Jan 2026 11:36:28 -0300 Subject: [PATCH 16/78] Refactor AuthContext to improve authentication state management - Removed the useEffect for loading tokens and replaced it with a synchronous function to initialize authentication state, preventing a flash of the login page. - Updated the AuthProvider to manage token and refresh token state more efficiently using a single authState object. - Simplified login, updateAccessToken, and logout functions to directly manipulate the authState, enhancing code clarity and maintainability. --- .../src/contexts/auth-context.tsx | 36 ++++++++++--------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/packages/duron-dashboard/src/contexts/auth-context.tsx b/packages/duron-dashboard/src/contexts/auth-context.tsx index d4516c2..04bf2e7 100644 --- a/packages/duron-dashboard/src/contexts/auth-context.tsx +++ b/packages/duron-dashboard/src/contexts/auth-context.tsx @@ -1,4 +1,4 @@ -import { createContext, type ReactNode, useContext, useEffect, useState } from 'react' +import { createContext, type ReactNode, useContext, useState } from 'react' interface AuthContextType { isAuthenticated: boolean @@ -14,36 +14,38 @@ const AuthContext = createContext(undefined) const ACCESS_TOKEN_KEY = 'auth_token' const REFRESH_TOKEN_KEY = 'refresh_token' -export function AuthProvider({ children }: { children: ReactNode }) { - const [token, setToken] = useState(null) - const [refreshToken, setRefreshToken] = useState(null) +function getInitialAuthState(): { token: string | null; refreshToken: string | null } { + if (typeof window === 'undefined') { + return { token: null, refreshToken: null } + } + const storedToken = localStorage.getItem(ACCESS_TOKEN_KEY) + const storedRefreshToken = localStorage.getItem(REFRESH_TOKEN_KEY) + if (storedToken && storedRefreshToken) { + return { token: storedToken, refreshToken: storedRefreshToken } + } + return { token: null, refreshToken: null } +} - useEffect(() => { - const storedToken = localStorage.getItem(ACCESS_TOKEN_KEY) - const storedRefreshToken = localStorage.getItem(REFRESH_TOKEN_KEY) - if (storedToken && storedRefreshToken) { - setToken(storedToken) - setRefreshToken(storedRefreshToken) - } - }, []) +export function AuthProvider({ children }: { children: ReactNode }) { + // Load auth state synchronously to avoid flash of login page + const [authState, setAuthState] = useState(() => getInitialAuthState()) + const { token, refreshToken } = authState const login = (newToken: string, newRefreshToken: string) => { localStorage.setItem(ACCESS_TOKEN_KEY, newToken) localStorage.setItem(REFRESH_TOKEN_KEY, newRefreshToken) - setToken(newToken) - setRefreshToken(newRefreshToken) + setAuthState({ token: newToken, refreshToken: newRefreshToken }) } const updateAccessToken = (newToken: string) => { localStorage.setItem(ACCESS_TOKEN_KEY, newToken) - setToken(newToken) + setAuthState((prev) => ({ ...prev, token: newToken })) } const logout = () => { localStorage.removeItem(ACCESS_TOKEN_KEY) localStorage.removeItem(REFRESH_TOKEN_KEY) - setToken(null) - setRefreshToken(null) + setAuthState({ token: null, refreshToken: null }) } return ( From c85d00d8dd3a3e614f2828d0c4bd90907aacfff4 Mon Sep 17 00:00:00 2001 From: Martin Acosta Date: Sun, 18 Jan 2026 16:29:11 -0300 Subject: [PATCH 17/78] Enhance PostgresBaseAdapter with updated_at timestamps and improve DataTable and Dashboard styling - Added updated_at timestamps to various job status updates in PostgresBaseAdapter to track changes more accurately. - Adjusted border styles in DataTable and Dashboard components for improved visual consistency and layout clarity. --- .../src/components/data-table/data-table.tsx | 2 +- .../duron-dashboard/src/views/dashboard.tsx | 4 ++-- packages/duron/src/adapters/postgres/base.ts | 22 ++++++++++++++----- 3 files changed, 20 insertions(+), 8 deletions(-) diff --git a/packages/duron-dashboard/src/components/data-table/data-table.tsx b/packages/duron-dashboard/src/components/data-table/data-table.tsx index 79d7ed7..6da26e8 100644 --- a/packages/duron-dashboard/src/components/data-table/data-table.tsx +++ b/packages/duron-dashboard/src/components/data-table/data-table.tsx @@ -87,7 +87,7 @@ export function DataTable({
{/* Fixed pagination footer */} -
+
{actionBar && table.getFilteredSelectedRowModel().rows.length > 0 && actionBar}
diff --git a/packages/duron-dashboard/src/views/dashboard.tsx b/packages/duron-dashboard/src/views/dashboard.tsx index 6949393..43b15ba 100644 --- a/packages/duron-dashboard/src/views/dashboard.tsx +++ b/packages/duron-dashboard/src/views/dashboard.tsx @@ -247,7 +247,7 @@ export function Dashboard({ showLogo = true, enableLogin = true }: DashboardProp -
+
setJobDetailsVisible(false)} />
diff --git a/packages/duron/src/adapters/postgres/base.ts b/packages/duron/src/adapters/postgres/base.ts index a7a5176..ba4cd5e 100644 --- a/packages/duron/src/adapters/postgres/base.ts +++ b/packages/duron/src/adapters/postgres/base.ts @@ -169,6 +169,7 @@ export class PostgresBaseAdapter e status: JOB_STATUS_COMPLETED, output, finished_at: sql`now()`, + updated_at: sql`now()`, }) .where( and( @@ -195,6 +196,7 @@ export class PostgresBaseAdapter e status: JOB_STATUS_FAILED, error, finished_at: sql`now()`, + updated_at: sql`now()`, }) .where( and( @@ -219,6 +221,7 @@ export class PostgresBaseAdapter e .set({ status: JOB_STATUS_CANCELLED, finished_at: sql`now()`, + updated_at: sql`now()`, }) .where( and( @@ -462,7 +465,8 @@ export class PostgresBaseAdapter e expires_at = now() + (timeout_ms || ' milliseconds')::interval, retries_count = 0, delayed_ms = NULL, - history_failed_attempts = '{}'::jsonb + history_failed_attempts = '{}'::jsonb, + updated_at = now() WHERE id IN (SELECT id FROM ancestors) RETURNING id ), @@ -478,7 +482,8 @@ export class PostgresBaseAdapter e expires_at = now() + (timeout_ms || ' milliseconds')::interval, retries_count = 0, delayed_ms = NULL, - history_failed_attempts = '{}'::jsonb + history_failed_attempts = '{}'::jsonb, + updated_at = now() WHERE id = (SELECT id FROM target_step) RETURNING id ), @@ -492,7 +497,8 @@ export class PostgresBaseAdapter e started_at = NULL, finished_at = NULL, client_id = NULL, - expires_at = NULL + expires_at = NULL, + updated_at = now() WHERE id = ${jobId} AND EXISTS (SELECT 1 FROM target_step) RETURNING id @@ -637,7 +643,8 @@ export class PostgresBaseAdapter e SET status = ${JOB_STATUS_ACTIVE}, started_at = now(), expires_at = now() + (timeout_ms || ' milliseconds')::interval, - client_id = ${this.id} + client_id = ${this.id}, + updated_at = now() FROM verify_concurrency vc WHERE j.id = vc.id AND vc.current_active < vc.concurrency_limit -- Final concurrency check using job's concurrency limit @@ -722,7 +729,8 @@ export class PostgresBaseAdapter e expires_at = NULL, finished_at = NULL, output = NULL, - error = NULL + error = NULL, + updated_at = now() WHERE EXISTS (SELECT 1 FROM locked_jobs lj WHERE lj.id = j.id) RETURNING id, checksum ), @@ -873,6 +881,7 @@ export class PostgresBaseAdapter e status: STEP_STATUS_COMPLETED, output, finished_at: sql`now()`, + updated_at: sql`now()`, }) .from(this.tables.jobsTable) .where( @@ -901,6 +910,7 @@ export class PostgresBaseAdapter e status: STEP_STATUS_FAILED, error, finished_at: sql`now()`, + updated_at: sql`now()`, }) .from(this.tables.jobsTable) .where( @@ -939,6 +949,7 @@ export class PostgresBaseAdapter e 'delayedMs', ${delayMs}::integer ) )`, + updated_at: sql`now()`, }) .from(jobsTable) .where( @@ -965,6 +976,7 @@ export class PostgresBaseAdapter e .set({ status: STEP_STATUS_CANCELLED, finished_at: sql`now()`, + updated_at: sql`now()`, }) .from(this.tables.jobsTable) .where( From 4c8ed95f4332ee6724da534ba926fda0124cb6b3 Mon Sep 17 00:00:00 2001 From: Martin Acosta Date: Sun, 18 Jan 2026 16:31:28 -0300 Subject: [PATCH 18/78] Add duration display to StepList component - Integrated duration calculation and formatting for job steps in the StepList component. - Displayed the formatted duration next to each step, enhancing user visibility of execution times. --- packages/duron-dashboard/src/views/step-list.tsx | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/duron-dashboard/src/views/step-list.tsx b/packages/duron-dashboard/src/views/step-list.tsx index b0a3602..67e27b5 100644 --- a/packages/duron-dashboard/src/views/step-list.tsx +++ b/packages/duron-dashboard/src/views/step-list.tsx @@ -4,6 +4,7 @@ import { ChevronRight, Clock, GitBranch, History, List, Search } from 'lucide-re import { useCallback, useMemo, useState } from 'react' import { Timeline } from '@/components/timeline' +import { calculateDurationSeconds, formatDurationSeconds } from '@/lib/duration' import { Accordion, AccordionContent, AccordionItem, AccordionTrigger } from '@/components/ui/accordion' import { Button } from '@/components/ui/button' import { Input } from '@/components/ui/input' @@ -204,6 +205,7 @@ export function StepList({ jobId, selectedStepId, onStepSelect }: StepListProps) const isParallel = (step as any).parallel === true // Calculate left padding based on depth (16px per level) const paddingLeft = depth * 16 + const duration = calculateDurationSeconds(step.startedAt, step.finishedAt) return ( {step.name}
+ {formatDurationSeconds(duration)} {canTimeTravel && ( From ac7d8e50e25bf8d6a962b3a5e8c0ab8f1bde4da1 Mon Sep 17 00:00:00 2001 From: Martin Acosta Date: Sun, 18 Jan 2026 18:49:41 -0300 Subject: [PATCH 19/78] added telemetry --- bun.lock | 1 + .../duron-dashboard/src/DuronDashboard.tsx | 5 +- .../src/components/metrics-panel.tsx | 132 ++ .../src/contexts/metrics-context.tsx | 47 + packages/duron-dashboard/src/dev.tsx | 3 + .../src/hooks/use-job-metrics.ts | 75 + .../duron-dashboard/src/views/job-details.tsx | 19 +- .../src/views/step-details-content.tsx | 23 + .../duron-dashboard/src/views/step-list.tsx | 6 +- .../migration.sql | 24 + .../snapshot.json | 1362 +++++++++++++++++ packages/duron/package.json | 1 + packages/duron/src/action-job.ts | 35 + packages/duron/src/action-manager.ts | 5 + packages/duron/src/action.ts | 14 + packages/duron/src/adapters/adapter.ts | 110 ++ packages/duron/src/adapters/postgres/base.ts | 127 ++ .../src/adapters/postgres/schema.default.ts | 4 +- .../duron/src/adapters/postgres/schema.ts | 47 +- packages/duron/src/adapters/schemas.ts | 70 + packages/duron/src/client.ts | 63 + packages/duron/src/index.ts | 1 + packages/duron/src/server.ts | 140 +- packages/duron/src/step-manager.ts | 89 +- packages/duron/src/telemetry/adapter.ts | 468 ++++++ packages/duron/src/telemetry/index.ts | 17 + packages/duron/src/telemetry/local.ts | 238 +++ packages/duron/src/telemetry/noop.ts | 95 ++ packages/duron/src/telemetry/opentelemetry.ts | 310 ++++ packages/duron/test/telemetry.test.ts | 454 ++++++ 30 files changed, 3975 insertions(+), 10 deletions(-) create mode 100644 packages/duron-dashboard/src/components/metrics-panel.tsx create mode 100644 packages/duron-dashboard/src/contexts/metrics-context.tsx create mode 100644 packages/duron-dashboard/src/hooks/use-job-metrics.ts create mode 100644 packages/duron/migrations/postgres/20260118202533_wealthy_mysterio/migration.sql create mode 100644 packages/duron/migrations/postgres/20260118202533_wealthy_mysterio/snapshot.json create mode 100644 packages/duron/src/telemetry/adapter.ts create mode 100644 packages/duron/src/telemetry/index.ts create mode 100644 packages/duron/src/telemetry/local.ts create mode 100644 packages/duron/src/telemetry/noop.ts create mode 100644 packages/duron/src/telemetry/opentelemetry.ts create mode 100644 packages/duron/test/telemetry.test.ts diff --git a/bun.lock b/bun.lock index eb71665..2f8c760 100644 --- a/bun.lock +++ b/bun.lock @@ -58,6 +58,7 @@ }, "devDependencies": { "@electric-sql/pglite": "^0.3.14", + "@opentelemetry/api": "^1.9.0", "@types/bun": "latest", "@types/node": "^24.0.15", "drizzle-kit": "^1.0.0-beta.11-05230d9", diff --git a/packages/duron-dashboard/src/DuronDashboard.tsx b/packages/duron-dashboard/src/DuronDashboard.tsx index 828a2a6..62b470d 100644 --- a/packages/duron-dashboard/src/DuronDashboard.tsx +++ b/packages/duron-dashboard/src/DuronDashboard.tsx @@ -4,6 +4,7 @@ import { NuqsAdapter } from 'nuqs/adapters/react' import { ApiProvider } from './contexts/api-context' import { AuthProvider, useAuth } from './contexts/auth-context' import { LayoutProvider } from './contexts/layout-context' +import { MetricsProvider } from './contexts/metrics-context' import { ThemeProvider } from './contexts/theme-context' import { Dashboard } from './views/dashboard' import Login from './views/login' @@ -58,7 +59,9 @@ export function DuronDashboard({ url, enableLogin = false, showLogo = true }: Du - + + + diff --git a/packages/duron-dashboard/src/components/metrics-panel.tsx b/packages/duron-dashboard/src/components/metrics-panel.tsx new file mode 100644 index 0000000..e5ecf19 --- /dev/null +++ b/packages/duron-dashboard/src/components/metrics-panel.tsx @@ -0,0 +1,132 @@ +'use client' + +import { Activity, Clock, Hash, Tag } from 'lucide-react' + +import { Badge } from '@/components/ui/badge' +import { ScrollArea, ScrollBar } from '@/components/ui/scroll-area' +import { type Metric, useJobMetrics, useStepMetrics } from '@/hooks/use-job-metrics' +import { formatDate } from '@/lib/format' +import { JsonView } from './json-view' + +interface MetricItemProps { + metric: Metric +} + +function MetricItem({ metric }: MetricItemProps) { + return ( +
+
+
+ + {metric.name} +
+ + {metric.type} + +
+ +
+
+ + {metric.value} +
+
+ + {formatDate(metric.timestamp)} +
+
+ + {Object.keys(metric.attributes).length > 0 && ( +
+
+ + Attributes +
+
+ +
+
+ )} +
+ ) +} + +interface JobMetricsPanelProps { + jobId: string +} + +export function JobMetricsPanel({ jobId }: JobMetricsPanelProps) { + const { data, isLoading, error } = useJobMetrics({ jobId, enabled: true }) + + if (isLoading) { + return
Loading metrics...
+ } + + if (error) { + return
Failed to load metrics: {(error as Error).message}
+ } + + if (!data?.metrics || data.metrics.length === 0) { + return
No metrics recorded for this job
+ } + + return ( + +
+
+

+ + Job Metrics +

+ {data.total} total +
+
+ {data.metrics.map((metric) => ( + + ))} +
+
+ + + ) +} + +interface StepMetricsPanelProps { + stepId: string +} + +export function StepMetricsPanel({ stepId }: StepMetricsPanelProps) { + const { data, isLoading, error } = useStepMetrics({ stepId, enabled: true }) + + if (isLoading) { + return
Loading metrics...
+ } + + if (error) { + return
Failed to load metrics: {(error as Error).message}
+ } + + if (!data?.metrics || data.metrics.length === 0) { + return
No metrics recorded for this step
+ } + + return ( + +
+
+

+ + Step Metrics +

+ {data.total} total +
+
+ {data.metrics.map((metric) => ( + + ))} +
+
+ + + ) +} diff --git a/packages/duron-dashboard/src/contexts/metrics-context.tsx b/packages/duron-dashboard/src/contexts/metrics-context.tsx new file mode 100644 index 0000000..b64352d --- /dev/null +++ b/packages/duron-dashboard/src/contexts/metrics-context.tsx @@ -0,0 +1,47 @@ +import { useQuery } from '@tanstack/react-query' +import { createContext, type ReactNode, useContext } from 'react' + +import { useApi } from './api-context' +import { useAuth } from './auth-context' + +interface MetricsContextType { + metricsEnabled: boolean + isLoading: boolean +} + +const MetricsContext = createContext(undefined) + +export function MetricsProvider({ children }: { children: ReactNode }) { + const { baseUrl } = useApi() + const { token } = useAuth() + + const { data, isLoading } = useQuery({ + queryKey: ['config'], + queryFn: async () => { + const headers: HeadersInit = {} + if (token) { + headers.Authorization = `Bearer ${token}` + } + const response = await fetch(`${baseUrl}/config`, { headers }) + if (!response.ok) { + throw new Error('Failed to fetch config') + } + return response.json() as Promise<{ metricsEnabled: boolean; authEnabled: boolean }> + }, + staleTime: Number.POSITIVE_INFINITY, // Config rarely changes + }) + + return ( + + {children} + + ) +} + +export function useMetrics() { + const context = useContext(MetricsContext) + if (context === undefined) { + throw new Error('useMetrics must be used within a MetricsProvider') + } + return context +} diff --git a/packages/duron-dashboard/src/dev.tsx b/packages/duron-dashboard/src/dev.tsx index ac7201d..58a789e 100644 --- a/packages/duron-dashboard/src/dev.tsx +++ b/packages/duron-dashboard/src/dev.tsx @@ -3,6 +3,7 @@ import { serve } from 'bun' import { getWeather, openaiChat, processOrder, sendEmail, variables } from '@shared-actions/index' import { postgresAdapter } from 'duron/adapters/postgres/postgres' import { createServer, duron } from 'duron/index' +import { localTelemetryAdapter } from 'duron/telemetry' import index from './index.html' @@ -21,6 +22,7 @@ const client = duron({ }, variables, logger: 'info', + telemetry: localTelemetryAdapter(), }) const app = createServer({ @@ -32,6 +34,7 @@ const app = createServer({ jwtSecret: process.env.JWT_SECRET || 'dev-secret-key-change-in-production', expirationTime: '1d', }, + metricsEnabled: true, }) const server = serve({ diff --git a/packages/duron-dashboard/src/hooks/use-job-metrics.ts b/packages/duron-dashboard/src/hooks/use-job-metrics.ts new file mode 100644 index 0000000..3e8fa40 --- /dev/null +++ b/packages/duron-dashboard/src/hooks/use-job-metrics.ts @@ -0,0 +1,75 @@ +import { useQuery } from '@tanstack/react-query' + +import { useApiRequest } from '@/lib/api' + +interface Metric { + id: string + jobId: string + stepId: string | null + name: string + value: number + attributes: Record + type: 'gauge' | 'counter' | 'histogram' | 'summary' + timestamp: string +} + +interface MetricsResult { + metrics: Metric[] + total: number + page?: number + pageSize?: number +} + +interface UseJobMetricsOptions { + jobId: string | null + enabled?: boolean + page?: number + pageSize?: number +} + +export function useJobMetrics({ jobId, enabled = true, page = 1, pageSize = 50 }: UseJobMetricsOptions) { + const apiRequest = useApiRequest() + + return useQuery({ + queryKey: ['job-metrics', jobId, page, pageSize], + queryFn: async () => { + if (!jobId) { + return { metrics: [], total: 0 } + } + const params = new URLSearchParams({ + page: String(page), + pageSize: String(pageSize), + }) + return apiRequest(`/jobs/${jobId}/metrics?${params}`) + }, + enabled: enabled && !!jobId, + }) +} + +interface UseStepMetricsOptions { + stepId: string | null + enabled?: boolean + page?: number + pageSize?: number +} + +export function useStepMetrics({ stepId, enabled = true, page = 1, pageSize = 50 }: UseStepMetricsOptions) { + const apiRequest = useApiRequest() + + return useQuery({ + queryKey: ['step-metrics', stepId, page, pageSize], + queryFn: async () => { + if (!stepId) { + return { metrics: [], total: 0 } + } + const params = new URLSearchParams({ + page: String(page), + pageSize: String(pageSize), + }) + return apiRequest(`/steps/${stepId}/metrics?${params}`) + }, + enabled: enabled && !!stepId, + }) +} + +export type { Metric, MetricsResult } diff --git a/packages/duron-dashboard/src/views/job-details.tsx b/packages/duron-dashboard/src/views/job-details.tsx index 27ff9e9..e6f4104 100644 --- a/packages/duron-dashboard/src/views/job-details.tsx +++ b/packages/duron-dashboard/src/views/job-details.tsx @@ -1,16 +1,18 @@ 'use client' -import { MoreVertical, Play, X } from 'lucide-react' +import { Activity, MoreVertical, Play, X } from 'lucide-react' import { useCallback, useEffect, useState } from 'react' import { Button } from '@/components/ui/button' import { DropdownMenu, DropdownMenuContent, DropdownMenuItem, DropdownMenuTrigger } from '@/components/ui/dropdown-menu' import { ScrollArea, ScrollBar } from '@/components/ui/scroll-area' +import { useMetrics } from '@/contexts/metrics-context' import { useJobStatusPolling } from '@/hooks/use-job-status-polling' import { useCancelJob, useDeleteJob, useJob, useRetryJob } from '@/lib/api' import { formatDate } from '@/lib/format' import { BadgeStatus } from '../components/badge-status' import { JsonView } from '../components/json-view' +import { JobMetricsPanel } from '../components/metrics-panel' import { isExpiring } from '../lib/is-expiring' interface JobDetailsProps { @@ -20,6 +22,8 @@ interface JobDetailsProps { export function JobDetails({ jobId, onClose }: JobDetailsProps) { const { data: job, isLoading: jobLoading } = useJob(jobId) + const { metricsEnabled } = useMetrics() + const [showMetrics, setShowMetrics] = useState(false) // Enable polling for job status updates - refetches entire job detail when status changes useJobStatusPolling(jobId, true) @@ -131,6 +135,12 @@ export function JobDetails({ jobId, onClose }: JobDetailsProps) { Retry + {metricsEnabled && ( + setShowMetrics(!showMetrics)}> + + {showMetrics ? 'Hide Metrics' : 'Show Metrics'} + + )} No output available
} + + {/* Metrics Panel */} + {metricsEnabled && showMetrics && ( +
+ +
+ )}
diff --git a/packages/duron-dashboard/src/views/step-details-content.tsx b/packages/duron-dashboard/src/views/step-details-content.tsx index 978bd3f..5b7e389 100644 --- a/packages/duron-dashboard/src/views/step-details-content.tsx +++ b/packages/duron-dashboard/src/views/step-details-content.tsx @@ -1,9 +1,13 @@ 'use client' +import { Activity } from 'lucide-react' import { useCallback, useEffect, useState } from 'react' import { JsonView } from '@/components/json-view' +import { StepMetricsPanel } from '@/components/metrics-panel' +import { Button } from '@/components/ui/button' import { Tooltip, TooltipContent, TooltipTrigger } from '@/components/ui/tooltip' +import { useMetrics } from '@/contexts/metrics-context' import { useStepStatusPolling } from '@/hooks/use-step-status-polling' import { useStep } from '@/lib/api' import { formatDate } from '@/lib/format' @@ -18,6 +22,8 @@ interface StepDetailsContentProps { export function StepDetailsContent({ stepId, jobId }: StepDetailsContentProps) { // Fetch the full step data including output const { data: step, isLoading, error } = useStep(stepId) + const { metricsEnabled } = useMetrics() + const [showMetrics, setShowMetrics] = useState(false) // Enable polling for individual step status updates useStepStatusPolling(stepId, jobId, true) @@ -195,6 +201,23 @@ export function StepDetailsContent({ stepId, jobId }: StepDetailsContentProps) {
)} + + {/* Metrics Toggle Button */} + {metricsEnabled && ( +
+ +
+ )} + + {/* Metrics Panel */} + {metricsEnabled && showMetrics && ( +
+ +
+ )}
) } diff --git a/packages/duron-dashboard/src/views/step-list.tsx b/packages/duron-dashboard/src/views/step-list.tsx index 67e27b5..2608dad 100644 --- a/packages/duron-dashboard/src/views/step-list.tsx +++ b/packages/duron-dashboard/src/views/step-list.tsx @@ -4,7 +4,6 @@ import { ChevronRight, Clock, GitBranch, History, List, Search } from 'lucide-re import { useCallback, useMemo, useState } from 'react' import { Timeline } from '@/components/timeline' -import { calculateDurationSeconds, formatDurationSeconds } from '@/lib/duration' import { Accordion, AccordionContent, AccordionItem, AccordionTrigger } from '@/components/ui/accordion' import { Button } from '@/components/ui/button' import { Input } from '@/components/ui/input' @@ -14,6 +13,7 @@ import { useStepView } from '@/contexts/layout-context' import { useDebouncedCallback } from '@/hooks/use-debounced-callback' import { useStepsPolling } from '@/hooks/use-steps-polling' import { type GetJobStepsResponse, useJob, useJobSteps, useTimeTravelJob } from '@/lib/api' +import { calculateDurationSeconds, formatDurationSeconds } from '@/lib/duration' import { BadgeStatus } from '../components/badge-status' // Step type from the API response (without output field) @@ -232,7 +232,9 @@ export function StepList({ jobId, selectedStepId, onStepSelect }: StepListProps) {step.name}
- {formatDurationSeconds(duration)} + + {formatDurationSeconds(duration)} + {canTimeTravel && ( diff --git a/packages/duron/migrations/postgres/20260118202533_wealthy_mysterio/migration.sql b/packages/duron/migrations/postgres/20260118202533_wealthy_mysterio/migration.sql new file mode 100644 index 0000000..4d9d4ca --- /dev/null +++ b/packages/duron/migrations/postgres/20260118202533_wealthy_mysterio/migration.sql @@ -0,0 +1,24 @@ +CREATE TABLE "duron"."metrics" ( + "id" uuid PRIMARY KEY DEFAULT gen_random_uuid(), + "job_id" uuid NOT NULL, + "step_id" uuid, + "name" text NOT NULL, + "value" double precision NOT NULL, + "attributes" jsonb DEFAULT '{}' NOT NULL, + "type" text NOT NULL, + "timestamp" timestamp with time zone DEFAULT now() NOT NULL, + "created_at" timestamp with time zone DEFAULT now() NOT NULL, + CONSTRAINT "metrics_type_check" CHECK ("type" IN ('metric', 'span_event', 'span_attribute')) +); +--> statement-breakpoint +CREATE INDEX "idx_metrics_job_id" ON "duron"."metrics" ("job_id");--> statement-breakpoint +CREATE INDEX "idx_metrics_step_id" ON "duron"."metrics" ("step_id");--> statement-breakpoint +CREATE INDEX "idx_metrics_name" ON "duron"."metrics" ("name");--> statement-breakpoint +CREATE INDEX "idx_metrics_type" ON "duron"."metrics" ("type");--> statement-breakpoint +CREATE INDEX "idx_metrics_timestamp" ON "duron"."metrics" ("timestamp");--> statement-breakpoint +CREATE INDEX "idx_metrics_job_step" ON "duron"."metrics" ("job_id","step_id");--> statement-breakpoint +CREATE INDEX "idx_metrics_job_name" ON "duron"."metrics" ("job_id","name");--> statement-breakpoint +CREATE INDEX "idx_metrics_job_type" ON "duron"."metrics" ("job_id","type");--> statement-breakpoint +CREATE INDEX "idx_metrics_attributes" ON "duron"."metrics" USING gin ("attributes");--> statement-breakpoint +ALTER TABLE "duron"."metrics" ADD CONSTRAINT "metrics_job_id_jobs_id_fkey" FOREIGN KEY ("job_id") REFERENCES "duron"."jobs"("id") ON DELETE CASCADE;--> statement-breakpoint +ALTER TABLE "duron"."metrics" ADD CONSTRAINT "metrics_step_id_job_steps_id_fkey" FOREIGN KEY ("step_id") REFERENCES "duron"."job_steps"("id") ON DELETE CASCADE; \ No newline at end of file diff --git a/packages/duron/migrations/postgres/20260118202533_wealthy_mysterio/snapshot.json b/packages/duron/migrations/postgres/20260118202533_wealthy_mysterio/snapshot.json new file mode 100644 index 0000000..c63d6dd --- /dev/null +++ b/packages/duron/migrations/postgres/20260118202533_wealthy_mysterio/snapshot.json @@ -0,0 +1,1362 @@ +{ + "version": "8", + "dialect": "postgres", + "id": "677e77a4-16f7-4931-a58b-2ca2184ce1be", + "prevIds": ["4dab3234-6e6b-4a65-a5c5-c898bf2f3911"], + "ddl": [ + { + "name": "duron", + "entityType": "schemas" + }, + { + "isRlsEnabled": false, + "name": "job_steps", + "entityType": "tables", + "schema": "duron" + }, + { + "isRlsEnabled": false, + "name": "jobs", + "entityType": "tables", + "schema": "duron" + }, + { + "isRlsEnabled": false, + "name": "metrics", + "entityType": "tables", + "schema": "duron" + }, + { + "type": "uuid", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "gen_random_uuid()", + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "duron", + "table": "job_steps" + }, + { + "type": "uuid", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "job_id", + "entityType": "columns", + "schema": "duron", + "table": "job_steps" + }, + { + "type": "uuid", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "parent_step_id", + "entityType": "columns", + "schema": "duron", + "table": "job_steps" + }, + { + "type": "boolean", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "false", + "generated": null, + "identity": null, + "name": "branch", + "entityType": "columns", + "schema": "duron", + "table": "job_steps" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "name", + "entityType": "columns", + "schema": "duron", + "table": "job_steps" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'active'", + "generated": null, + "identity": null, + "name": "status", + "entityType": "columns", + "schema": "duron", + "table": "job_steps" + }, + { + "type": "jsonb", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "output", + "entityType": "columns", + "schema": "duron", + "table": "job_steps" + }, + { + "type": "jsonb", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "error", + "entityType": "columns", + "schema": "duron", + "table": "job_steps" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "started_at", + "entityType": "columns", + "schema": "duron", + "table": "job_steps" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "finished_at", + "entityType": "columns", + "schema": "duron", + "table": "job_steps" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "timeout_ms", + "entityType": "columns", + "schema": "duron", + "table": "job_steps" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "expires_at", + "entityType": "columns", + "schema": "duron", + "table": "job_steps" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "0", + "generated": null, + "identity": null, + "name": "retries_limit", + "entityType": "columns", + "schema": "duron", + "table": "job_steps" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "0", + "generated": null, + "identity": null, + "name": "retries_count", + "entityType": "columns", + "schema": "duron", + "table": "job_steps" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "delayed_ms", + "entityType": "columns", + "schema": "duron", + "table": "job_steps" + }, + { + "type": "jsonb", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'{}'", + "generated": null, + "identity": null, + "name": "history_failed_attempts", + "entityType": "columns", + "schema": "duron", + "table": "job_steps" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "duron", + "table": "job_steps" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "duron", + "table": "job_steps" + }, + { + "type": "uuid", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "gen_random_uuid()", + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "duron", + "table": "jobs" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "action_name", + "entityType": "columns", + "schema": "duron", + "table": "jobs" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "group_key", + "entityType": "columns", + "schema": "duron", + "table": "jobs" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'created'", + "generated": null, + "identity": null, + "name": "status", + "entityType": "columns", + "schema": "duron", + "table": "jobs" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "checksum", + "entityType": "columns", + "schema": "duron", + "table": "jobs" + }, + { + "type": "jsonb", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'{}'", + "generated": null, + "identity": null, + "name": "input", + "entityType": "columns", + "schema": "duron", + "table": "jobs" + }, + { + "type": "jsonb", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "output", + "entityType": "columns", + "schema": "duron", + "table": "jobs" + }, + { + "type": "jsonb", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "error", + "entityType": "columns", + "schema": "duron", + "table": "jobs" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "timeout_ms", + "entityType": "columns", + "schema": "duron", + "table": "jobs" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "expires_at", + "entityType": "columns", + "schema": "duron", + "table": "jobs" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "started_at", + "entityType": "columns", + "schema": "duron", + "table": "jobs" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "finished_at", + "entityType": "columns", + "schema": "duron", + "table": "jobs" + }, + { + "type": "text", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "client_id", + "entityType": "columns", + "schema": "duron", + "table": "jobs" + }, + { + "type": "integer", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "10", + "generated": null, + "identity": null, + "name": "concurrency_limit", + "entityType": "columns", + "schema": "duron", + "table": "jobs" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "duron", + "table": "jobs" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "updated_at", + "entityType": "columns", + "schema": "duron", + "table": "jobs" + }, + { + "type": "uuid", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "gen_random_uuid()", + "generated": null, + "identity": null, + "name": "id", + "entityType": "columns", + "schema": "duron", + "table": "metrics" + }, + { + "type": "uuid", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "job_id", + "entityType": "columns", + "schema": "duron", + "table": "metrics" + }, + { + "type": "uuid", + "typeSchema": null, + "notNull": false, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "step_id", + "entityType": "columns", + "schema": "duron", + "table": "metrics" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "name", + "entityType": "columns", + "schema": "duron", + "table": "metrics" + }, + { + "type": "double precision", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "value", + "entityType": "columns", + "schema": "duron", + "table": "metrics" + }, + { + "type": "jsonb", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "'{}'", + "generated": null, + "identity": null, + "name": "attributes", + "entityType": "columns", + "schema": "duron", + "table": "metrics" + }, + { + "type": "text", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": null, + "generated": null, + "identity": null, + "name": "type", + "entityType": "columns", + "schema": "duron", + "table": "metrics" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "timestamp", + "entityType": "columns", + "schema": "duron", + "table": "metrics" + }, + { + "type": "timestamp with time zone", + "typeSchema": null, + "notNull": true, + "dimensions": 0, + "default": "now()", + "generated": null, + "identity": null, + "name": "created_at", + "entityType": "columns", + "schema": "duron", + "table": "metrics" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "job_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_job_steps_job_id", + "entityType": "indexes", + "schema": "duron", + "table": "job_steps" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "status", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_job_steps_status", + "entityType": "indexes", + "schema": "duron", + "table": "job_steps" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "name", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_job_steps_name", + "entityType": "indexes", + "schema": "duron", + "table": "job_steps" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "expires_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_job_steps_expires_at", + "entityType": "indexes", + "schema": "duron", + "table": "job_steps" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "parent_step_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_job_steps_parent_step_id", + "entityType": "indexes", + "schema": "duron", + "table": "job_steps" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "job_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "status", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_job_steps_job_status", + "entityType": "indexes", + "schema": "duron", + "table": "job_steps" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "job_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "name", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_job_steps_job_name", + "entityType": "indexes", + "schema": "duron", + "table": "job_steps" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "to_tsvector('english', \"output\"::text)", + "isExpression": true, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "gin", + "concurrently": false, + "name": "idx_job_steps_output_fts", + "entityType": "indexes", + "schema": "duron", + "table": "job_steps" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "action_name", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_jobs_action_name", + "entityType": "indexes", + "schema": "duron", + "table": "jobs" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "status", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_jobs_status", + "entityType": "indexes", + "schema": "duron", + "table": "jobs" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "group_key", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_jobs_group_key", + "entityType": "indexes", + "schema": "duron", + "table": "jobs" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "started_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_jobs_started_at", + "entityType": "indexes", + "schema": "duron", + "table": "jobs" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "finished_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_jobs_finished_at", + "entityType": "indexes", + "schema": "duron", + "table": "jobs" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "expires_at", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_jobs_expires_at", + "entityType": "indexes", + "schema": "duron", + "table": "jobs" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "client_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_jobs_client_id", + "entityType": "indexes", + "schema": "duron", + "table": "jobs" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "checksum", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_jobs_checksum", + "entityType": "indexes", + "schema": "duron", + "table": "jobs" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "concurrency_limit", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_jobs_concurrency_limit", + "entityType": "indexes", + "schema": "duron", + "table": "jobs" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "action_name", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "status", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_jobs_action_status", + "entityType": "indexes", + "schema": "duron", + "table": "jobs" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "action_name", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "group_key", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_jobs_action_group", + "entityType": "indexes", + "schema": "duron", + "table": "jobs" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "to_tsvector('english', \"input\"::text)", + "isExpression": true, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "gin", + "concurrently": false, + "name": "idx_jobs_input_fts", + "entityType": "indexes", + "schema": "duron", + "table": "jobs" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "to_tsvector('english', \"output\"::text)", + "isExpression": true, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "gin", + "concurrently": false, + "name": "idx_jobs_output_fts", + "entityType": "indexes", + "schema": "duron", + "table": "jobs" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "job_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_metrics_job_id", + "entityType": "indexes", + "schema": "duron", + "table": "metrics" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "step_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_metrics_step_id", + "entityType": "indexes", + "schema": "duron", + "table": "metrics" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "name", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_metrics_name", + "entityType": "indexes", + "schema": "duron", + "table": "metrics" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "type", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_metrics_type", + "entityType": "indexes", + "schema": "duron", + "table": "metrics" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "timestamp", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_metrics_timestamp", + "entityType": "indexes", + "schema": "duron", + "table": "metrics" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "job_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "step_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_metrics_job_step", + "entityType": "indexes", + "schema": "duron", + "table": "metrics" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "job_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "name", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_metrics_job_name", + "entityType": "indexes", + "schema": "duron", + "table": "metrics" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "job_id", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + }, + { + "value": "type", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "btree", + "concurrently": false, + "name": "idx_metrics_job_type", + "entityType": "indexes", + "schema": "duron", + "table": "metrics" + }, + { + "nameExplicit": true, + "columns": [ + { + "value": "attributes", + "isExpression": false, + "asc": true, + "nullsFirst": false, + "opclass": null + } + ], + "isUnique": false, + "where": null, + "with": "", + "method": "gin", + "concurrently": false, + "name": "idx_metrics_attributes", + "entityType": "indexes", + "schema": "duron", + "table": "metrics" + }, + { + "nameExplicit": false, + "columns": ["job_id"], + "schemaTo": "duron", + "tableTo": "jobs", + "columnsTo": ["id"], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "job_steps_job_id_jobs_id_fkey", + "entityType": "fks", + "schema": "duron", + "table": "job_steps" + }, + { + "nameExplicit": false, + "columns": ["job_id"], + "schemaTo": "duron", + "tableTo": "jobs", + "columnsTo": ["id"], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "metrics_job_id_jobs_id_fkey", + "entityType": "fks", + "schema": "duron", + "table": "metrics" + }, + { + "nameExplicit": false, + "columns": ["step_id"], + "schemaTo": "duron", + "tableTo": "job_steps", + "columnsTo": ["id"], + "onUpdate": "NO ACTION", + "onDelete": "CASCADE", + "name": "metrics_step_id_job_steps_id_fkey", + "entityType": "fks", + "schema": "duron", + "table": "metrics" + }, + { + "columns": ["id"], + "nameExplicit": false, + "name": "job_steps_pkey", + "schema": "duron", + "table": "job_steps", + "entityType": "pks" + }, + { + "columns": ["id"], + "nameExplicit": false, + "name": "jobs_pkey", + "schema": "duron", + "table": "jobs", + "entityType": "pks" + }, + { + "columns": ["id"], + "nameExplicit": false, + "name": "metrics_pkey", + "schema": "duron", + "table": "metrics", + "entityType": "pks" + }, + { + "nameExplicit": true, + "columns": ["job_id", "name"], + "nullsNotDistinct": false, + "name": "unique_job_step_name", + "entityType": "uniques", + "schema": "duron", + "table": "job_steps" + }, + { + "value": "\"status\" IN ('active','completed','failed','cancelled')", + "name": "job_steps_status_check", + "entityType": "checks", + "schema": "duron", + "table": "job_steps" + }, + { + "value": "\"status\" IN ('created','active','completed','failed','cancelled')", + "name": "jobs_status_check", + "entityType": "checks", + "schema": "duron", + "table": "jobs" + }, + { + "value": "\"type\" IN ('metric', 'span_event', 'span_attribute')", + "name": "metrics_type_check", + "entityType": "checks", + "schema": "duron", + "table": "metrics" + } + ], + "renames": [] +} diff --git a/packages/duron/package.json b/packages/duron/package.json index ed65d7b..86ad0be 100644 --- a/packages/duron/package.json +++ b/packages/duron/package.json @@ -67,6 +67,7 @@ }, "devDependencies": { "@electric-sql/pglite": "^0.3.14", + "@opentelemetry/api": "^1.9.0", "@types/bun": "latest", "@types/node": "^24.0.15", "drizzle-kit": "^1.0.0-beta.11-05230d9", diff --git a/packages/duron/src/action-job.ts b/packages/duron/src/action-job.ts index f07b6e9..6354bbd 100644 --- a/packages/duron/src/action-job.ts +++ b/packages/duron/src/action-job.ts @@ -4,12 +4,14 @@ import type { Action } from './action.js' import type { Adapter } from './adapters/adapter.js' import { ActionCancelError, ActionTimeoutError, isCancelError, StepTimeoutError, serializeError } from './errors.js' import { StepManager } from './step-manager.js' +import type { Span, TelemetryAdapter } from './telemetry/adapter.js' import waitForAbort from './utils/wait-for-abort.js' export interface ActionJobOptions> { job: { id: string; input: any; groupKey: string; timeoutMs: number; actionName: string } action: TAction database: Adapter + telemetry: TelemetryAdapter variables: Record logger: Logger } @@ -24,6 +26,7 @@ export class ActionJob> { #job: { id: string; input: any; groupKey: string; timeoutMs: number; actionName: string } #action: TAction #database: Adapter + #telemetry: TelemetryAdapter #variables: Record #logger: Logger #stepManager: StepManager @@ -31,6 +34,7 @@ export class ActionJob> { #timeoutId: NodeJS.Timeout | null = null #done: Promise #resolve: (() => void) | null = null + #jobSpan: Span | null = null // ============================================================================ // Constructor @@ -45,6 +49,7 @@ export class ActionJob> { this.#job = options.job this.#action = options.action this.#database = options.database + this.#telemetry = options.telemetry this.#variables = options.variables this.#logger = options.logger this.#abortController = new AbortController() @@ -54,6 +59,7 @@ export class ActionJob> { jobId: options.job.id, actionName: options.job.actionName, adapter: options.database, + telemetry: options.telemetry, logger: options.logger, concurrencyLimit: options.action.concurrency, }) @@ -78,6 +84,17 @@ export class ActionJob> { * @throws Error if the job fails or output validation fails */ async execute() { + // Start job telemetry span + this.#jobSpan = await this.#telemetry.startJobSpan({ + jobId: this.#job.id, + actionName: this.#action.name, + groupKey: this.#job.groupKey, + input: this.#job.input, + }) + + // Set the job span on the step manager + this.#stepManager.setJobSpan(this.#jobSpan) + try { // Create a child logger for this job const jobLogger = this.#logger.child({ @@ -85,6 +102,9 @@ export class ActionJob> { actionName: this.#action.name, }) + // Create observe context for the action handler + const observeContext = this.#telemetry.createObserveContext(this.#job.id, null, this.#jobSpan) + // Create action context with step manager const ctx = this.#stepManager.createActionContext( this.#job, @@ -92,6 +112,7 @@ export class ActionJob> { this.#variables as any, this.#abortController.signal, jobLogger, + observeContext, ) this.#timeoutId = setTimeout(() => { @@ -138,6 +159,9 @@ export class ActionJob> { '[ActionJob] Action finished executing', ) + // End job span successfully + await this.#telemetry.endJobSpan(this.#jobSpan, { status: 'ok' }) + return result } catch (error) { if ( @@ -146,6 +170,11 @@ export class ActionJob> { ) { this.#logger.warn({ jobId: this.#job.id, actionName: this.#action.name }, '[ActionJob] Job cancelled') await this.#database.cancelJob({ jobId: this.#job.id }) + + // End job span as cancelled + if (this.#jobSpan) { + await this.#telemetry.endJobSpan(this.#jobSpan, { status: 'cancelled' }) + } return } @@ -158,6 +187,12 @@ export class ActionJob> { this.#logger.error({ jobId: this.#job.id, actionName: this.#action.name }, message) await this.#database.failJob({ jobId: this.#job.id, error: serializeError(error) }) + + // End job span with error + if (this.#jobSpan) { + await this.#telemetry.endJobSpan(this.#jobSpan, { status: 'error', error }) + } + throw error } finally { this.#clear() diff --git a/packages/duron/src/action-manager.ts b/packages/duron/src/action-manager.ts index 596559b..72a596d 100644 --- a/packages/duron/src/action-manager.ts +++ b/packages/duron/src/action-manager.ts @@ -4,10 +4,12 @@ import type { Logger } from 'pino' import type { Action } from './action.js' import { ActionJob } from './action-job.js' import type { Adapter, Job } from './adapters/adapter.js' +import type { TelemetryAdapter } from './telemetry/adapter.js' export interface ActionManagerOptions> { action: TAction database: Adapter + telemetry: TelemetryAdapter variables: Record logger: Logger concurrencyLimit: number @@ -22,6 +24,7 @@ export interface ActionManagerOptions> { export class ActionManager> { #action: TAction #database: Adapter + #telemetry: TelemetryAdapter #variables: Record #logger: Logger #queue: fastq.queueAsPromised @@ -41,6 +44,7 @@ export class ActionManager> { constructor(options: ActionManagerOptions) { this.#action = options.action this.#database = options.database + this.#telemetry = options.telemetry this.#variables = options.variables this.#logger = options.logger this.#concurrencyLimit = options.concurrencyLimit @@ -149,6 +153,7 @@ export class ActionManager> { }, action: this.#action, database: this.#database, + telemetry: this.#telemetry, variables: this.#variables, logger: this.#logger, }) diff --git a/packages/duron/src/action.ts b/packages/duron/src/action.ts index d4263fd..f9a6a62 100644 --- a/packages/duron/src/action.ts +++ b/packages/duron/src/action.ts @@ -1,6 +1,7 @@ import type { Logger } from 'pino' import * as z from 'zod' +import type { ObserveContext } from './telemetry/adapter.js' import generateChecksum from './utils/checksum.js' export type RetryOptions = z.infer @@ -13,6 +14,13 @@ export interface ActionHandlerContext( name: string, cb: (ctx: StepHandlerContext) => Promise, @@ -40,6 +48,12 @@ export interface StepHandlerContext { */ parentStepId: string | null + /** + * Observability context for recording metrics and span data. + * Allows recording custom metrics, span attributes, and events. + */ + observe: ObserveContext + /** * Create a nested child step. * Child steps inherit the abort signal chain from their parent. diff --git a/packages/duron/src/adapters/adapter.ts b/packages/duron/src/adapters/adapter.ts index d587abb..78552d5 100644 --- a/packages/duron/src/adapters/adapter.ts +++ b/packages/duron/src/adapters/adapter.ts @@ -24,6 +24,7 @@ import type { DelayJobStepOptions, DeleteJobOptions, DeleteJobsOptions, + DeleteMetricsOptions, FailJobOptions, FailJobStepOptions, FetchOptions, @@ -32,6 +33,9 @@ import type { GetJobStepsResult, GetJobsOptions, GetJobsResult, + GetMetricsOptions, + GetMetricsResult, + InsertMetricOptions, Job, JobStatusResult, JobStep, @@ -52,6 +56,7 @@ import { DelayJobStepOptionsSchema, DeleteJobOptionsSchema, DeleteJobsOptionsSchema, + DeleteMetricsOptionsSchema, FailJobOptionsSchema, FailJobStepOptionsSchema, FetchOptionsSchema, @@ -60,6 +65,9 @@ import { GetJobStepsResultSchema, GetJobsOptionsSchema, GetJobsResultSchema, + GetMetricsOptionsSchema, + GetMetricsResultSchema, + InsertMetricOptionsSchema, JobIdResultSchema, JobSchema, JobStatusResultSchema, @@ -85,6 +93,7 @@ export type { DelayJobStepOptions, DeleteJobOptions, DeleteJobsOptions, + DeleteMetricsOptions, FailJobOptions, FailJobStepOptions, FetchOptions, @@ -93,6 +102,9 @@ export type { GetJobStepsResult, GetJobsOptions, GetJobsResult, + GetMetricsOptions, + GetMetricsResult, + InsertMetricOptions, Job, JobFilters, JobSort, @@ -100,6 +112,11 @@ export type { JobStatusResult, JobStep, JobStepStatusResult, + Metric, + MetricFilters, + MetricSort, + MetricSortField, + MetricType, RecoverJobsOptions, RetryJobOptions, SortOrder, @@ -975,6 +992,99 @@ export abstract class Adapter extends EventEmitter { */ protected abstract _getActions(): Promise + // ============================================================================ + // Metrics Methods + // ============================================================================ + + /** + * Insert a metric record. + * Note: This method bypasses telemetry tracing to prevent infinite loops. + * + * @param options - The metric data to insert + * @returns Promise resolving to the metric ID + */ + async insertMetric(options: InsertMetricOptions): Promise { + try { + await this.start() + const parsedOptions = InsertMetricOptionsSchema.parse(options) + const result = await this._insertMetric(parsedOptions) + return z.string().parse(result) + } catch (error) { + this.#logger?.error(error, 'Error in Adapter.insertMetric()') + throw error + } + } + + /** + * Get metrics for a job or step. + * Note: This method bypasses telemetry tracing to prevent infinite loops. + * + * @param options - Query options including jobId/stepId, filters, sort, and pagination + * @returns Promise resolving to metrics result with pagination info + */ + async getMetrics(options: GetMetricsOptions): Promise { + try { + await this.start() + const parsedOptions = GetMetricsOptionsSchema.parse(options) + // Validate that at least one of jobId or stepId is provided + if (!parsedOptions.jobId && !parsedOptions.stepId) { + throw new Error('At least one of jobId or stepId must be provided') + } + const result = await this._getMetrics(parsedOptions) + return GetMetricsResultSchema.parse(result) + } catch (error) { + this.#logger?.error(error, 'Error in Adapter.getMetrics()') + throw error + } + } + + /** + * Delete all metrics for a job. + * Note: This method bypasses telemetry tracing to prevent infinite loops. + * + * @param options - Options containing the jobId + * @returns Promise resolving to the number of metrics deleted + */ + async deleteMetrics(options: DeleteMetricsOptions): Promise { + try { + await this.start() + const parsedOptions = DeleteMetricsOptionsSchema.parse(options) + const result = await this._deleteMetrics(parsedOptions) + return NumberResultSchema.parse(result) + } catch (error) { + this.#logger?.error(error, 'Error in Adapter.deleteMetrics()') + throw error + } + } + + // ============================================================================ + // Private Metrics Methods (to be implemented by adapters) + // ============================================================================ + + /** + * Internal method to insert a metric record. + * + * @param options - Validated metric data + * @returns Promise resolving to the metric ID + */ + protected abstract _insertMetric(options: InsertMetricOptions): Promise + + /** + * Internal method to get metrics for a job or step. + * + * @param options - Validated query options + * @returns Promise resolving to metrics result with pagination info + */ + protected abstract _getMetrics(options: GetMetricsOptions): Promise + + /** + * Internal method to delete all metrics for a job. + * + * @param options - Validated options containing the jobId + * @returns Promise resolving to the number of metrics deleted + */ + protected abstract _deleteMetrics(options: DeleteMetricsOptions): Promise + // ============================================================================ // Protected Abstract Methods (to be implemented by adapters) // ============================================================================ diff --git a/packages/duron/src/adapters/postgres/base.ts b/packages/duron/src/adapters/postgres/base.ts index ba4cd5e..5db6e7d 100644 --- a/packages/duron/src/adapters/postgres/base.ts +++ b/packages/duron/src/adapters/postgres/base.ts @@ -24,6 +24,7 @@ import { type DelayJobStepOptions, type DeleteJobOptions, type DeleteJobsOptions, + type DeleteMetricsOptions, type FailJobOptions, type FailJobStepOptions, type FetchOptions, @@ -32,11 +33,15 @@ import { type GetJobStepsResult, type GetJobsOptions, type GetJobsResult, + type GetMetricsOptions, + type GetMetricsResult, + type InsertMetricOptions, type Job, type JobSort, type JobStatusResult, type JobStep, type JobStepStatusResult, + type MetricSort, type RecoverJobsOptions, type RetryJobOptions, type TimeTravelJobOptions, @@ -1349,6 +1354,128 @@ export class PostgresBaseAdapter e } } + // ============================================================================ + // Metrics Methods + // ============================================================================ + + /** + * Internal method to insert a metric record. + */ + protected async _insertMetric(options: InsertMetricOptions): Promise { + const [result] = await this.db + .insert(this.tables.metricsTable) + .values({ + job_id: options.jobId, + step_id: options.stepId ?? null, + name: options.name, + value: options.value, + attributes: options.attributes ?? {}, + type: options.type, + }) + .returning({ id: this.tables.metricsTable.id }) + + return result!.id + } + + /** + * Internal method to get metrics for a job or step. + */ + protected async _getMetrics(options: GetMetricsOptions): Promise { + const metricsTable = this.tables.metricsTable + const page = options.page ?? 1 + const pageSize = options.pageSize ?? 100 + const filters = options.filters ?? {} + + // Build WHERE clause + const where = this._buildMetricsWhereClause(options.jobId, options.stepId, filters) + + // Build sort + const sortInput = options.sort ?? { field: 'timestamp', order: 'desc' } + const sortFieldMap: Record = { + name: metricsTable.name, + value: metricsTable.value, + timestamp: metricsTable.timestamp, + createdAt: metricsTable.created_at, + } + + // Get total count + const total = await this.db.$count(metricsTable, where) + if (!total) { + return { + metrics: [], + total: 0, + page, + pageSize, + } + } + + const sortField = sortFieldMap[sortInput.field] + const orderByClause = sortInput.order === 'asc' ? asc(sortField) : desc(sortField) + + const metrics = await this.db + .select({ + id: metricsTable.id, + jobId: metricsTable.job_id, + stepId: metricsTable.step_id, + name: metricsTable.name, + value: metricsTable.value, + attributes: metricsTable.attributes, + type: metricsTable.type, + timestamp: metricsTable.timestamp, + createdAt: metricsTable.created_at, + }) + .from(metricsTable) + .where(where) + .orderBy(orderByClause) + .limit(pageSize) + .offset((page - 1) * pageSize) + + return { + metrics, + total, + page, + pageSize, + } + } + + /** + * Internal method to delete all metrics for a job. + */ + protected async _deleteMetrics(options: DeleteMetricsOptions): Promise { + const result = await this.db + .delete(this.tables.metricsTable) + .where(eq(this.tables.metricsTable.job_id, options.jobId)) + .returning({ id: this.tables.metricsTable.id }) + + return result.length + } + + /** + * Build WHERE clause for metrics queries. + */ + protected _buildMetricsWhereClause(jobId?: string, stepId?: string, filters?: GetMetricsOptions['filters']) { + const metricsTable = this.tables.metricsTable + + return and( + jobId ? eq(metricsTable.job_id, jobId) : undefined, + stepId ? eq(metricsTable.step_id, stepId) : undefined, + filters?.name + ? Array.isArray(filters.name) + ? or(...filters.name.map((n) => ilike(metricsTable.name, `%${n}%`))) + : ilike(metricsTable.name, `%${filters.name}%`) + : undefined, + filters?.type + ? inArray(metricsTable.type, Array.isArray(filters.type) ? filters.type : [filters.type]) + : undefined, + filters?.timestampRange && filters.timestampRange.length === 2 + ? between(metricsTable.timestamp, filters.timestampRange[0]!, filters.timestampRange[1]!) + : undefined, + ...(filters?.attributesFilter && Object.keys(filters.attributesFilter).length > 0 + ? this.#buildJsonbWhereConditions(filters.attributesFilter, metricsTable.attributes) + : []), + ) + } + // ============================================================================ // Private Methods // ============================================================================ diff --git a/packages/duron/src/adapters/postgres/schema.default.ts b/packages/duron/src/adapters/postgres/schema.default.ts index 320a464..300a14b 100644 --- a/packages/duron/src/adapters/postgres/schema.default.ts +++ b/packages/duron/src/adapters/postgres/schema.default.ts @@ -1,5 +1,5 @@ import createSchema from './schema.js' -const { schema, jobsTable, jobStepsTable } = createSchema('duron') +const { schema, jobsTable, jobStepsTable, metricsTable } = createSchema('duron') -export { schema, jobsTable, jobStepsTable } +export { schema, jobsTable, jobStepsTable, metricsTable } diff --git a/packages/duron/src/adapters/postgres/schema.ts b/packages/duron/src/adapters/postgres/schema.ts index fe0f060..57eb454 100644 --- a/packages/duron/src/adapters/postgres/schema.ts +++ b/packages/duron/src/adapters/postgres/schema.ts @@ -1,5 +1,17 @@ import { sql } from 'drizzle-orm' -import { boolean, check, index, integer, jsonb, pgSchema, text, timestamp, unique, uuid } from 'drizzle-orm/pg-core' +import { + boolean, + check, + doublePrecision, + index, + integer, + jsonb, + pgSchema, + text, + timestamp, + unique, + uuid, +} from 'drizzle-orm/pg-core' import { JOB_STATUSES, type JobStatus, STEP_STATUS_ACTIVE, STEP_STATUSES, type StepStatus } from '../../constants.js' import type { SerializableError } from '../../errors.js' @@ -114,9 +126,42 @@ export default function createSchema(schemaName: string) { ], ) + const metricsTable = schema.table( + 'metrics', + { + id: uuid('id').primaryKey().defaultRandom(), + job_id: uuid('job_id') + .notNull() + .references(() => jobsTable.id, { onDelete: 'cascade' }), + step_id: uuid('step_id').references(() => jobStepsTable.id, { onDelete: 'cascade' }), + name: text('name').notNull(), + value: doublePrecision('value').notNull(), + attributes: jsonb('attributes').$type>().notNull().default({}), + type: text('type').$type<'metric' | 'span_event' | 'span_attribute'>().notNull(), + timestamp: timestamp('timestamp', { withTimezone: true }).notNull().defaultNow(), + created_at: timestamp('created_at', { withTimezone: true }).notNull().defaultNow(), + }, + (table) => [ + // Single column indexes + index('idx_metrics_job_id').on(table.job_id), + index('idx_metrics_step_id').on(table.step_id), + index('idx_metrics_name').on(table.name), + index('idx_metrics_type').on(table.type), + index('idx_metrics_timestamp').on(table.timestamp), + // Composite indexes + index('idx_metrics_job_step').on(table.job_id, table.step_id), + index('idx_metrics_job_name').on(table.job_id, table.name), + index('idx_metrics_job_type').on(table.job_id, table.type), + // GIN index for JSONB attributes filtering + index('idx_metrics_attributes').using('gin', table.attributes), + check('metrics_type_check', sql`${table.type} IN ('metric', 'span_event', 'span_attribute')`), + ], + ) + return { schema, jobsTable, jobStepsTable, + metricsTable, } } diff --git a/packages/duron/src/adapters/schemas.ts b/packages/duron/src/adapters/schemas.ts index 717043b..60bfd2b 100644 --- a/packages/duron/src/adapters/schemas.ts +++ b/packages/duron/src/adapters/schemas.ts @@ -294,6 +294,67 @@ export const JobStepStatusResultSchema = z.object({ updatedAt: DateSchema, }) +// ============================================================================ +// Metrics Schemas +// ============================================================================ + +export const MetricTypeSchema = z.enum(['metric', 'span_event', 'span_attribute']) + +export const MetricSchema = z.object({ + id: z.string(), + jobId: z.string(), + stepId: z.string().nullable(), + name: z.string(), + value: z.number(), + attributes: z.record(z.string(), z.any()), + type: MetricTypeSchema, + timestamp: DateSchema, + createdAt: DateSchema, +}) + +export const MetricSortFieldSchema = z.enum(['name', 'value', 'timestamp', 'createdAt']) + +export const MetricSortSchema = z.object({ + field: MetricSortFieldSchema, + order: SortOrderSchema, +}) + +export const MetricFiltersSchema = z.object({ + name: z.union([z.string(), z.array(z.string())]).optional(), + type: z.union([MetricTypeSchema, z.array(MetricTypeSchema)]).optional(), + attributesFilter: z.record(z.string(), z.any()).optional(), + timestampRange: z.array(DateSchema).length(2).optional(), +}) + +export const InsertMetricOptionsSchema = z.object({ + jobId: z.string(), + stepId: z.string().optional(), + name: z.string(), + value: z.number(), + attributes: z.record(z.string(), z.any()).optional(), + type: MetricTypeSchema, +}) + +export const GetMetricsOptionsSchema = z.object({ + jobId: z.string().optional(), + stepId: z.string().optional(), + filters: MetricFiltersSchema.optional(), + sort: MetricSortSchema.optional(), + page: z.number().int().positive().optional(), + pageSize: z.number().int().positive().optional(), +}) + +export const GetMetricsResultSchema = z.object({ + metrics: z.array(MetricSchema), + total: z.number().int().nonnegative(), + page: z.number().int().positive().optional(), + pageSize: z.number().int().positive().optional(), +}) + +export const DeleteMetricsOptionsSchema = z.object({ + jobId: z.string(), +}) + // ============================================================================ // Type Exports // ============================================================================ @@ -329,3 +390,12 @@ export type DelayJobStepOptions = z.infer export type CancelJobStepOptions = z.infer export type CreateOrRecoverJobStepResult = z.infer export type TimeTravelJobOptions = z.infer +export type MetricType = z.infer +export type Metric = z.infer +export type MetricSortField = z.infer +export type MetricSort = z.infer +export type MetricFilters = z.infer +export type InsertMetricOptions = z.infer +export type GetMetricsOptions = z.infer +export type GetMetricsResult = z.infer +export type DeleteMetricsOptions = z.infer diff --git a/packages/duron/src/client.ts b/packages/duron/src/client.ts index 84e092b..b80922b 100644 --- a/packages/duron/src/client.ts +++ b/packages/duron/src/client.ts @@ -11,11 +11,14 @@ import type { GetJobStepsResult, GetJobsOptions, GetJobsResult, + GetMetricsOptions, + GetMetricsResult, Job, JobStep, } from './adapters/adapter.js' import type { JobStatusResult, JobStepStatusResult } from './adapters/schemas.js' import { JOB_STATUS_CANCELLED, JOB_STATUS_COMPLETED, JOB_STATUS_FAILED, type JobStatus } from './constants.js' +import { LocalTelemetryAdapter, noopTelemetryAdapter, type TelemetryAdapter } from './telemetry/index.js' const BaseOptionsSchema = z.object({ /** @@ -136,6 +139,17 @@ export interface ClientOptions< * These can be accessed in action handlers using `ctx.var`. */ variables?: TVariables + + /** + * Optional telemetry adapter for observability. + * When provided, traces job and step execution with spans and allows recording custom metrics. + * + * Available adapters: + * - `openTelemetryAdapter()` - Export traces to external systems (Jaeger, OTLP, etc.) + * - `localTelemetryAdapter({ database })` - Store metrics in the Duron database + * - `noopTelemetryAdapter()` - No-op adapter (default) + */ + telemetry?: TelemetryAdapter } interface FetchOptions { @@ -157,6 +171,7 @@ export class Client< #id: string #actions: TActions | null #database: Adapter + #telemetry: TelemetryAdapter #variables: Record #logger: Logger #started: boolean = false @@ -190,11 +205,14 @@ export class Client< this.#options = BaseOptionsSchema.parse(options) this.#id = options.id ?? globalThis.crypto.randomUUID() this.#database = options.database + this.#telemetry = options.telemetry ?? noopTelemetryAdapter() this.#actions = options.actions ?? null this.#variables = options?.variables ?? {} this.#logger = this.#normalizeLogger(options?.logger) this.#database.setId(this.#id) this.#database.setLogger(this.#logger) + this.#telemetry.setLogger(this.#logger) + this.#telemetry.setClient(this) } #normalizeLogger(logger?: Logger | 'fatal' | 'error' | 'warn' | 'info' | 'debug' | 'trace' | 'silent'): Logger { @@ -217,6 +235,28 @@ export class Client< return this.#logger } + /** + * Get the telemetry adapter instance. + */ + get telemetry(): TelemetryAdapter { + return this.#telemetry + } + + /** + * Get the database adapter instance. + */ + get database(): Adapter { + return this.#database + } + + /** + * Check if local telemetry is enabled. + * Returns true if using LocalTelemetryAdapter. + */ + get metricsEnabled(): boolean { + return this.#telemetry instanceof LocalTelemetryAdapter + } + /** * Get the current configuration of this Duron instance. * @@ -563,6 +603,22 @@ export class Client< return this.#database.getActions() } + /** + * Get metrics for a job or step. + * Only available when using LocalTelemetryAdapter. + * + * @param options - Query options including jobId/stepId, filters, sort, and pagination + * @returns Promise resolving to metrics result with pagination info + * @throws Error if not using LocalTelemetryAdapter + */ + async getMetrics(options: GetMetricsOptions): Promise { + await this.start() + if (!this.metricsEnabled) { + throw new Error('Metrics are only available when using LocalTelemetryAdapter') + } + return this.#database.getMetrics(options) + } + /** * Get action metadata including input schemas and mock data. * This is useful for generating UI forms or mock data. @@ -625,6 +681,9 @@ export class Client< return false } + // Start telemetry adapter + await this.#telemetry.start() + if (this.#actions) { if (this.#options.recoverJobsOnStart) { await this.#database.recoverJobs({ @@ -695,6 +754,9 @@ export class Client< }), ) + // Stop telemetry adapter + await this.#telemetry.stop() + const dbStopped = await this.#database.stop() if (!dbStopped) { return false @@ -810,6 +872,7 @@ export class Client< actionManager = new ActionManager({ action, database: this.#database, + telemetry: this.#telemetry, variables: this.#variables, logger: this.#logger, concurrencyLimit: this.#options.actionConcurrencyLimit, diff --git a/packages/duron/src/index.ts b/packages/duron/src/index.ts index 8263b23..5d4e585 100644 --- a/packages/duron/src/index.ts +++ b/packages/duron/src/index.ts @@ -5,6 +5,7 @@ export { defineAction } from './action.js' export * from './constants.js' export { NonRetriableError, UnhandledChildStepsError } from './errors.js' export * from './server.js' +export * from './telemetry/index.js' export const duron = < TActions extends Record>, diff --git a/packages/duron/src/server.ts b/packages/duron/src/server.ts index 1d3ecfb..d914cd3 100644 --- a/packages/duron/src/server.ts +++ b/packages/duron/src/server.ts @@ -2,17 +2,20 @@ import { Elysia } from 'elysia' import { jwtVerify, SignJWT } from 'jose' import { z } from 'zod' -import type { GetJobStepsOptions, GetJobsOptions } from './adapters/adapter.js' +import type { GetJobStepsOptions, GetJobsOptions, GetMetricsOptions } from './adapters/adapter.js' import { GetActionsResultSchema, GetJobStepsResultSchema, GetJobsResultSchema, + GetMetricsResultSchema, JobSchema, JobSortFieldSchema, JobStatusResultSchema, JobStatusSchema, JobStepSchema, JobStepStatusResultSchema, + MetricSortFieldSchema, + MetricTypeSchema, SortOrderSchema, } from './adapters/schemas.js' import type { Client } from './client.js' @@ -173,6 +176,56 @@ export const GetActionsMetadataResponseSchema = z.array( export type GetJobsQueryInput = z.input export type GetJobStepsQueryInput = z.input +// Metrics query schema +export const GetMetricsQuerySchema = z + .object({ + // Pagination + page: z.coerce.number().int().min(1).optional(), + pageSize: z.coerce.number().int().min(1).max(1000).optional(), + + // Filters + fName: z.union([z.string(), z.array(z.string())]).optional(), + fType: z.union([MetricTypeSchema, z.array(MetricTypeSchema)]).optional(), + fTimestampRange: z.array(z.coerce.date()).length(2).optional(), + fAttributesFilter: z.record(z.string(), z.any()).optional(), + + // Sort - format: "field:asc" or "field:desc" + sort: z.string().optional(), + }) + .transform((data) => { + const filters: any = {} + + if (data.fName) filters.name = data.fName + if (data.fType) filters.type = data.fType + if (data.fTimestampRange) filters.timestampRange = data.fTimestampRange + if (data.fAttributesFilter) filters.attributesFilter = data.fAttributesFilter + + // Parse sort string: "field:asc" -> { field: 'field', order: 'asc' } + let sort: { field: z.infer; order: z.infer } | undefined + if (data.sort) { + const [field, order] = data.sort.split(':').map((s) => s.trim()) + if (field && order) { + const fieldResult = MetricSortFieldSchema.safeParse(field) + const orderResult = SortOrderSchema.safeParse(order.toLowerCase()) + if (fieldResult.success && orderResult.success) { + sort = { + field: fieldResult.data, + order: orderResult.data, + } + } + } + } + + return { + page: data.page, + pageSize: data.pageSize, + filters: Object.keys(filters).length > 0 ? filters : undefined, + sort, + } + }) + +export type GetMetricsQueryInput = z.input + export const ErrorResponseSchema = z.object({ error: z.string(), message: z.string().optional(), @@ -221,6 +274,14 @@ export interface CreateServerOptions

{ */ prefix?: P + /** + * Enable metrics endpoints (/jobs/:id/metrics, /steps/:id/metrics). + * Only works when client is configured with LocalTelemetryAdapter. + * When true, enables the dashboard to show metrics buttons. + * @default auto-detected from client.metricsEnabled + */ + metricsEnabled?: boolean + login?: { onLogin: (body: { email: string; password: string }) => Promise jwtSecret: string | Uint8Array @@ -242,12 +303,15 @@ export interface CreateServerOptions

{ * @param options - Configuration options * @returns Elysia server instance */ -export function createServer

({ client, prefix, login }: CreateServerOptions

) { +export function createServer

({ client, prefix, login, metricsEnabled }: CreateServerOptions

) { // Convert string secret to Uint8Array if needed const secretKey = typeof login?.jwtSecret === 'string' ? new TextEncoder().encode(login?.jwtSecret) : login?.jwtSecret const routePrefix = (prefix ?? '/api') as P + // Auto-detect metricsEnabled from client if not explicitly set + const isMetricsEnabled = metricsEnabled ?? client.metricsEnabled + return new Elysia({ prefix: routePrefix, }) @@ -608,6 +672,78 @@ export function createServer

({ client, prefix, login }: Create auth: true, }, ) + .get( + '/config', + async () => { + return { + metricsEnabled: isMetricsEnabled, + authEnabled: !!login, + } + }, + { + response: { + 200: z.object({ + metricsEnabled: z.boolean(), + authEnabled: z.boolean(), + }), + 500: ErrorResponseSchema, + }, + }, + ) + .get( + '/jobs/:id/metrics', + async ({ params, query }) => { + if (!isMetricsEnabled) { + throw new Error('Metrics are not enabled. Use LocalTelemetryAdapter to enable metrics.') + } + const options: GetMetricsOptions = { + jobId: params.id, + page: query.page, + pageSize: query.pageSize, + filters: query.filters, + sort: query.sort, + } + return client.getMetrics(options) + }, + { + params: JobIdParamsSchema, + query: GetMetricsQuerySchema, + response: { + 200: GetMetricsResultSchema, + 400: ErrorResponseSchema, + 500: ErrorResponseSchema, + 401: ErrorResponseSchema, + }, + auth: true, + }, + ) + .get( + '/steps/:id/metrics', + async ({ params, query }) => { + if (!isMetricsEnabled) { + throw new Error('Metrics are not enabled. Use LocalTelemetryAdapter to enable metrics.') + } + const options: GetMetricsOptions = { + stepId: params.id, + page: query.page, + pageSize: query.pageSize, + filters: query.filters, + sort: query.sort, + } + return client.getMetrics(options) + }, + { + params: StepIdParamsSchema, + query: GetMetricsQuerySchema, + response: { + 200: GetMetricsResultSchema, + 400: ErrorResponseSchema, + 500: ErrorResponseSchema, + 401: ErrorResponseSchema, + }, + auth: true, + }, + ) .post( '/actions/:actionName/run', async ({ params, body }) => { diff --git a/packages/duron/src/step-manager.ts b/packages/duron/src/step-manager.ts index 4b86c19..bdfd477 100644 --- a/packages/duron/src/step-manager.ts +++ b/packages/duron/src/step-manager.ts @@ -21,6 +21,7 @@ import { serializeError, UnhandledChildStepsError, } from './errors.js' +import type { ObserveContext, Span, TelemetryAdapter } from './telemetry/adapter.js' import pRetry from './utils/p-retry.js' import waitForAbort from './utils/wait-for-abort.js' @@ -129,6 +130,7 @@ export interface StepManagerOptions { jobId: string actionName: string adapter: Adapter + telemetry: TelemetryAdapter logger: Logger concurrencyLimit: number } @@ -141,10 +143,15 @@ export class StepManager { #jobId: string #actionName: string #stepStore: StepStore + #telemetry: TelemetryAdapter #queue: fastq.queueAsPromised #logger: Logger // each step name should be executed only once per action job #historySteps = new Set() + // Store step spans for nested step tracking + #stepSpans = new Map() + // Store the job span for creating step spans + #jobSpan: Span | null = null // ============================================================================ // Constructor @@ -159,6 +166,7 @@ export class StepManager { this.#jobId = options.jobId this.#actionName = options.actionName this.#logger = options.logger + this.#telemetry = options.telemetry this.#stepStore = new StepStore(options.adapter) this.#queue = fastq.promise(async (task: TaskStep) => { if (this.#historySteps.has(task.name)) { @@ -169,6 +177,14 @@ export class StepManager { }, options.concurrencyLimit) } + /** + * Set the job span for this step manager. + * Called from ActionJob after the job span is created. + */ + setJobSpan(span: Span): void { + this.#jobSpan = span + } + // ============================================================================ // Public API Methods // ============================================================================ @@ -182,6 +198,7 @@ export class StepManager { * @param variables - Variables available to the action * @param abortSignal - Abort signal for cancelling the action * @param logger - Pino child logger for this job + * @param observeContext - Observability context for telemetry * @returns ActionHandlerContext instance */ createActionContext>( @@ -190,8 +207,35 @@ export class StepManager { variables: TVariables, abortSignal: AbortSignal, logger: Logger, + observeContext: ObserveContext, ): ActionHandlerContext { - return new ActionContext(this, job, action, variables, abortSignal, logger) + return new ActionContext(this, job, action, variables, abortSignal, logger, observeContext) + } + + /** + * Create an observe context for a step. + */ + createStepObserveContext(stepId: string): ObserveContext { + const stepSpan = this.#stepSpans.get(stepId) + if (stepSpan) { + return this.#telemetry.createObserveContext(this.#jobId, stepId, stepSpan) + } + // Fallback to job span if step span not found + if (this.#jobSpan) { + return this.#telemetry.createObserveContext(this.#jobId, stepId, this.#jobSpan) + } + // No-op observe context + return { + recordMetric: () => { + // No-op + }, + addSpanAttribute: () => { + // No-op + }, + addSpanEvent: () => { + // No-op + }, + } } /** @@ -263,6 +307,17 @@ export class StepManager { step = newStep + // Start step telemetry span + const parentSpan = parentStepId ? this.#stepSpans.get(parentStepId) : this.#jobSpan + const stepSpan = await this.#telemetry.startStepSpan({ + jobId: this.#jobId, + stepId: step.id, + stepName: name, + parentSpan: parentSpan ?? undefined, + parentStepId, + }) + this.#stepSpans.set(step.id, stepSpan) + if (abortSignal.aborted) { throw new ActionCancelError(this.#actionName, this.#jobId, { cause: 'step cancelled after create step' }) } @@ -318,11 +373,15 @@ export class StepManager { const childAbortController = new AbortController() const childSignal = AbortSignal.any([stepSignal, childAbortController.signal]) + // Create observe context for this step + const stepObserveContext = this.createStepObserveContext(step.id) + // Create StepHandlerContext with nested step support const stepContext: StepHandlerContext = { signal: stepSignal, stepId: step.id, parentStepId, + observe: stepObserveContext, step: ( childName: string, childCb: (ctx: StepHandlerContext) => Promise, @@ -431,6 +490,13 @@ export class StepManager { throw new Error(`Failed to complete step "${name}" for job "${this.#jobId}" action "${this.#actionName}"`) } + // End step telemetry span successfully + const stepSpan = this.#stepSpans.get(step.id) + if (stepSpan) { + await this.#telemetry.endStepSpan(stepSpan, { status: 'ok' }) + this.#stepSpans.delete(step.id) + } + // Log step completion this.#logger.debug( { jobId: this.#jobId, actionName: this.#actionName, stepName: name, stepId: step.id }, @@ -471,6 +537,17 @@ export class StepManager { }, }).catch(async (error) => { if (step) { + // End step telemetry span with error/cancelled status + const stepSpan = this.#stepSpans.get(step.id) + if (stepSpan) { + if (isCancelError(error)) { + await this.#telemetry.endStepSpan(stepSpan, { status: 'cancelled' }) + } else { + await this.#telemetry.endStepSpan(stepSpan, { status: 'error', error }) + } + this.#stepSpans.delete(step.id) + } + if (isCancelError(error)) { await this.#stepStore.updateStatus(step.id, 'cancelled') } else { @@ -501,6 +578,7 @@ class ActionContext + #observeContext: ObserveContext // ============================================================================ // Constructor @@ -513,6 +591,7 @@ class ActionContext 'Error parsing action input', @@ -574,6 +654,13 @@ class ActionContext +} + +/** + * Options for adding a span event. + */ +export interface AddSpanEventOptions { + span: Span + name: string + attributes?: Record +} + +/** + * Options for adding a span attribute. + */ +export interface AddSpanAttributeOptions { + span: Span + key: string + value: string | number | boolean +} + +/** + * Observe context provided to action and step handlers. + */ +export interface ObserveContext { + /** + * Record a custom metric. + * + * @param name - The metric name (e.g., 'ai.tokens.total', 'processing.duration_ms') + * @param value - The metric value + * @param attributes - Optional attributes for the metric + */ + recordMetric(name: string, value: number, attributes?: Record): void + + /** + * Add an attribute to the current span. + * + * @param key - The attribute key + * @param value - The attribute value + */ + addSpanAttribute(key: string, value: string | number | boolean): void + + /** + * Add an event to the current span. + * + * @param name - The event name + * @param attributes - Optional event attributes + */ + addSpanEvent(name: string, attributes?: Record): void +} + +// ============================================================================ +// Abstract Telemetry Adapter +// ============================================================================ + +/** + * Abstract base class for telemetry adapters. + * All telemetry adapters must extend this class and implement its abstract methods. + */ +export abstract class TelemetryAdapter { + #logger: Logger | null = null + #client: TelemetryClient | null = null + #started: boolean = false + #stopped: boolean = false + #starting: Promise | null = null + #stopping: Promise | null = null + + // ============================================================================ + // Lifecycle Methods + // ============================================================================ + + /** + * Start the telemetry adapter. + * Performs any necessary initialization. + * + * @returns Promise resolving to `true` if started successfully, `false` otherwise + */ + async start(): Promise { + try { + if (this.#stopping || this.#stopped) { + return false + } + + if (this.#started) { + return true + } + + if (this.#starting) { + return this.#starting + } + + this.#starting = (async () => { + await this._start() + this.#started = true + this.#starting = null + return true + })() + + return this.#starting + } catch (error) { + this.#logger?.error(error, 'Error in TelemetryAdapter.start()') + throw error + } + } + + /** + * Stop the telemetry adapter. + * Performs cleanup. + * + * @returns Promise resolving to `true` if stopped successfully, `false` otherwise + */ + async stop(): Promise { + try { + if (this.#stopped) { + return true + } + + if (this.#stopping) { + return this.#stopping + } + + this.#stopping = (async () => { + await this._stop() + this.#stopped = true + this.#stopping = null + return true + })() + + return this.#stopping + } catch (error) { + this.#logger?.error(error, 'Error in TelemetryAdapter.stop()') + throw error + } + } + + // ============================================================================ + // Configuration Methods + // ============================================================================ + + /** + * Set the logger instance for this adapter. + * + * @param logger - The logger instance to use for logging + */ + setLogger(logger: Logger): void { + this.#logger = logger + } + + /** + * Get the logger instance for this adapter. + * + * @returns The logger instance, or `null` if not set + */ + get logger(): Logger | null { + return this.#logger + } + + /** + * Set the Duron client instance for this adapter. + * This is called automatically by the Duron client during initialization. + * + * @param client - The Duron client instance + */ + setClient(client: TelemetryClient): void { + this.#client = client + } + + /** + * Get the Duron client instance. + * Available to subclasses for accessing the database adapter. + * + * @returns The Duron client instance, or `null` if not set + */ + protected get client(): TelemetryClient | null { + return this.#client + } + + // ============================================================================ + // Span Methods + // ============================================================================ + + /** + * Start a span for job execution. + * + * @param options - Options for the job span + * @returns The created span + */ + async startJobSpan(options: StartJobSpanOptions): Promise { + await this.start() + return this._startJobSpan(options) + } + + /** + * End a job span. + * + * @param span - The span to end + * @param options - End options including status and error + */ + async endJobSpan(span: Span, options: EndSpanOptions): Promise { + await this.start() + return this._endJobSpan(span, options) + } + + /** + * Start a span for step execution. + * + * @param options - Options for the step span + * @returns The created span + */ + async startStepSpan(options: StartStepSpanOptions): Promise { + await this.start() + return this._startStepSpan(options) + } + + /** + * End a step span. + * + * @param span - The span to end + * @param options - End options including status and error + */ + async endStepSpan(span: Span, options: EndSpanOptions): Promise { + await this.start() + return this._endStepSpan(span, options) + } + + /** + * Start a span for database operation (optional tracing). + * + * @param options - Options for the database span + * @returns The created span, or null if database tracing is disabled + */ + async startDatabaseSpan(options: StartDatabaseSpanOptions): Promise { + await this.start() + return this._startDatabaseSpan(options) + } + + /** + * End a database span. + * + * @param span - The span to end + * @param options - End options including status and error + */ + async endDatabaseSpan(span: Span | null, options: EndSpanOptions): Promise { + if (!span) return + await this.start() + return this._endDatabaseSpan(span, options) + } + + // ============================================================================ + // Metrics Methods + // ============================================================================ + + /** + * Record a metric. + * + * @param options - Options for recording the metric + */ + async recordMetric(options: RecordMetricOptions): Promise { + await this.start() + return this._recordMetric(options) + } + + /** + * Add an event to a span. + * + * @param options - Options for the span event + */ + async addSpanEvent(options: AddSpanEventOptions): Promise { + await this.start() + return this._addSpanEvent(options) + } + + /** + * Add an attribute to a span. + * + * @param options - Options for the span attribute + */ + async addSpanAttribute(options: AddSpanAttributeOptions): Promise { + await this.start() + return this._addSpanAttribute(options) + } + + // ============================================================================ + // Context Methods + // ============================================================================ + + /** + * Create an observe context for action/step handlers. + * + * @param jobId - The job ID + * @param stepId - The step ID (optional) + * @param span - The current span + * @returns ObserveContext for use in handlers + */ + createObserveContext(jobId: string, stepId: string | null, span: Span): ObserveContext { + return { + recordMetric: (name: string, value: number, attributes?: Record) => { + this.recordMetric({ + jobId, + stepId: stepId ?? undefined, + name, + value, + attributes, + }).catch((err) => { + this.#logger?.error(err, 'Error recording metric') + }) + }, + addSpanAttribute: (key: string, value: string | number | boolean) => { + this.addSpanAttribute({ span, key, value }).catch((err) => { + this.#logger?.error(err, 'Error adding span attribute') + }) + }, + addSpanEvent: (name: string, attributes?: Record) => { + this.addSpanEvent({ span, name, attributes }).catch((err) => { + this.#logger?.error(err, 'Error adding span event') + }) + }, + } + } + + // ============================================================================ + // Protected Abstract Methods (to be implemented by adapters) + // ============================================================================ + + /** + * Start the adapter. + */ + protected abstract _start(): Promise + + /** + * Stop the adapter. + */ + protected abstract _stop(): Promise + + /** + * Internal method to start a job span. + */ + protected abstract _startJobSpan(options: StartJobSpanOptions): Promise + + /** + * Internal method to end a job span. + */ + protected abstract _endJobSpan(span: Span, options: EndSpanOptions): Promise + + /** + * Internal method to start a step span. + */ + protected abstract _startStepSpan(options: StartStepSpanOptions): Promise + + /** + * Internal method to end a step span. + */ + protected abstract _endStepSpan(span: Span, options: EndSpanOptions): Promise + + /** + * Internal method to start a database span. + */ + protected abstract _startDatabaseSpan(options: StartDatabaseSpanOptions): Promise + + /** + * Internal method to end a database span. + */ + protected abstract _endDatabaseSpan(span: Span, options: EndSpanOptions): Promise + + /** + * Internal method to record a metric. + */ + protected abstract _recordMetric(options: RecordMetricOptions): Promise + + /** + * Internal method to add a span event. + */ + protected abstract _addSpanEvent(options: AddSpanEventOptions): Promise + + /** + * Internal method to add a span attribute. + */ + protected abstract _addSpanAttribute(options: AddSpanAttributeOptions): Promise +} diff --git a/packages/duron/src/telemetry/index.ts b/packages/duron/src/telemetry/index.ts new file mode 100644 index 0000000..066f593 --- /dev/null +++ b/packages/duron/src/telemetry/index.ts @@ -0,0 +1,17 @@ +// Re-export telemetry adapters and types + +export { + type AddSpanAttributeOptions, + type AddSpanEventOptions, + type EndSpanOptions, + type ObserveContext, + type RecordMetricOptions, + type Span, + type StartDatabaseSpanOptions, + type StartJobSpanOptions, + type StartStepSpanOptions, + TelemetryAdapter, +} from './adapter.js' +export { LocalTelemetryAdapter, type LocalTelemetryAdapterOptions, localTelemetryAdapter } from './local.js' +export { NoopTelemetryAdapter, noopTelemetryAdapter } from './noop.js' +export { OpenTelemetryAdapter, type OpenTelemetryAdapterOptions, openTelemetryAdapter } from './opentelemetry.js' diff --git a/packages/duron/src/telemetry/local.ts b/packages/duron/src/telemetry/local.ts new file mode 100644 index 0000000..edb76f5 --- /dev/null +++ b/packages/duron/src/telemetry/local.ts @@ -0,0 +1,238 @@ +import type { Adapter } from '../adapters/adapter.js' +import { + type AddSpanAttributeOptions, + type AddSpanEventOptions, + type EndSpanOptions, + type RecordMetricOptions, + type Span, + type StartDatabaseSpanOptions, + type StartJobSpanOptions, + type StartStepSpanOptions, + TelemetryAdapter, +} from './adapter.js' + +// ============================================================================ +// Types +// ============================================================================ + +// Note: This interface is intentionally empty as the database is obtained from the Duron client +export type LocalTelemetryAdapterOptions = Record + +// ============================================================================ +// Local Telemetry Adapter +// ============================================================================ + +/** + * Local telemetry adapter that stores metrics directly in the Duron database. + * Perfect for development and self-hosted deployments. + * + * This adapter automatically uses the database adapter configured in the Duron client. + * No additional configuration is required. + * + * @example + * ```typescript + * const client = duron({ + * database: postgresAdapter({ connection: 'postgres://...' }), + * telemetry: localTelemetryAdapter(), + * actions: { ... } + * }) + * ``` + */ +export class LocalTelemetryAdapter extends TelemetryAdapter { + #spanStartTimes = new Map() + + /** + * Get the database adapter from the Duron client. + * @throws Error if the client is not set + */ + get #database(): Adapter { + const client = this.client + if (!client) { + throw new Error( + 'LocalTelemetryAdapter requires the Duron client to be set. This is done automatically by the Duron client.', + ) + } + return client.database + } + + // ============================================================================ + // Lifecycle Methods + // ============================================================================ + + protected async _start(): Promise { + // Database adapter should already be started by the client + } + + protected async _stop(): Promise { + this.#spanStartTimes.clear() + } + + // ============================================================================ + // Span Methods + // ============================================================================ + + protected async _startJobSpan(options: StartJobSpanOptions): Promise { + const spanId = `job:${options.jobId}` + this.#spanStartTimes.set(spanId, Date.now()) + + // Record span start as a metric + await this.#database.insertMetric({ + jobId: options.jobId, + name: 'duron.job.span.start', + value: Date.now(), + type: 'span_event', + attributes: { + actionName: options.actionName, + groupKey: options.groupKey, + spanId, + }, + }) + + return { + id: spanId, + jobId: options.jobId, + stepId: null, + parentSpanId: null, + } + } + + protected async _endJobSpan(span: Span, options: EndSpanOptions): Promise { + const startTime = this.#spanStartTimes.get(span.id) + const duration = startTime ? Date.now() - startTime : 0 + this.#spanStartTimes.delete(span.id) + + // Record span end with duration + await this.#database.insertMetric({ + jobId: span.jobId, + name: 'duron.job.span.end', + value: duration, + type: 'span_event', + attributes: { + spanId: span.id, + status: options.status, + error: options.error?.message ?? null, + durationMs: duration, + }, + }) + } + + protected async _startStepSpan(options: StartStepSpanOptions): Promise { + const spanId = `step:${options.stepId}` + this.#spanStartTimes.set(spanId, Date.now()) + + // Record span start as a metric + await this.#database.insertMetric({ + jobId: options.jobId, + stepId: options.stepId, + name: 'duron.step.span.start', + value: Date.now(), + type: 'span_event', + attributes: { + stepName: options.stepName, + parentStepId: options.parentStepId, + parentSpanId: options.parentSpan?.id ?? null, + spanId, + }, + }) + + return { + id: spanId, + jobId: options.jobId, + stepId: options.stepId, + parentSpanId: options.parentSpan?.id ?? null, + } + } + + protected async _endStepSpan(span: Span, options: EndSpanOptions): Promise { + const startTime = this.#spanStartTimes.get(span.id) + const duration = startTime ? Date.now() - startTime : 0 + this.#spanStartTimes.delete(span.id) + + // Record span end with duration + await this.#database.insertMetric({ + jobId: span.jobId, + stepId: span.stepId ?? undefined, + name: 'duron.step.span.end', + value: duration, + type: 'span_event', + attributes: { + spanId: span.id, + status: options.status, + error: options.error?.message ?? null, + durationMs: duration, + }, + }) + } + + protected async _startDatabaseSpan(_options: StartDatabaseSpanOptions): Promise { + // Local adapter doesn't trace database operations to avoid infinite loops + return null + } + + protected async _endDatabaseSpan(_span: Span, _options: EndSpanOptions): Promise { + // No-op for local adapter + } + + // ============================================================================ + // Metrics Methods + // ============================================================================ + + protected async _recordMetric(options: RecordMetricOptions): Promise { + await this.#database.insertMetric({ + jobId: options.jobId, + stepId: options.stepId, + name: options.name, + value: options.value, + type: 'metric', + attributes: options.attributes, + }) + } + + protected async _addSpanEvent(options: AddSpanEventOptions): Promise { + await this.#database.insertMetric({ + jobId: options.span.jobId, + stepId: options.span.stepId ?? undefined, + name: options.name, + value: Date.now(), + type: 'span_event', + attributes: { + spanId: options.span.id, + ...options.attributes, + }, + }) + } + + protected async _addSpanAttribute(options: AddSpanAttributeOptions): Promise { + await this.#database.insertMetric({ + jobId: options.span.jobId, + stepId: options.span.stepId ?? undefined, + name: `attribute:${options.key}`, + value: typeof options.value === 'number' ? options.value : 0, + type: 'span_attribute', + attributes: { + spanId: options.span.id, + key: options.key, + value: String(options.value), + }, + }) + } +} + +/** + * Create a local telemetry adapter that stores metrics in the Duron database. + * Perfect for development and self-hosted deployments. + * + * The database adapter is automatically obtained from the Duron client. + * + * @returns LocalTelemetryAdapter instance + * + * @example + * ```typescript + * const client = duron({ + * database: postgresAdapter({ connection: 'postgres://...' }), + * telemetry: localTelemetryAdapter(), + * actions: { ... } + * }) + * ``` + */ +export const localTelemetryAdapter = () => new LocalTelemetryAdapter() diff --git a/packages/duron/src/telemetry/noop.ts b/packages/duron/src/telemetry/noop.ts new file mode 100644 index 0000000..8c2f706 --- /dev/null +++ b/packages/duron/src/telemetry/noop.ts @@ -0,0 +1,95 @@ +import { + type AddSpanAttributeOptions, + type AddSpanEventOptions, + type EndSpanOptions, + type RecordMetricOptions, + type Span, + type StartDatabaseSpanOptions, + type StartJobSpanOptions, + type StartStepSpanOptions, + TelemetryAdapter, +} from './adapter.js' + +// ============================================================================ +// Noop Telemetry Adapter +// ============================================================================ + +/** + * No-operation telemetry adapter. + * Used when telemetry is disabled. All methods are no-ops. + */ +export class NoopTelemetryAdapter extends TelemetryAdapter { + // ============================================================================ + // Lifecycle Methods + // ============================================================================ + + protected async _start(): Promise { + // No-op + } + + protected async _stop(): Promise { + // No-op + } + + // ============================================================================ + // Span Methods + // ============================================================================ + + protected async _startJobSpan(options: StartJobSpanOptions): Promise { + return { + id: 'noop', + jobId: options.jobId, + stepId: null, + parentSpanId: null, + } + } + + protected async _endJobSpan(_span: Span, _options: EndSpanOptions): Promise { + // No-op + } + + protected async _startStepSpan(options: StartStepSpanOptions): Promise { + return { + id: 'noop', + jobId: options.jobId, + stepId: options.stepId, + parentSpanId: options.parentSpan?.id ?? null, + } + } + + protected async _endStepSpan(_span: Span, _options: EndSpanOptions): Promise { + // No-op + } + + protected async _startDatabaseSpan(_options: StartDatabaseSpanOptions): Promise { + return null + } + + protected async _endDatabaseSpan(_span: Span, _options: EndSpanOptions): Promise { + // No-op + } + + // ============================================================================ + // Metrics Methods + // ============================================================================ + + protected async _recordMetric(_options: RecordMetricOptions): Promise { + // No-op + } + + protected async _addSpanEvent(_options: AddSpanEventOptions): Promise { + // No-op + } + + protected async _addSpanAttribute(_options: AddSpanAttributeOptions): Promise { + // No-op + } +} + +/** + * Create a no-operation telemetry adapter. + * Use this when telemetry should be disabled. + * + * @returns NoopTelemetryAdapter instance + */ +export const noopTelemetryAdapter = () => new NoopTelemetryAdapter() diff --git a/packages/duron/src/telemetry/opentelemetry.ts b/packages/duron/src/telemetry/opentelemetry.ts new file mode 100644 index 0000000..03f417e --- /dev/null +++ b/packages/duron/src/telemetry/opentelemetry.ts @@ -0,0 +1,310 @@ +import type { Span as OTelSpan, Tracer, TracerProvider } from '@opentelemetry/api' + +import { + type AddSpanAttributeOptions, + type AddSpanEventOptions, + type EndSpanOptions, + type RecordMetricOptions, + type Span, + type StartDatabaseSpanOptions, + type StartJobSpanOptions, + type StartStepSpanOptions, + TelemetryAdapter, +} from './adapter.js' + +// ============================================================================ +// Types +// ============================================================================ + +export interface OpenTelemetryAdapterOptions { + /** + * Service name for telemetry. + * Used as the tracer name. + */ + serviceName?: string + + /** + * Optional TracerProvider to use. + * If not provided, uses the global tracer provider. + */ + tracerProvider?: TracerProvider + + /** + * Whether to trace database queries. + * @default false + */ + traceDatabaseQueries?: boolean +} + +interface ExtendedSpan extends Span { + otelSpan: OTelSpan +} + +// ============================================================================ +// OpenTelemetry Adapter +// ============================================================================ + +/** + * OpenTelemetry telemetry adapter. + * Exports traces to external systems like Jaeger, OTLP, etc. + */ +export class OpenTelemetryAdapter extends TelemetryAdapter { + #serviceName: string + #tracerProvider: TracerProvider | null + #traceDatabaseQueries: boolean + #tracer: Tracer | null = null + #spanMap = new Map() + + constructor(options: OpenTelemetryAdapterOptions = {}) { + super() + this.#serviceName = options.serviceName ?? 'duron' + this.#tracerProvider = options.tracerProvider ?? null + this.#traceDatabaseQueries = options.traceDatabaseQueries ?? false + } + + // ============================================================================ + // Lifecycle Methods + // ============================================================================ + + protected async _start(): Promise { + // Dynamically import OpenTelemetry API to make it optional + const api = await import('@opentelemetry/api') + + // Get tracer from provider or global + if (this.#tracerProvider) { + this.#tracer = this.#tracerProvider.getTracer(this.#serviceName) + } else { + this.#tracer = api.trace.getTracer(this.#serviceName) + } + } + + protected async _stop(): Promise { + this.#spanMap.clear() + this.#tracer = null + } + + // ============================================================================ + // Span Methods + // ============================================================================ + + protected async _startJobSpan(options: StartJobSpanOptions): Promise { + if (!this.#tracer) { + throw new Error('OpenTelemetry tracer not initialized') + } + + const api = await import('@opentelemetry/api') + + const otelSpan = this.#tracer.startSpan(`job:${options.actionName}`, { + kind: api.SpanKind.INTERNAL, + attributes: { + 'duron.job.id': options.jobId, + 'duron.job.action_name': options.actionName, + 'duron.job.group_key': options.groupKey, + }, + }) + + const spanId = `job:${options.jobId}` + this.#spanMap.set(spanId, otelSpan) + + const span: ExtendedSpan = { + id: spanId, + jobId: options.jobId, + stepId: null, + parentSpanId: null, + otelSpan, + } + + return span + } + + protected async _endJobSpan(span: Span, options: EndSpanOptions): Promise { + const api = await import('@opentelemetry/api') + const extSpan = span as ExtendedSpan + const otelSpan = extSpan.otelSpan + + if (options.status === 'error') { + otelSpan.setStatus({ + code: api.SpanStatusCode.ERROR, + message: options.error?.message ?? 'Unknown error', + }) + if (options.error) { + otelSpan.recordException(options.error) + } + } else if (options.status === 'cancelled') { + otelSpan.setStatus({ + code: api.SpanStatusCode.OK, + message: 'Cancelled', + }) + otelSpan.setAttribute('duron.job.cancelled', true) + } else { + otelSpan.setStatus({ code: api.SpanStatusCode.OK }) + } + + otelSpan.end() + this.#spanMap.delete(span.id) + } + + protected async _startStepSpan(options: StartStepSpanOptions): Promise { + if (!this.#tracer) { + throw new Error('OpenTelemetry tracer not initialized') + } + + const api = await import('@opentelemetry/api') + + // Get parent span context + let parentContext = api.context.active() + if (options.parentSpan) { + const parentExtSpan = options.parentSpan as ExtendedSpan + if (parentExtSpan.otelSpan) { + parentContext = api.trace.setSpan(api.context.active(), parentExtSpan.otelSpan) + } + } + + const otelSpan = this.#tracer.startSpan( + `step:${options.stepName}`, + { + kind: api.SpanKind.INTERNAL, + attributes: { + 'duron.job.id': options.jobId, + 'duron.step.id': options.stepId, + 'duron.step.name': options.stepName, + 'duron.step.parent_step_id': options.parentStepId ?? undefined, + }, + }, + parentContext, + ) + + const spanId = `step:${options.stepId}` + this.#spanMap.set(spanId, otelSpan) + + const span: ExtendedSpan = { + id: spanId, + jobId: options.jobId, + stepId: options.stepId, + parentSpanId: options.parentSpan?.id ?? null, + otelSpan, + } + + return span + } + + protected async _endStepSpan(span: Span, options: EndSpanOptions): Promise { + const api = await import('@opentelemetry/api') + const extSpan = span as ExtendedSpan + const otelSpan = extSpan.otelSpan + + if (options.status === 'error') { + otelSpan.setStatus({ + code: api.SpanStatusCode.ERROR, + message: options.error?.message ?? 'Unknown error', + }) + if (options.error) { + otelSpan.recordException(options.error) + } + } else if (options.status === 'cancelled') { + otelSpan.setStatus({ + code: api.SpanStatusCode.OK, + message: 'Cancelled', + }) + otelSpan.setAttribute('duron.step.cancelled', true) + } else { + otelSpan.setStatus({ code: api.SpanStatusCode.OK }) + } + + otelSpan.end() + this.#spanMap.delete(span.id) + } + + protected async _startDatabaseSpan(options: StartDatabaseSpanOptions): Promise { + if (!this.#traceDatabaseQueries || !this.#tracer) { + return null + } + + const api = await import('@opentelemetry/api') + + const otelSpan = this.#tracer.startSpan(`db:${options.operation}`, { + kind: api.SpanKind.CLIENT, + attributes: { + 'db.system': 'postgresql', + 'db.operation': options.operation, + 'db.statement': options.query, + }, + }) + + const spanId = `db:${globalThis.crypto.randomUUID()}` + this.#spanMap.set(spanId, otelSpan) + + const span: ExtendedSpan = { + id: spanId, + jobId: '', + stepId: null, + parentSpanId: null, + otelSpan, + } + + return span + } + + protected async _endDatabaseSpan(span: Span, options: EndSpanOptions): Promise { + const api = await import('@opentelemetry/api') + const extSpan = span as ExtendedSpan + const otelSpan = extSpan.otelSpan + + if (options.status === 'error') { + otelSpan.setStatus({ + code: api.SpanStatusCode.ERROR, + message: options.error?.message ?? 'Unknown error', + }) + if (options.error) { + otelSpan.recordException(options.error) + } + } else { + otelSpan.setStatus({ code: api.SpanStatusCode.OK }) + } + + otelSpan.end() + this.#spanMap.delete(span.id) + } + + // ============================================================================ + // Metrics Methods + // ============================================================================ + + protected async _recordMetric(options: RecordMetricOptions): Promise { + // OpenTelemetry metrics would require MeterProvider + // For now, we record as span events on the current active span + const api = await import('@opentelemetry/api') + const activeSpan = api.trace.getActiveSpan() + + if (activeSpan) { + activeSpan.addEvent('metric', { + 'metric.name': options.name, + 'metric.value': options.value, + ...options.attributes, + }) + } + } + + protected async _addSpanEvent(options: AddSpanEventOptions): Promise { + const extSpan = options.span as ExtendedSpan + if (extSpan.otelSpan) { + extSpan.otelSpan.addEvent(options.name, options.attributes) + } + } + + protected async _addSpanAttribute(options: AddSpanAttributeOptions): Promise { + const extSpan = options.span as ExtendedSpan + if (extSpan.otelSpan) { + extSpan.otelSpan.setAttribute(options.key, options.value) + } + } +} + +/** + * Create an OpenTelemetry telemetry adapter. + * Exports traces to external systems like Jaeger, OTLP, etc. + * + * @param options - Configuration options + * @returns OpenTelemetryAdapter instance + */ +export const openTelemetryAdapter = (options?: OpenTelemetryAdapterOptions) => new OpenTelemetryAdapter(options) diff --git a/packages/duron/test/telemetry.test.ts b/packages/duron/test/telemetry.test.ts new file mode 100644 index 0000000..9d4e18c --- /dev/null +++ b/packages/duron/test/telemetry.test.ts @@ -0,0 +1,454 @@ +import { afterEach, beforeEach, describe, expect, it } from 'bun:test' + +import { defineAction } from '../src/action.js' +import type { Adapter } from '../src/adapters/adapter.js' +import { Client } from '../src/client.js' +import { JOB_STATUS_COMPLETED } from '../src/constants.js' +import { createServer } from '../src/server.js' +import { LocalTelemetryAdapter, localTelemetryAdapter } from '../src/telemetry/local.js' +import { NoopTelemetryAdapter, noopTelemetryAdapter } from '../src/telemetry/noop.js' +import { type AdapterFactory, pgliteFactory, postgresFactory } from './adapters.js' + +function runTelemetryTests(adapterFactory: AdapterFactory) { + describe(`Telemetry Tests with ${adapterFactory.name}`, () => { + let adapter: Adapter + let deleteDb: () => Promise + + beforeEach( + async () => { + const adapterInstance = await adapterFactory.create({}) + adapter = adapterInstance.adapter + deleteDb = adapterInstance.deleteDb + }, + { timeout: 60_000 }, + ) + + afterEach(async () => { + if (deleteDb) { + await deleteDb() + } + }) + + describe('NoopTelemetryAdapter', () => { + it('should create a noop adapter', async () => { + const telemetry = noopTelemetryAdapter() + expect(telemetry).toBeInstanceOf(NoopTelemetryAdapter) + + await telemetry.start() + + // Noop methods should not throw + const span = await telemetry.startJobSpan({ jobId: 'test', actionName: 'test', groupKey: '@default' }) + + const observeContext = telemetry.createObserveContext('test', null, span) + observeContext.recordMetric('test', 1) + observeContext.addSpanAttribute('test', 'value') + observeContext.addSpanEvent('test') + + await telemetry.endJobSpan(span, { status: 'ok' }) + + await telemetry.stop() + }) + + it('should work with client without storing metrics', async () => { + const action = defineAction()({ + name: 'test-action', + version: '1.0.0', + handler: async (ctx) => { + // Record a metric + ctx.observe.recordMetric('custom.metric', 42) + ctx.observe.addSpanAttribute('custom.attr', 'test') + + await ctx.step('step-1', async (stepCtx) => { + stepCtx.observe.recordMetric('step.metric', 10) + return { done: true } + }) + + return { success: true } + }, + }) + + const client = new Client({ + database: adapter, + actions: { action }, + syncPattern: false, + recoverJobsOnStart: false, + logger: 'error', + telemetry: noopTelemetryAdapter(), + }) + + await client.start() + + const jobId = await client.runAction('action', {}) + await client.fetch({ batchSize: 10 }) + await new Promise((resolve) => setTimeout(resolve, 500)) + + const job = await client.getJobById(jobId) + expect(job?.status).toBe(JOB_STATUS_COMPLETED) + + // metricsEnabled should be false with noop adapter + expect(client.metricsEnabled).toBe(false) + + await client.stop() + }) + }) + + describe('LocalTelemetryAdapter', () => { + it('should create a local adapter', async () => { + const telemetry = localTelemetryAdapter() + expect(telemetry).toBeInstanceOf(LocalTelemetryAdapter) + }) + + it('should store and retrieve metrics', async () => { + const action = defineAction()({ + name: 'metrics-action', + version: '1.0.0', + handler: async (ctx) => { + // Record job-level metrics + ctx.observe.recordMetric('tokens.input', 150) + ctx.observe.recordMetric('tokens.output', 50) + ctx.observe.recordMetric('latency.ms', 1234, { model: 'gpt-4' }) + + await ctx.step('ai-step', async (stepCtx) => { + // Record step-level metrics + stepCtx.observe.recordMetric('step.tokens', 100) + stepCtx.observe.addSpanAttribute('model', 'gpt-4') + stepCtx.observe.addSpanEvent('api.call.start') + return { response: 'AI response' } + }) + + return { processed: true } + }, + }) + + const telemetry = localTelemetryAdapter() + + const client = new Client({ + database: adapter, + actions: { action }, + syncPattern: false, + recoverJobsOnStart: false, + logger: 'error', + telemetry, + }) + + await client.start() + expect(client.metricsEnabled).toBe(true) + + const jobId = await client.runAction('action', {}) + await client.fetch({ batchSize: 10 }) + await new Promise((resolve) => setTimeout(resolve, 1000)) + + const job = await client.getJobById(jobId) + expect(job?.status).toBe(JOB_STATUS_COMPLETED) + + // Retrieve metrics + const result = await client.getMetrics({ jobId }) + expect(result.metrics.length).toBeGreaterThan(0) + + // Check for our custom metrics + const tokenMetric = result.metrics.find((m) => m.name === 'tokens.input') + expect(tokenMetric).toBeTruthy() + expect(tokenMetric?.value).toBe(150) + + const latencyMetric = result.metrics.find((m) => m.name === 'latency.ms') + expect(latencyMetric).toBeTruthy() + expect(latencyMetric?.value).toBe(1234) + // biome-ignore lint/complexity/useLiteralKeys: TypeScript requires bracket notation for index signatures + expect((latencyMetric?.attributes as Record)?.['model']).toBe('gpt-4') + + await client.stop() + }) + + it('should not throw when getMetrics is called on noop adapter', async () => { + const client = new Client({ + database: adapter, + actions: {}, + syncPattern: false, + recoverJobsOnStart: false, + logger: 'error', + telemetry: noopTelemetryAdapter(), + }) + + await client.start() + expect(client.metricsEnabled).toBe(false) + + // This should throw because metrics are not enabled + let error: Error | null = null + try { + await client.getMetrics({ jobId: 'test' }) + } catch (e) { + error = e as Error + } + + expect(error).toBeTruthy() + expect(error?.message).toContain('Metrics are only available when using LocalTelemetryAdapter') + + await client.stop() + }) + + it('should handle step metrics separately', async () => { + let capturedStepId: string | null = null + + const action = defineAction()({ + name: 'step-metrics-action', + version: '1.0.0', + handler: async (ctx) => { + await ctx.step('tracked-step', async (stepCtx) => { + capturedStepId = stepCtx.stepId + stepCtx.observe.recordMetric('step.duration.ms', 500) + stepCtx.observe.recordMetric('step.items.processed', 10) + return { done: true } + }) + return { success: true } + }, + }) + + const telemetry = localTelemetryAdapter() + + const client = new Client({ + database: adapter, + actions: { action }, + syncPattern: false, + recoverJobsOnStart: false, + logger: 'error', + telemetry, + }) + + await client.start() + + const jobId = await client.runAction('action', {}) + await client.fetch({ batchSize: 10 }) + await new Promise((resolve) => setTimeout(resolve, 1000)) + + const job = await client.getJobById(jobId) + expect(job?.status).toBe(JOB_STATUS_COMPLETED) + + // Verify step ID was captured + expect(capturedStepId).toBeTruthy() + + // Get metrics for the specific step + const stepMetrics = await client.getMetrics({ stepId: capturedStepId! }) + expect(stepMetrics.metrics.length).toBeGreaterThan(0) + + const durationMetric = stepMetrics.metrics.find((m) => m.name === 'step.duration.ms') + expect(durationMetric).toBeTruthy() + expect(durationMetric?.stepId).toBe(capturedStepId) + + await client.stop() + }) + }) + + describe('Server metrics endpoints', () => { + it('should return metricsEnabled in config', async () => { + const action = defineAction()({ + name: 'test-action', + version: '1.0.0', + handler: async () => ({ done: true }), + }) + + const telemetry = localTelemetryAdapter() + + const client = new Client({ + database: adapter, + actions: { action }, + syncPattern: false, + recoverJobsOnStart: false, + logger: 'error', + telemetry, + }) + + await client.start() + + const server = createServer({ client }) + + // Check config endpoint + const configResponse = await server.handle(new Request('http://localhost/api/config')) + expect(configResponse.status).toBe(200) + + const config = (await configResponse.json()) as { metricsEnabled: boolean; authEnabled: boolean } + expect(config.metricsEnabled).toBe(true) + expect(config.authEnabled).toBe(false) + + await client.stop() + }) + + it('should get job metrics via API', async () => { + const action = defineAction()({ + name: 'api-test-action', + version: '1.0.0', + handler: async (ctx) => { + ctx.observe.recordMetric('api.test.metric', 999) + return { done: true } + }, + }) + + const telemetry = localTelemetryAdapter() + + const client = new Client({ + database: adapter, + actions: { action }, + syncPattern: false, + recoverJobsOnStart: false, + logger: 'error', + telemetry, + }) + + await client.start() + + const server = createServer({ client }) + + // Run a job + const jobId = await client.runAction('action', {}) + await client.fetch({ batchSize: 10 }) + await new Promise((resolve) => setTimeout(resolve, 1000)) + + // Get metrics via API + const response = await server.handle(new Request(`http://localhost/api/jobs/${jobId}/metrics`)) + expect(response.status).toBe(200) + + const result = (await response.json()) as { metrics: any[]; total: number } + expect(result.metrics).toBeInstanceOf(Array) + expect(result.metrics.length).toBeGreaterThan(0) + + const testMetric = result.metrics.find((m) => m.name === 'api.test.metric') + expect(testMetric).toBeTruthy() + expect(testMetric?.value).toBe(999) + + await client.stop() + }) + + it('should return error when metrics not enabled', async () => { + const action = defineAction()({ + name: 'test-action', + version: '1.0.0', + handler: async () => ({ done: true }), + }) + + const client = new Client({ + database: adapter, + actions: { action }, + syncPattern: false, + recoverJobsOnStart: false, + logger: 'error', + telemetry: noopTelemetryAdapter(), + }) + + await client.start() + + const server = createServer({ client }) + + // Try to get metrics without local telemetry - should return 500 + const response = await server.handle( + new Request('http://localhost/api/jobs/123e4567-e89b-12d3-a456-426614174000/metrics'), + ) + expect(response.status).toBe(500) + + // The error message is wrapped in a generic "Internal server error" by Elysia + // Just verify the status code is correct + const error = (await response.json()) as { error: string } + expect(error.error).toBeTruthy() + + await client.stop() + }) + }) + + describe('Observe context', () => { + it('should support span attributes and events', async () => { + // Create a custom spy on the telemetry + const baseTelemetry = localTelemetryAdapter() + + const action = defineAction()({ + name: 'observe-test', + version: '1.0.0', + handler: async (ctx) => { + ctx.observe.addSpanAttribute('job.custom', 'value1') + ctx.observe.addSpanEvent('job.started', { custom: true }) + + await ctx.step('step-1', async (stepCtx) => { + stepCtx.observe.addSpanAttribute('step.model', 'gpt-4o') + stepCtx.observe.addSpanEvent('step.processing') + return { ok: true } + }) + + ctx.observe.addSpanEvent('job.completed') + return { done: true } + }, + }) + + const client = new Client({ + database: adapter, + actions: { action }, + syncPattern: false, + recoverJobsOnStart: false, + logger: 'error', + telemetry: baseTelemetry, + }) + + await client.start() + + const jobId = await client.runAction('action', {}) + await client.fetch({ batchSize: 10 }) + await new Promise((resolve) => setTimeout(resolve, 1000)) + + const job = await client.getJobById(jobId) + expect(job?.status).toBe(JOB_STATUS_COMPLETED) + + await client.stop() + }) + + it('should handle different metric types', async () => { + const action = defineAction()({ + name: 'metric-types-test', + version: '1.0.0', + handler: async (ctx) => { + ctx.observe.recordMetric('gauge.metric', 100) + ctx.observe.recordMetric('counter.metric', 1) + ctx.observe.recordMetric('histogram.metric', 50) + ctx.observe.recordMetric('summary.metric', 75) + return { done: true } + }, + }) + + const telemetry = localTelemetryAdapter() + + const client = new Client({ + database: adapter, + actions: { action }, + syncPattern: false, + recoverJobsOnStart: false, + logger: 'error', + telemetry, + }) + + await client.start() + + const jobId = await client.runAction('action', {}) + await client.fetch({ batchSize: 10 }) + await new Promise((resolve) => setTimeout(resolve, 1000)) + + const result = await client.getMetrics({ jobId }) + + const gaugeMetric = result.metrics.find((m) => m.name === 'gauge.metric') + expect(gaugeMetric).toBeTruthy() + expect(gaugeMetric?.value).toBe(100) + + const counterMetric = result.metrics.find((m) => m.name === 'counter.metric') + expect(counterMetric).toBeTruthy() + expect(counterMetric?.value).toBe(1) + + const histogramMetric = result.metrics.find((m) => m.name === 'histogram.metric') + expect(histogramMetric).toBeTruthy() + expect(histogramMetric?.value).toBe(50) + + const summaryMetric = result.metrics.find((m) => m.name === 'summary.metric') + expect(summaryMetric).toBeTruthy() + expect(summaryMetric?.value).toBe(75) + + await client.stop() + }) + }) + }) +} + +runTelemetryTests(postgresFactory) +runTelemetryTests(pgliteFactory) From 336a5b71777cd330fabbd0c152474fe79ddf8fa3 Mon Sep 17 00:00:00 2001 From: Martin Acosta Date: Sun, 18 Jan 2026 20:51:32 -0300 Subject: [PATCH 20/78] Enhance documentation and add telemetry features - Updated CLAUDE.md to include a new section on Telemetry & Observability, detailing built-in support for metrics and tracing with pluggable adapters. - Expanded the getting started guide to reference the new Telemetry documentation. - Enhanced the jobs-and-steps documentation to include observability context and nested step handling. - Updated meta.json to include telemetry in the documentation structure. --- CLAUDE.md | 74 +++++ .../docs/content/docs/getting-started.mdx | 1 + packages/docs/content/docs/jobs-and-steps.mdx | 107 +++++- packages/docs/content/docs/meta.json | 1 + packages/docs/content/docs/telemetry.mdx | 309 ++++++++++++++++++ 5 files changed, 488 insertions(+), 4 deletions(-) create mode 100644 packages/docs/content/docs/telemetry.mdx diff --git a/CLAUDE.md b/CLAUDE.md index 188fe3e..f3dc345 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -20,6 +20,7 @@ Duron is a modern, type-safe background job processing system built with TypeScr - **Database Adapters** - PostgreSQL (production) and PGLite (development/testing) - **REST API Server** - Built-in Elysia-based API with advanced filtering and pagination - **Dashboard UI** - Beautiful React dashboard for real-time job monitoring +- **Telemetry & Observability** - Built-in support for metrics, tracing, and custom observability with pluggable adapters ## Runtime Environment @@ -86,6 +87,10 @@ The main library providing: - **Adapters**: - `duron/adapters/postgres` - PostgreSQL adapter for production - `duron/adapters/pglite` - PGLite adapter for development/testing +- **Telemetry** (`duron/telemetry`): + - `localTelemetryAdapter()` - Store metrics in the database + - `openTelemetryAdapter()` - Export to OpenTelemetry backends + - `noopTelemetryAdapter()` - Disable telemetry (default) **Key Dependencies:** - `zod` - Schema validation @@ -94,6 +99,7 @@ The main library providing: - `pino` - Logging - `fastq` - Queue implementation - `jose` - JWT handling +- `@opentelemetry/api` - OpenTelemetry integration (optional) ### `duron-dashboard` (React Dashboard) @@ -272,6 +278,73 @@ const jobId = await client.runAction('send-email', { const job = await client.waitForJob(jobId) ``` +### Telemetry & Observability + +Duron provides built-in telemetry support with pluggable adapters: + +```typescript +import { duron, localTelemetryAdapter } from 'duron' +import { postgresAdapter } from 'duron/adapters/postgres' + +const client = duron({ + database: postgresAdapter({ + connection: process.env.DATABASE_URL, + }), + // Enable local telemetry - stores metrics in the database + telemetry: localTelemetryAdapter(), + actions: { sendEmail }, +}) +``` + +**Available Telemetry Adapters:** + +- `localTelemetryAdapter()` - Stores metrics in the Duron database (great for development and self-hosted) +- `openTelemetryAdapter({ serviceName, exporterUrl })` - Exports to OpenTelemetry-compatible backends (Jaeger, OTLP, etc.) +- `noopTelemetryAdapter()` - No-op adapter, disables telemetry (default) + +**Recording Custom Metrics:** + +The `observe` context is available in action and step handlers for recording custom metrics: + +```typescript +const processAI = defineAction()({ + name: 'process-ai', + handler: async (ctx) => { + const startTime = Date.now() + + // Record job-level metrics + ctx.observe.recordMetric('ai.request.start', 1) + ctx.observe.addSpanAttribute('model', 'gpt-4') + ctx.observe.addSpanEvent('processing.started') + + const result = await ctx.step('call-api', async ({ observe }) => { + const response = await callAI(ctx.input) + + // Record step-level metrics + observe.recordMetric('ai.tokens.input', response.inputTokens) + observe.recordMetric('ai.tokens.output', response.outputTokens) + observe.recordMetric('ai.latency.ms', Date.now() - startTime) + observe.addSpanEvent('api.call.complete', { status: 'success' }) + + return response + }) + + return result + }, +}) +``` + +**Accessing Metrics via API:** + +When using `localTelemetryAdapter()`, metrics are stored in the database and accessible via the REST API: + +``` +GET /api/jobs/:id/metrics +GET /api/steps/:id/metrics +``` + +The dashboard also shows metrics when local telemetry is enabled. + ### Creating a Server with Dashboard ```typescript @@ -385,6 +458,7 @@ Uses Bun's bundler mode with: | `packages/duron/src/adapters/adapter.ts` | Base adapter class | | `packages/duron/src/adapters/postgres/` | PostgreSQL adapter | | `packages/duron/src/step-manager.ts` | Step execution and nested step handling | +| `packages/duron/src/telemetry/` | Telemetry adapters (local, opentelemetry, noop) | | `packages/duron-dashboard/src/DuronDashboard.tsx` | Dashboard root | | `packages/duron-dashboard/src/views/` | Dashboard pages | | `packages/examples/basic/start.ts` | Basic example | diff --git a/packages/docs/content/docs/getting-started.mdx b/packages/docs/content/docs/getting-started.mdx index 95f633c..0fedc4d 100644 --- a/packages/docs/content/docs/getting-started.mdx +++ b/packages/docs/content/docs/getting-started.mdx @@ -136,4 +136,5 @@ console.log(`Job created with ID: ${jobId}`) - Learn about [Actions](/docs/actions) - the building blocks of Duron - Understand [Jobs and Steps](/docs/jobs-and-steps) - how work is organized - Explore [Adapters](/docs/adapters) - database backends +- Add [Telemetry](/docs/telemetry) - observability for your job queue - Check out [Examples](/docs/examples) - real-world use cases diff --git a/packages/docs/content/docs/jobs-and-steps.mdx b/packages/docs/content/docs/jobs-and-steps.mdx index c0d82b4..cb48a08 100644 --- a/packages/docs/content/docs/jobs-and-steps.mdx +++ b/packages/docs/content/docs/jobs-and-steps.mdx @@ -125,15 +125,109 @@ const processOrder = defineAction()({ The step handler receives a context object with: - **`signal`** - AbortSignal for cancellation support +- **`stepId`** - Unique identifier for the current step +- **`parentStepId`** - ID of the parent step (null for root steps) +- **`step`** - Function to create nested child steps +- **`observe`** - Observability context for metrics and tracing (see [Telemetry](/docs/telemetry)) ```ts -await ctx.step('fetchData', async ({ signal }) => { +await ctx.step('fetchData', async ({ signal, stepId, parentStepId }) => { + console.log(`Executing step ${stepId}, parent: ${parentStepId}`) + // Use signal for cancellation const response = await fetch(url, { signal }) return response.json() }) ``` +### Nested Steps + +Steps can create child steps using the `step` function available in the step handler context. This is useful for breaking down complex operations into smaller, trackable units. + +```ts +const processOrder = defineAction()({ + name: 'processOrder', + input: z.object({ orderId: z.string() }), + handler: async (ctx) => { + const result = await ctx.step('process', async ({ step, stepId }) => { + console.log('Processing step:', stepId) + + // Create child steps - they inherit the parent's abort signal + const validation = await step('validate', async ({ parentStepId }) => { + // parentStepId links back to the 'process' step + console.log('Parent step:', parentStepId) + return { valid: true } + }) + + // Child steps can also be nested further + const payment = await step('charge', async ({ step: nestedStep }) => { + const auth = await nestedStep('authorize', async () => { + return { authCode: '123' } + }) + return { charged: true, authCode: auth.authCode } + }) + + return { success: validation.valid && payment.charged } + }) + + return result + }, +}) +``` + +#### Abort Signal Propagation + +Child steps automatically inherit the abort signal chain from their parent. When a parent step is cancelled or times out, all child steps receive the abort signal. + +```ts +await ctx.step('parent', async ({ step, signal }) => { + // If this step times out, all children are also aborted + const child1 = await step('child1', async ({ signal: childSignal }) => { + // childSignal is linked to the parent's signal + return await fetch(url, { signal: childSignal }) + }) + + return child1 +}) +``` + +#### Important: Await All Child Steps + +All child steps **must** be awaited before the parent step returns. If a parent step completes with unawaited children, Duron will: + +1. Abort all pending child steps +2. Wait for them to settle +3. Throw an `UnhandledChildStepsError` + +```ts +// ❌ Bad: Unawaited child steps +await ctx.step('parent', async ({ step }) => { + step('child1', async () => { /* ... */ }) // Not awaited! + step('child2', async () => { /* ... */ }) // Not awaited! + return { done: true } // Throws UnhandledChildStepsError +}) + +// ✅ Good: All children awaited +await ctx.step('parent', async ({ step }) => { + await step('child1', async () => { /* ... */ }) + await step('child2', async () => { /* ... */ }) + return { done: true } +}) + +// ✅ Good: Parallel execution with Promise.all +await ctx.step('parent', async ({ step }) => { + const [result1, result2] = await Promise.all([ + step('child1', async () => { /* ... */ }), + step('child2', async () => { /* ... */ }), + ]) + return { result1, result2 } +}) +``` + +#### Step Hierarchy in the Dashboard + +Nested steps are displayed hierarchically in the Duron Dashboard. Each step shows its parent relationship, making it easy to understand the execution flow of complex workflows. + ### Step Options You can configure steps with options: @@ -196,16 +290,21 @@ Each step has the following properties: - **`id`** - Unique identifier (UUID) - **`jobId`** - ID of the parent job +- **`parentStepId`** - ID of the parent step (null for root steps) - **`name`** - Name of the step - **`status`** - Current status -- **`input`** - Input data (serialized) - **`output`** - Output data (if completed) - **`error`** - Error information (if failed) -- **`createdAt`** - Timestamp when step was created +- **`timeoutMs`** - Timeout in milliseconds +- **`expiresAt`** - Expiration timestamp +- **`retriesLimit`** - Maximum retry attempts +- **`retriesCount`** - Current retry count +- **`delayedMs`** - Delay before next retry (if retrying) +- **`historyFailedAttempts`** - Record of previous failed attempts - **`startedAt`** - Timestamp when step started - **`finishedAt`** - Timestamp when step finished +- **`createdAt`** - Timestamp when step was created - **`updatedAt`** - Timestamp when step was last updated -- **`clientId`** - ID of the Duron instance that owns the step ### Querying Steps diff --git a/packages/docs/content/docs/meta.json b/packages/docs/content/docs/meta.json index 563fde5..4fa7561 100644 --- a/packages/docs/content/docs/meta.json +++ b/packages/docs/content/docs/meta.json @@ -7,6 +7,7 @@ "client-api", "server-api", "adapters", + "telemetry", "retries", "error-handling", "multi-worker", diff --git a/packages/docs/content/docs/telemetry.mdx b/packages/docs/content/docs/telemetry.mdx new file mode 100644 index 0000000..5635939 --- /dev/null +++ b/packages/docs/content/docs/telemetry.mdx @@ -0,0 +1,309 @@ +--- +title: Telemetry +description: Add observability to your job queue with telemetry adapters +icon: Activity +--- + +Duron provides a pluggable telemetry system that gives you visibility into job and step execution. You can integrate with OpenTelemetry for external tracing systems or use the built-in local telemetry for database-backed metrics. + +## Telemetry Adapters + +Duron supports three telemetry adapters: + +| Adapter | Use Case | +|---------|----------| +| `localTelemetryAdapter()` | Store metrics in the Duron database. Perfect for development and self-hosted deployments. | +| `openTelemetryAdapter()` | Export traces to OpenTelemetry backends (Jaeger, Zipkin, OTLP collectors). | +| `noopTelemetryAdapter()` | Disable telemetry (default behavior). | + +## Installation + +For OpenTelemetry support, install the optional peer dependencies: + +```bash +# Using npm +npm install @opentelemetry/api @opentelemetry/sdk-trace-node @opentelemetry/exporter-trace-otlp-http + +# Using bun +bun add @opentelemetry/api @opentelemetry/sdk-trace-node @opentelemetry/exporter-trace-otlp-http +``` + +For local telemetry, no additional dependencies are needed. + +## Configuration + +Configure the `telemetry` option when creating a Duron client: + +### Local Telemetry + +Store metrics directly in the Duron database. This is the simplest option and works great for development and self-hosted deployments. + +```ts +import { duron } from 'duron' +import { postgresAdapter } from 'duron/adapters/postgres' +import { localTelemetryAdapter } from 'duron/telemetry' + +const client = duron({ + database: postgresAdapter({ + connection: process.env.DATABASE_URL, + }), + actions: { /* your actions */ }, + telemetry: localTelemetryAdapter(), +}) +``` + +The local telemetry adapter automatically uses the same database as your Duron client. + +### OpenTelemetry + +Export traces to external systems like Jaeger, Zipkin, or any OTLP-compatible collector: + +```ts +import { duron } from 'duron' +import { postgresAdapter } from 'duron/adapters/postgres' +import { openTelemetryAdapter } from 'duron/telemetry' + +const client = duron({ + database: postgresAdapter({ + connection: process.env.DATABASE_URL, + }), + actions: { /* your actions */ }, + telemetry: openTelemetryAdapter({ + serviceName: 'my-job-queue', + exporterUrl: 'http://localhost:4318/v1/traces', // OTLP HTTP endpoint + }), +}) +``` + +### Disable Telemetry + +To explicitly disable telemetry (this is the default behavior): + +```ts +import { noopTelemetryAdapter } from 'duron/telemetry' + +const client = duron({ + // ... + telemetry: noopTelemetryAdapter(), +}) +``` + +## Recording Custom Metrics + +The `ctx.observe` object is available in both action and step handlers. Use it to record custom metrics, add span attributes, and add span events. + +### In Action Handlers + +```ts +const processOrder = defineAction()({ + name: 'processOrder', + input: z.object({ orderId: z.string() }), + handler: async (ctx) => { + const startTime = Date.now() + + // Add attributes to the job span + ctx.observe.addSpanAttribute('order.id', ctx.input.orderId) + ctx.observe.addSpanAttribute('order.priority', 'high') + + // Add events to mark important points + ctx.observe.addSpanEvent('order.processing.started') + + const result = await ctx.step('process', async () => { + // ... processing logic + return { success: true } + }) + + // Record custom metrics + const duration = Date.now() - startTime + ctx.observe.recordMetric('order.processing.duration_ms', duration) + ctx.observe.recordMetric('order.total_value', 99.99, { + currency: 'USD', + }) + + ctx.observe.addSpanEvent('order.processing.completed') + + return result + }, +}) +``` + +### In Step Handlers + +```ts +const aiAction = defineAction()({ + name: 'ai-generation', + input: z.object({ prompt: z.string() }), + handler: async (ctx) => { + const response = await ctx.step('call-openai', async (stepCtx) => { + // Add step-specific attributes + stepCtx.observe.addSpanAttribute('ai.model', 'gpt-4o') + stepCtx.observe.addSpanAttribute('ai.prompt.length', ctx.input.prompt.length) + + // Record token usage as metrics + stepCtx.observe.recordMetric('ai.tokens.input', 150, { model: 'gpt-4o' }) + stepCtx.observe.recordMetric('ai.tokens.output', 300, { model: 'gpt-4o' }) + stepCtx.observe.recordMetric('ai.cost.usd', 0.005, { model: 'gpt-4o' }) + + // Add events for timing + stepCtx.observe.addSpanEvent('openai.request.sent') + + // Simulate API call + const result = await callOpenAI(ctx.input.prompt) + + stepCtx.observe.addSpanEvent('openai.response.received') + + return result + }) + + return response + }, +}) +``` + +## Observe API Reference + +The `observe` object provides three methods: + +### `recordMetric(name, value, attributes?)` + +Record a numeric metric with optional attributes: + +```ts +ctx.observe.recordMetric('api.request.duration_ms', 150) +ctx.observe.recordMetric('tokens.consumed', 500, { model: 'gpt-4' }) +``` + +### `addSpanAttribute(key, value)` + +Add an attribute to the current span: + +```ts +ctx.observe.addSpanAttribute('user.id', 'user-123') +ctx.observe.addSpanAttribute('order.items.count', 5) +ctx.observe.addSpanAttribute('payment.method', 'card') +``` + +### `addSpanEvent(name, attributes?)` + +Add an event to the current span: + +```ts +ctx.observe.addSpanEvent('payment.authorized') +ctx.observe.addSpanEvent('email.sent', { recipient: 'user@example.com' }) +``` + +## Accessing Metrics via Dashboard + +When using `localTelemetryAdapter()`, metrics are stored in the database and can be viewed in the Duron Dashboard. The dashboard will show a "View Metrics" button on job and step detail views. + +## Accessing Metrics via API + +When local telemetry is enabled, you can access metrics through the REST API: + +### Get Job Metrics + +```http +GET /api/jobs/:jobId/metrics +``` + +Query parameters: +- `page` - Page number (default: 1) +- `pageSize` - Items per page (default: 10) +- `sort` - Sort format: `field:order` (e.g., `timestamp:desc`) +- `fName` - Filter by metric name +- `fType` - Filter by metric type + +### Get Step Metrics + +```http +GET /api/steps/:stepId/metrics +``` + +Same query parameters as job metrics. + +### Example Response + +```json +{ + "metrics": [ + { + "id": "550e8400-e29b-41d4-a716-446655440000", + "jobId": "123e4567-e89b-12d3-a456-426614174000", + "stepId": null, + "name": "order.processing.duration_ms", + "value": 1523, + "type": "metric", + "attributes": {}, + "timestamp": "2024-01-15T10:30:00.000Z" + } + ], + "total": 1, + "page": 1, + "pageSize": 10 +} +``` + +## Automatic Tracing + +Duron automatically creates spans for: + +- **Job execution** - A root span for each job with action name, group key, and status +- **Step execution** - Child spans for each step linked to the parent job span + +These spans include: +- Start and end timestamps +- Duration +- Status (ok, error, cancelled) +- Error details when applicable + +## Best Practices + +### Use Meaningful Metric Names + +Follow a consistent naming convention: + +```ts +// Good: Namespaced and descriptive +ctx.observe.recordMetric('order.processing.duration_ms', duration) +ctx.observe.recordMetric('ai.tokens.total', tokens) + +// Bad: Generic and unclear +ctx.observe.recordMetric('time', duration) +ctx.observe.recordMetric('count', tokens) +``` + +### Add Context with Attributes + +Use attributes to add context that helps with filtering and debugging: + +```ts +ctx.observe.recordMetric('api.request.duration_ms', 250, { + endpoint: '/users', + method: 'GET', + statusCode: 200, +}) +``` + +### Use Events for Milestones + +Mark important points in your job execution: + +```ts +ctx.observe.addSpanEvent('validation.started') +// ... validation logic +ctx.observe.addSpanEvent('validation.completed', { valid: true }) +``` + +### Track Costs and Resources + +Record resource consumption for billing and optimization: + +```ts +// AI token usage +stepCtx.observe.recordMetric('ai.tokens.input', inputTokens) +stepCtx.observe.recordMetric('ai.tokens.output', outputTokens) +stepCtx.observe.recordMetric('ai.cost.usd', cost) + +// API rate limits +stepCtx.observe.recordMetric('api.ratelimit.remaining', remaining) +``` From e2fa4ee0e4347218bcc52f4cbb16e172a22e37be Mon Sep 17 00:00:00 2001 From: Martin Acosta Date: Sun, 18 Jan 2026 21:14:38 -0300 Subject: [PATCH 21/78] Update dependencies and refactor metrics handling - Upgraded "@tanstack/react-virtual" to version 3.13.18 in bun.lock and package.json for improved performance. - Removed pagination parameters from metrics query schemas and related functions to simplify the metrics retrieval process. - Refactored metrics display components to utilize modals instead of panels, enhancing user experience with search functionality and improved layout. - Updated Job and Step metrics handling to align with the new modal structure, ensuring consistent data presentation. --- bun.lock | 6 +- packages/duron-dashboard/package.json | 2 +- .../src/components/metrics-panel.tsx | 289 ++++++++++++++---- .../src/hooks/use-job-metrics.ts | 26 +- .../duron-dashboard/src/views/job-details.tsx | 12 +- .../src/views/step-details-content.tsx | 16 +- packages/duron/src/adapters/postgres/base.ts | 8 - packages/duron/src/adapters/schemas.ts | 4 - packages/duron/src/server.ts | 10 - 9 files changed, 243 insertions(+), 130 deletions(-) diff --git a/bun.lock b/bun.lock index 2f8c760..30d8c81 100644 --- a/bun.lock +++ b/bun.lock @@ -92,7 +92,7 @@ "@radix-ui/react-tooltip": "^1.2.8", "@tanstack/react-query": "^5.90.10", "@tanstack/react-table": "^8.21.3", - "@tanstack/react-virtual": "^3.13.12", + "@tanstack/react-virtual": "^3.13.18", "@uiw/react-json-view": "2.0.0-alpha.37", "bun-plugin-tailwind": "^0.1.2", "class-variance-authority": "^0.7.1", @@ -653,7 +653,7 @@ "@tanstack/react-table": ["@tanstack/react-table@8.21.3", "", { "dependencies": { "@tanstack/table-core": "8.21.3" }, "peerDependencies": { "react": ">=16.8", "react-dom": ">=16.8" } }, "sha512-5nNMTSETP4ykGegmVkhjcS8tTLW6Vl4axfEGQN3v0zdHYbK4UfoqfPChclTrJ4EoK9QynqAu9oUf8VEmrpZ5Ww=="], - "@tanstack/react-virtual": ["@tanstack/react-virtual@3.13.12", "", { "dependencies": { "@tanstack/virtual-core": "3.13.12" }, "peerDependencies": { "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0", "react-dom": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" } }, "sha512-Gd13QdxPSukP8ZrkbgS2RwoZseTTbQPLnQEn7HY/rqtM+8Zt95f7xKC7N0EsKs7aoz0WzZ+fditZux+F8EzYxA=="], + "@tanstack/react-virtual": ["@tanstack/react-virtual@3.13.18", "", { "dependencies": { "@tanstack/virtual-core": "3.13.18" }, "peerDependencies": { "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0", "react-dom": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" } }, "sha512-dZkhyfahpvlaV0rIKnvQiVoWPyURppl6w4m9IwMDpuIjcJ1sD9YGWrt0wISvgU7ewACXx2Ct46WPgI6qAD4v6A=="], "@tanstack/router-core": ["@tanstack/router-core@1.139.12", "", { "dependencies": { "@tanstack/history": "1.139.0", "@tanstack/store": "^0.8.0", "cookie-es": "^2.0.0", "seroval": "^1.4.0", "seroval-plugins": "^1.4.0", "tiny-invariant": "^1.3.3", "tiny-warning": "^1.0.3" } }, "sha512-HCDi4fpnAFeDDogT0C61yd2nJn0FrIyFDhyHG3xJji8emdn8Ni4rfyrN4Av46xKkXTPUGdbsqih45+uuNtunew=="], @@ -681,7 +681,7 @@ "@tanstack/table-core": ["@tanstack/table-core@8.21.3", "", {}, "sha512-ldZXEhOBb8Is7xLs01fR3YEc3DERiz5silj8tnGkFZytt1abEvl/GhUmCE0PMLaMPTa3Jk4HbKmRlHmu+gCftg=="], - "@tanstack/virtual-core": ["@tanstack/virtual-core@3.13.12", "", {}, "sha512-1YBOJfRHV4sXUmWsFSf5rQor4Ss82G8dQWLRbnk3GA4jeP8hQt1hxXh0tmflpC0dz3VgEv/1+qwPyLeWkQuPFA=="], + "@tanstack/virtual-core": ["@tanstack/virtual-core@3.13.18", "", {}, "sha512-Mx86Hqu1k39icq2Zusq+Ey2J6dDWTjDvEv43PJtRCoEYTLyfaPnxIQ6iy7YAOK0NV/qOEmZQ/uCufrppZxTgcg=="], "@tanstack/virtual-file-routes": ["@tanstack/virtual-file-routes@1.139.0", "", {}, "sha512-9PImF1d1tovTUIpjFVa0W7Fwj/MHif7BaaczgJJfbv3sDt1Gh+oW9W9uCw9M3ndEJynnp5ZD/TTs0RGubH5ssg=="], diff --git a/packages/duron-dashboard/package.json b/packages/duron-dashboard/package.json index b21da03..3faff3c 100644 --- a/packages/duron-dashboard/package.json +++ b/packages/duron-dashboard/package.json @@ -55,7 +55,7 @@ "@radix-ui/react-tooltip": "^1.2.8", "@tanstack/react-query": "^5.90.10", "@tanstack/react-table": "^8.21.3", - "@tanstack/react-virtual": "^3.13.12", + "@tanstack/react-virtual": "^3.13.18", "@uiw/react-json-view": "2.0.0-alpha.37", "bun-plugin-tailwind": "^0.1.2", "class-variance-authority": "^0.7.1", diff --git a/packages/duron-dashboard/src/components/metrics-panel.tsx b/packages/duron-dashboard/src/components/metrics-panel.tsx index e5ecf19..704e64c 100644 --- a/packages/duron-dashboard/src/components/metrics-panel.tsx +++ b/packages/duron-dashboard/src/components/metrics-panel.tsx @@ -1,26 +1,50 @@ 'use client' -import { Activity, Clock, Hash, Tag } from 'lucide-react' +import { useVirtualizer } from '@tanstack/react-virtual' +import { Activity, Clock, Hash, Search, Tag, X } from 'lucide-react' +import { useMemo, useRef, useState } from 'react' import { Badge } from '@/components/ui/badge' -import { ScrollArea, ScrollBar } from '@/components/ui/scroll-area' +import { Button } from '@/components/ui/button' +import { Dialog, DialogContent, DialogHeader, DialogTitle } from '@/components/ui/dialog' +import { Input } from '@/components/ui/input' import { type Metric, useJobMetrics, useStepMetrics } from '@/hooks/use-job-metrics' import { formatDate } from '@/lib/format' import { JsonView } from './json-view' interface MetricItemProps { metric: Metric + searchTerm: string } -function MetricItem({ metric }: MetricItemProps) { +function highlightText(text: string, searchTerm: string): React.ReactNode { + if (!searchTerm.trim()) { + return text + } + + const regex = new RegExp(`(${searchTerm.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')})`, 'gi') + const parts = text.split(regex) + + return parts.map((part, index) => + regex.test(part) ? ( + + {part} + + ) : ( + part + ), + ) +} + +function MetricItem({ metric, searchTerm }: MetricItemProps) { return (

-
- - {metric.name} +
+ + {highlightText(metric.name, searchTerm)}
- + {metric.type}
@@ -51,82 +75,215 @@ function MetricItem({ metric }: MetricItemProps) { ) } -interface JobMetricsPanelProps { - jobId: string +interface VirtualizedMetricsListProps { + metrics: Metric[] + searchTerm: string } -export function JobMetricsPanel({ jobId }: JobMetricsPanelProps) { - const { data, isLoading, error } = useJobMetrics({ jobId, enabled: true }) +function VirtualizedMetricsList({ metrics, searchTerm }: VirtualizedMetricsListProps) { + const parentRef = useRef(null) - if (isLoading) { - return
Loading metrics...
- } + const virtualizer = useVirtualizer({ + count: metrics.length, + getScrollElement: () => parentRef.current, + estimateSize: () => 100, // Initial estimate, will be measured dynamically + overscan: 5, + }) - if (error) { - return
Failed to load metrics: {(error as Error).message}
- } + const virtualItems = virtualizer.getVirtualItems() - if (!data?.metrics || data.metrics.length === 0) { - return
No metrics recorded for this job
- } + return ( +
+
+ {virtualItems.map((virtualItem) => { + const metric = metrics[virtualItem.index]! + return ( +
+
+ +
+
+ ) + })} +
+
+ ) +} + +interface MetricsModalProps { + open: boolean + onClose: () => void + title: string + metrics: Metric[] + total: number + isLoading: boolean + error: Error | null +} + +function MetricsModal({ open, onClose, title, metrics, total, isLoading, error }: MetricsModalProps) { + const [searchTerm, setSearchTerm] = useState('') + + // Filter metrics based on search term + const filteredMetrics = useMemo(() => { + if (!searchTerm.trim()) { + return metrics + } + + const lowerSearch = searchTerm.toLowerCase() + return metrics.filter((metric) => { + // Search in name + if (metric.name.toLowerCase().includes(lowerSearch)) { + return true + } + // Search in type + if (metric.type.toLowerCase().includes(lowerSearch)) { + return true + } + // Search in value (as string) + if (String(metric.value).includes(lowerSearch)) { + return true + } + // Search in attributes + const attributesStr = JSON.stringify(metric.attributes).toLowerCase() + if (attributesStr.includes(lowerSearch)) { + return true + } + return false + }) + }, [metrics, searchTerm]) return ( - -
-
-

- - Job Metrics -

- {data.total} total -
-
- {data.metrics.map((metric) => ( - - ))} + !isOpen && onClose()}> + + + + + {title} + + + +
+ {/* Search Input */} +
+ + setSearchTerm(e.target.value)} + className="pl-10 pr-10" + /> + {searchTerm && ( + + )} +
+ + {/* Results count */} +
+ {searchTerm ? ( + <> + Showing {filteredMetrics.length} of {total} metrics + + ) : ( + <>{total} metrics total + )} +
+ + {/* Content */} +
+ {isLoading && ( +
+ Loading metrics... +
+ )} + + {error && ( +
+ Failed to load metrics: {error.message} +
+ )} + + {!isLoading && !error && filteredMetrics.length === 0 && ( +
+ {searchTerm ? 'No metrics match your search' : 'No metrics recorded'} +
+ )} + + {!isLoading && !error && filteredMetrics.length > 0 && ( +
+ +
+ )} +
-
- - + + ) } -interface StepMetricsPanelProps { - stepId: string +interface JobMetricsModalProps { + jobId: string | null + open: boolean + onClose: () => void } -export function StepMetricsPanel({ stepId }: StepMetricsPanelProps) { - const { data, isLoading, error } = useStepMetrics({ stepId, enabled: true }) +export function JobMetricsModal({ jobId, open, onClose }: JobMetricsModalProps) { + const { data, isLoading, error } = useJobMetrics({ jobId, enabled: open && !!jobId }) - if (isLoading) { - return
Loading metrics...
- } + return ( + + ) +} - if (error) { - return
Failed to load metrics: {(error as Error).message}
- } +interface StepMetricsModalProps { + stepId: string | null + open: boolean + onClose: () => void +} - if (!data?.metrics || data.metrics.length === 0) { - return
No metrics recorded for this step
- } +export function StepMetricsModal({ stepId, open, onClose }: StepMetricsModalProps) { + const { data, isLoading, error } = useStepMetrics({ stepId, enabled: open && !!stepId }) return ( - -
-
-

- - Step Metrics -

- {data.total} total -
-
- {data.metrics.map((metric) => ( - - ))} -
-
- - + ) } diff --git a/packages/duron-dashboard/src/hooks/use-job-metrics.ts b/packages/duron-dashboard/src/hooks/use-job-metrics.ts index 3e8fa40..a2ecc97 100644 --- a/packages/duron-dashboard/src/hooks/use-job-metrics.ts +++ b/packages/duron-dashboard/src/hooks/use-job-metrics.ts @@ -16,31 +16,23 @@ interface Metric { interface MetricsResult { metrics: Metric[] total: number - page?: number - pageSize?: number } interface UseJobMetricsOptions { jobId: string | null enabled?: boolean - page?: number - pageSize?: number } -export function useJobMetrics({ jobId, enabled = true, page = 1, pageSize = 50 }: UseJobMetricsOptions) { +export function useJobMetrics({ jobId, enabled = true }: UseJobMetricsOptions) { const apiRequest = useApiRequest() return useQuery({ - queryKey: ['job-metrics', jobId, page, pageSize], + queryKey: ['job-metrics', jobId], queryFn: async () => { if (!jobId) { return { metrics: [], total: 0 } } - const params = new URLSearchParams({ - page: String(page), - pageSize: String(pageSize), - }) - return apiRequest(`/jobs/${jobId}/metrics?${params}`) + return apiRequest(`/jobs/${jobId}/metrics`) }, enabled: enabled && !!jobId, }) @@ -49,24 +41,18 @@ export function useJobMetrics({ jobId, enabled = true, page = 1, pageSize = 50 } interface UseStepMetricsOptions { stepId: string | null enabled?: boolean - page?: number - pageSize?: number } -export function useStepMetrics({ stepId, enabled = true, page = 1, pageSize = 50 }: UseStepMetricsOptions) { +export function useStepMetrics({ stepId, enabled = true }: UseStepMetricsOptions) { const apiRequest = useApiRequest() return useQuery({ - queryKey: ['step-metrics', stepId, page, pageSize], + queryKey: ['step-metrics', stepId], queryFn: async () => { if (!stepId) { return { metrics: [], total: 0 } } - const params = new URLSearchParams({ - page: String(page), - pageSize: String(pageSize), - }) - return apiRequest(`/steps/${stepId}/metrics?${params}`) + return apiRequest(`/steps/${stepId}/metrics`) }, enabled: enabled && !!stepId, }) diff --git a/packages/duron-dashboard/src/views/job-details.tsx b/packages/duron-dashboard/src/views/job-details.tsx index e6f4104..f79f923 100644 --- a/packages/duron-dashboard/src/views/job-details.tsx +++ b/packages/duron-dashboard/src/views/job-details.tsx @@ -12,7 +12,7 @@ import { useCancelJob, useDeleteJob, useJob, useRetryJob } from '@/lib/api' import { formatDate } from '@/lib/format' import { BadgeStatus } from '../components/badge-status' import { JsonView } from '../components/json-view' -import { JobMetricsPanel } from '../components/metrics-panel' +import { JobMetricsModal } from '../components/metrics-panel' import { isExpiring } from '../lib/is-expiring' interface JobDetailsProps { @@ -282,17 +282,13 @@ export function JobDetails({ jobId, onClose }: JobDetailsProps) { )} {!job.output &&
No output available
} - - {/* Metrics Panel */} - {metricsEnabled && showMetrics && ( -
- -
- )}
+ + {/* Metrics Modal */} + {metricsEnabled && setShowMetrics(false)} />}
) } diff --git a/packages/duron-dashboard/src/views/step-details-content.tsx b/packages/duron-dashboard/src/views/step-details-content.tsx index 5b7e389..84ae77e 100644 --- a/packages/duron-dashboard/src/views/step-details-content.tsx +++ b/packages/duron-dashboard/src/views/step-details-content.tsx @@ -4,7 +4,7 @@ import { Activity } from 'lucide-react' import { useCallback, useEffect, useState } from 'react' import { JsonView } from '@/components/json-view' -import { StepMetricsPanel } from '@/components/metrics-panel' +import { StepMetricsModal } from '@/components/metrics-panel' import { Button } from '@/components/ui/button' import { Tooltip, TooltipContent, TooltipTrigger } from '@/components/ui/tooltip' import { useMetrics } from '@/contexts/metrics-context' @@ -202,22 +202,18 @@ export function StepDetailsContent({ stepId, jobId }: StepDetailsContentProps) {
)} - {/* Metrics Toggle Button */} + {/* Metrics Button */} {metricsEnabled && (
-
)} - {/* Metrics Panel */} - {metricsEnabled && showMetrics && ( -
- -
- )} + {/* Metrics Modal */} + {metricsEnabled && setShowMetrics(false)} />}
) } diff --git a/packages/duron/src/adapters/postgres/base.ts b/packages/duron/src/adapters/postgres/base.ts index 5db6e7d..8605af4 100644 --- a/packages/duron/src/adapters/postgres/base.ts +++ b/packages/duron/src/adapters/postgres/base.ts @@ -1382,8 +1382,6 @@ export class PostgresBaseAdapter e */ protected async _getMetrics(options: GetMetricsOptions): Promise { const metricsTable = this.tables.metricsTable - const page = options.page ?? 1 - const pageSize = options.pageSize ?? 100 const filters = options.filters ?? {} // Build WHERE clause @@ -1404,8 +1402,6 @@ export class PostgresBaseAdapter e return { metrics: [], total: 0, - page, - pageSize, } } @@ -1427,14 +1423,10 @@ export class PostgresBaseAdapter e .from(metricsTable) .where(where) .orderBy(orderByClause) - .limit(pageSize) - .offset((page - 1) * pageSize) return { metrics, total, - page, - pageSize, } } diff --git a/packages/duron/src/adapters/schemas.ts b/packages/duron/src/adapters/schemas.ts index 60bfd2b..eee251f 100644 --- a/packages/duron/src/adapters/schemas.ts +++ b/packages/duron/src/adapters/schemas.ts @@ -340,15 +340,11 @@ export const GetMetricsOptionsSchema = z.object({ stepId: z.string().optional(), filters: MetricFiltersSchema.optional(), sort: MetricSortSchema.optional(), - page: z.number().int().positive().optional(), - pageSize: z.number().int().positive().optional(), }) export const GetMetricsResultSchema = z.object({ metrics: z.array(MetricSchema), total: z.number().int().nonnegative(), - page: z.number().int().positive().optional(), - pageSize: z.number().int().positive().optional(), }) export const DeleteMetricsOptionsSchema = z.object({ diff --git a/packages/duron/src/server.ts b/packages/duron/src/server.ts index d914cd3..66a8d8d 100644 --- a/packages/duron/src/server.ts +++ b/packages/duron/src/server.ts @@ -179,10 +179,6 @@ export type GetJobStepsQueryInput = z.input // Metrics query schema export const GetMetricsQuerySchema = z .object({ - // Pagination - page: z.coerce.number().int().min(1).optional(), - pageSize: z.coerce.number().int().min(1).max(1000).optional(), - // Filters fName: z.union([z.string(), z.array(z.string())]).optional(), fType: z.union([MetricTypeSchema, z.array(MetricTypeSchema)]).optional(), @@ -217,8 +213,6 @@ export const GetMetricsQuerySchema = z } return { - page: data.page, - pageSize: data.pageSize, filters: Object.keys(filters).length > 0 ? filters : undefined, sort, } @@ -698,8 +692,6 @@ export function createServer

({ client, prefix, login, metricsE } const options: GetMetricsOptions = { jobId: params.id, - page: query.page, - pageSize: query.pageSize, filters: query.filters, sort: query.sort, } @@ -725,8 +717,6 @@ export function createServer

({ client, prefix, login, metricsE } const options: GetMetricsOptions = { stepId: params.id, - page: query.page, - pageSize: query.pageSize, filters: query.filters, sort: query.sort, } From 498af6fd3599d26bba48ca8f19dceb730ebea90c Mon Sep 17 00:00:00 2001 From: Martin Acosta Date: Sun, 18 Jan 2026 21:25:32 -0300 Subject: [PATCH 22/78] Refactor metrics handling in Adapter and LocalTelemetryAdapter - Updated Adapter class to support batch insertion of multiple metric records, improving performance and reducing database load. - Refactored PostgresBaseAdapter to handle batch inserts, returning the number of metrics inserted. - Enhanced LocalTelemetryAdapter to queue metrics for batch insertion after a delay, ensuring efficient database operations. - Updated telemetry tests to accommodate the new batching behavior and verify metrics insertion. --- packages/duron/src/adapters/adapter.ts | 27 +++--- packages/duron/src/adapters/postgres/base.ts | 30 ++++--- packages/duron/src/telemetry/local.ts | 95 ++++++++++++++++++-- packages/duron/test/telemetry.test.ts | 12 +++ 4 files changed, 132 insertions(+), 32 deletions(-) diff --git a/packages/duron/src/adapters/adapter.ts b/packages/duron/src/adapters/adapter.ts index 78552d5..45c4f6c 100644 --- a/packages/duron/src/adapters/adapter.ts +++ b/packages/duron/src/adapters/adapter.ts @@ -997,20 +997,23 @@ export abstract class Adapter extends EventEmitter { // ============================================================================ /** - * Insert a metric record. + * Insert multiple metric records in a single batch operation. * Note: This method bypasses telemetry tracing to prevent infinite loops. * - * @param options - The metric data to insert - * @returns Promise resolving to the metric ID + * @param metrics - Array of metric data to insert + * @returns Promise resolving to the number of metrics inserted */ - async insertMetric(options: InsertMetricOptions): Promise { + async insertMetrics(metrics: InsertMetricOptions[]): Promise { try { + if (metrics.length === 0) { + return 0 + } await this.start() - const parsedOptions = InsertMetricOptionsSchema.parse(options) - const result = await this._insertMetric(parsedOptions) - return z.string().parse(result) + const parsedMetrics = metrics.map((m) => InsertMetricOptionsSchema.parse(m)) + const result = await this._insertMetrics(parsedMetrics) + return NumberResultSchema.parse(result) } catch (error) { - this.#logger?.error(error, 'Error in Adapter.insertMetric()') + this.#logger?.error(error, 'Error in Adapter.insertMetrics()') throw error } } @@ -1062,12 +1065,12 @@ export abstract class Adapter extends EventEmitter { // ============================================================================ /** - * Internal method to insert a metric record. + * Internal method to insert multiple metric records in a single batch. * - * @param options - Validated metric data - * @returns Promise resolving to the metric ID + * @param metrics - Array of validated metric data + * @returns Promise resolving to the number of metrics inserted */ - protected abstract _insertMetric(options: InsertMetricOptions): Promise + protected abstract _insertMetrics(metrics: InsertMetricOptions[]): Promise /** * Internal method to get metrics for a job or step. diff --git a/packages/duron/src/adapters/postgres/base.ts b/packages/duron/src/adapters/postgres/base.ts index 8605af4..30506c1 100644 --- a/packages/duron/src/adapters/postgres/base.ts +++ b/packages/duron/src/adapters/postgres/base.ts @@ -1359,22 +1359,28 @@ export class PostgresBaseAdapter e // ============================================================================ /** - * Internal method to insert a metric record. + * Internal method to insert multiple metric records in a single batch. */ - protected async _insertMetric(options: InsertMetricOptions): Promise { - const [result] = await this.db + protected async _insertMetrics(metrics: InsertMetricOptions[]): Promise { + if (metrics.length === 0) { + return 0 + } + + const values = metrics.map((m) => ({ + job_id: m.jobId, + step_id: m.stepId ?? null, + name: m.name, + value: m.value, + attributes: m.attributes ?? {}, + type: m.type, + })) + + const result = await this.db .insert(this.tables.metricsTable) - .values({ - job_id: options.jobId, - step_id: options.stepId ?? null, - name: options.name, - value: options.value, - attributes: options.attributes ?? {}, - type: options.type, - }) + .values(values) .returning({ id: this.tables.metricsTable.id }) - return result!.id + return result.length } /** diff --git a/packages/duron/src/telemetry/local.ts b/packages/duron/src/telemetry/local.ts index edb76f5..f93e444 100644 --- a/packages/duron/src/telemetry/local.ts +++ b/packages/duron/src/telemetry/local.ts @@ -1,4 +1,4 @@ -import type { Adapter } from '../adapters/adapter.js' +import type { Adapter, InsertMetricOptions } from '../adapters/adapter.js' import { type AddSpanAttributeOptions, type AddSpanEventOptions, @@ -18,6 +18,12 @@ import { // Note: This interface is intentionally empty as the database is obtained from the Duron client export type LocalTelemetryAdapterOptions = Record +// ============================================================================ +// Constants +// ============================================================================ + +const METRICS_FLUSH_DELAY_MS = 1000 + // ============================================================================ // Local Telemetry Adapter // ============================================================================ @@ -29,6 +35,8 @@ export type LocalTelemetryAdapterOptions = Record * This adapter automatically uses the database adapter configured in the Duron client. * No additional configuration is required. * + * Metrics are batched and inserted after 1 second of inactivity to reduce database load. + * * @example * ```typescript * const client = duron({ @@ -40,6 +48,9 @@ export type LocalTelemetryAdapterOptions = Record */ export class LocalTelemetryAdapter extends TelemetryAdapter { #spanStartTimes = new Map() + #metricsQueue: InsertMetricOptions[] = [] + #flushTimer: ReturnType | null = null + #flushPromise: Promise | null = null /** * Get the database adapter from the Duron client. @@ -55,6 +66,71 @@ export class LocalTelemetryAdapter extends TelemetryAdapter { return client.database } + // ============================================================================ + // Queue Management + // ============================================================================ + + /** + * Queue a metric for batch insertion. + * The metric will be inserted after 1 second of inactivity. + */ + #queueMetric(options: InsertMetricOptions): void { + this.#metricsQueue.push(options) + this.#scheduleFlush() + } + + /** + * Schedule a flush of the metrics queue. + * Resets the timer on each call (debounce behavior). + */ + #scheduleFlush(): void { + if (this.#flushTimer) { + clearTimeout(this.#flushTimer) + } + + this.#flushTimer = setTimeout(() => { + this.#flushTimer = null + this.#flushPromise = this.#flushQueue().finally(() => { + this.#flushPromise = null + }) + }, METRICS_FLUSH_DELAY_MS) + } + + /** + * Flush all queued metrics to the database. + */ + async #flushQueue(): Promise { + if (this.#metricsQueue.length === 0) { + return + } + + // Take all metrics from the queue + const metrics = this.#metricsQueue.splice(0, this.#metricsQueue.length) + + // Batch insert all metrics in a single database operation + await this.#database.insertMetrics(metrics) + } + + /** + * Force flush the queue immediately. + * Used during shutdown to ensure all metrics are persisted. + */ + async #forceFlush(): Promise { + // Clear any pending timer + if (this.#flushTimer) { + clearTimeout(this.#flushTimer) + this.#flushTimer = null + } + + // Wait for any in-progress flush + if (this.#flushPromise) { + await this.#flushPromise + } + + // Flush remaining metrics + await this.#flushQueue() + } + // ============================================================================ // Lifecycle Methods // ============================================================================ @@ -64,6 +140,8 @@ export class LocalTelemetryAdapter extends TelemetryAdapter { } protected async _stop(): Promise { + // Flush any remaining metrics before stopping + await this.#forceFlush() this.#spanStartTimes.clear() } @@ -76,7 +154,7 @@ export class LocalTelemetryAdapter extends TelemetryAdapter { this.#spanStartTimes.set(spanId, Date.now()) // Record span start as a metric - await this.#database.insertMetric({ + this.#queueMetric({ jobId: options.jobId, name: 'duron.job.span.start', value: Date.now(), @@ -102,7 +180,7 @@ export class LocalTelemetryAdapter extends TelemetryAdapter { this.#spanStartTimes.delete(span.id) // Record span end with duration - await this.#database.insertMetric({ + this.#queueMetric({ jobId: span.jobId, name: 'duron.job.span.end', value: duration, @@ -121,7 +199,7 @@ export class LocalTelemetryAdapter extends TelemetryAdapter { this.#spanStartTimes.set(spanId, Date.now()) // Record span start as a metric - await this.#database.insertMetric({ + this.#queueMetric({ jobId: options.jobId, stepId: options.stepId, name: 'duron.step.span.start', @@ -149,7 +227,7 @@ export class LocalTelemetryAdapter extends TelemetryAdapter { this.#spanStartTimes.delete(span.id) // Record span end with duration - await this.#database.insertMetric({ + this.#queueMetric({ jobId: span.jobId, stepId: span.stepId ?? undefined, name: 'duron.step.span.end', @@ -178,7 +256,7 @@ export class LocalTelemetryAdapter extends TelemetryAdapter { // ============================================================================ protected async _recordMetric(options: RecordMetricOptions): Promise { - await this.#database.insertMetric({ + this.#queueMetric({ jobId: options.jobId, stepId: options.stepId, name: options.name, @@ -189,7 +267,7 @@ export class LocalTelemetryAdapter extends TelemetryAdapter { } protected async _addSpanEvent(options: AddSpanEventOptions): Promise { - await this.#database.insertMetric({ + this.#queueMetric({ jobId: options.span.jobId, stepId: options.span.stepId ?? undefined, name: options.name, @@ -203,7 +281,7 @@ export class LocalTelemetryAdapter extends TelemetryAdapter { } protected async _addSpanAttribute(options: AddSpanAttributeOptions): Promise { - await this.#database.insertMetric({ + this.#queueMetric({ jobId: options.span.jobId, stepId: options.span.stepId ?? undefined, name: `attribute:${options.key}`, @@ -223,6 +301,7 @@ export class LocalTelemetryAdapter extends TelemetryAdapter { * Perfect for development and self-hosted deployments. * * The database adapter is automatically obtained from the Duron client. + * Metrics are batched and inserted after 1 second of inactivity to reduce database load. * * @returns LocalTelemetryAdapter instance * diff --git a/packages/duron/test/telemetry.test.ts b/packages/duron/test/telemetry.test.ts index 9d4e18c..13b9eb0 100644 --- a/packages/duron/test/telemetry.test.ts +++ b/packages/duron/test/telemetry.test.ts @@ -141,6 +141,9 @@ function runTelemetryTests(adapterFactory: AdapterFactory) { const job = await client.getJobById(jobId) expect(job?.status).toBe(JOB_STATUS_COMPLETED) + // Wait for metrics debounce flush (1 second debounce + buffer) + await new Promise((resolve) => setTimeout(resolve, 1500)) + // Retrieve metrics const result = await client.getMetrics({ jobId }) expect(result.metrics.length).toBeGreaterThan(0) @@ -226,6 +229,9 @@ function runTelemetryTests(adapterFactory: AdapterFactory) { // Verify step ID was captured expect(capturedStepId).toBeTruthy() + // Wait for metrics debounce flush (1 second debounce + buffer) + await new Promise((resolve) => setTimeout(resolve, 1500)) + // Get metrics for the specific step const stepMetrics = await client.getMetrics({ stepId: capturedStepId! }) expect(stepMetrics.metrics.length).toBeGreaterThan(0) @@ -302,6 +308,9 @@ function runTelemetryTests(adapterFactory: AdapterFactory) { await client.fetch({ batchSize: 10 }) await new Promise((resolve) => setTimeout(resolve, 1000)) + // Wait for metrics debounce flush (1 second debounce + buffer) + await new Promise((resolve) => setTimeout(resolve, 1500)) + // Get metrics via API const response = await server.handle(new Request(`http://localhost/api/jobs/${jobId}/metrics`)) expect(response.status).toBe(200) @@ -426,6 +435,9 @@ function runTelemetryTests(adapterFactory: AdapterFactory) { await client.fetch({ batchSize: 10 }) await new Promise((resolve) => setTimeout(resolve, 1000)) + // Wait for metrics debounce flush (1 second debounce + buffer) + await new Promise((resolve) => setTimeout(resolve, 1500)) + const result = await client.getMetrics({ jobId }) const gaugeMetric = result.metrics.find((m) => m.name === 'gauge.metric') From 3ca51768f6eeeb4c1c9e4c37fdb150ea552f0f51 Mon Sep 17 00:00:00 2001 From: Martin Acosta Date: Sun, 18 Jan 2026 22:42:30 -0300 Subject: [PATCH 23/78] Add jsonata and usehooks-ts dependencies; enhance LocalTelemetryAdapter options - Added `jsonata` and `usehooks-ts` dependencies to `bun.lock` and `package.json` for improved functionality. - Updated `LocalTelemetryAdapter` to accept a configurable `flushDelayMs` option, allowing customization of the metrics flushing delay. - Refactored related documentation and examples to reflect the new configuration options for better clarity. --- bun.lock | 8 ++++ packages/duron-dashboard/package.json | 2 + .../src/components/job-search.tsx | 24 +++++------ .../src/components/metrics-panel.tsx | 10 +++-- .../duron-dashboard/src/views/step-list.tsx | 25 +++-------- packages/duron/src/telemetry/local.ts | 41 ++++++++++++++----- 6 files changed, 62 insertions(+), 48 deletions(-) diff --git a/bun.lock b/bun.lock index 30d8c81..dab66ee 100644 --- a/bun.lock +++ b/bun.lock @@ -99,6 +99,7 @@ "clsx": "^2.1.1", "cmdk": "^1.1.1", "date-fns": "^4.1.0", + "jsonata": "^2.1.0", "lucide-react": "^0.555.0", "motion": "^12.23.24", "nanoid": "^5.1.6", @@ -109,6 +110,7 @@ "react-dom": "^19", "react-resizable-panels": "^4.4.1", "tailwind-merge": "^3.4.0", + "usehooks-ts": "^3.1.1", "zod": "^4.1.12", }, "devDependencies": { @@ -1129,6 +1131,8 @@ "json5": ["json5@2.2.3", "", { "bin": { "json5": "lib/cli.js" } }, "sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg=="], + "jsonata": ["jsonata@2.1.0", "", {}, "sha512-OCzaRMK8HobtX8fp37uIVmL8CY1IGc/a6gLsDqz3quExFR09/U78HUzWYr7T31UEB6+Eu0/8dkVD5fFDOl9a8w=="], + "jsonwebtoken": ["jsonwebtoken@9.0.2", "", { "dependencies": { "jws": "^3.2.2", "lodash.includes": "^4.3.0", "lodash.isboolean": "^3.0.3", "lodash.isinteger": "^4.0.4", "lodash.isnumber": "^3.0.3", "lodash.isplainobject": "^4.0.6", "lodash.isstring": "^4.0.1", "lodash.once": "^4.0.0", "ms": "^2.1.1", "semver": "^7.5.4" } }, "sha512-PRp66vJ865SSqOlgqS8hujT5U4AOgMfhrwYIuIhfKaoSCZcirrmASQr8CX7cUg+RMih+hgznrjp99o+W4pJLHQ=="], "jwa": ["jwa@1.4.2", "", { "dependencies": { "buffer-equal-constant-time": "^1.0.1", "ecdsa-sig-formatter": "1.0.11", "safe-buffer": "^5.0.1" } }, "sha512-eeH5JO+21J78qMvTIDdBXidBd6nG2kZjg5Ohz/1fpa28Z4CcsWUzJ1ZZyFq/3z3N17aZy+ZuBoHljASbL1WfOw=="], @@ -1159,6 +1163,8 @@ "lightningcss-win32-x64-msvc": ["lightningcss-win32-x64-msvc@1.30.2", "", { "os": "win32", "cpu": "x64" }, "sha512-5g1yc73p+iAkid5phb4oVFMB45417DkRevRbt/El/gKXJk4jid+vPFF/AXbxn05Aky8PapwzZrdJShv5C0avjw=="], + "lodash.debounce": ["lodash.debounce@4.0.8", "", {}, "sha512-FT1yDzDYEoYWhnSGnpE/4Kj1fLZkDFyqRb7fNt6FdYOSxlUWAtp42Eh6Wb0rGIv/m9Bgo7x4GhQbm5Ys4SG5ow=="], + "lodash.includes": ["lodash.includes@4.3.0", "", {}, "sha512-W3Bx6mdkRTGtlJISOvVD/lbqjTlPPUDTMnlXZFnVwi9NKJ6tiAk6LVdlhZMm17VZisqhKcgzpO5Wz91PCt5b0w=="], "lodash.isboolean": ["lodash.isboolean@3.0.3", "", {}, "sha512-Bz5mupy2SVbPHURB98VAcw+aHh4vRV5IPNhILUCsOzRmsTmSQ17jIuqopAentWoehktxGd9e/hbIXq980/1QJg=="], @@ -1627,6 +1633,8 @@ "use-sync-external-store": ["use-sync-external-store@1.6.0", "", { "peerDependencies": { "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" } }, "sha512-Pp6GSwGP/NrPIrxVFAIkOQeyw8lFenOHijQWkUTrDvrF4ALqylP2C/KCkeS9dpUM3KvYRQhna5vt7IL95+ZQ9w=="], + "usehooks-ts": ["usehooks-ts@3.1.1", "", { "dependencies": { "lodash.debounce": "^4.0.8" }, "peerDependencies": { "react": "^16.8.0 || ^17 || ^18 || ^19 || ^19.0.0-rc" } }, "sha512-I4diPp9Cq6ieSUH2wu+fDAVQO43xwtulo+fKEidHUwZPnYImbtkTjzIJYcDcJqxgmX31GVqNFURodvcgHcW0pA=="], + "util-deprecate": ["util-deprecate@1.0.2", "", {}, "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw=="], "uuid": ["uuid@8.3.2", "", { "bin": { "uuid": "dist/bin/uuid" } }, "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg=="], diff --git a/packages/duron-dashboard/package.json b/packages/duron-dashboard/package.json index 3faff3c..66ea974 100644 --- a/packages/duron-dashboard/package.json +++ b/packages/duron-dashboard/package.json @@ -62,6 +62,7 @@ "clsx": "^2.1.1", "cmdk": "^1.1.1", "date-fns": "^4.1.0", + "jsonata": "^2.1.0", "lucide-react": "^0.555.0", "motion": "^12.23.24", "nanoid": "^5.1.6", @@ -72,6 +73,7 @@ "react-dom": "^19", "react-resizable-panels": "^4.4.1", "tailwind-merge": "^3.4.0", + "usehooks-ts": "^3.1.1", "zod": "^4.1.12" }, "devDependencies": { diff --git a/packages/duron-dashboard/src/components/job-search.tsx b/packages/duron-dashboard/src/components/job-search.tsx index 150aca3..0a94f5b 100644 --- a/packages/duron-dashboard/src/components/job-search.tsx +++ b/packages/duron-dashboard/src/components/job-search.tsx @@ -3,9 +3,9 @@ import { Search, X } from 'lucide-react' import { parseAsString, useQueryState } from 'nuqs' import { useCallback, useEffect, useRef, useState } from 'react' +import { useDebounceValue } from 'usehooks-ts' import { Input } from '@/components/ui/input' -import { useDebouncedCallback } from '@/hooks/use-debounced-callback' import { cn } from '@/lib/utils' interface JobSearchProps { @@ -15,26 +15,22 @@ interface JobSearchProps { export function JobSearch({ className }: JobSearchProps) { const [search, setSearch] = useQueryState('search', parseAsString.withDefault('')) const [inputValue, setInputValue] = useState(search) + const [debouncedInputValue] = useDebounceValue(inputValue, 300) const inputRef = useRef(null) - // Debounce the search query update with 1000ms delay - const debouncedSetSearch = useDebouncedCallback((value: string | null) => { - setSearch(value) - }, 1000) + // Sync debounced input value to query state + useEffect(() => { + setSearch(debouncedInputValue || null) + }, [debouncedInputValue, setSearch]) - // Sync input value with query state when it changes externally (e.g., clear button) + // Sync input value with query state when it changes externally (e.g., browser back/forward) useEffect(() => { setInputValue(search) }, [search]) - const handleChange = useCallback( - (e: React.ChangeEvent) => { - const value = e.target.value || null - setInputValue(value || '') - debouncedSetSearch(value) - }, - [debouncedSetSearch], - ) + const handleChange = useCallback((e: React.ChangeEvent) => { + setInputValue(e.target.value) + }, []) const handleClear = useCallback(() => { setInputValue('') diff --git a/packages/duron-dashboard/src/components/metrics-panel.tsx b/packages/duron-dashboard/src/components/metrics-panel.tsx index 704e64c..6c5ef3f 100644 --- a/packages/duron-dashboard/src/components/metrics-panel.tsx +++ b/packages/duron-dashboard/src/components/metrics-panel.tsx @@ -3,6 +3,7 @@ import { useVirtualizer } from '@tanstack/react-virtual' import { Activity, Clock, Hash, Search, Tag, X } from 'lucide-react' import { useMemo, useRef, useState } from 'react' +import { useDebounceValue } from 'usehooks-ts' import { Badge } from '@/components/ui/badge' import { Button } from '@/components/ui/button' @@ -139,14 +140,15 @@ interface MetricsModalProps { function MetricsModal({ open, onClose, title, metrics, total, isLoading, error }: MetricsModalProps) { const [searchTerm, setSearchTerm] = useState('') + const [debouncedSearchTerm] = useDebounceValue(searchTerm, 300) - // Filter metrics based on search term + // Filter metrics based on debounced search term const filteredMetrics = useMemo(() => { - if (!searchTerm.trim()) { + if (!debouncedSearchTerm.trim()) { return metrics } - const lowerSearch = searchTerm.toLowerCase() + const lowerSearch = debouncedSearchTerm.toLowerCase() return metrics.filter((metric) => { // Search in name if (metric.name.toLowerCase().includes(lowerSearch)) { @@ -167,7 +169,7 @@ function MetricsModal({ open, onClose, title, metrics, total, isLoading, error } } return false }) - }, [metrics, searchTerm]) + }, [metrics, debouncedSearchTerm]) return (

!isOpen && onClose()}> diff --git a/packages/duron-dashboard/src/views/step-list.tsx b/packages/duron-dashboard/src/views/step-list.tsx index 2608dad..185e4a7 100644 --- a/packages/duron-dashboard/src/views/step-list.tsx +++ b/packages/duron-dashboard/src/views/step-list.tsx @@ -2,6 +2,7 @@ import { ChevronRight, Clock, GitBranch, History, List, Search } from 'lucide-react' import { useCallback, useMemo, useState } from 'react' +import { useDebounceValue } from 'usehooks-ts' import { Timeline } from '@/components/timeline' import { Accordion, AccordionContent, AccordionItem, AccordionTrigger } from '@/components/ui/accordion' @@ -10,7 +11,6 @@ import { Input } from '@/components/ui/input' import { ScrollArea } from '@/components/ui/scroll-area' import { Tooltip, TooltipContent, TooltipTrigger } from '@/components/ui/tooltip' import { useStepView } from '@/contexts/layout-context' -import { useDebouncedCallback } from '@/hooks/use-debounced-callback' import { useStepsPolling } from '@/hooks/use-steps-polling' import { type GetJobStepsResponse, useJob, useJobSteps, useTimeTravelJob } from '@/lib/api' import { calculateDurationSeconds, formatDurationSeconds } from '@/lib/duration' @@ -84,26 +84,13 @@ function flattenStepTree(nodes: StepNode[]): Array<{ step: JobStepWithoutOutput; } export function StepList({ jobId, selectedStepId, onStepSelect }: StepListProps) { - const [inputValue, setInputValue] = useState('') const [searchTerm, setSearchTerm] = useState('') + const [debouncedSearchTerm] = useDebounceValue(searchTerm, 300) const { viewType, setViewType } = useStepView() - // Debounce the search term update with 1000ms delay - const debouncedSetSearchTerm = useDebouncedCallback((value: string) => { - setSearchTerm(value) - }, 1000) - - const handleSearchChange = useCallback( - (value: string) => { - setInputValue(value) - debouncedSetSearchTerm(value) - }, - [debouncedSetSearchTerm], - ) - // Fetch all steps (no pagination) const { data: stepsData, isLoading: stepsLoading } = useJobSteps(jobId, { - search: searchTerm || undefined, + search: debouncedSearchTerm || undefined, }) const { data: job } = useJob(jobId) @@ -152,8 +139,8 @@ export function StepList({ jobId, selectedStepId, onStepSelect }: StepListProps) handleSearchChange(e.target.value)} + value={searchTerm} + onChange={(e) => setSearchTerm(e.target.value)} className="pl-8" />
@@ -190,7 +177,7 @@ export function StepList({ jobId, selectedStepId, onStepSelect }: StepListProps)
Loading steps...
) : orderedSteps.length === 0 ? (
- {inputValue ? 'No steps found matching your search' : 'No steps found'} + {searchTerm ? 'No steps found matching your search' : 'No steps found'}
) : ( +export interface LocalTelemetryAdapterOptions { + /** + * Delay in milliseconds before flushing queued metrics to the database. + * Metrics are batched and inserted after this delay of inactivity. + * @default 1000 + */ + flushDelayMs?: number +} // ============================================================================ // Constants // ============================================================================ -const METRICS_FLUSH_DELAY_MS = 1000 +const DEFAULT_FLUSH_DELAY_MS = 1000 // ============================================================================ // Local Telemetry Adapter @@ -33,15 +39,13 @@ const METRICS_FLUSH_DELAY_MS = 1000 * Perfect for development and self-hosted deployments. * * This adapter automatically uses the database adapter configured in the Duron client. - * No additional configuration is required. - * - * Metrics are batched and inserted after 1 second of inactivity to reduce database load. + * Metrics are batched and inserted after a configurable delay of inactivity to reduce database load. * * @example * ```typescript * const client = duron({ * database: postgresAdapter({ connection: 'postgres://...' }), - * telemetry: localTelemetryAdapter(), + * telemetry: localTelemetryAdapter({ flushDelayMs: 500 }), // Custom 500ms delay * actions: { ... } * }) * ``` @@ -51,6 +55,12 @@ export class LocalTelemetryAdapter extends TelemetryAdapter { #metricsQueue: InsertMetricOptions[] = [] #flushTimer: ReturnType | null = null #flushPromise: Promise | null = null + #flushDelayMs: number + + constructor(options?: LocalTelemetryAdapterOptions) { + super() + this.#flushDelayMs = options?.flushDelayMs ?? DEFAULT_FLUSH_DELAY_MS + } /** * Get the database adapter from the Duron client. @@ -93,7 +103,7 @@ export class LocalTelemetryAdapter extends TelemetryAdapter { this.#flushPromise = this.#flushQueue().finally(() => { this.#flushPromise = null }) - }, METRICS_FLUSH_DELAY_MS) + }, this.#flushDelayMs) } /** @@ -301,17 +311,26 @@ export class LocalTelemetryAdapter extends TelemetryAdapter { * Perfect for development and self-hosted deployments. * * The database adapter is automatically obtained from the Duron client. - * Metrics are batched and inserted after 1 second of inactivity to reduce database load. + * Metrics are batched and inserted after a configurable delay of inactivity to reduce database load. * + * @param options - Configuration options + * @param options.flushDelayMs - Delay in milliseconds before flushing queued metrics (default: 1000) * @returns LocalTelemetryAdapter instance * * @example * ```typescript * const client = duron({ * database: postgresAdapter({ connection: 'postgres://...' }), - * telemetry: localTelemetryAdapter(), + * telemetry: localTelemetryAdapter(), // Uses default 1 second delay + * actions: { ... } + * }) + * + * // Or with custom flush delay + * const client = duron({ + * database: postgresAdapter({ connection: 'postgres://...' }), + * telemetry: localTelemetryAdapter({ flushDelayMs: 500 }), // 500ms delay * actions: { ... } * }) * ``` */ -export const localTelemetryAdapter = () => new LocalTelemetryAdapter() +export const localTelemetryAdapter = (options?: LocalTelemetryAdapterOptions) => new LocalTelemetryAdapter(options) From 75d14e62f83f10d3ef65de336c18673527b51545 Mon Sep 17 00:00:00 2001 From: Martin Acosta Date: Sun, 18 Jan 2026 22:58:09 -0300 Subject: [PATCH 24/78] Refactor metrics panel to integrate JSONata for advanced filtering - Replaced search functionality with a JSONata expression input for filtering metrics in the MetricsModal component. - Updated MetricItem and VirtualizedMetricsList components to remove search term dependency. - Added error handling and display for JSONata evaluation results, enhancing user feedback. - Improved layout and styling for the JSONata query input and results display. --- .../src/components/metrics-panel.tsx | 199 +++++++++++------- 1 file changed, 127 insertions(+), 72 deletions(-) diff --git a/packages/duron-dashboard/src/components/metrics-panel.tsx b/packages/duron-dashboard/src/components/metrics-panel.tsx index 6c5ef3f..c7a3595 100644 --- a/packages/duron-dashboard/src/components/metrics-panel.tsx +++ b/packages/duron-dashboard/src/components/metrics-panel.tsx @@ -1,49 +1,30 @@ 'use client' import { useVirtualizer } from '@tanstack/react-virtual' -import { Activity, Clock, Hash, Search, Tag, X } from 'lucide-react' -import { useMemo, useRef, useState } from 'react' +import jsonata from 'jsonata' +import { Activity, Clock, Code, Hash, Tag, X } from 'lucide-react' +import { useEffect, useRef, useState } from 'react' import { useDebounceValue } from 'usehooks-ts' import { Badge } from '@/components/ui/badge' import { Button } from '@/components/ui/button' import { Dialog, DialogContent, DialogHeader, DialogTitle } from '@/components/ui/dialog' -import { Input } from '@/components/ui/input' +import { Textarea } from '@/components/ui/textarea' import { type Metric, useJobMetrics, useStepMetrics } from '@/hooks/use-job-metrics' import { formatDate } from '@/lib/format' import { JsonView } from './json-view' interface MetricItemProps { metric: Metric - searchTerm: string } -function highlightText(text: string, searchTerm: string): React.ReactNode { - if (!searchTerm.trim()) { - return text - } - - const regex = new RegExp(`(${searchTerm.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')})`, 'gi') - const parts = text.split(regex) - - return parts.map((part, index) => - regex.test(part) ? ( - - {part} - - ) : ( - part - ), - ) -} - -function MetricItem({ metric, searchTerm }: MetricItemProps) { +function MetricItem({ metric }: MetricItemProps) { return (
- {highlightText(metric.name, searchTerm)} + {metric.name}
{metric.type} @@ -78,16 +59,15 @@ function MetricItem({ metric, searchTerm }: MetricItemProps) { interface VirtualizedMetricsListProps { metrics: Metric[] - searchTerm: string } -function VirtualizedMetricsList({ metrics, searchTerm }: VirtualizedMetricsListProps) { +function VirtualizedMetricsList({ metrics }: VirtualizedMetricsListProps) { const parentRef = useRef(null) const virtualizer = useVirtualizer({ count: metrics.length, getScrollElement: () => parentRef.current, - estimateSize: () => 100, // Initial estimate, will be measured dynamically + estimateSize: () => 100, overscan: 5, }) @@ -118,7 +98,7 @@ function VirtualizedMetricsList({ metrics, searchTerm }: VirtualizedMetricsListP }} >
- +
) @@ -128,6 +108,65 @@ function VirtualizedMetricsList({ metrics, searchTerm }: VirtualizedMetricsListP ) } +interface JsonataResult { + type: 'metrics' | 'primitive' | 'error' | 'empty' + metrics?: Metric[] + primitiveValue?: unknown + error?: string +} + +function isMetricLike(item: unknown): item is Metric { + return ( + typeof item === 'object' && + item !== null && + 'id' in item && + 'name' in item && + 'value' in item && + 'type' in item && + 'jobId' in item && + 'timestamp' in item + ) +} + +async function evaluateJsonata(expression: string, metrics: Metric[]): Promise { + if (!expression.trim()) { + return { type: 'empty' } + } + + try { + const compiled = jsonata(expression) + const result = await compiled.evaluate(metrics) + + // Check if result is undefined/null + if (result === undefined || result === null) { + return { type: 'primitive', primitiveValue: result } + } + + // Check if result is an array + if (Array.isArray(result)) { + // Check if it looks like an array of metrics + const isMetricsArray = result.every(isMetricLike) + + if (isMetricsArray) { + return { type: 'metrics', metrics: result } + } + + // It's an array but not metrics - show as primitive + return { type: 'primitive', primitiveValue: result } + } + + // Check if it's a single metric object + if (isMetricLike(result)) { + return { type: 'metrics', metrics: [result] } + } + + // It's a primitive value (string, number, boolean, object without metric shape) + return { type: 'primitive', primitiveValue: result } + } catch (err) { + return { type: 'error', error: err instanceof Error ? err.message : 'Unknown error' } + } +} + interface MetricsModalProps { open: boolean onClose: () => void @@ -139,37 +178,27 @@ interface MetricsModalProps { } function MetricsModal({ open, onClose, title, metrics, total, isLoading, error }: MetricsModalProps) { - const [searchTerm, setSearchTerm] = useState('') - const [debouncedSearchTerm] = useDebounceValue(searchTerm, 300) + const [query, setQuery] = useState('') + const [debouncedQuery] = useDebounceValue(query, 300) + const [jsonataResult, setJsonataResult] = useState({ type: 'empty' }) - // Filter metrics based on debounced search term - const filteredMetrics = useMemo(() => { - if (!debouncedSearchTerm.trim()) { - return metrics - } + // Evaluate JSONata expression asynchronously + useEffect(() => { + let cancelled = false - const lowerSearch = debouncedSearchTerm.toLowerCase() - return metrics.filter((metric) => { - // Search in name - if (metric.name.toLowerCase().includes(lowerSearch)) { - return true - } - // Search in type - if (metric.type.toLowerCase().includes(lowerSearch)) { - return true - } - // Search in value (as string) - if (String(metric.value).includes(lowerSearch)) { - return true - } - // Search in attributes - const attributesStr = JSON.stringify(metric.attributes).toLowerCase() - if (attributesStr.includes(lowerSearch)) { - return true + evaluateJsonata(debouncedQuery, metrics).then((result) => { + if (!cancelled) { + setJsonataResult(result) } - return false }) - }, [metrics, debouncedSearchTerm]) + + return () => { + cancelled = true + } + }, [metrics, debouncedQuery]) + + // Determine which metrics to display + const displayMetrics = jsonataResult.type === 'metrics' ? jsonataResult.metrics! : metrics return ( !isOpen && onClose()}> @@ -182,32 +211,58 @@ function MetricsModal({ open, onClose, title, metrics, total, isLoading, error }
- {/* Search Input */} + {/* JSONata Query Input */}
- - setSearchTerm(e.target.value)} - className="pl-10 pr-10" +
+ + JSONata Query +
+