From 3100db4d4b7214ad6a104f243fac859cf98b8b62 Mon Sep 17 00:00:00 2001 From: KeKs0r Date: Fri, 18 Sep 2026 18:13:00 -0700 Subject: [PATCH 01/11] =?UTF-8?q?=E2=9C=A8=20Add=20@chkit/plugin-ingest:?= =?UTF-8?q?=20journaled=20pull=20ingestion=20runtime?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .changeset/config.json | 1 + .changeset/ingestion-runtime.md | 14 + CLAUDE.md | 1 + README.md | 1 + .../content/docs/configuration/overview.md | 12 + apps/docs/src/content/docs/plugins/ingest.md | 135 +++++ .../docs/src/content/docs/plugins/overview.md | 1 + bun.lock | 34 +- packages/cli/src/runtime/config-merge.ts | 1 + packages/clickhouse/src/index.ts | 3 + packages/core/src/model-types.ts | 13 +- packages/core/src/model.ts | 10 +- packages/core/src/on-cluster.test.ts | 12 + packages/plugin-ingest/README.md | 22 + packages/plugin-ingest/package.json | 55 ++ packages/plugin-ingest/src/destination.ts | 41 ++ packages/plugin-ingest/src/errors.ts | 139 +++++ packages/plugin-ingest/src/executor.test.ts | 307 ++++++++++ packages/plugin-ingest/src/executor.ts | 531 ++++++++++++++++++ packages/plugin-ingest/src/incremental.ts | 87 +++ packages/plugin-ingest/src/index.ts | 28 + packages/plugin-ingest/src/ingest.e2e.test.ts | 93 +++ packages/plugin-ingest/src/journal.ts | 238 ++++++++ packages/plugin-ingest/src/loader.ts | 46 ++ packages/plugin-ingest/src/paginate.ts | 44 ++ packages/plugin-ingest/src/plugin.ts | 291 ++++++++++ packages/plugin-ingest/src/queue.ts | 56 ++ packages/plugin-ingest/src/registry.ts | 197 +++++++ packages/plugin-ingest/src/retry.ts | 112 ++++ packages/plugin-ingest/src/semaphore.ts | 51 ++ packages/plugin-ingest/src/testing.ts | 54 ++ packages/plugin-ingest/src/types.ts | 249 ++++++++ packages/plugin-ingest/tsconfig.json | 9 + 33 files changed, 2877 insertions(+), 11 deletions(-) create mode 100644 .changeset/ingestion-runtime.md create mode 100644 apps/docs/src/content/docs/plugins/ingest.md create mode 100644 packages/plugin-ingest/README.md create mode 100644 packages/plugin-ingest/package.json create mode 100644 packages/plugin-ingest/src/destination.ts create mode 100644 packages/plugin-ingest/src/errors.ts create mode 100644 packages/plugin-ingest/src/executor.test.ts create mode 100644 packages/plugin-ingest/src/executor.ts create mode 100644 packages/plugin-ingest/src/incremental.ts create mode 100644 packages/plugin-ingest/src/index.ts create mode 100644 packages/plugin-ingest/src/ingest.e2e.test.ts create mode 100644 packages/plugin-ingest/src/journal.ts create mode 100644 packages/plugin-ingest/src/loader.ts create mode 100644 packages/plugin-ingest/src/paginate.ts create mode 100644 packages/plugin-ingest/src/plugin.ts create mode 100644 packages/plugin-ingest/src/queue.ts create mode 100644 packages/plugin-ingest/src/registry.ts create mode 100644 packages/plugin-ingest/src/retry.ts create mode 100644 packages/plugin-ingest/src/semaphore.ts create mode 100644 packages/plugin-ingest/src/testing.ts create mode 100644 packages/plugin-ingest/src/types.ts create mode 100644 packages/plugin-ingest/tsconfig.json diff --git a/.changeset/config.json b/.changeset/config.json index 488b07f8..efd130ea 100644 --- a/.changeset/config.json +++ b/.changeset/config.json @@ -12,6 +12,7 @@ "@chkit/plugin-codegen", "@chkit/plugin-pull", "@chkit/plugin-backfill", + "@chkit/plugin-ingest", "@chkit/plugin-obsessiondb" ] ], diff --git a/.changeset/ingestion-runtime.md b/.changeset/ingestion-runtime.md new file mode 100644 index 00000000..bf660798 --- /dev/null +++ b/.changeset/ingestion-runtime.md @@ -0,0 +1,14 @@ +--- +"@chkit/plugin-ingest": patch +"@chkit/clickhouse": patch +"@chkit/core": patch +"chkit": patch +--- + +Add `@chkit/plugin-ingest`, the first cut of scheduled pull ingestion into ClickHouse. Streams are ordinary TypeScript: a `read` async generator fetches, maps, and yields destination-shaped rows, and `definePipeline` registers a tagged, non-durable group of streams from the project entry. `chkit ingest run` executes the selected streams (`--tag` is repeatable with exact AND semantics; an explicit empty selection fails), `chkit ingest list` shows the loaded graph, and `chkit ingest status` prints committed checkpoints. + +Progress follows one rule: rows are saved before the bookmark advances. Every batch is written with a stable `insert_deduplication_token`, and only after the ClickHouse acknowledgement does the executor append a `batch_committed` fact to the append-only ingestion journal. Checkpoints are a projection of that journal, so a crashed or lost-acknowledgement run replays from the last durable boundary with the same batch identity instead of skipping rows. Bundled strategies are `timestampWindow`, `cursorState` for provider-owned state, and the full-sync fallback; `--backfill ` runs an explicit range in an isolated checkpoint namespace. + +Source operations run through `context.attempt`, which owns fetch permits, p-retry-shaped retry policy, `Retry-After`, cancellation, and failure classification (`HttpError.fromResponse` is the canonical boundary for fetch-based readers). Pipelines carry separate `maxStreams`, `maxFetches`, and `maxLoads` ceilings, executions have a duration budget, and the executor emits OpenTelemetry spans. + +`@chkit/core` gains a singular `entry` config field, mutually exclusive with `schema` globs: the module is imported once, its exported schema definitions are collected, and plugin-domain definitions self-register while it loads. `@chkit/clickhouse` `insert()` accepts per-insert `settings`. diff --git a/CLAUDE.md b/CLAUDE.md index d2adff38..93dbac37 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -21,6 +21,7 @@ This is a monorepo managed with Bun workspaces and Turborepo. | `packages/plugin-codegen` | `@chkit/plugin-codegen` | Plugin: TypeScript type + Zod schema generation | | `packages/plugin-pull` | `@chkit/plugin-pull` | Plugin: introspect live ClickHouse into schema files | | `packages/plugin-backfill` | `@chkit/plugin-backfill` | Plugin: time-windowed data backfill with checkpoints | +| `packages/plugin-ingest` | `@chkit/plugin-ingest` | Plugin: scheduled pull ingestion with journaled checkpoints | | `packages/plugin-obsessiondb` | `@chkit/plugin-obsessiondb` | Plugin: ObsessionDB integration; auto-rewrites `Shared` engines for non-ObsessionDB targets | ### Documentation diff --git a/README.md b/README.md index 4054d8f4..0bfdf8d6 100644 --- a/README.md +++ b/README.md @@ -114,6 +114,7 @@ See the [configuration docs](https://chkit.obsessiondb.com/configuration/overvie | [`@chkit/plugin-pull`](packages/plugin-pull) | Pull live schema into local files | | [`@chkit/plugin-codegen`](packages/plugin-codegen) | Codegen plugin for the CLI | | [`@chkit/plugin-backfill`](packages/plugin-backfill) | Backfill plugin for data migrations | +| [`@chkit/plugin-ingest`](packages/plugin-ingest) | Ingestion plugin: scheduled API pulls with journaled checkpoints | | [`@chkit/plugin-obsessiondb`](packages/plugin-obsessiondb) | ObsessionDB integration: auto-rewrite `Shared` engines for ClickHouse targets | ## Documentation diff --git a/apps/docs/src/content/docs/configuration/overview.md b/apps/docs/src/content/docs/configuration/overview.md index 5762535f..66a63aaf 100644 --- a/apps/docs/src/content/docs/configuration/overview.md +++ b/apps/docs/src/content/docs/configuration/overview.md @@ -37,6 +37,18 @@ export default defineConfig({ }) ``` +## Project entry (`entry`) + +Instead of `schema` globs you can point chkit at one entry module: + +```ts +export default defineConfig({ + entry: './src/chkit.ts', +}) +``` + +chkit imports the module once. Schema definitions it exports (directly or re-exported from other files) are collected exactly like glob-matched schema files, and plugin-domain definitions such as [ingestion pipelines](/plugins/ingest/) register themselves while it loads. `entry` and `schema` are mutually exclusive. + ## Cluster mode (`ON CLUSTER`) For self-managed multi-node ClickHouse clusters, set `clickhouse.cluster` to the cluster name from your server's `remote_servers` config: diff --git a/apps/docs/src/content/docs/plugins/ingest.md b/apps/docs/src/content/docs/plugins/ingest.md new file mode 100644 index 00000000..23673e78 --- /dev/null +++ b/apps/docs/src/content/docs/plugins/ingest.md @@ -0,0 +1,135 @@ +--- +title: Ingest Plugin +description: Scheduled pull ingestion from application APIs into ClickHouse with journaled checkpoints. +sidebar: + order: 5 +--- + +This document covers practical usage of the optional `ingest` plugin. + +## What it does + +- Runs finite, scheduled pulls from application APIs into chkit-managed tables. +- Keeps progress in an append-only ingestion journal inside the target database. Checkpoints are a projection of that journal. +- Saves rows before advancing the bookmark. A crash may cause rereading; it never causes unsaved rows to be skipped. +- Retries source requests with backoff, `Retry-After`, and failure classification. +- Selects streams by exact tags so any external scheduler (cron, CI, Kubernetes) can drive it. + +The plugin never creates or changes destination tables. Your chkit schema stays the only DDL authority. + +## Plugin setup + +Ingestion uses the singular `entry` config field instead of `schema` globs. The entry module is imported once: exported tables are collected as schema, and pipelines register themselves while it loads. + +```ts +// clickhouse.config.ts +import { defineConfig } from '@chkit/core' +import { ingest } from '@chkit/plugin-ingest' + +export default defineConfig({ + entry: './src/chkit.ts', + plugins: [ingest()], + clickhouse: { url: process.env.CLICKHOUSE_URL ?? '' }, +}) +``` + +## Writing a stream + +A stream is a destination table plus a `read` generator. Fetching and mapping are ordinary code inside `read`; each yielded chunk carries rows already shaped for the table. + +```ts +// src/chkit.ts +import { table } from '@chkit/core' +import { HttpError, definePipeline, defineStream, ingestionColumns, paginate, timestampWindow } from '@chkit/plugin-ingest' + +export const tickets = table({ + database: 'crm', + name: 'tickets', + columns: [ + { name: 'id', type: 'String' }, + { name: 'updated_at', type: "DateTime64(3, 'UTC')" }, + { name: 'raw', type: 'String' }, + ...ingestionColumns, + ], + engine: 'ReplacingMergeTree(updated_at)', + primaryKey: ['id'], + orderBy: ['id'], +}) + +const ticketStream = defineStream({ + id: 'helpdesk.tickets', + destination: tickets, + tags: ['schedule:1h'], + incremental: timestampWindow({ + // Re-read one hour of overlap; ReplacingMergeTree reconciles repeats. + from: ({ watermark }) => (watermark ? new Date(watermark.getTime() - 3_600_000) : new Date(0)), + }), + async *read(context) { + const pages = paginate({ + context, + label: 'GET /tickets', + fetchPage: async (cursor: string | undefined, signal) => { + const url = new URL('https://api.example.com/tickets') + url.searchParams.set('updated_since', context.selection.from.toISOString()) + if (cursor) url.searchParams.set('cursor', cursor) + const response = await fetch(url, { signal, headers: { Authorization: `Bearer ${process.env.HELPDESK_TOKEN}` } }) + if (!response.ok) throw await HttpError.fromResponse(response) + const body = await response.json() + return { items: body.data, next: body.next_cursor ?? undefined } + }, + }) + for await (const items of pages) { + yield { rows: items.map((item) => ({ id: item.id, updated_at: item.updated_at, raw: JSON.stringify(item) })) } + } + }, +}) + +definePipeline({ id: 'helpdesk', streams: [ticketStream], maxFetches: 4 }) +``` + +Spread `ingestionColumns` into every destination table. The loader fills `_chkit_batch_id` and `_chkit_run_id`; `_chkit_ingested_at` is set by ClickHouse at the physical insert. + +Rows must be deterministic for a given source page. Batch identity includes a content hash, so a field like `synced_at: new Date()` in a row defeats retry deduplication. + +## Progress and checkpoints + +| Strategy | Use when | Bookmark advances | +|---|---|---| +| none (full sync) | The source is small or has no change filter | Never; every run reads everything | +| `timestampWindow({ from })` | The API filters by an updated-since timestamp | To the run cutoff, after the whole window loaded | +| `cursorState({ id, version, parse })` | The provider owns the state: compound cursor, change token, page position | Whenever a yielded chunk carries `state` and its rows have been saved | + +With `cursorState`, `state` on a chunk must be the complete state that is safe to resume from once every row up to that chunk is saved. Omit it when you cannot make that claim; the run then restarts from the previous checkpoint after a failure. + +A checkpoint records its strategy id and version. Changing either makes the next run fail rather than reinterpret old state. + +## Commands + +```sh +chkit ingest list # streams in the loaded graph +chkit ingest run # run every stream +chkit ingest run --tag schedule:1h # exact tag match; repeat --tag for AND +chkit ingest run --tag stream:helpdesk.tickets +chkit ingest status # committed checkpoint per stream +chkit ingest run --backfill jan --from 2026-01-01 --to 2026-02-01 +``` + +Every stream also carries the derived tags `pipeline:` and `stream:`. `schedule:` is a convention only: chkit never interprets it. A `--tag` filter that matches nothing fails before any work runs. + +A backfill uses its own checkpoint namespace, so it never moves the scheduled bookmark. Rerunning the same `--backfill` id resumes it. + +`chkit check` verifies that every stream destination carries the ingestion metadata columns. + +## Delivery guarantee + +Ingestion is at-least-once. Each batch is inserted with a stable `insert_deduplication_token`, so a retry after a lost acknowledgement is suppressed while the table's deduplication window covers it. Pick a destination engine that reconciles repeats for your data, for example `ReplacingMergeTree` keyed by the provider id. + +Run at most one ingestion process per project and target at a time. Use your scheduler's concurrency control (for example a GitHub Actions concurrency group) to enforce it. + +## Options + +| Option | Default | Description | +|---|---|---| +| `journalTable` | `_chkit_ingestion_journal` | Journal table name in the configured database | +| `maxDurationSeconds` | `3600` | Execution budget. Exhausting it ends the run as incomplete and keeps committed progress | +| `prefetchBatches` | `1` | Mapped batches buffered between fetching and loading | diff --git a/apps/docs/src/content/docs/plugins/overview.md b/apps/docs/src/content/docs/plugins/overview.md index f58a2058..fe322ebe 100644 --- a/apps/docs/src/content/docs/plugins/overview.md +++ b/apps/docs/src/content/docs/plugins/overview.md @@ -38,3 +38,4 @@ If you deploy to [ObsessionDB](https://obsessiondb.com), start at the dedicated - [`@chkit/plugin-codegen`](/plugins/codegen/) — TypeScript row types and optional Zod schemas, generated from your schema files. - [`@chkit/plugin-pull`](/plugins/pull/) — introspect a live ClickHouse database into local schema files. Useful for adopting chkit on an existing database. - [`@chkit/plugin-backfill`](/plugins/backfill/) — time-windowed data backfill with checkpoints, for materialized views and historical data loads. +- [`@chkit/plugin-ingest`](/plugins/ingest/) — scheduled pull ingestion from application APIs with journaled checkpoints. diff --git a/bun.lock b/bun.lock index 4cc04496..4adcbb26 100644 --- a/bun.lock +++ b/bun.lock @@ -47,7 +47,7 @@ }, "packages/cli": { "name": "chkit", - "version": "0.1.2-beta.4", + "version": "0.1.2-beta.7", "bin": { "chkit": "./dist/bin/chkit.js", }, @@ -64,7 +64,7 @@ }, "packages/clickhouse": { "name": "@chkit/clickhouse", - "version": "0.1.2-beta.4", + "version": "0.1.2-beta.7", "dependencies": { "@chkit/core": "workspace:*", "@clickhouse/client": "^1.18.0", @@ -74,14 +74,14 @@ }, "packages/codegen": { "name": "@chkit/codegen", - "version": "0.1.2-beta.4", + "version": "0.1.2-beta.7", "dependencies": { "@chkit/core": "0.1.0-beta.26", }, }, "packages/core": { "name": "@chkit/core", - "version": "0.1.2-beta.4", + "version": "0.1.2-beta.7", "dependencies": { "fast-glob": "^3.3.2", "jiti": "^2.7.0", @@ -92,7 +92,7 @@ }, "packages/create-chkit": { "name": "create-chkit", - "version": "0.1.2-beta.4", + "version": "0.1.2-beta.7", "bin": { "create-chkit": "./dist/bin/create-chkit.js", }, @@ -109,7 +109,7 @@ }, "packages/plugin-backfill": { "name": "@chkit/plugin-backfill", - "version": "0.1.2-beta.4", + "version": "0.1.2-beta.7", "dependencies": { "@chkit/clickhouse": "workspace:*", "@chkit/core": "workspace:*", @@ -120,7 +120,7 @@ }, "packages/plugin-codegen": { "name": "@chkit/plugin-codegen", - "version": "0.1.2-beta.4", + "version": "0.1.2-beta.7", "dependencies": { "@chkit/core": "workspace:*", }, @@ -131,9 +131,19 @@ "zod": "^4.0.0", }, }, + "packages/plugin-ingest": { + "name": "@chkit/plugin-ingest", + "version": "0.1.0-beta.0", + "dependencies": { + "@chkit/clickhouse": "workspace:*", + "@chkit/core": "workspace:*", + "@opentelemetry/api": "^1.9.1", + "zod": "^4.3.6", + }, + }, "packages/plugin-obsessiondb": { "name": "@chkit/plugin-obsessiondb", - "version": "0.1.2-beta.4", + "version": "0.1.2-beta.7", "dependencies": { "@chkit/clickhouse": "workspace:*", "@chkit/core": "workspace:*", @@ -146,7 +156,7 @@ }, "packages/plugin-pull": { "name": "@chkit/plugin-pull", - "version": "0.1.2-beta.4", + "version": "0.1.2-beta.7", "dependencies": { "@chkit/clickhouse": "workspace:*", "@chkit/core": "workspace:*", @@ -281,6 +291,8 @@ "@chkit/plugin-codegen": ["@chkit/plugin-codegen@workspace:packages/plugin-codegen"], + "@chkit/plugin-ingest": ["@chkit/plugin-ingest@workspace:packages/plugin-ingest"], + "@chkit/plugin-obsessiondb": ["@chkit/plugin-obsessiondb@workspace:packages/plugin-obsessiondb"], "@chkit/plugin-pull": ["@chkit/plugin-pull@workspace:packages/plugin-pull"], @@ -451,6 +463,8 @@ "@nodelib/fs.walk": ["@nodelib/fs.walk@1.2.8", "", { "dependencies": { "@nodelib/fs.scandir": "2.1.5", "fastq": "^1.6.0" } }, "sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg=="], + "@opentelemetry/api": ["@opentelemetry/api@1.9.1", "", {}, "sha512-gLyJlPHPZYdAk1JENA9LeHejZe1Ti77/pTeFm/nMXmQH/HFZlcS/O2XJB+L8fkbrNSqhdtlvjBVjxwUYanNH5Q=="], + "@orpc/client": ["@orpc/client@1.13.4", "", { "dependencies": { "@orpc/shared": "1.13.4", "@orpc/standard-server": "1.13.4", "@orpc/standard-server-fetch": "1.13.4", "@orpc/standard-server-peer": "1.13.4" } }, "sha512-s13GPMeoooJc5Th2EaYT5HMFtWG8S03DUVytYfJv8pIhP87RYKl94w52A36denH6r/B4LaAgBeC9nTAOslK+Og=="], "@orpc/contract": ["@orpc/contract@1.13.4", "", { "dependencies": { "@orpc/client": "1.13.4", "@orpc/shared": "1.13.4", "@standard-schema/spec": "^1.1.0", "openapi-types": "^12.1.3" } }, "sha512-TIxyaF67uOlihCRcasjHZxguZpbqfNK7aMrDLnhoufmQBE4OKvguNzmrOFHgsuM0OXoopX0Nuhun1ccaxKP10A=="], @@ -1519,6 +1533,8 @@ "@chkit/plugin-codegen/zod": ["zod@4.3.6", "", {}, "sha512-rftlrkhHZOcjDwkGlnUtZZkvaPHCsDATp4pGpuOOMDaTdDDXF91wuVDJoWoPsKX/3YPQ5fHuF3STjcYyKr+Qhg=="], + "@chkit/plugin-ingest/zod": ["zod@4.3.6", "", {}, "sha512-rftlrkhHZOcjDwkGlnUtZZkvaPHCsDATp4pGpuOOMDaTdDDXF91wuVDJoWoPsKX/3YPQ5fHuF3STjcYyKr+Qhg=="], + "@chkit/plugin-obsessiondb/zod": ["zod@4.3.6", "", {}, "sha512-rftlrkhHZOcjDwkGlnUtZZkvaPHCsDATp4pGpuOOMDaTdDDXF91wuVDJoWoPsKX/3YPQ5fHuF3STjcYyKr+Qhg=="], "@chkit/plugin-pull/zod": ["zod@4.3.6", "", {}, "sha512-rftlrkhHZOcjDwkGlnUtZZkvaPHCsDATp4pGpuOOMDaTdDDXF91wuVDJoWoPsKX/3YPQ5fHuF3STjcYyKr+Qhg=="], diff --git a/packages/cli/src/runtime/config-merge.ts b/packages/cli/src/runtime/config-merge.ts index 20b9705a..2b7f0aff 100644 --- a/packages/cli/src/runtime/config-merge.ts +++ b/packages/cli/src/runtime/config-merge.ts @@ -71,6 +71,7 @@ export function mergeUserConfig( ): ChxUserConfig { return { schema: overlay.schema ?? base.schema, + entry: overlay.entry ?? base.entry, outDir: overlay.outDir ?? base.outDir, migrationsDir: overlay.migrationsDir ?? base.migrationsDir, metaDir: overlay.metaDir ?? base.metaDir, diff --git a/packages/clickhouse/src/index.ts b/packages/clickhouse/src/index.ts index ca3d6270..8a20c013 100644 --- a/packages/clickhouse/src/index.ts +++ b/packages/clickhouse/src/index.ts @@ -42,6 +42,8 @@ export interface ClickHouseInsertParams> { table: string values: T[] compressed?: boolean + /** Per-insert settings, e.g. a stable `insert_deduplication_token`. */ + settings?: ClickHouseSettings } export interface ClickHouseJsonQueryResult< @@ -735,6 +737,7 @@ export function createExecutorWithClient( table: params.table, values: params.values, format: 'JSONEachRow', + ...(params.settings ? { clickhouse_settings: params.settings } : {}), }) assertStreamedQuerySucceeded({ response_headers: result.response_headers, diff --git a/packages/core/src/model-types.ts b/packages/core/src/model-types.ts index 8ecf54c7..86c86078 100644 --- a/packages/core/src/model-types.ts +++ b/packages/core/src/model-types.ts @@ -266,7 +266,17 @@ export interface ChxInlinePluginRegistration< export type ChxPluginRegistration = ChxInlinePluginRegistration export interface ChxUserConfig { - schema: string | string[] + /** + * Glob patterns for schema files. Mutually exclusive with `entry`. + */ + schema?: string | string[] + /** + * Single project entry module. It is imported once: exported schema + * definitions are collected from it, and plugin-domain definitions (for + * example ingestion pipelines) self-register while it loads. Mutually + * exclusive with `schema`. + */ + entry?: string outDir?: string migrationsDir?: string metaDir?: string @@ -278,6 +288,7 @@ export interface ChxUserConfig { export interface ChxResolvedConfig { schema: string[] + entry?: string outDir: string migrationsDir: string metaDir: string diff --git a/packages/core/src/model.ts b/packages/core/src/model.ts index 7d46afb0..6fb4cb20 100644 --- a/packages/core/src/model.ts +++ b/packages/core/src/model.ts @@ -53,8 +53,16 @@ export function resolveConfig(config: ChxUserConfig): ChxResolvedConfig { const migrationsDir = config.migrationsDir ?? join(outDir, 'migrations') const metaDir = config.metaDir ?? join(outDir, 'meta') + const schemaGlobs = config.schema === undefined ? [] : Array.isArray(config.schema) ? config.schema : [config.schema] + if (config.entry !== undefined && schemaGlobs.length > 0) { + throw new Error('Config fields "entry" and "schema" are mutually exclusive. Use one project entry module or schema globs, not both.') + } + return { - schema: Array.isArray(config.schema) ? config.schema : [config.schema], + // In entry mode the entry module is the only schema source: its exported + // definitions are collected exactly like any other schema file. + schema: config.entry !== undefined ? [config.entry] : schemaGlobs, + entry: config.entry, outDir, migrationsDir, metaDir, diff --git a/packages/core/src/on-cluster.test.ts b/packages/core/src/on-cluster.test.ts index 163771b1..788b43f1 100644 --- a/packages/core/src/on-cluster.test.ts +++ b/packages/core/src/on-cluster.test.ts @@ -209,6 +209,18 @@ describe('applyOnClusterToPlan', () => { }) }) +describe('resolveConfig project entry', () => { + test('entry mode makes the entry module the only schema source', () => { + const resolved = resolveConfig({ entry: './src/chkit.ts' }) + expect(resolved.entry).toBe('./src/chkit.ts') + expect(resolved.schema).toEqual(['./src/chkit.ts']) + }) + + test('entry and schema globs are mutually exclusive', () => { + expect(() => resolveConfig({ entry: './src/chkit.ts', schema: './src/schema/**/*.ts' })).toThrow('mutually exclusive') + }) +}) + describe('resolveConfig cluster validation', () => { test('passes through an identifier and a macro', () => { expect(resolveConfig({ schema: 's', clickhouse: { url: 'u', cluster: 'my_cluster' } }).clickhouse?.cluster).toBe( diff --git a/packages/plugin-ingest/README.md b/packages/plugin-ingest/README.md new file mode 100644 index 00000000..001c968d --- /dev/null +++ b/packages/plugin-ingest/README.md @@ -0,0 +1,22 @@ +# @chkit/plugin-ingest + +Scheduled pull ingestion from application APIs into ClickHouse for [chkit](https://www.npmjs.com/package/chkit), with journaled checkpoints. + +Rows are saved before the bookmark advances: a crash may cause rereading, never skipped rows. + +```ts +import { defineConfig } from '@chkit/core' +import { ingest } from '@chkit/plugin-ingest' + +export default defineConfig({ + entry: './src/chkit.ts', + plugins: [ingest()], + clickhouse: { url: process.env.CLICKHOUSE_URL ?? '' }, +}) +``` + +```sh +chkit ingest run --tag schedule:1h +``` + +Documentation: https://chkit.obsessiondb.com/plugins/ingest/ diff --git a/packages/plugin-ingest/package.json b/packages/plugin-ingest/package.json new file mode 100644 index 00000000..6e244551 --- /dev/null +++ b/packages/plugin-ingest/package.json @@ -0,0 +1,55 @@ +{ + "name": "@chkit/plugin-ingest", + "version": "0.1.2-beta.7", + "description": "Scheduled pull ingestion into ClickHouse with journaled checkpoints for chkit", + "license": "MIT", + "author": "ObsessionDB", + "keywords": [ + "clickhouse", + "ingestion", + "etl", + "chkit-plugin" + ], + "homepage": "https://chkit.obsessiondb.com", + "bugs": { + "url": "https://github.com/obsessiondb/chkit/issues" + }, + "repository": { + "type": "git", + "url": "https://github.com/obsessiondb/chkit.git", + "directory": "packages/plugin-ingest" + }, + "type": "module", + "main": "./dist/index.js", + "types": "./dist/index.d.ts", + "exports": { + ".": { + "source": "./src/index.ts", + "types": "./dist/index.d.ts", + "default": "./dist/index.js" + }, + "./testing": { + "source": "./src/testing.ts", + "types": "./dist/testing.d.ts", + "default": "./dist/testing.js" + } + }, + "files": [ + "dist" + ], + "scripts": { + "build": "tsc -p tsconfig.json", + "dev": "tsc -p tsconfig.json --watch", + "typecheck": "tsc -p tsconfig.json --noEmit", + "lint": "biome lint src", + "test": "bun test src", + "test:env": "doppler run --project chkit --config ci -- bun test src", + "clean": "rm -rf dist" + }, + "dependencies": { + "@chkit/clickhouse": "workspace:*", + "@chkit/core": "workspace:*", + "@opentelemetry/api": "^1.9.1", + "zod": "^4.3.6" + } +} diff --git a/packages/plugin-ingest/src/destination.ts b/packages/plugin-ingest/src/destination.ts new file mode 100644 index 00000000..491757fe --- /dev/null +++ b/packages/plugin-ingest/src/destination.ts @@ -0,0 +1,41 @@ +import type { ClickHouseExecutor } from '@chkit/clickhouse' +import type { ColumnDefinition } from '@chkit/core' + +import type { DestinationAdapter } from './types.js' + +export const BATCH_ID_COLUMN = '_chkit_batch_id' +export const RUN_ID_COLUMN = '_chkit_run_id' +export const INGESTED_AT_COLUMN = '_chkit_ingested_at' + +/** + * Runtime-owned metadata every ingestion destination table carries. Spread it + * into the table's `columns`. `_chkit_ingested_at` is destination-owned: it is + * never sent by the loader, so it records the physical publication time even + * when a retry recomputes it under the same deduplication token. + */ +export const ingestionColumns: readonly ColumnDefinition[] = [ + { name: BATCH_ID_COLUMN, type: 'String' }, + { name: RUN_ID_COLUMN, type: 'String' }, + { name: INGESTED_AT_COLUMN, type: "DateTime64(6, 'UTC')", default: 'fn:now64(6)' }, +] + +/** + * A successful synchronous insert response (or an awaited async insert) is the + * sink evidence ChKit trusts. A missing response stays ambiguous and is retried + * with the same token. + */ +export function createClickHouseDestination(executor: ClickHouseExecutor): DestinationAdapter { + return { + async insert({ table, rows, token }) { + if (rows.length === 0) return + await executor.insert({ + table: `${table.database}.${table.name}`, + values: [...rows], + settings: { + insert_deduplication_token: token, + wait_for_async_insert: 1, + }, + }) + }, + } +} diff --git a/packages/plugin-ingest/src/errors.ts b/packages/plugin-ingest/src/errors.ts new file mode 100644 index 00000000..fbdb0682 --- /dev/null +++ b/packages/plugin-ingest/src/errors.ts @@ -0,0 +1,139 @@ +import type { ErrorClassifier, FailureClass } from './types.js' + +const DIAGNOSTIC_BODY_LIMIT = 2048 +const TRANSIENT_NETWORK_CODES = new Set([ + 'ECONNRESET', + 'ECONNREFUSED', + 'ECONNABORTED', + 'ETIMEDOUT', + 'EPIPE', + 'EAI_AGAIN', + 'ENOTFOUND', + 'UND_ERR_SOCKET', + 'UND_ERR_CONNECT_TIMEOUT', + 'UND_ERR_HEADERS_TIMEOUT', + 'UND_ERR_BODY_TIMEOUT', +]) + +export class IngestConfigError extends Error { + constructor(message: string) { + super(message) + this.name = 'IngestConfigError' + } +} + +/** + * Canonical boundary for fetch-based readers when `response.ok` is false. + * Keeps normalized HTTP facts plus the original Response for raw inspection. + */ +export class HttpError extends Error { + readonly status: number + readonly statusText: string + readonly url: string + readonly retryAfterMs: number | undefined + readonly body: string + readonly response: Response + + private constructor(response: Response, body: string) { + super(`HTTP ${response.status} ${response.statusText} for ${redactUrl(response.url)}${body ? `: ${body}` : ''}`) + this.name = 'HttpError' + this.status = response.status + this.statusText = response.statusText + this.url = redactUrl(response.url) + this.retryAfterMs = parseRetryAfter(response.headers.get('retry-after')) + this.body = body + this.response = response + } + + static async fromResponse(response: Response): Promise { + const body = await readDiagnosticBody(response) + return new HttpError(response, body) + } +} + +/** Executor-owned failure that always preserves the exact thrown value. */ +export class FetchFailure extends Error { + readonly classification: FailureClass + override readonly cause: unknown + + constructor(cause: unknown, classification: FailureClass) { + super(cause instanceof Error ? cause.message : String(cause)) + this.name = 'FetchFailure' + this.cause = cause + this.classification = classification + } +} + +export class BudgetExhausted extends Error { + constructor(message: string) { + super(message) + this.name = 'BudgetExhausted' + } +} + +/** + * Cancellation is authoritative, then the common HTTP/network fallback is + * derived, then an optional provider classifier may enrich or override it. + */ +export function classifyFailure( + cause: unknown, + signal: AbortSignal, + classifier: ErrorClassifier | undefined +): FailureClass { + if (signal.aborted || isAbortError(cause)) return { kind: 'cancelled' } + const fallback = classifyCommon(cause) + return classifier?.(cause, fallback) ?? fallback +} + +export function classifyCommon(cause: unknown): FailureClass { + if (cause instanceof HttpError) { + if (cause.status === 429) return { kind: 'rate_limited', retryAfterMs: cause.retryAfterMs } + if (cause.status === 408 || cause.status === 425 || cause.status >= 500) return { kind: 'transient' } + if (cause.status >= 400) return { kind: 'permanent' } + return { kind: 'unknown' } + } + const code = errorCode(cause) + if (code !== undefined && TRANSIENT_NETWORK_CODES.has(code)) return { kind: 'transient' } + if (cause instanceof TypeError && /fetch failed|network|socket/i.test(cause.message)) return { kind: 'transient' } + if (cause instanceof Error && cause.name === 'TimeoutError') return { kind: 'transient' } + return { kind: 'unknown' } +} + +export function isAbortError(value: unknown): boolean { + return value instanceof Error && value.name === 'AbortError' +} + +function errorCode(value: unknown): string | undefined { + if (typeof value !== 'object' || value === null) return undefined + if ('code' in value && typeof value.code === 'string') return value.code + if ('cause' in value) return errorCode(value.cause) + return undefined +} + +function parseRetryAfter(header: string | null): number | undefined { + if (header === null) return undefined + const seconds = Number(header) + if (Number.isFinite(seconds)) return Math.max(0, seconds * 1000) + const date = Date.parse(header) + if (Number.isNaN(date)) return undefined + return Math.max(0, date - Date.now()) +} + +async function readDiagnosticBody(response: Response): Promise { + try { + const text = await response.clone().text() + return text.length > DIAGNOSTIC_BODY_LIMIT ? `${text.slice(0, DIAGNOSTIC_BODY_LIMIT)}…` : text + } catch { + return '' + } +} + +// Query strings may carry API keys; keep only origin and path in diagnostics. +function redactUrl(url: string): string { + try { + const parsed = new URL(url) + return `${parsed.origin}${parsed.pathname}` + } catch { + return url + } +} diff --git a/packages/plugin-ingest/src/executor.test.ts b/packages/plugin-ingest/src/executor.test.ts new file mode 100644 index 00000000..836ddc3f --- /dev/null +++ b/packages/plugin-ingest/src/executor.test.ts @@ -0,0 +1,307 @@ +import { beforeEach, describe, expect, test } from 'bun:test' + +import { table } from '@chkit/core' + +import { ingestionColumns } from './destination.js' +import { HttpError } from './errors.js' +import { runIngestion } from './executor.js' +import { cursorState, timestampWindow } from './incremental.js' +import { paginate } from './paginate.js' +import { definePipeline, defineStream, resetRegistry, selectStreams } from './registry.js' +import { createMemoryDestination, createMemoryJournal } from './testing.js' +import type { DestinationAdapter } from './types.js' + +const events = table({ + database: 'app', + name: 'events', + columns: [{ name: 'id', type: 'UInt64' }, ...ingestionColumns], + engine: 'MergeTree()', + primaryKey: ['id'], + orderBy: ['id'], +}) + +const noSleep = async () => undefined +const pages = (count: number, size: number) => + Array.from({ length: count }, (_, page) => Array.from({ length: size }, (_, index) => ({ id: page * size + index }))) + +function httpError(status: number, headers: Record = {}) { + return HttpError.fromResponse(new Response('nope', { status, headers })) +} + +beforeEach(() => resetRegistry()) + +describe('runIngestion', () => { + test('loads rows with runtime metadata and journals progress only after sink evidence', async () => { + const journal = createMemoryJournal() + const destination = createMemoryDestination() + const order: string[] = [] + const observed: DestinationAdapter = { + insert: async (input) => { + await destination.insert(input) + order.push('insert') + }, + } + const stream = defineStream({ + id: 'app.events', + destination: events, + async *read() { + yield { rows: [{ id: 1 }, { id: 2 }] } + }, + }) + const pipeline = definePipeline({ id: 'app', streams: [stream] }) + const append = journal.append.bind(journal) + journal.append = async (event) => { + if (event.eventKind === 'batch_committed') order.push('commit') + await append(event) + } + + const result = await runIngestion({ selected: selectStreams([pipeline], []), backfill: undefined }, { journal, destination: observed }) + + expect(result.ok).toBe(true) + expect(order).toEqual(['insert', 'commit']) + const rows = destination.tables.get('app.events') ?? [] + expect(rows.map((row) => row.id)).toEqual([1, 2]) + expect(rows[0]?._chkit_run_id).toBe(result.runId) + expect(typeof rows[0]?._chkit_batch_id).toBe('string') + expect(rows[0]).not.toHaveProperty('_chkit_ingested_at') + expect(journal.events.map((event) => event.eventKind)).toEqual([ + 'run_started', + 'work_planned', + 'attempt_started', + 'batch_committed', + 'work_finished', + 'run_finished', + ]) + }) + + test('a crash before sink evidence never advances the bookmark, and the restart neither skips nor duplicates rows', async () => { + const journal = createMemoryJournal() + const destination = createMemoryDestination() + const source = pages(3, 2) + const stream = defineStream({ + id: 'app.cursor', + destination: events, + batchSize: 2, + incremental: cursorState({ + id: 'test.page', + version: 1, + parse: (raw) => Number(raw), + }), + async *read({ selection }) { + for (let page = selection ?? 0; page < source.length; page += 1) { + yield { rows: source[page] ?? [], state: page + 1 } + } + }, + }) + const pipeline = definePipeline({ id: 'app', streams: [stream], retry: { retries: 0 } }) + let inserts = 0 + const crashing: DestinationAdapter = { + insert: async (input) => { + inserts += 1 + // Second batch: the write lands but the acknowledgement is lost. + if (inserts === 2) { + await destination.insert(input) + throw new Error('socket hang up') + } + if (inserts > 2 && inserts <= 4) throw new Error('still down') + await destination.insert(input) + }, + } + + const first = await runIngestion({ selected: selectStreams([pipeline], []), backfill: undefined }, { journal, destination: crashing, sleep: noSleep }) + expect(first.ok).toBe(false) + expect((await journal.readCheckpoint('app.cursor')).envelope?.state).toBe(1) + + const second = await runIngestion({ selected: selectStreams([pipeline], []), backfill: undefined }, { journal, destination, sleep: noSleep }) + expect(second.ok).toBe(true) + expect((await journal.readCheckpoint('app.cursor')).envelope?.state).toBe(3) + const ids = (destination.tables.get('app.events') ?? []).map((row) => row.id) + expect(ids).toEqual([0, 1, 2, 3, 4, 5]) + }) + + test('timestampWindow commits the cutoff as watermark only after the whole window loaded', async () => { + const journal = createMemoryJournal() + const destination = createMemoryDestination() + const selections: Array<{ from: string; to: string }> = [] + const stream = defineStream({ + id: 'app.window', + destination: events, + incremental: timestampWindow({ from: ({ watermark }) => watermark ?? new Date('2026-01-01T00:00:00Z') }), + async *read({ selection }) { + selections.push({ from: selection.from.toISOString(), to: selection.to.toISOString() }) + yield { rows: [{ id: selections.length }] } + }, + }) + const pipeline = definePipeline({ id: 'app', streams: [stream] }) + const times = [new Date('2026-02-01T00:00:00Z'), new Date('2026-03-01T00:00:00Z')] + + for (const time of times) { + await runIngestion({ selected: selectStreams([pipeline], []), backfill: undefined }, { journal, destination, now: () => time }) + } + + expect(selections).toEqual([ + { from: '2026-01-01T00:00:00.000Z', to: '2026-02-01T00:00:00.000Z' }, + { from: '2026-02-01T00:00:00.000Z', to: '2026-03-01T00:00:00.000Z' }, + ]) + expect((await journal.readCheckpoint('app.window')).envelope?.state).toEqual({ watermark: '2026-03-01T00:00:00.000Z' }) + }) + + test('a failed window leaves the watermark untouched', async () => { + const journal = createMemoryJournal() + const stream = defineStream({ + id: 'app.window', + destination: events, + retry: { retries: 0 }, + incremental: timestampWindow({ from: ({ watermark }) => watermark ?? new Date(0) }), + async *read() { + yield { rows: [{ id: 1 }] } + throw new Error('provider exploded') + }, + }) + const pipeline = definePipeline({ id: 'app', streams: [stream] }) + + const result = await runIngestion({ selected: selectStreams([pipeline], []), backfill: undefined }, { journal, destination: createMemoryDestination(), sleep: noSleep }) + + expect(result.streams[0]?.outcome).toBe('failed') + expect(result.streams[0]?.error).toContain('provider exploded') + expect((await journal.readCheckpoint('app.window')).envelope).toBeUndefined() + }) + + test('attempt retries rate limits and transient failures but not permanent ones', async () => { + const journal = createMemoryJournal() + const destination = createMemoryDestination() + const delays: number[] = [] + let calls = 0 + const flaky = defineStream({ + id: 'app.flaky', + destination: events, + retry: { randomize: false, minTimeout: 10 }, + async *read(context) { + const rows = await context.attempt(async () => { + calls += 1 + if (calls === 1) throw await httpError(429, { 'retry-after': '7' }) + if (calls === 2) throw await httpError(503) + return [{ id: 1 }] + }) + yield { rows } + }, + }) + const denied = defineStream({ + id: 'app.denied', + destination: events, + async *read(context) { + yield { rows: await context.attempt(async () => { throw await httpError(401) }) } + }, + }) + const pipeline = definePipeline({ id: 'app', streams: [flaky, denied] }) + + const result = await runIngestion( + { selected: selectStreams([pipeline], []), backfill: undefined }, + { journal, destination, sleep: async (ms) => { delays.push(ms) } } + ) + + expect(calls).toBe(3) + expect(delays).toEqual([7000, 20]) + // One failing stream does not stop its sibling. + expect(result.streams.map((stream) => stream.outcome)).toEqual(['succeeded', 'failed']) + expect(journal.events.filter((event) => event.eventKind === 'retry_scheduled').map((event) => event.errorClass)).toEqual(['rate_limited', 'transient']) + }) + + test('refuses to reinterpret a checkpoint written by another strategy version', async () => { + const journal = createMemoryJournal() + const destination = createMemoryDestination() + const build = (version: number) => { + resetRegistry() + const stream = defineStream({ + id: 'app.versioned', + destination: events, + incremental: cursorState({ id: 'test.cursor', version, parse: (raw) => String(raw) }), + async *read() { + yield { rows: [{ id: 1 }], state: 'c1' } + }, + }) + return definePipeline({ id: 'app', streams: [stream] }) + } + + await runIngestion({ selected: selectStreams([build(1)], []), backfill: undefined }, { journal, destination }) + const result = await runIngestion({ selected: selectStreams([build(2)], []), backfill: undefined }, { journal, destination }) + + expect(result.streams[0]?.outcome).toBe('failed') + expect(result.streams[0]?.error).toContain('never silently reinterprets') + }) + + test('budget exhaustion is incomplete but preserves committed progress', async () => { + const journal = createMemoryJournal() + const destination = createMemoryDestination() + const stream = defineStream({ + id: 'app.budget', + destination: events, + batchSize: 1, + budget: { maxChunks: 2 }, + incremental: cursorState({ id: 'test.page', version: 1, parse: (raw) => Number(raw) }), + async *read({ selection }) { + for (let page = selection ?? 0; page < 5; page += 1) yield { rows: [{ id: page }], state: page + 1 } + }, + }) + const pipeline = definePipeline({ id: 'app', streams: [stream] }) + + const result = await runIngestion({ selected: selectStreams([pipeline], []), backfill: undefined }, { journal, destination }) + + expect(result.ok).toBe(false) + expect(result.streams[0]?.outcome).toBe('budget_exhausted') + expect((await journal.readCheckpoint('app.budget')).envelope?.state).toBe(2) + }) + + test('a backfill uses an isolated checkpoint namespace', async () => { + const journal = createMemoryJournal() + const destination = createMemoryDestination() + const stream = defineStream({ + id: 'app.window', + destination: events, + incremental: timestampWindow({ from: ({ watermark }) => watermark ?? new Date(0) }), + async *read() { + yield { rows: [{ id: 1 }] } + }, + }) + const pipeline = definePipeline({ id: 'app', streams: [stream] }) + + await runIngestion( + { selected: selectStreams([pipeline], []), backfill: { id: 'jan', from: new Date('2026-01-01'), to: new Date('2026-02-01') } }, + { journal, destination } + ) + + expect((await journal.readCheckpoint('app.window')).envelope).toBeUndefined() + expect((await journal.readCheckpoint('app.window#backfill:jan')).envelope?.state).toEqual({ watermark: '2026-02-01T00:00:00.000Z' }) + }) +}) + +describe('selectStreams', () => { + test('repeated tags use exact AND semantics and an explicit empty selection fails', () => { + const hourly = defineStream({ id: 'crm.people', destination: events, tags: ['schedule:1h'], async *read() {} }) + const daily = defineStream({ id: 'crm.deals', destination: events, tags: ['schedule:1d'], async *read() {} }) + const pipeline = definePipeline({ id: 'crm', tags: ['crm'], streams: [hourly, daily] }) + + expect(selectStreams([pipeline], []).map((entry) => entry.stream.id)).toEqual(['crm.people', 'crm.deals']) + expect(selectStreams([pipeline], ['crm', 'schedule:1h']).map((entry) => entry.stream.id)).toEqual(['crm.people']) + expect(selectStreams([pipeline], ['stream:crm.deals']).map((entry) => entry.stream.id)).toEqual(['crm.deals']) + expect(() => selectStreams([pipeline], ['schedule'])).toThrow('No stream matches every requested tag') + }) + + test('stream ids are globally unique across pipelines', () => { + const stream = defineStream({ id: 'crm.people', destination: events, async *read() {} }) + definePipeline({ id: 'a', streams: [stream] }) + expect(() => definePipeline({ id: 'b', streams: [stream] })).toThrow('globally unique') + }) +}) + +describe('paginate', () => { + test('rejects a repeated continuation instead of looping forever', async () => { + const context = { signal: new AbortController().signal, attempt: (operation: (signal: AbortSignal) => Promise) => operation(new AbortController().signal) } + const iterate = async () => { + for await (const _ of paginate({ context, fetchPage: async () => ({ items: [1], next: 'same' }) })) { + // drain + } + } + await expect(iterate()).rejects.toThrow('repeated continuation') + }) +}) diff --git a/packages/plugin-ingest/src/executor.ts b/packages/plugin-ingest/src/executor.ts new file mode 100644 index 00000000..0a3348ef --- /dev/null +++ b/packages/plugin-ingest/src/executor.ts @@ -0,0 +1,531 @@ +import { randomUUID } from 'node:crypto' + +import { SpanStatusCode, trace, type Span } from '@opentelemetry/api' + +import { BudgetExhausted, classifyFailure, FetchFailure, IngestConfigError, isAbortError } from './errors.js' +import { canonicalJson, digest } from './journal.js' +import { simpleLoader } from './loader.js' +import { createBoundedQueue } from './queue.js' +import type { SelectedStream } from './registry.js' +import { DEFAULT_RETRY, mergeRetry, runAttempt, sleep } from './retry.js' +import { createSemaphore, type Semaphore } from './semaphore.js' +import type { + AnyStreamDefinition, + CheckpointEnvelope, + DestinationAdapter, + ExecutionResult, + Journal, + JournalEvent, + PipelineDefinition, + Row, + SinkReceipt, + StreamOutcome, + StreamResult, +} from './types.js' + +const RUN_NAMESPACE = '@run' +const DEFAULT_BATCH_SIZE = 10_000 +const DEFAULT_PREFETCH_BATCHES = 1 +const DEFAULT_MAX_DURATION_MS = 60 * 60_000 +const LOAD_ATTEMPTS = 3 +const tracer = trace.getTracer('@chkit/plugin-ingest') + +export interface BackfillRequest { + /** Stable identity: resuming the same backfill reuses its isolated checkpoint namespace. */ + id: string + from: Date | undefined + to: Date | undefined +} + +export interface ExecutionRequest { + selected: readonly SelectedStream[] + backfill: BackfillRequest | undefined +} + +export interface ExecutionEnv { + journal: Journal + destination: DestinationAdapter + signal?: AbortSignal + /** Host-default execution budget. Exhaustion preserves committed progress. */ + maxDurationMs?: number + /** Finite number of mapped batches buffered between fetch and load. */ + prefetchBatches?: number + now?: () => Date + sleep?: (ms: number, signal: AbortSignal) => Promise + random?: () => number + log?: (message: string) => void +} + +interface ResolvedEnv extends Required> { + signal: AbortSignal + deadline: number +} + +interface PipelinePermits { + streams: Semaphore + fetches: Semaphore + loads: Semaphore +} + +interface PendingBatch { + rows: Row[] + /** Candidate provider state that becomes safe once these rows have sink evidence. */ + state: { value: unknown } | undefined +} + +interface StreamProgress { + seq: number + version: number + envelope: CheckpointEnvelope | undefined + rows: number + batches: number + chunks: number +} + +/** + * One transient execution context for one exact stream selection. The run id + * correlates journal and telemetry evidence only: every stream plans and + * recovers independently from its own journal-backed checkpoint. + */ +export async function runIngestion(request: ExecutionRequest, input: ExecutionEnv): Promise { + const env = resolveEnv(input) + const runId = randomUUID() + const cutoff = env.now() + const streamIds = request.selected.map((entry) => entry.stream.id) + + return tracer.startActiveSpan('chkit.ingest.execution', async (span) => { + span.setAttribute('chkit.ingest.run_id', runId) + span.setAttribute('chkit.ingest.stream_ids', streamIds) + try { + await env.journal.ensure() + const runHead = (await env.journal.readCheckpoint(RUN_NAMESPACE)).headSeq + await env.journal.append( + runEvent(runHead + 1, 'run_started', runId, '', { + streamIds, + cutoff: cutoff.toISOString(), + backfill: request.backfill?.id, + }) + ) + + const permits = new Map() + const streams = await Promise.all( + request.selected.map((entry) => + executeSelectedStream(entry, permitsFor(permits, entry.pipeline), request.backfill, runId, cutoff, env) + ) + ) + + const ok = streams.every((stream) => stream.outcome === 'succeeded') + await env.journal.append( + runEvent(runHead + 2, 'run_finished', runId, ok ? 'succeeded' : 'failed', { + outcomes: Object.fromEntries(streams.map((stream) => [stream.namespaceId, stream.outcome])), + }) + ) + if (!ok) span.setStatus({ code: SpanStatusCode.ERROR }) + return { runId, cutoff: cutoff.toISOString(), streams, ok } + } finally { + span.end() + } + }) +} + +// An ordinary failure in one stream never prevents unrelated selected streams +// from being attempted: every failure is folded into that stream's result. +async function executeSelectedStream( + entry: SelectedStream, + permits: PipelinePermits, + backfill: BackfillRequest | undefined, + runId: string, + cutoff: Date, + env: ResolvedEnv +): Promise { + const { stream, pipeline } = entry + const namespaceId = backfill ? `${stream.id}#backfill:${backfill.id}` : stream.id + const progress: StreamProgress = { seq: 0, version: 0, envelope: undefined, rows: 0, batches: 0, chunks: 0 } + const result = (outcome: StreamOutcome, error: string | undefined): StreamResult => ({ + streamId: stream.id, + pipelineId: pipeline.id, + namespaceId, + outcome, + rows: progress.rows, + batches: progress.batches, + chunks: progress.chunks, + checkpointVersion: progress.version, + error, + }) + + let release: (() => void) | undefined + try { + release = await permits.streams.acquire(env.signal) + } catch { + return result('cancelled', 'cancelled before start') + } + + return tracer.startActiveSpan('chkit.ingest.stream', async (span) => { + span.setAttribute('chkit.ingest.stream_id', stream.id) + span.setAttribute('chkit.ingest.namespace_id', namespaceId) + try { + const outcome = await executeStream({ stream, pipeline, namespaceId, backfill, runId, cutoff, permits, progress, env }) + env.log?.(`${namespaceId}: ${outcome} (${progress.rows} rows, ${progress.batches} batches, checkpoint v${progress.version})`) + return result(outcome, undefined) + } catch (error) { + const message = error instanceof Error ? error.message : String(error) + const outcome: StreamOutcome = env.signal.aborted ? 'cancelled' : 'failed' + recordFailure(span, error) + env.log?.(`${namespaceId}: ${outcome} — ${message}`) + return result(outcome, message) + } finally { + span.end() + release?.() + } + }) +} + +async function executeStream(input: { + stream: AnyStreamDefinition + pipeline: PipelineDefinition + namespaceId: string + backfill: BackfillRequest | undefined + runId: string + cutoff: Date + permits: PipelinePermits + progress: StreamProgress + env: ResolvedEnv +}): Promise { + const { stream, pipeline, namespaceId, progress, env } = input + const committed = await env.journal.readCheckpoint(namespaceId) + progress.seq = committed.headSeq + progress.version = committed.version + progress.envelope = committed.envelope + + const retry = mergeRetry(pipeline.retry, stream.retry) + const readerAttempts = (retry.retries ?? DEFAULT_RETRY.retries) + 1 + let workId = '' + let outcome: StreamOutcome = 'failed' + let failure: unknown + + for (let attemptNo = 1; attemptNo <= readerAttempts; attemptNo += 1) { + try { + // Re-plan from the latest durable boundary on every reader (re)creation. + const state = restoreState(stream, progress.envelope, namespaceId) + const selection: unknown = stream.incremental.plan({ + state, + cutoff: input.cutoff, + range: input.backfill ? { from: input.backfill.from, to: input.backfill.to } : undefined, + }) + const nextWorkId = digest([namespaceId, String(progress.version), canonicalJson(selection)]).slice(0, 24) + if (nextWorkId !== workId) { + workId = nextWorkId + await append(input, 'work_planned', { workId, workState: 'planned', detail: { selection, strategy: stream.incremental.id, strategyVersion: stream.incremental.version, pipelineId: pipeline.id } }) + } + await append(input, 'attempt_started', { workId, attemptNo, workState: 'running' }) + outcome = await readAndLoad({ ...input, workId, attemptNo, state, selection, retry }) + failure = undefined + break + } catch (error) { + failure = error + if (error instanceof IngestConfigError) break + const classification = error instanceof FetchFailure ? error.classification : classifyFailure(error, env.signal, stream.classifyError) + if (classification.kind === 'cancelled') { + outcome = 'cancelled' + break + } + // A FetchFailure already exhausted its fine-grained retries; an opaque + // iterator failure gets coarse reader recreation from the last checkpoint. + if (error instanceof FetchFailure || classification.kind === 'permanent' || attemptNo === readerAttempts) break + const retryDelay = Math.min(retry.maxTimeout ?? DEFAULT_RETRY.maxTimeout, (retry.minTimeout ?? DEFAULT_RETRY.minTimeout) * 2 ** (attemptNo - 1)) + await append(input, 'retry_scheduled', { workId, attemptNo, retryAt: new Date(env.now().getTime() + retryDelay), errorClass: classification.kind, detail: { error: messageOf(error) } }) + await env.sleep(retryDelay, env.signal) + } + } + + await append(input, 'work_finished', { + workId, + workState: failure === undefined ? outcome : outcome === 'cancelled' ? 'cancelled' : 'failed', + errorClass: failure === undefined ? '' : failureKind(failure, env.signal, stream), + detail: { rows: progress.rows, batches: progress.batches, chunks: progress.chunks, error: failure === undefined ? undefined : messageOf(failure) }, + }) + if (failure !== undefined) throw failure + return outcome +} + +/** + * Pull the reader through a finite buffer into the loader. Exactly one next() + * is pending at a time, and source progress is appended to the journal only + * after the covering rows have sink evidence. + */ +async function readAndLoad(input: { + stream: AnyStreamDefinition + namespaceId: string + runId: string + cutoff: Date + permits: PipelinePermits + progress: StreamProgress + env: ResolvedEnv + workId: string + attemptNo: number + state: unknown + selection: unknown + retry: ReturnType +}): Promise { + const { stream, namespaceId, progress, env } = input + const local = new AbortController() + const signal = AbortSignal.any([env.signal, local.signal]) + const queue = createBoundedQueue(env.prefetchBatches) + const batchSize = stream.batchSize ?? DEFAULT_BATCH_SIZE + let budgetExhausted = false + + const produce = async () => { + let pending: PendingBatch = { rows: [], state: undefined } + const reader = stream.read({ + streamId: stream.id, + selection: input.selection, + state: input.state, + cutoff: input.cutoff, + signal, + attempt: (operation, options) => + runAttempt(operation, options?.label ?? 'source', input.retry, { + signal, + fetchPermits: input.permits.fetches, + classifier: stream.classifyError, + sleep: env.sleep, + random: env.random, + now: () => env.now().getTime(), + onAttempt: () => undefined, + onRetry: ({ label, context }) => + append(input, 'retry_scheduled', { + workId: input.workId, + attemptNo: input.attemptNo, + retryAt: new Date(env.now().getTime() + context.retryDelay), + errorClass: context.error.classification.kind, + detail: { label, sourceAttempt: context.attemptNumber, error: context.error.message }, + }), + }), + }) + + // for-await calls iterator.return() on every early exit, so the reader's + // own finally blocks own cursor and connection cleanup. + for await (const chunk of reader) { + signal.throwIfAborted() + assertValidChunk(stream.id, chunk) + progress.chunks += 1 + pending.rows.push(...chunk.rows) + if (chunk.state !== undefined) pending.state = { value: chunk.state } + if (pending.rows.length >= batchSize) { + await queue.push(pending, signal) + pending = { rows: [], state: undefined } + } + if (stream.budget?.maxChunks !== undefined && progress.chunks >= stream.budget.maxChunks) budgetExhausted = true + if (env.now().getTime() >= env.deadline) budgetExhausted = true + if (budgetExhausted) break + } + + // Only a fully consumed selection may claim the strategy's completion state. + if (!budgetExhausted) { + const completed: unknown = stream.incremental.complete?.({ state: input.state, selection: input.selection }) + if (completed !== undefined) pending.state = { value: completed } + } + if (pending.rows.length > 0 || pending.state !== undefined) await queue.push(pending, signal) + queue.close() + } + + const consume = async () => { + // Batch identity is anchored to the last durable boundary: the committed + // checkpoint version plus the batch's position since that version. A replay + // after a crash starts from that same boundary, so identical rows reproduce + // the identical id (and deduplication token) even in a fresh process. + let sinceBoundary = 0 + for (let batch = await queue.pop(signal); batch !== undefined; batch = await queue.pop(signal)) { + const batchId = digest([namespaceId, String(progress.version), String(sinceBoundary), canonicalJson(batch.rows)]).slice(0, 32) + const receipt = await loadBatch(input, batchId, batch.rows, signal) + const envelope: CheckpointEnvelope | undefined = batch.state + ? { strategy: stream.incremental.id, version: stream.incremental.version, state: batch.state.value } + : progress.envelope + const advanced = canonicalJson(envelope ?? null) !== canonicalJson(progress.envelope ?? null) + const expected = progress.version + const version = advanced ? expected + 1 : expected + await append(input, 'batch_committed', { + workId: input.workId, + attemptNo: input.attemptNo, + batchId, + expectedCheckpointVersion: expected, + checkpointVersion: version, + checkpoint: envelope, + sinkEvidence: receipt.evidence, + detail: { rows: receipt.rows, writeUnits: receipt.writeUnits }, + }) + sinceBoundary = advanced ? 0 : sinceBoundary + 1 + progress.version = version + progress.envelope = envelope + progress.rows += receipt.rows + progress.batches += 1 + } + } + + // The first failure is the root cause; the sibling only fails because of the + // induced abort, so it must not mask what actually went wrong. + let rootCause: { error: unknown } | undefined + await Promise.allSettled( + [produce(), consume()].map((task) => + task.catch((error: unknown) => { + rootCause ??= { error } + local.abort(error) + }) + ) + ) + if (rootCause) throw rootCause.error + if (budgetExhausted) { + env.log?.(`${namespaceId}: ${new BudgetExhausted('execution budget exhausted; committed progress is preserved').message}`) + return 'budget_exhausted' + } + return 'succeeded' +} + +// A fresh Loader instance per attempt. An ambiguous acknowledgement replays the +// same stable batch identity and accepts a possible duplicate over data loss. +async function loadBatch( + input: { stream: AnyStreamDefinition; runId: string; permits: PipelinePermits; env: ResolvedEnv }, + batchId: string, + rows: readonly Row[], + signal: AbortSignal +): Promise { + const factory = input.stream.loader ?? simpleLoader() + const release = await input.permits.loads.acquire(signal) + try { + return await tracer.startActiveSpan('chkit.ingest.load', async (span) => { + span.setAttribute('chkit.ingest.batch_id', batchId) + span.setAttribute('chkit.ingest.rows', rows.length) + try { + for (let attempt = 1; ; attempt += 1) { + const loader = factory({ + streamId: input.stream.id, + runId: input.runId, + table: input.stream.destination, + destination: input.env.destination, + signal, + }) + try { + await loader.write({ batchId, rows }) + return await loader.finalize() + } catch (error) { + await loader.abort(error) + if (signal.aborted || isAbortError(error) || attempt >= LOAD_ATTEMPTS) throw error + await input.env.sleep(1000 * 2 ** (attempt - 1), signal) + } + } + } catch (error) { + recordFailure(span, error) + throw error + } finally { + span.end() + } + }) + } finally { + release() + } +} + +function restoreState(stream: AnyStreamDefinition, envelope: CheckpointEnvelope | undefined, namespaceId: string): unknown { + if (!envelope) return undefined + if (envelope.strategy !== stream.incremental.id || envelope.version !== stream.incremental.version) { + throw new IngestConfigError( + `Checkpoint for "${namespaceId}" was written by strategy ${envelope.strategy}@${envelope.version}, but the stream now declares ${stream.incremental.id}@${stream.incremental.version}. ` + + 'ChKit never silently reinterprets a checkpoint: migrate the state explicitly, use a new stream id, or run a separately namespaced backfill.' + ) + } + return stream.incremental.parseState(envelope.state) +} + +async function append( + input: { namespaceId: string; runId: string; progress: StreamProgress; env: ResolvedEnv }, + eventKind: JournalEvent['eventKind'], + fields: Partial> +): Promise { + input.progress.seq += 1 + await input.env.journal.append({ + namespaceId: input.namespaceId, + eventSeq: input.progress.seq, + eventKind, + runId: input.runId, + workId: fields.workId ?? '', + attemptNo: fields.attemptNo ?? 0, + batchId: fields.batchId ?? '', + expectedCheckpointVersion: fields.expectedCheckpointVersion ?? input.progress.version, + checkpointVersion: fields.checkpointVersion ?? input.progress.version, + checkpoint: fields.checkpoint, + workState: fields.workState ?? '', + sinkEvidence: fields.sinkEvidence ?? '', + retryAt: fields.retryAt, + errorClass: fields.errorClass ?? '', + detail: fields.detail ?? {}, + }) +} + +function runEvent(seq: number, eventKind: 'run_started' | 'run_finished', runId: string, workState: JournalEvent['workState'], detail: Record): JournalEvent { + return { + namespaceId: RUN_NAMESPACE, + eventSeq: seq, + eventKind, + runId, + workId: runId, + attemptNo: 0, + batchId: '', + expectedCheckpointVersion: 0, + checkpointVersion: 0, + checkpoint: undefined, + workState, + sinkEvidence: '', + retryAt: undefined, + errorClass: '', + detail, + } +} + +function assertValidChunk(streamId: string, chunk: unknown): asserts chunk is { rows: readonly Row[]; state?: unknown } { + if (typeof chunk !== 'object' || chunk === null || !('rows' in chunk) || !Array.isArray(chunk.rows)) { + throw new IngestConfigError(`Stream "${streamId}" yielded a chunk without a "rows" array.`) + } +} + +function permitsFor(cache: Map, pipeline: PipelineDefinition): PipelinePermits { + const existing = cache.get(pipeline.id) + if (existing) return existing + const created: PipelinePermits = { + streams: createSemaphore(pipeline.maxStreams), + fetches: createSemaphore(pipeline.maxFetches), + loads: createSemaphore(pipeline.maxLoads), + } + cache.set(pipeline.id, created) + return created +} + +function resolveEnv(input: ExecutionEnv): ResolvedEnv { + const now = input.now ?? (() => new Date()) + const maxDurationMs = input.maxDurationMs ?? DEFAULT_MAX_DURATION_MS + return { + journal: input.journal, + destination: input.destination, + signal: input.signal ?? new AbortController().signal, + maxDurationMs, + prefetchBatches: input.prefetchBatches ?? DEFAULT_PREFETCH_BATCHES, + now, + sleep: input.sleep ?? sleep, + random: input.random ?? Math.random, + log: input.log ?? (() => undefined), + deadline: now().getTime() + maxDurationMs, + } +} + +function failureKind(error: unknown, signal: AbortSignal, stream: AnyStreamDefinition): string { + if (error instanceof IngestConfigError) return 'config' + if (error instanceof FetchFailure) return error.classification.kind + return classifyFailure(error, signal, stream.classifyError).kind +} + +function recordFailure(span: Span, error: unknown): void { + span.setStatus({ code: SpanStatusCode.ERROR, message: messageOf(error) }) + if (error instanceof Error) span.recordException(error) +} + +function messageOf(error: unknown): string { + return error instanceof Error ? error.message : String(error) +} diff --git a/packages/plugin-ingest/src/incremental.ts b/packages/plugin-ingest/src/incremental.ts new file mode 100644 index 00000000..d10e74b7 --- /dev/null +++ b/packages/plugin-ingest/src/incremental.ts @@ -0,0 +1,87 @@ +import { IngestConfigError } from './errors.js' +import type { IncrementalStrategy } from './types.js' + +export interface TimestampWindowState { + watermark: string +} + +export interface TimestampRange { + from: Date + to: Date +} + +export interface TimestampWindowOptions { + /** + * Lower bound for the next read. Bootstrap and overlap are ordinary branches + * here; the result is selection input and is never persisted as the watermark. + */ + from(input: { watermark: Date | undefined; cutoff: Date }): Date +} + +export interface CursorStateOptions { + /** Stable strategy identifier stored in the checkpoint envelope. */ + id: string + version: number + /** Validate committed state. Throw when it must not be reinterpreted. */ + parse(raw: unknown): TState +} + +/** No incremental selection: every execution reads the whole source. */ +export function fullSync(): IncrementalStrategy { + return { + id: 'chkit.full_sync', + version: 1, + parseState: () => undefined, + plan: () => undefined, + } +} + +/** + * Bundled timestamp strategy. The upper bound is fixed to the execution cutoff + * and `range.to` becomes the committed watermark only after the whole window + * has sink evidence. + */ +export function timestampWindow( + options: TimestampWindowOptions +): IncrementalStrategy { + return { + id: 'chkit.timestamp_window', + version: 1, + parseState(raw) { + if (typeof raw !== 'object' || raw === null || !('watermark' in raw) || typeof raw.watermark !== 'string') { + throw new IngestConfigError('timestampWindow: committed state has no string "watermark".') + } + if (Number.isNaN(Date.parse(raw.watermark))) { + throw new IngestConfigError(`timestampWindow: committed watermark "${raw.watermark}" is not a timestamp.`) + } + return { watermark: raw.watermark } + }, + plan({ state, cutoff, range }) { + const to = range?.to ?? cutoff + const from = range?.from ?? options.from({ watermark: state ? new Date(state.watermark) : undefined, cutoff }) + if (from.getTime() > to.getTime()) { + throw new IngestConfigError( + `timestampWindow: planned lower bound ${from.toISOString()} is after upper bound ${to.toISOString()}.` + ) + } + return { from, to } + }, + complete({ selection }) { + return { watermark: selection.to.toISOString() } + }, + } +} + +/** + * Generic provider-owned state (compound cursor, change token, snapshot id…). + * The reader receives the committed state as its selection and advances it by + * yielding chunks that carry complete candidate state. + */ +export function cursorState(options: CursorStateOptions): IncrementalStrategy { + return { + id: options.id, + version: options.version, + parseState: options.parse, + plan: ({ state }) => state, + } +} diff --git a/packages/plugin-ingest/src/index.ts b/packages/plugin-ingest/src/index.ts new file mode 100644 index 00000000..b178043a --- /dev/null +++ b/packages/plugin-ingest/src/index.ts @@ -0,0 +1,28 @@ +export { ingest, createIngestPlugin, checkGraph, type IngestPlugin, type IngestPluginOptions } from './plugin.js' +export { defineStream, definePipeline, listPipelines, selectStreams, type SelectedStream } from './registry.js' +export { fullSync, timestampWindow, cursorState, type TimestampRange, type TimestampWindowState } from './incremental.js' +export { paginate, type Page } from './paginate.js' +export { simpleLoader } from './loader.js' +export { ingestionColumns, createClickHouseDestination } from './destination.js' +export { createClickHouseJournal } from './journal.js' +export { runIngestion, type BackfillRequest, type ExecutionEnv, type ExecutionRequest } from './executor.js' +export { HttpError, FetchFailure, IngestConfigError } from './errors.js' +export type { + DestinationAdapter, + ErrorClassifier, + ExecutionResult, + FailureClass, + IncrementalStrategy, + Journal, + Loader, + LoaderContext, + LoaderFactory, + PipelineDefinition, + ReadContext, + RetryOptions, + Row, + SinkReceipt, + SourceChunk, + StreamDefinition, + StreamResult, +} from './types.js' diff --git a/packages/plugin-ingest/src/ingest.e2e.test.ts b/packages/plugin-ingest/src/ingest.e2e.test.ts new file mode 100644 index 00000000..19e13e22 --- /dev/null +++ b/packages/plugin-ingest/src/ingest.e2e.test.ts @@ -0,0 +1,93 @@ +import { afterAll, beforeAll, describe, expect, test } from 'bun:test' + +import { table, toCreateSQL } from '@chkit/core' +import type { ClickHouseExecutor } from '@chkit/clickhouse' +import { createLiveExecutor, createPrefix, getRequiredEnv, quoteIdent, waitForTable } from '@chkit/clickhouse/e2e-testkit' + +import { createClickHouseDestination, ingestionColumns } from './destination.js' +import { runIngestion } from './executor.js' +import { cursorState } from './incremental.js' +import { createClickHouseJournal } from './journal.js' +import { definePipeline, defineStream, resetRegistry, selectStreams } from './registry.js' +import type { DestinationAdapter } from './types.js' + +describe('@chkit/plugin-ingest live env e2e', () => { + const liveEnv = getRequiredEnv() + const prefix = createPrefix('ingest') + const database = liveEnv.clickhouseDatabase + const journalTable = `${prefix}journal` + const destinationTable = table({ + database, + name: `${prefix}items`, + columns: [{ name: 'id', type: 'UInt64' }, { name: 'label', type: 'String' }, ...ingestionColumns], + engine: 'MergeTree()', + primaryKey: ['id'], + orderBy: ['id'], + settings: { non_replicated_deduplication_window: '100' }, + }) + let executor: ClickHouseExecutor + + beforeAll(async () => { + executor = createLiveExecutor(liveEnv) + await executor.command(toCreateSQL(destinationTable)) + await waitForTable(executor, database, destinationTable.name) + }) + + afterAll(async () => { + await executor.command(`DROP TABLE IF EXISTS ${quoteIdent(database)}.${quoteIdent(destinationTable.name)}`) + await executor.command(`DROP TABLE IF EXISTS ${quoteIdent(database)}.${quoteIdent(journalTable)}`) + await executor.close() + }) + + test('a lost acknowledgement replays from the journaled checkpoint without skipping or duplicating rows', async () => { + resetRegistry() + const source = Array.from({ length: 3 }, (_, page) => [0, 1].map((offset) => ({ id: page * 2 + offset, label: `row-${page * 2 + offset}` }))) + const stream = defineStream({ + id: `${prefix}items`, + destination: destinationTable, + batchSize: 2, + incremental: cursorState({ id: 'e2e.page', version: 1, parse: (raw) => Number(raw) }), + async *read({ selection }) { + for (let page = selection ?? 0; page < source.length; page += 1) yield { rows: source[page] ?? [], state: page + 1 } + }, + }) + const pipeline = definePipeline({ id: `${prefix}pipeline`, streams: [stream], retry: { retries: 0 } }) + const journal = () => createClickHouseJournal({ executor, database, targetId: `e2e/${prefix}`, table: journalTable }) + const destination = createClickHouseDestination(executor) + let inserts = 0 + const lossy: DestinationAdapter = { + insert: async (input) => { + inserts += 1 + if (inserts === 2) { + await destination.insert(input) + throw new Error('acknowledgement lost') + } + if (inserts > 2) throw new Error('target unavailable') + await destination.insert(input) + }, + } + const noSleep = async () => undefined + + const first = await runIngestion({ selected: selectStreams([pipeline], []), backfill: undefined }, { journal: journal(), destination: lossy, sleep: noSleep }) + expect(first.ok).toBe(false) + expect((await journal().readCheckpoint(stream.id)).envelope?.state).toBe(1) + + // A fresh executor process reconstructs everything from the journal. + const second = await runIngestion({ selected: selectStreams([pipeline], []), backfill: undefined }, { journal: journal(), destination, sleep: noSleep }) + expect(second.ok).toBe(true) + const checkpoint = await journal().readCheckpoint(stream.id) + expect(checkpoint.envelope).toEqual({ strategy: 'e2e.page', version: 1, state: 3 }) + expect(checkpoint.version).toBe(3) + + const rows = await executor.query<{ id: string; run_ids: string }>( + `SELECT id, uniqExact(_chkit_run_id) AS run_ids FROM ${quoteIdent(database)}.${quoteIdent(destinationTable.name)} GROUP BY id ORDER BY id`, + { select_sequential_consistency: '1' } + ) + expect(rows.map((row) => Number(row.id))).toEqual([0, 1, 2, 3, 4, 5]) + const physical = await executor.query<{ rows: string }>( + `SELECT count() AS rows FROM ${quoteIdent(database)}.${quoteIdent(destinationTable.name)}`, + { select_sequential_consistency: '1' } + ) + expect(Number(physical[0]?.rows)).toBe(6) + }, 120_000) +}) diff --git a/packages/plugin-ingest/src/journal.ts b/packages/plugin-ingest/src/journal.ts new file mode 100644 index 00000000..f86f7ba5 --- /dev/null +++ b/packages/plugin-ingest/src/journal.ts @@ -0,0 +1,238 @@ +import { createHash } from 'node:crypto' + +import type { ClickHouseExecutor } from '@chkit/clickhouse' + +import { IngestConfigError } from './errors.js' +import type { CheckpointEnvelope, CommittedCheckpoint, Journal, JournalEvent } from './types.js' + +export const DEFAULT_JOURNAL_TABLE = '_chkit_ingestion_journal' +const TABLE_NAME_PATTERN = /^[A-Za-z_][A-Za-z0-9_]*$/ + +// A type alias (not an interface) so it is assignable to the executor's Record-based insert values. +export type JournalRow = { + target_id: string + namespace_id: string + event_seq: string + event_id: string + payload_hash: string + event_at: string + event_kind: string + run_id: string + work_id: string + attempt_no: number + batch_id: string + expected_checkpoint_version: string + checkpoint_version: string + checkpoint_json: string + work_state: string + sink_evidence: string + retry_at: string | null + error_class: string + detail_json: string +} + +export interface ClickHouseJournalOptions { + executor: ClickHouseExecutor + database: string + targetId: string + table?: string + now?: () => Date +} + +/** + * Append-only, target-linked journal. It is the sole authority for durable + * ingestion control state: checkpoints are read as a projection over + * `batch_committed` facts and never stored anywhere else. + */ +export function createClickHouseJournal(options: ClickHouseJournalOptions): Journal { + const table = options.table ?? DEFAULT_JOURNAL_TABLE + if (!TABLE_NAME_PATTERN.test(table)) throw new IngestConfigError(`Invalid journal table name "${table}".`) + if (!TABLE_NAME_PATTERN.test(options.database)) { + throw new IngestConfigError(`Invalid journal database name "${options.database}".`) + } + const qualified = `\`${options.database}\`.\`${table}\`` + const now = options.now ?? (() => new Date()) + + return { + async ensure() { + await options.executor.command(journalTableSql(qualified)) + }, + + async append(event) { + const row = toJournalRow(event, options.targetId, now()) + await options.executor.insert({ + table: `${options.database}.${table}`, + values: [row], + // A retried append of the same deterministic fact is suppressed while + // the deduplication window lasts; readers canonicalize by event_id anyway. + settings: { insert_deduplication_token: row.event_id, async_insert: 0 }, + }) + }, + + async readCheckpoint(namespaceId) { + // Physical retry duplicates are allowed; canonicalize by event_id and + // refuse to continue when one deterministic id carries different payloads. + const rows = await options.executor.query<{ + head_seq: string + drifted: string + checkpoint_version: string + checkpoint_json: string + }>( + `SELECT + max(event_seq) AS head_seq, + countIf(distinct_payloads > 1) AS drifted, + argMaxIf(fact_version, (fact_version, event_seq, event_id), fact_kind = 'batch_committed') AS checkpoint_version, + argMaxIf(fact_checkpoint, (fact_version, event_seq, event_id), fact_kind = 'batch_committed') AS checkpoint_json +FROM ( + SELECT + event_seq, + event_id, + any(event_kind) AS fact_kind, + any(checkpoint_version) AS fact_version, + any(checkpoint_json) AS fact_checkpoint, + uniqExact(payload_hash) AS distinct_payloads + FROM ${qualified} + WHERE target_id = ${sqlString(options.targetId)} AND namespace_id = ${sqlString(namespaceId)} + GROUP BY event_seq, event_id +)`, + { select_sequential_consistency: '1' } + ) + const row = rows[0] + if (!row) return { version: 0, envelope: undefined, headSeq: 0 } + if (Number(row.drifted) > 0) { + throw new Error( + `Ingestion journal payload drift detected for namespace "${namespaceId}": a deterministic event id has conflicting payloads. Refusing to continue.` + ) + } + return { + version: Number(row.checkpoint_version), + envelope: parseEnvelope(row.checkpoint_json), + headSeq: Number(row.head_seq), + } + }, + } +} + +export function toJournalRow(event: JournalEvent, targetId: string, at: Date): JournalRow { + const checkpointJson = event.checkpoint ? canonicalJson(event.checkpoint) : '' + const detailJson = canonicalJson(event.detail) + // Identity covers what makes the fact unique; the payload hash covers what a + // replay of that same fact must reproduce. Timestamps are excluded from both. + const eventId = digest([targetId, event.namespaceId, String(event.eventSeq), event.eventKind, event.workId, event.batchId, String(event.attemptNo)]) + const payload = digest([ + eventId, + String(event.expectedCheckpointVersion), + String(event.checkpointVersion), + checkpointJson, + event.workState, + event.sinkEvidence, + event.errorClass, + ]) + return { + target_id: targetId, + namespace_id: event.namespaceId, + event_seq: String(event.eventSeq), + event_id: eventId, + payload_hash: BigInt(`0x${payload.slice(0, 16)}`).toString(), + event_at: toClickHouseDateTime(at), + event_kind: event.eventKind, + run_id: event.runId, + work_id: event.workId, + attempt_no: event.attemptNo, + batch_id: event.batchId, + expected_checkpoint_version: String(event.expectedCheckpointVersion), + checkpoint_version: String(event.checkpointVersion), + checkpoint_json: checkpointJson, + work_state: event.workState, + sink_evidence: event.sinkEvidence, + retry_at: event.retryAt ? toClickHouseDateTime(event.retryAt) : null, + error_class: event.errorClass, + detail_json: detailJson, + } +} + +export function parseEnvelope(json: string): CheckpointEnvelope | undefined { + if (json === '') return undefined + const parsed: unknown = JSON.parse(json) + if ( + typeof parsed !== 'object' || + parsed === null || + !('strategy' in parsed) || + typeof parsed.strategy !== 'string' || + !('version' in parsed) || + typeof parsed.version !== 'number' + ) { + throw new Error('Committed checkpoint is not a valid ChKit checkpoint envelope.') + } + return { strategy: parsed.strategy, version: parsed.version, state: 'state' in parsed ? parsed.state : undefined } +} + +/** Stable key order so equal values always serialize identically. */ +export function canonicalJson(value: unknown): string { + // JSON.stringify(undefined) is undefined, not a string. + return JSON.stringify(sortKeys(value)) ?? 'null' +} + +export function digest(parts: readonly string[]): string { + const hash = createHash('sha256') + for (const part of parts) { + hash.update(String(part.length)) + hash.update(':') + hash.update(part) + } + return hash.digest('hex') +} + +export function emptyCheckpoint(): CommittedCheckpoint { + return { version: 0, envelope: undefined, headSeq: 0 } +} + +function journalTableSql(qualified: string): string { + return `CREATE TABLE IF NOT EXISTS ${qualified} +( + target_id LowCardinality(String), + namespace_id String, + event_seq UInt64, + event_id String, + payload_hash UInt64, + event_at DateTime64(6, 'UTC'), + event_kind LowCardinality(String), + run_id String, + work_id String, + attempt_no UInt32, + batch_id String, + expected_checkpoint_version UInt64, + checkpoint_version UInt64, + checkpoint_json String, + work_state LowCardinality(String), + sink_evidence LowCardinality(String), + retry_at Nullable(DateTime64(6, 'UTC')), + error_class LowCardinality(String), + detail_json String +) +ENGINE = MergeTree +PARTITION BY toYYYYMM(event_at) +ORDER BY (target_id, namespace_id, event_seq, event_id)` +} + +function sortKeys(value: unknown): unknown { + if (Array.isArray(value)) return value.map(sortKeys) + if (value instanceof Date) return value.toISOString() + if (typeof value === 'object' && value !== null) { + return Object.fromEntries( + Object.entries(value) + .filter(([, entry]) => entry !== undefined) + .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)) + .map(([key, entry]) => [key, sortKeys(entry)]) + ) + } + return value +} + +function toClickHouseDateTime(date: Date): string { + return date.toISOString().replace('T', ' ').replace('Z', '') +} + +function sqlString(value: string): string { + return `'${value.replace(/\\/g, '\\\\').replace(/'/g, "\\'")}'` +} diff --git a/packages/plugin-ingest/src/loader.ts b/packages/plugin-ingest/src/loader.ts new file mode 100644 index 00000000..db08ac74 --- /dev/null +++ b/packages/plugin-ingest/src/loader.ts @@ -0,0 +1,46 @@ +import { BATCH_ID_COLUMN, RUN_ID_COLUMN } from './destination.js' +import type { LoaderFactory } from './types.js' + +export interface SimpleLoaderOptions { + /** Rows per physical INSERT. A larger batch is split into deterministic chunks. */ + maxRowsPerInsert?: number +} + +const DEFAULT_MAX_ROWS_PER_INSERT = 50_000 + +/** + * Direct at-least-once destination writes. Every write unit reuses one stable + * token across retries: a batch written by one INSERT has one token, a + * deterministically chunked batch has one distinct token per chunk. + */ +export function simpleLoader(options: SimpleLoaderOptions = {}): LoaderFactory { + const maxRows = options.maxRowsPerInsert ?? DEFAULT_MAX_ROWS_PER_INSERT + + return (ctx) => { + let rows = 0 + let writeUnits = 0 + + return { + ctx, + async write(batch) { + for (let offset = 0, unit = 0; offset < batch.rows.length; offset += maxRows, unit += 1) { + ctx.signal.throwIfAborted() + const slice = batch.rows.slice(offset, offset + maxRows).map((row) => ({ + ...row, + [BATCH_ID_COLUMN]: batch.batchId, + [RUN_ID_COLUMN]: ctx.runId, + })) + await ctx.destination.insert({ table: ctx.table, rows: slice, token: `${batch.batchId}:${unit}` }) + rows += slice.length + writeUnits += 1 + } + }, + async finalize() { + return { evidence: writeUnits === 0 ? 'none_required' : 'clickhouse_ack', rows, writeUnits } + }, + async abort() { + // Direct writes cannot be withdrawn; replay reuses the same tokens. + }, + } + } +} diff --git a/packages/plugin-ingest/src/paginate.ts b/packages/plugin-ingest/src/paginate.ts new file mode 100644 index 00000000..d46a1c63 --- /dev/null +++ b/packages/plugin-ingest/src/paginate.ts @@ -0,0 +1,44 @@ +import type { AttemptOptions, ReadContext } from './types.js' + +export interface Page { + items: readonly TItem[] + /** Continuation for the next request; `undefined` ends the sequence. */ + next: TCursor | undefined +} + +export interface PaginateOptions { + // biome-ignore lint/suspicious/noExplicitAny: pagination only needs the attempt capability, not the stream generics + context: Pick, 'attempt' | 'signal'> + fetchPage(cursor: TCursor | undefined, signal: AbortSignal): Promise> + initial?: TCursor + label?: AttemptOptions['label'] +} + +/** + * Pull-based pagination over one retryable Promise step per page. Every request + * runs through the executor attempt capability, and repeated continuations are + * rejected so a cyclic provider cursor cannot loop forever. + */ +export async function* paginate( + options: PaginateOptions +): AsyncGenerator { + const seen = new Set() + let cursor = options.initial + + while (true) { + options.context.signal.throwIfAborted() + const current = cursor + const page = await options.context.attempt((signal) => options.fetchPage(current, signal), { + label: options.label, + }) + if (page.items.length > 0) yield page.items + if (page.next === undefined || page.next === null) return + + const key = JSON.stringify(page.next) + if (seen.has(key)) { + throw new Error(`paginate: provider returned a repeated continuation ${key}; refusing to loop.`) + } + seen.add(key) + cursor = page.next + } +} diff --git a/packages/plugin-ingest/src/plugin.ts b/packages/plugin-ingest/src/plugin.ts new file mode 100644 index 00000000..2b237edc --- /dev/null +++ b/packages/plugin-ingest/src/plugin.ts @@ -0,0 +1,291 @@ +import process from 'node:process' + +import { createClickHouseExecutor, type ClickHouseExecutor } from '@chkit/clickhouse' +import { + createPluginRunner, + defineFlags, + withFactoryDefaults, + type ChxInlinePluginRegistration, + type ResolvedChxConfig, +} from '@chkit/core' +import { loadSchemaDefinitions } from '@chkit/core/schema-loader' +import { z } from 'zod' + +import { BATCH_ID_COLUMN, createClickHouseDestination, INGESTED_AT_COLUMN, RUN_ID_COLUMN } from './destination.js' +import { IngestConfigError } from './errors.js' +import { runIngestion, type BackfillRequest } from './executor.js' +import { createClickHouseJournal, DEFAULT_JOURNAL_TABLE } from './journal.js' +import { listPipelines, selectStreams, type SelectedStream } from './registry.js' +import type { PipelineDefinition } from './types.js' + +const REQUIRED_COLUMNS = [BATCH_ID_COLUMN, RUN_ID_COLUMN, INGESTED_AT_COLUMN] + +const IngestOptionsSchema = z.object({ + journalTable: z.string().min(1).default(DEFAULT_JOURNAL_TABLE), + /** Host-default execution budget in seconds. */ + maxDurationSeconds: z.number().positive().default(3600), + prefetchBatches: z.number().int().positive().default(1), +}) +type IngestOptions = z.infer + +export type IngestPluginOptions = Partial + +const SELECTION_FLAGS = defineFlags([ + { name: '--tag', type: 'string[]', description: 'Exact tag every selected stream must carry (repeatable, AND)', placeholder: '' }, +] as const) + +const RUN_FLAGS = defineFlags([ + ...SELECTION_FLAGS, + { name: '--backfill', type: 'string', description: 'Stable backfill id; uses an isolated checkpoint namespace', placeholder: '' }, + { name: '--from', type: 'string', description: 'Backfill range lower bound (ISO timestamp)', placeholder: '' }, + { name: '--to', type: 'string', description: 'Backfill range upper bound (ISO timestamp)', placeholder: '' }, + { name: '--max-duration', type: 'string', description: 'Execution budget in seconds', placeholder: '' }, +] as const) + +export interface IngestPluginCommandContext { + args: string[] + flags: Record + jsonMode: boolean + options: IngestOptions + config: ResolvedChxConfig + configPath: string + print: (value: unknown) => void + pluginContext?: { executor: ClickHouseExecutor; hasExecutor: boolean } +} + +const runCommand = createPluginRunner({ configErrorClass: IngestConfigError }) + +export function createIngestPlugin(options: IngestPluginOptions = {}) { + const optionsSchema = withFactoryDefaults(IngestOptionsSchema, options) + + return { + manifest: { name: 'ingest' as const, apiVersion: 1 as const }, + optionsSchema, + commands: [ + { + name: 'run', + description: 'Execute the selected ingestion streams from their journaled checkpoints', + flags: RUN_FLAGS, + optionsSchema, + run: runCommand({ + command: 'run', + label: 'Ingest run', + fn: async (context) => { + const selected = await loadSelection(context) + const backfill = parseBackfill(context.flags) + const maxDurationSeconds = parseMaxDuration(context.flags['--max-duration']) ?? context.options.maxDurationSeconds + const target = openTarget(context) + const abort = new AbortController() + const onSignal = () => abort.abort(new DOMException('Interrupted', 'AbortError')) + process.once('SIGINT', onSignal) + process.once('SIGTERM', onSignal) + + try { + const result = await runIngestion( + { selected, backfill }, + { + journal: createClickHouseJournal({ + executor: target.executor, + database: target.database, + targetId: target.targetId, + table: context.options.journalTable, + }), + destination: createClickHouseDestination(target.executor), + signal: abort.signal, + maxDurationMs: maxDurationSeconds * 1000, + prefetchBatches: context.options.prefetchBatches, + log: context.jsonMode ? undefined : (message) => context.print(message), + } + ) + if (context.jsonMode) { + context.print({ command: 'run', ...result }) + } else { + const rows = result.streams.reduce((sum, stream) => sum + stream.rows, 0) + context.print(`Ingest run ${result.runId}: ${result.ok ? 'ok' : 'incomplete'} (${result.streams.length} streams, ${rows} rows)`) + } + return result.ok ? 0 : 1 + } finally { + process.off('SIGINT', onSignal) + process.off('SIGTERM', onSignal) + await target.close() + } + }, + }), + }, + { + name: 'list', + description: 'List the ingestion streams in the loaded definition graph', + flags: SELECTION_FLAGS, + optionsSchema, + run: runCommand({ + command: 'list', + label: 'Ingest list', + fn: async (context) => { + const selected = await loadSelection(context) + const streams = selected.map(describeStream) + if (context.jsonMode) { + context.print({ ok: true, command: 'list', streams }) + } else { + for (const stream of streams) { + context.print(`${stream.streamId} -> ${stream.destination} [${stream.tags.join(', ')}]`) + } + } + return 0 + }, + }), + }, + { + name: 'status', + description: 'Show the committed checkpoint of each selected stream', + flags: SELECTION_FLAGS, + optionsSchema, + run: runCommand({ + command: 'status', + label: 'Ingest status', + fn: async (context) => { + const selected = await loadSelection(context) + const target = openTarget(context) + try { + const journal = createClickHouseJournal({ + executor: target.executor, + database: target.database, + targetId: target.targetId, + table: context.options.journalTable, + }) + await journal.ensure() + const streams = await Promise.all( + selected.map(async (entry) => { + const checkpoint = await journal.readCheckpoint(entry.stream.id) + return { ...describeStream(entry), checkpointVersion: checkpoint.version, checkpoint: checkpoint.envelope ?? null } + }) + ) + if (context.jsonMode) { + context.print({ ok: true, command: 'status', targetId: target.targetId, streams }) + } else { + for (const stream of streams) { + context.print(`${stream.streamId}: v${stream.checkpointVersion} ${stream.checkpoint ? JSON.stringify(stream.checkpoint.state) : '(no checkpoint)'}`) + } + } + return 0 + } finally { + await target.close() + } + }, + }), + }, + ], + hooks: { + async onCheck(context: { config: ResolvedChxConfig }) { + const findings = checkGraph(await loadGraph(context.config)) + return { + plugin: 'ingest', + evaluated: true, + ok: findings.every((finding) => finding.severity !== 'error'), + findings, + } + }, + }, + } +} + +export type IngestPlugin = ReturnType + +export function ingest(options: IngestPluginOptions = {}): ChxInlinePluginRegistration { + return { plugin: createIngestPlugin(options), name: 'ingest', enabled: true, options } +} + +/** Local, zero-network graph checks: they hold in `--offline` mode too. */ +export function checkGraph(pipelines: readonly PipelineDefinition[]) { + const findings: Array<{ code: string; message: string; severity: 'info' | 'warn' | 'error' }> = [] + if (pipelines.length === 0) { + findings.push({ code: 'ingest_no_pipelines', message: 'No ingestion pipeline is registered by the project entry.', severity: 'warn' }) + } + for (const pipeline of pipelines) { + for (const stream of pipeline.streams) { + const columns = new Set(stream.destination.columns.map((column) => column.name)) + const missing = REQUIRED_COLUMNS.filter((name) => !columns.has(name)) + if (missing.length > 0) { + findings.push({ + code: 'ingest_missing_metadata_columns', + message: `Stream "${stream.id}" destination ${stream.destination.database}.${stream.destination.name} is missing ${missing.join(', ')}. Spread ingestionColumns into its columns.`, + severity: 'error', + }) + } + } + } + return findings +} + +// Importing the entry (or legacy schema files) is what lets definePipeline +// self-register; the module cache guarantees this happens once per process. +async function loadGraph(config: ResolvedChxConfig): Promise { + if (config.schema.length > 0) await loadSchemaDefinitions(config.schema, { cwd: process.cwd() }) + return listPipelines() +} + +async function loadSelection(context: IngestPluginCommandContext): Promise { + const pipelines = await loadGraph(context.config) + if (pipelines.length === 0) { + throw new IngestConfigError('No ingestion pipeline is registered. Call definePipeline(...) from the module configured as "entry".') + } + const tags = context.flags['--tag'] + return selectStreams(pipelines, Array.isArray(tags) ? tags : typeof tags === 'string' ? [tags] : []) +} + +function openTarget(context: IngestPluginCommandContext) { + const clickhouse = context.config.clickhouse + // A direct connection carries per-insert settings (the deduplication token); + // fall back to the host-provided executor only when no URL is configured. + if (clickhouse) { + const executor = createClickHouseExecutor(clickhouse) + return { executor, database: clickhouse.database, targetId: targetIdOf(clickhouse.url, clickhouse.database), close: () => executor.close() } + } + if (context.pluginContext?.hasExecutor) { + return { executor: context.pluginContext.executor, database: 'default', targetId: 'host-executor/default', close: async () => undefined } + } + throw new IngestConfigError('Ingestion needs a ClickHouse target. Configure clickhouse in your clickhouse.config.ts.') +} + +function targetIdOf(url: string, database: string): string { + try { + return `${new URL(url).host}/${database}` + } catch { + return `${url}/${database}` + } +} + +function parseBackfill(flags: IngestPluginCommandContext['flags']): BackfillRequest | undefined { + const id = flags['--backfill'] + const from = parseTimestamp(flags['--from'], '--from') + const to = parseTimestamp(flags['--to'], '--to') + if (typeof id !== 'string') { + if (from || to) throw new IngestConfigError('--from and --to require --backfill : explicit ranges never touch the scheduled checkpoint.') + return undefined + } + if (!/^[A-Za-z0-9][A-Za-z0-9_.-]*$/.test(id)) throw new IngestConfigError(`Invalid --backfill id "${id}".`) + return { id, from, to } +} + +function parseTimestamp(raw: string | string[] | boolean | undefined, flag: string): Date | undefined { + if (typeof raw !== 'string') return undefined + const date = new Date(raw) + if (Number.isNaN(date.getTime())) throw new IngestConfigError(`Invalid timestamp for ${flag}: ${raw}`) + return date +} + +function parseMaxDuration(raw: string | string[] | boolean | undefined): number | undefined { + if (typeof raw !== 'string') return undefined + const seconds = Number(raw) + if (!Number.isFinite(seconds) || seconds <= 0) throw new IngestConfigError(`Invalid value for --max-duration: ${raw}`) + return seconds +} + +function describeStream(entry: SelectedStream) { + return { + streamId: entry.stream.id, + pipelineId: entry.pipeline.id, + destination: `${entry.stream.destination.database}.${entry.stream.destination.name}`, + strategy: `${entry.stream.incremental.id}@${entry.stream.incremental.version}`, + tags: [...entry.effectiveTags], + } +} diff --git a/packages/plugin-ingest/src/queue.ts b/packages/plugin-ingest/src/queue.ts new file mode 100644 index 00000000..0b1bb0a6 --- /dev/null +++ b/packages/plugin-ingest/src/queue.ts @@ -0,0 +1,56 @@ +export interface BoundedQueue { + /** Waits while the queue is full, which is what applies backpressure to the reader. */ + push(item: T, signal: AbortSignal): Promise + /** Resolves `undefined` once the queue is closed and drained. */ + pop(signal: AbortSignal): Promise + close(): void +} + +export function createBoundedQueue(capacity: number): BoundedQueue { + const limit = Math.max(1, capacity) + const items: T[] = [] + let closed = false + let wake: Array<() => void> = [] + + const notify = () => { + const waiting = wake + wake = [] + for (const resolve of waiting) resolve() + } + + const wait = (signal: AbortSignal) => + new Promise((resolve, reject) => { + const onAbort = () => reject(signal.reason) + signal.addEventListener('abort', onAbort, { once: true }) + wake.push(() => { + signal.removeEventListener('abort', onAbort) + resolve() + }) + }) + + return { + async push(item, signal) { + while (items.length >= limit) { + signal.throwIfAborted() + await wait(signal) + } + signal.throwIfAborted() + items.push(item) + notify() + }, + async pop(signal) { + while (items.length === 0) { + if (closed) return undefined + signal.throwIfAborted() + await wait(signal) + } + const item = items.shift() + notify() + return item + }, + close() { + closed = true + notify() + }, + } +} diff --git a/packages/plugin-ingest/src/registry.ts b/packages/plugin-ingest/src/registry.ts new file mode 100644 index 00000000..291386fe --- /dev/null +++ b/packages/plugin-ingest/src/registry.ts @@ -0,0 +1,197 @@ +import type { TableDefinition } from '@chkit/core' + +import { IngestConfigError } from './errors.js' +import { fullSync } from './incremental.js' +import type { + AnyStreamDefinition, + ErrorClassifier, + IncrementalStrategy, + LoaderFactory, + PipelineDefinition, + ReadContext, + RetryOptions, + Row, + SourceChunk, + StreamBudget, + StreamDefinition, +} from './types.js' + +// The registry lives on globalThis so that definePipeline calls made from a +// project entry reach the plugin even when the package is resolved twice +// (for example `source` vs `default` export conditions). +const REGISTRY_KEY = Symbol.for('chkit.ingest.registry') +const STREAM_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9_.:-]*$/ + +interface Registry { + pipelines: Map +} + +export interface StreamInput { + id: string + destination: TableDefinition + tags?: readonly string[] + incremental: IncrementalStrategy + read: (context: ReadContext) => AsyncIterable> + loader?: LoaderFactory + retry?: RetryOptions + batchSize?: number + budget?: StreamBudget + classifyError?: ErrorClassifier +} + +export interface FullSyncStreamInput + extends Omit, 'incremental'> { + incremental?: undefined +} + +export interface PipelineInput { + id: string + tags?: readonly string[] + streams: readonly AnyStreamDefinition[] + maxStreams?: number + maxFetches?: number + maxLoads?: number + retry?: RetryOptions +} + +export interface SelectedStream { + stream: AnyStreamDefinition + pipeline: PipelineDefinition + effectiveTags: readonly string[] +} + +export function defineStream(input: FullSyncStreamInput): StreamDefinition +export function defineStream( + input: StreamInput +): StreamDefinition +export function defineStream( + input: StreamInput | FullSyncStreamInput +): AnyStreamDefinition { + assertValidId('stream', input.id) + if (input.batchSize !== undefined && (!Number.isInteger(input.batchSize) || input.batchSize <= 0)) { + throw new IngestConfigError(`Stream "${input.id}": batchSize must be a positive integer.`) + } + return { + kind: 'ingest_stream', + id: input.id, + destination: input.destination, + tags: dedupe(input.tags ?? []), + incremental: input.incremental ?? fullSync(), + read: input.read, + loader: input.loader, + retry: input.retry, + batchSize: input.batchSize, + budget: input.budget, + classifyError: input.classifyError, + } +} + +/** + * Register a non-durable named group of streams. Pipeline identity never + * participates in checkpoint or batch identity, so moving a stream between + * pipelines does not reset its state. + */ +export function definePipeline(input: PipelineInput): PipelineDefinition { + assertValidId('pipeline', input.id) + const pipeline: PipelineDefinition = { + kind: 'ingest_pipeline', + id: input.id, + tags: dedupe(input.tags ?? []), + streams: [...input.streams], + maxStreams: positiveCeiling(input.id, 'maxStreams', input.maxStreams, 4), + maxFetches: positiveCeiling(input.id, 'maxFetches', input.maxFetches, 4), + maxLoads: positiveCeiling(input.id, 'maxLoads', input.maxLoads, 2), + retry: input.retry, + } + + const registry = getRegistry() + const owners = streamOwners(registry) + for (const stream of pipeline.streams) { + const owner = owners.get(stream.id) + if (owner !== undefined && owner !== pipeline.id) { + throw new IngestConfigError( + `Stream id "${stream.id}" is registered by both pipeline "${owner}" and pipeline "${pipeline.id}". Stream ids must be globally unique.` + ) + } + } + const seen = new Set() + for (const stream of pipeline.streams) { + if (seen.has(stream.id)) { + throw new IngestConfigError(`Pipeline "${pipeline.id}" lists stream "${stream.id}" more than once.`) + } + seen.add(stream.id) + } + + registry.pipelines.set(pipeline.id, pipeline) + return pipeline +} + +export function listPipelines(): PipelineDefinition[] { + return [...getRegistry().pipelines.values()] +} + +export function resetRegistry(): void { + getRegistry().pipelines.clear() +} + +/** + * Repeated exact case-sensitive tags with AND semantics over the union of + * pipeline and stream tags plus derived `pipeline:` and `stream:`. + * No filter selects the complete graph; an explicit empty selection throws. + */ +export function selectStreams(pipelines: readonly PipelineDefinition[], tags: readonly string[]): SelectedStream[] { + const wanted = dedupe(tags) + const all: SelectedStream[] = pipelines.flatMap((pipeline) => + pipeline.streams.map((stream) => ({ + stream, + pipeline, + effectiveTags: dedupe([...pipeline.tags, ...stream.tags, `pipeline:${pipeline.id}`, `stream:${stream.id}`]), + })) + ) + if (wanted.length === 0) return all + + const selected = all.filter((entry) => wanted.every((tag) => entry.effectiveTags.includes(tag))) + if (selected.length === 0) { + throw new IngestConfigError( + `No stream matches every requested tag: ${wanted.map((tag) => `--tag ${tag}`).join(' ')}. Nothing was executed.` + ) + } + return selected +} + +function getRegistry(): Registry { + const holder = globalThis as { [REGISTRY_KEY]?: Registry } + const existing = holder[REGISTRY_KEY] + if (existing) return existing + const created: Registry = { pipelines: new Map() } + holder[REGISTRY_KEY] = created + return created +} + +function streamOwners(registry: Registry): Map { + const owners = new Map() + for (const pipeline of registry.pipelines.values()) { + for (const stream of pipeline.streams) owners.set(stream.id, pipeline.id) + } + return owners +} + +function assertValidId(kind: 'stream' | 'pipeline', id: string): void { + if (!STREAM_ID_PATTERN.test(id)) { + throw new IngestConfigError( + `Invalid ${kind} id "${id}". Use letters, digits, "_", ".", ":" or "-", starting with a letter or digit.` + ) + } +} + +function positiveCeiling(pipelineId: string, name: string, value: number | undefined, fallback: number): number { + if (value === undefined) return fallback + if (!Number.isInteger(value) || value <= 0) { + throw new IngestConfigError(`Pipeline "${pipelineId}": ${name} must be a positive integer.`) + } + return value +} + +function dedupe(values: readonly string[]): string[] { + return [...new Set(values)] +} diff --git a/packages/plugin-ingest/src/retry.ts b/packages/plugin-ingest/src/retry.ts new file mode 100644 index 00000000..152beefd --- /dev/null +++ b/packages/plugin-ingest/src/retry.ts @@ -0,0 +1,112 @@ +import { classifyFailure, FetchFailure } from './errors.js' +import type { Semaphore } from './semaphore.js' +import type { ErrorClassifier, RetryContext, RetryOptions } from './types.js' + +export const DEFAULT_RETRY: Required> = { + retries: 5, + factor: 2, + minTimeout: 1000, + maxTimeout: 60_000, + randomize: true, + maxRetryTime: 10 * 60_000, +} + +export interface AttemptEnv { + signal: AbortSignal + fetchPermits: Semaphore + classifier: ErrorClassifier | undefined + sleep: (ms: number, signal: AbortSignal) => Promise + random: () => number + now: () => number + onAttempt: (input: { label: string; attemptNumber: number }) => Promise | void + onRetry: (input: { label: string; context: RetryContext }) => Promise | void +} + +/** Pipeline default, partially overridden by the stream. */ +export function mergeRetry(...layers: Array): RetryOptions { + const merged: RetryOptions = {} + for (const layer of layers) { + if (!layer) continue + for (const [key, value] of Object.entries(layer)) { + if (value !== undefined) Object.assign(merged, { [key]: value }) + } + } + return merged +} + +/** + * Run one source operation. A fetch permit is acquired per attempt and released + * while waiting for a retry timer, so backoff never occupies fetch capacity. + */ +export async function runAttempt( + operation: (signal: AbortSignal) => Promise, + label: string, + policy: RetryOptions, + env: AttemptEnv +): Promise { + const retries = policy.retries ?? DEFAULT_RETRY.retries + const maxRetryTime = policy.maxRetryTime ?? DEFAULT_RETRY.maxRetryTime + const startedAt = env.now() + let attemptNumber = 0 + let retriesConsumed = 0 + + while (true) { + attemptNumber += 1 + env.signal.throwIfAborted() + await env.onAttempt({ label, attemptNumber }) + + const release = await env.fetchPermits.acquire(env.signal) + try { + return await operation(env.signal) + } catch (cause) { + const classification = classifyFailure(cause, env.signal, env.classifier) + const failure = new FetchFailure(cause, classification) + if (classification.kind === 'cancelled' || classification.kind === 'permanent') throw failure + + const retriesLeft = retries - retriesConsumed + const backoff = backoffDelay(policy, retriesConsumed, env.random) + const hinted = classification.kind === 'rate_limited' ? classification.retryAfterMs : undefined + const retryDelay = hinted === undefined ? backoff : Math.max(hinted, backoff) + const context: RetryContext = { error: failure, attemptNumber, retriesLeft, retriesConsumed, retryDelay } + + if (retriesLeft <= 0) throw failure + if (env.now() - startedAt + retryDelay > maxRetryTime) throw failure + if (policy.shouldRetry && !(await policy.shouldRetry(context))) throw failure + const consume = policy.shouldConsumeRetry ? await policy.shouldConsumeRetry(context) : true + if (consume) retriesConsumed += 1 + + await env.onRetry({ label, context }) + release() + await env.sleep(retryDelay, env.signal) + } finally { + release() + } + } +} + +export function sleep(ms: number, signal: AbortSignal): Promise { + return new Promise((resolve, reject) => { + if (signal.aborted) { + reject(signal.reason) + return + } + const onAbort = () => { + clearTimeout(timer) + reject(signal.reason) + } + const timer = setTimeout(() => { + signal.removeEventListener('abort', onAbort) + resolve() + }, ms) + signal.addEventListener('abort', onAbort, { once: true }) + }) +} + +function backoffDelay(policy: RetryOptions, retriesConsumed: number, random: () => number): number { + const factor = policy.factor ?? DEFAULT_RETRY.factor + const minTimeout = policy.minTimeout ?? DEFAULT_RETRY.minTimeout + const maxTimeout = policy.maxTimeout ?? DEFAULT_RETRY.maxTimeout + const randomize = policy.randomize ?? DEFAULT_RETRY.randomize + const jitter = randomize ? 1 + random() : 1 + return Math.min(maxTimeout, Math.round(jitter * minTimeout * factor ** retriesConsumed)) +} diff --git a/packages/plugin-ingest/src/semaphore.ts b/packages/plugin-ingest/src/semaphore.ts new file mode 100644 index 00000000..dcf7e15e --- /dev/null +++ b/packages/plugin-ingest/src/semaphore.ts @@ -0,0 +1,51 @@ +export interface Semaphore { + /** Resolves with an idempotent release function. */ + acquire(signal: AbortSignal): Promise<() => void> +} + +export function createSemaphore(limit: number): Semaphore { + let active = 0 + const waiters: Array<{ grant: () => void; cancel: (reason: unknown) => void }> = [] + + const release = () => { + active -= 1 + const next = waiters.shift() + if (next) next.grant() + } + + const onceRelease = () => { + let released = false + return () => { + if (released) return + released = true + release() + } + } + + return { + acquire(signal) { + if (signal.aborted) return Promise.reject(signal.reason) + if (active < limit) { + active += 1 + return Promise.resolve(onceRelease()) + } + return new Promise((resolve, reject) => { + const waiter = { + grant: () => { + signal.removeEventListener('abort', onAbort) + active += 1 + resolve(onceRelease()) + }, + cancel: reject, + } + const onAbort = () => { + const index = waiters.indexOf(waiter) + if (index >= 0) waiters.splice(index, 1) + waiter.cancel(signal.reason) + } + signal.addEventListener('abort', onAbort, { once: true }) + waiters.push(waiter) + }) + }, + } +} diff --git a/packages/plugin-ingest/src/testing.ts b/packages/plugin-ingest/src/testing.ts new file mode 100644 index 00000000..2eb22fa9 --- /dev/null +++ b/packages/plugin-ingest/src/testing.ts @@ -0,0 +1,54 @@ +import { emptyCheckpoint, toJournalRow, type JournalRow } from './journal.js' +import type { CommittedCheckpoint, DestinationAdapter, Journal, JournalEvent, Row } from './types.js' + +export interface MemoryJournal extends Journal { + readonly events: JournalEvent[] + readonly rows: JournalRow[] +} + +export interface MemoryDestination extends DestinationAdapter { + /** Physical rows per `database.table`, after token deduplication. */ + readonly tables: Map + readonly tokens: string[] +} + +/** In-memory journal with the same projection semantics as the ClickHouse one. */ +export function createMemoryJournal(): MemoryJournal { + const events: JournalEvent[] = [] + const rows: JournalRow[] = [] + return { + events, + rows, + async ensure() {}, + async append(event) { + events.push(event) + rows.push(toJournalRow(event, 'memory', new Date(0))) + }, + async readCheckpoint(namespaceId): Promise { + const scoped = events.filter((event) => event.namespaceId === namespaceId) + if (scoped.length === 0) return emptyCheckpoint() + const headSeq = Math.max(...scoped.map((event) => event.eventSeq)) + const committed = scoped + .filter((event) => event.eventKind === 'batch_committed') + .sort((a, b) => a.checkpointVersion - b.checkpointVersion || a.eventSeq - b.eventSeq) + .at(-1) + return { version: committed?.checkpointVersion ?? 0, envelope: committed?.checkpoint, headSeq } + }, + } +} + +/** In-memory destination that honours `insert_deduplication_token` like ClickHouse. */ +export function createMemoryDestination(): MemoryDestination { + const tables = new Map() + const tokens: string[] = [] + return { + tables, + tokens, + async insert({ table, rows, token }) { + const key = `${table.database}.${table.name}` + if (tokens.includes(`${key}:${token}`)) return + tokens.push(`${key}:${token}`) + tables.set(key, [...(tables.get(key) ?? []), ...rows]) + }, + } +} diff --git a/packages/plugin-ingest/src/types.ts b/packages/plugin-ingest/src/types.ts new file mode 100644 index 00000000..312d58f5 --- /dev/null +++ b/packages/plugin-ingest/src/types.ts @@ -0,0 +1,249 @@ +import type { TableDefinition } from '@chkit/core' + +import type { FetchFailure } from './errors.js' + +// ───── Authoring: streams and pipelines ───── + +export type Row = Record + +/** + * One bounded unit produced by a stream reader. Rows are already shaped for the + * destination table: mapping is ordinary code inside the reader. + * + * `state` is optional and, when present, must be the COMPLETE candidate + * provider state that is safe to commit once every row up to and including + * this chunk has sink evidence. The executor never infers ordering: a reader + * that cannot claim a safe frontier simply omits `state`. + */ +export interface SourceChunk { + rows: readonly TRow[] + state?: TState +} + +export interface AttemptOptions { + /** Short non-secret label used in journal and telemetry, e.g. `GET /v2/objects`. */ + label?: string +} + +export interface ReadContext { + streamId: string + /** What to read this execution, planned by the incremental strategy. */ + selection: TSelection + /** Last committed provider state, already validated by the strategy. */ + state: TState | undefined + /** Immutable cutoff shared by every stream selected in this execution. */ + cutoff: Date + signal: AbortSignal + /** + * Run one source operation under executor authority: fetch permit, retry + * policy, failure classification and cancellation. Operations that bypass it + * only get coarse recovery (reader recreation from the last checkpoint). + */ + attempt(operation: (signal: AbortSignal) => Promise, options?: AttemptOptions): Promise +} + +/** + * Provider-owned incremental strategy inside the ChKit-owned checkpoint + * envelope `{ strategy, version, state }`. Core treats `state` as opaque. + */ +export interface IncrementalStrategy { + readonly id: string + readonly version: number + /** Validate previously committed state. Throw when it cannot be trusted. */ + parseState(raw: unknown): TState + plan(input: PlanInput): TSelection + /** + * Candidate state once the WHOLE selection has sink evidence. Return + * `undefined` to leave the checkpoint untouched. + */ + complete?(input: { state: TState | undefined; selection: TSelection }): TState | undefined +} + +export interface PlanInput { + state: TState | undefined + cutoff: Date + /** Explicit historical range from a backfill invocation. */ + range: { from: Date | undefined; to: Date | undefined } | undefined +} + +export interface RetryContext { + error: FetchFailure + attemptNumber: number + retriesLeft: number + retriesConsumed: number + retryDelay: number +} + +/** Portable p-retry-shaped subset. */ +export interface RetryOptions { + retries?: number + factor?: number + minTimeout?: number + maxTimeout?: number + randomize?: boolean + maxRetryTime?: number + shouldRetry?: (context: RetryContext) => boolean | Promise + shouldConsumeRetry?: (context: RetryContext) => boolean | Promise +} + +export interface StreamBudget { + /** Maximum source chunks pulled in one execution of this stream. */ + maxChunks?: number +} + +export interface StreamDefinition { + readonly kind: 'ingest_stream' + /** Globally stable identity. Owns the checkpoint; never derived from the pipeline. */ + readonly id: string + readonly destination: TableDefinition + readonly tags: readonly string[] + readonly incremental: IncrementalStrategy + readonly read: (context: ReadContext) => AsyncIterable> + readonly loader: LoaderFactory | undefined + readonly retry: RetryOptions | undefined + readonly batchSize: number | undefined + readonly budget: StreamBudget | undefined + readonly classifyError: ErrorClassifier | undefined +} + +// biome-ignore lint/suspicious/noExplicitAny: a heterogeneous stream list cannot share row/state generics +export type AnyStreamDefinition = StreamDefinition + +export interface PipelineDefinition { + readonly kind: 'ingest_pipeline' + readonly id: string + readonly tags: readonly string[] + readonly streams: readonly AnyStreamDefinition[] + readonly maxStreams: number + readonly maxFetches: number + readonly maxLoads: number + readonly retry: RetryOptions | undefined +} + +// ───── Failure classification ───── + +export type FailureClass = + | { kind: 'cancelled' } + | { kind: 'rate_limited'; retryAfterMs: number | undefined } + | { kind: 'transient' } + | { kind: 'permanent' } + | { kind: 'unknown' } + +export type ErrorClassifier = (cause: unknown, fallback: FailureClass) => FailureClass | undefined + +// ───── Loading ───── + +export type SinkEvidenceKind = 'clickhouse_ack' | 'none_required' + +export interface SinkReceipt { + evidence: SinkEvidenceKind + rows: number + writeUnits: number +} + +export interface LoadBatch { + /** Stable across every retry and crash replay of the same logical batch. */ + batchId: string + rows: readonly Row[] +} + +export interface LoaderContext { + streamId: string + runId: string + table: TableDefinition + destination: DestinationAdapter + signal: AbortSignal +} + +export interface Loader { + readonly ctx: LoaderContext + write(batch: LoadBatch): Promise + finalize(): Promise + abort(reason: unknown): Promise +} + +export type LoaderFactory = (ctx: LoaderContext) => Loader + +/** Low-level destination writes performed on behalf of a portable Loader. */ +export interface DestinationAdapter { + insert(input: { + table: TableDefinition + rows: readonly Row[] + /** Deterministic `insert_deduplication_token` for this write unit. */ + token: string + }): Promise +} + +// ───── Journal ───── + +export type JournalEventKind = + | 'run_started' + | 'work_planned' + | 'attempt_started' + | 'retry_scheduled' + | 'batch_committed' + | 'work_finished' + | 'run_finished' + +export type WorkState = '' | 'planned' | 'running' | 'succeeded' | 'failed' | 'budget_exhausted' | 'cancelled' + +export interface CheckpointEnvelope { + strategy: string + version: number + state: unknown +} + +export interface JournalEvent { + namespaceId: string + eventSeq: number + eventKind: JournalEventKind + runId: string + workId: string + attemptNo: number + batchId: string + expectedCheckpointVersion: number + checkpointVersion: number + checkpoint: CheckpointEnvelope | undefined + workState: WorkState + sinkEvidence: SinkEvidenceKind | '' + retryAt: Date | undefined + errorClass: string + detail: Record +} + +export interface CommittedCheckpoint { + version: number + envelope: CheckpointEnvelope | undefined + /** Highest journal sequence observed for the namespace (any event kind). */ + headSeq: number +} + +/** Authoritative append-only control state. Checkpoints are projections of it. */ +export interface Journal { + ensure(): Promise + append(event: JournalEvent): Promise + readCheckpoint(namespaceId: string): Promise +} + +// ───── Execution ───── + +export type StreamOutcome = 'succeeded' | 'failed' | 'budget_exhausted' | 'cancelled' + +export interface StreamResult { + streamId: string + pipelineId: string + namespaceId: string + outcome: StreamOutcome + rows: number + batches: number + chunks: number + checkpointVersion: number + error: string | undefined +} + +export interface ExecutionResult { + runId: string + cutoff: string + streams: StreamResult[] + ok: boolean +} diff --git a/packages/plugin-ingest/tsconfig.json b/packages/plugin-ingest/tsconfig.json new file mode 100644 index 00000000..d38ac708 --- /dev/null +++ b/packages/plugin-ingest/tsconfig.json @@ -0,0 +1,9 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "rootDir": "src", + "outDir": "dist" + }, + "include": ["src/**/*.ts"], + "exclude": ["src/**/*.test.ts"] +} From 58e05ccc397b0c3d60cc91ccf09f6d4eef50f48c Mon Sep 17 00:00:00 2001 From: KeKs0r Date: Fri, 18 Sep 2026 18:23:20 -0700 Subject: [PATCH 02/11] =?UTF-8?q?=F0=9F=90=9B=20Harden=20ingestion=20runti?= =?UTF-8?q?me=20after=20architecture=20review?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Interval-based batch identity, validated journal history, enforced execution deadline, abandonable readers, load permit per attempt, serialized journal appends, stateless executor for concurrent work. --- packages/plugin-ingest/src/executor.test.ts | 126 +++++++++ packages/plugin-ingest/src/executor.ts | 248 ++++++++++++------ packages/plugin-ingest/src/ingest.e2e.test.ts | 5 +- packages/plugin-ingest/src/journal.ts | 88 +++++-- packages/plugin-ingest/src/loader.ts | 12 +- packages/plugin-ingest/src/paginate.ts | 8 +- packages/plugin-ingest/src/plugin.ts | 6 +- packages/plugin-ingest/src/types.ts | 11 + 8 files changed, 387 insertions(+), 117 deletions(-) diff --git a/packages/plugin-ingest/src/executor.test.ts b/packages/plugin-ingest/src/executor.test.ts index 836ddc3f..219dfe5d 100644 --- a/packages/plugin-ingest/src/executor.test.ts +++ b/packages/plugin-ingest/src/executor.test.ts @@ -275,6 +275,132 @@ describe('runIngestion', () => { }) }) +describe('runtime contracts', () => { + const run = (pipeline: ReturnType, env: Parameters[1]) => + runIngestion({ selected: selectStreams([pipeline], []), backfill: undefined }, { sleep: noSleep, ...env }) + + test('a declared interval id keeps batch identity stable when replayed rows changed', async () => { + const build = (label: string, withId: boolean) => { + resetRegistry() + const stream = defineStream({ + id: 'app.mutable', + destination: events, + async *read() { + yield { rows: [{ id: 1, label }], ...(withId ? { id: 'page:1' } : {}) } + }, + }) + return definePipeline({ id: 'app', streams: [stream] }) + } + + const declared = createMemoryDestination() + await run(build('before', true), { journal: createMemoryJournal(), destination: declared }) + await run(build('after', true), { journal: createMemoryJournal(), destination: declared }) + expect(declared.tables.get('app.events')).toHaveLength(1) + + // Without a declared interval the content hash wins: a duplicate over a suppressed change. + const undeclared = createMemoryDestination() + await run(build('before', false), { journal: createMemoryJournal(), destination: undeclared }) + await run(build('after', false), { journal: createMemoryJournal(), destination: undeclared }) + expect(undeclared.tables.get('app.events')).toHaveLength(2) + }) + + test('the execution budget interrupts a hung reader and preserves committed progress', async () => { + const journal = createMemoryJournal() + const stream = defineStream({ + id: 'app.hung', + destination: events, + batchSize: 1, + incremental: cursorState({ id: 'test.page', version: 1, parse: (raw) => Number(raw) }), + async *read() { + yield { rows: [{ id: 1 }], state: 1 } + await new Promise(() => undefined) + }, + }) + const pipeline = definePipeline({ id: 'app', streams: [stream] }) + + const result = await run(pipeline, { journal, destination: createMemoryDestination(), maxDurationMs: 50 }) + + expect(result.streams[0]?.outcome).toBe('budget_exhausted') + expect((await journal.readCheckpoint('app.hung')).envelope?.state).toBe(1) + expect(journal.events.at(-1)?.eventKind).toBe('run_finished') + }) + + test('an oversized chunk fails the stream instead of being buffered', async () => { + const stream = defineStream({ + id: 'app.big', + destination: events, + budget: { maxChunkRows: 2 }, + async *read() { + yield { rows: [{ id: 1 }, { id: 2 }, { id: 3 }] } + }, + }) + const result = await run(definePipeline({ id: 'app', streams: [stream] }), { journal: createMemoryJournal(), destination: createMemoryDestination() }) + expect(result.streams[0]?.error).toContain('above the 2-row bound') + }) + + test('a mapped row cannot supply the destination-owned publication time', async () => { + const destination = createMemoryDestination() + const stream = defineStream({ + id: 'app.stamped', + destination: events, + async *read() { + yield { rows: [{ id: 1, _chkit_ingested_at: '2000-01-01 00:00:00' }] } + }, + }) + await run(definePipeline({ id: 'app', streams: [stream] }), { journal: createMemoryJournal(), destination }) + expect(destination.tables.get('app.events')?.[0]).not.toHaveProperty('_chkit_ingested_at') + }) + + test('an ambiguous journal append retries the same fact without consuming a sequence number', async () => { + const journal = createMemoryJournal() + const append = journal.append.bind(journal) + let failures = 0 + journal.append = async (event) => { + if (event.eventKind === 'batch_committed' && failures === 0) { + failures += 1 + throw new Error('acknowledgement lost') + } + await append(event) + } + const stream = defineStream({ id: 'app.events', destination: events, async *read() { yield { rows: [{ id: 1 }] } } }) + + const result = await run(definePipeline({ id: 'app', streams: [stream] }), { journal, destination: createMemoryDestination() }) + + expect(result.ok).toBe(true) + const sequences = journal.events.filter((event) => event.namespaceId === 'app.events').map((event) => event.eventSeq) + expect(sequences).toEqual([1, 2, 3, 4]) + }) + + test('a failing loader cleanup does not mask the write failure or stop the retry', async () => { + const destination = createMemoryDestination() + let writes = 0 + const stream = defineStream({ + id: 'app.cleanup', + destination: events, + loader: (ctx) => ({ + ctx, + async write(batch) { + writes += 1 + if (writes === 1) throw new Error('write failed') + await ctx.destination.insert({ table: ctx.table, rows: batch.rows, token: batch.batchId }) + }, + finalize: async () => ({ evidence: 'clickhouse_ack', rows: 1, writeUnits: 1 }), + abort: async () => { + throw new Error('cleanup failed') + }, + }), + async *read() { + yield { rows: [{ id: 1 }] } + }, + }) + + const result = await run(definePipeline({ id: 'app', streams: [stream] }), { journal: createMemoryJournal(), destination }) + + expect(result.ok).toBe(true) + expect(writes).toBe(2) + }) +}) + describe('selectStreams', () => { test('repeated tags use exact AND semantics and an explicit empty selection fails', () => { const hourly = defineStream({ id: 'crm.people', destination: events, tags: ['schedule:1h'], async *read() {} }) diff --git a/packages/plugin-ingest/src/executor.ts b/packages/plugin-ingest/src/executor.ts index 0a3348ef..503f7f71 100644 --- a/packages/plugin-ingest/src/executor.ts +++ b/packages/plugin-ingest/src/executor.ts @@ -28,6 +28,10 @@ const DEFAULT_BATCH_SIZE = 10_000 const DEFAULT_PREFETCH_BATCHES = 1 const DEFAULT_MAX_DURATION_MS = 60 * 60_000 const LOAD_ATTEMPTS = 3 +const JOURNAL_APPEND_ATTEMPTS = 4 +const DEFAULT_MAX_CHUNK_ROWS = 100_000 +// How long a failed stream waits for an uncooperative reader before abandoning it. +const READER_SHUTDOWN_GRACE_MS = 5000 const tracer = trace.getTracer('@chkit/plugin-ingest') export interface BackfillRequest { @@ -57,8 +61,10 @@ export interface ExecutionEnv { } interface ResolvedEnv extends Required> { + /** Host cancellation combined with the execution deadline. */ signal: AbortSignal - deadline: number + /** Host cancellation only: distinguishes a cancelled run from an exhausted budget. */ + hostSignal: AbortSignal } interface PipelinePermits { @@ -69,12 +75,17 @@ interface PipelinePermits { interface PendingBatch { rows: Row[] + /** Declared source-interval ids, or `undefined` once any chunk omitted its id. */ + intervalIds: string[] | undefined /** Candidate provider state that becomes safe once these rows have sink evidence. */ state: { value: unknown } | undefined } interface StreamProgress { + /** Last journal sequence confirmed for this namespace. */ seq: number + /** Serializes appends so a sequence number is only consumed by a confirmed fact. */ + appendChain: Promise version: number envelope: CheckpointEnvelope | undefined rows: number @@ -88,7 +99,10 @@ interface StreamProgress { * recovers independently from its own journal-backed checkpoint. */ export async function runIngestion(request: ExecutionRequest, input: ExecutionEnv): Promise { - const env = resolveEnv(input) + const deadline = new AbortController() + const env = resolveEnv(input, deadline.signal) + // The budget is a real bound: it interrupts hung readers, retry timers and waits. + const deadlineTimer = setTimeout(() => deadline.abort(new BudgetExhausted('execution budget exhausted')), env.maxDurationMs) const runId = randomUUID() const cutoff = env.now() const streamIds = request.selected.map((entry) => entry.stream.id) @@ -123,6 +137,7 @@ export async function runIngestion(request: ExecutionRequest, input: ExecutionEn if (!ok) span.setStatus({ code: SpanStatusCode.ERROR }) return { runId, cutoff: cutoff.toISOString(), streams, ok } } finally { + clearTimeout(deadlineTimer) span.end() } }) @@ -140,7 +155,7 @@ async function executeSelectedStream( ): Promise { const { stream, pipeline } = entry const namespaceId = backfill ? `${stream.id}#backfill:${backfill.id}` : stream.id - const progress: StreamProgress = { seq: 0, version: 0, envelope: undefined, rows: 0, batches: 0, chunks: 0 } + const progress: StreamProgress = { seq: 0, appendChain: Promise.resolve(), version: 0, envelope: undefined, rows: 0, batches: 0, chunks: 0 } const result = (outcome: StreamOutcome, error: string | undefined): StreamResult => ({ streamId: stream.id, pipelineId: pipeline.id, @@ -169,7 +184,7 @@ async function executeSelectedStream( return result(outcome, undefined) } catch (error) { const message = error instanceof Error ? error.message : String(error) - const outcome: StreamOutcome = env.signal.aborted ? 'cancelled' : 'failed' + const outcome: StreamOutcome = env.hostSignal.aborted ? 'cancelled' : env.signal.aborted ? 'budget_exhausted' : 'failed' recordFailure(span, error) env.log?.(`${namespaceId}: ${outcome} — ${message}`) return result(outcome, message) @@ -226,7 +241,9 @@ async function executeStream(input: { if (error instanceof IngestConfigError) break const classification = error instanceof FetchFailure ? error.classification : classifyFailure(error, env.signal, stream.classifyError) if (classification.kind === 'cancelled') { - outcome = 'cancelled' + outcome = env.hostSignal.aborted ? 'cancelled' : 'budget_exhausted' + // Exhausting the budget is an incomplete result, not a failure: committed progress stands. + if (outcome === 'budget_exhausted') failure = undefined break } // A FetchFailure already exhausted its fine-grained retries; an opaque @@ -275,7 +292,7 @@ async function readAndLoad(input: { let budgetExhausted = false const produce = async () => { - let pending: PendingBatch = { rows: [], state: undefined } + let pending = emptyBatch() const reader = stream.read({ streamId: stream.id, selection: input.selection, @@ -302,21 +319,32 @@ async function readAndLoad(input: { }), }) - // for-await calls iterator.return() on every early exit, so the reader's - // own finally blocks own cursor and connection cleanup. - for await (const chunk of reader) { - signal.throwIfAborted() - assertValidChunk(stream.id, chunk) - progress.chunks += 1 - pending.rows.push(...chunk.rows) - if (chunk.state !== undefined) pending.state = { value: chunk.state } - if (pending.rows.length >= batchSize) { - await queue.push(pending, signal) - pending = { rows: [], state: undefined } + // Iterate by hand so a hung next() can be abandoned on abort; return() still + // runs on every exit so the reader's finally blocks own cursor cleanup. + const iterator = reader[Symbol.asyncIterator]() + const maxChunkRows = stream.budget?.maxChunkRows ?? DEFAULT_MAX_CHUNK_ROWS + try { + while (!budgetExhausted) { + const step = await abortable(iterator.next(), signal) + if (step.done) break + const chunk: unknown = step.value + assertValidChunk(stream.id, chunk, maxChunkRows) + progress.chunks += 1 + pending.rows.push(...chunk.rows) + if (pending.intervalIds) { + if (chunk.id === undefined) pending.intervalIds = undefined + else pending.intervalIds.push(chunk.id) + } + if (chunk.state !== undefined) pending.state = { value: chunk.state } + if (pending.rows.length >= batchSize) { + await queue.push(pending, signal) + pending = emptyBatch() + } + if (stream.budget?.maxChunks !== undefined && progress.chunks >= stream.budget.maxChunks) budgetExhausted = true } - if (stream.budget?.maxChunks !== undefined && progress.chunks >= stream.budget.maxChunks) budgetExhausted = true - if (env.now().getTime() >= env.deadline) budgetExhausted = true - if (budgetExhausted) break + } finally { + // Never await an uncooperative reader: cleanup is best effort once we leave. + void Promise.resolve(iterator.return?.()).catch(() => undefined) } // Only a fully consumed selection may claim the strategy's completion state. @@ -330,12 +358,15 @@ async function readAndLoad(input: { const consume = async () => { // Batch identity is anchored to the last durable boundary: the committed - // checkpoint version plus the batch's position since that version. A replay - // after a crash starts from that same boundary, so identical rows reproduce - // the identical id (and deduplication token) even in a fresh process. + // checkpoint version plus the batch's position since that version, so a + // replay in a fresh process reproduces the same id and deduplication token. + // Declared source-interval ids complete the identity; without them a + // content hash does, preferring a possible duplicate over suppressing rows + // that changed between attempts. let sinceBoundary = 0 for (let batch = await queue.pop(signal); batch !== undefined; batch = await queue.pop(signal)) { - const batchId = digest([namespaceId, String(progress.version), String(sinceBoundary), canonicalJson(batch.rows)]).slice(0, 32) + const discriminator = batch.intervalIds ? `interval:${canonicalJson(batch.intervalIds)}` : `content:${canonicalJson(batch.rows)}` + const batchId = digest([namespaceId, String(progress.version), String(sinceBoundary), discriminator]).slice(0, 32) const receipt = await loadBatch(input, batchId, batch.rows, signal) const envelope: CheckpointEnvelope | undefined = batch.state ? { strategy: stream.incremental.id, version: stream.incremental.version, state: batch.state.value } @@ -364,7 +395,7 @@ async function readAndLoad(input: { // The first failure is the root cause; the sibling only fails because of the // induced abort, so it must not mask what actually went wrong. let rootCause: { error: unknown } | undefined - await Promise.allSettled( + const settled = Promise.allSettled( [produce(), consume()].map((task) => task.catch((error: unknown) => { rootCause ??= { error } @@ -372,6 +403,9 @@ async function readAndLoad(input: { }) ) ) + // After a failure, a source operation that ignores its signal must not pin + // the stream (and its permits) forever. + await Promise.race([settled, graceAfterAbort(signal, READER_SHUTDOWN_GRACE_MS)]) if (rootCause) throw rootCause.error if (budgetExhausted) { env.log?.(`${namespaceId}: ${new BudgetExhausted('execution budget exhausted; committed progress is preserved').message}`) @@ -389,39 +423,41 @@ async function loadBatch( signal: AbortSignal ): Promise { const factory = input.stream.loader ?? simpleLoader() - const release = await input.permits.loads.acquire(signal) - try { - return await tracer.startActiveSpan('chkit.ingest.load', async (span) => { - span.setAttribute('chkit.ingest.batch_id', batchId) - span.setAttribute('chkit.ingest.rows', rows.length) - try { - for (let attempt = 1; ; attempt += 1) { - const loader = factory({ - streamId: input.stream.id, - runId: input.runId, - table: input.stream.destination, - destination: input.env.destination, - signal, + return tracer.startActiveSpan('chkit.ingest.load', async (span) => { + span.setAttribute('chkit.ingest.batch_id', batchId) + span.setAttribute('chkit.ingest.rows', rows.length) + try { + for (let attempt = 1; ; attempt += 1) { + // The load permit covers only the active write, never the backoff timer. + const release = await input.permits.loads.acquire(signal) + const loader = factory({ + streamId: input.stream.id, + runId: input.runId, + table: input.stream.destination, + destination: input.env.destination, + signal, + }) + try { + await loader.write({ batchId, rows }) + return await loader.finalize() + } catch (error) { + // A cleanup failure must not replace the write failure that decides the retry. + await loader.abort(error).catch((cleanupError: unknown) => { + span.recordException(cleanupError instanceof Error ? cleanupError : new Error(String(cleanupError))) }) - try { - await loader.write({ batchId, rows }) - return await loader.finalize() - } catch (error) { - await loader.abort(error) - if (signal.aborted || isAbortError(error) || attempt >= LOAD_ATTEMPTS) throw error - await input.env.sleep(1000 * 2 ** (attempt - 1), signal) - } + if (signal.aborted || isAbortError(error) || attempt >= LOAD_ATTEMPTS) throw error + } finally { + release() } - } catch (error) { - recordFailure(span, error) - throw error - } finally { - span.end() + await input.env.sleep(1000 * 2 ** (attempt - 1), signal) } - }) - } finally { - release() - } + } catch (error) { + recordFailure(span, error) + throw error + } finally { + span.end() + } + }) } function restoreState(stream: AnyStreamDefinition, envelope: CheckpointEnvelope | undefined, namespaceId: string): unknown { @@ -435,29 +471,47 @@ function restoreState(stream: AnyStreamDefinition, envelope: CheckpointEnvelope return stream.incremental.parseState(envelope.state) } -async function append( +// Appends for one namespace are serialized and the sequence number is taken +// only when the write starts, so a fact that never lands cannot leave a gap. +// An ambiguous failure retries the exact same deterministic fact. +function append( input: { namespaceId: string; runId: string; progress: StreamProgress; env: ResolvedEnv }, eventKind: JournalEvent['eventKind'], fields: Partial> ): Promise { - input.progress.seq += 1 - await input.env.journal.append({ - namespaceId: input.namespaceId, - eventSeq: input.progress.seq, - eventKind, - runId: input.runId, - workId: fields.workId ?? '', - attemptNo: fields.attemptNo ?? 0, - batchId: fields.batchId ?? '', - expectedCheckpointVersion: fields.expectedCheckpointVersion ?? input.progress.version, - checkpointVersion: fields.checkpointVersion ?? input.progress.version, - checkpoint: fields.checkpoint, - workState: fields.workState ?? '', - sinkEvidence: fields.sinkEvidence ?? '', - retryAt: fields.retryAt, - errorClass: fields.errorClass ?? '', - detail: fields.detail ?? {}, + const { progress, env } = input + const write = progress.appendChain.then(async () => { + const event: JournalEvent = { + namespaceId: input.namespaceId, + eventSeq: progress.seq + 1, + eventKind, + runId: input.runId, + workId: fields.workId ?? '', + attemptNo: fields.attemptNo ?? 0, + batchId: fields.batchId ?? '', + expectedCheckpointVersion: fields.expectedCheckpointVersion ?? progress.version, + checkpointVersion: fields.checkpointVersion ?? progress.version, + checkpoint: fields.checkpoint, + workState: fields.workState ?? '', + sinkEvidence: fields.sinkEvidence ?? '', + retryAt: fields.retryAt, + errorClass: fields.errorClass ?? '', + detail: fields.detail ?? {}, + } + for (let attempt = 1; ; attempt += 1) { + try { + await env.journal.append(event) + progress.seq = event.eventSeq + return + } catch (error) { + if (attempt >= JOURNAL_APPEND_ATTEMPTS) throw error + // Terminal facts must still land after cancellation, so this wait ignores the run signal. + await env.sleep(250 * 2 ** (attempt - 1), NEVER_ABORTED) + } + } }) + progress.appendChain = write.catch(() => undefined) + return write } function runEvent(seq: number, eventKind: 'run_started' | 'run_finished', runId: string, workState: JournalEvent['workState'], detail: Record): JournalEvent { @@ -480,10 +534,48 @@ function runEvent(seq: number, eventKind: 'run_started' | 'run_finished', runId: } } -function assertValidChunk(streamId: string, chunk: unknown): asserts chunk is { rows: readonly Row[]; state?: unknown } { +function assertValidChunk( + streamId: string, + chunk: unknown, + maxChunkRows: number +): asserts chunk is { rows: readonly Row[]; state?: unknown; id?: string } { if (typeof chunk !== 'object' || chunk === null || !('rows' in chunk) || !Array.isArray(chunk.rows)) { throw new IngestConfigError(`Stream "${streamId}" yielded a chunk without a "rows" array.`) } + if ('id' in chunk && chunk.id !== undefined && typeof chunk.id !== 'string') { + throw new IngestConfigError(`Stream "${streamId}" yielded a chunk whose "id" is not a string.`) + } + if (chunk.rows.length > maxChunkRows) { + throw new IngestConfigError( + `Stream "${streamId}" yielded a chunk of ${chunk.rows.length} rows, above the ${maxChunkRows}-row bound. Yield smaller chunks or raise budget.maxChunkRows.` + ) + } +} + +function emptyBatch(): PendingBatch { + return { rows: [], intervalIds: [], state: undefined } +} + +/** Resolves a fixed grace period after the signal aborts; never resolves otherwise. */ +function graceAfterAbort(signal: AbortSignal, graceMs: number): Promise { + return new Promise((resolve) => { + const start = () => { + const timer = setTimeout(resolve, graceMs) + if (typeof timer === 'object' && 'unref' in timer) timer.unref() + } + if (signal.aborted) start() + else signal.addEventListener('abort', start, { once: true }) + }) +} + +/** Settle with the promise, or reject as soon as the signal aborts. */ +function abortable(promise: Promise, signal: AbortSignal): Promise { + if (signal.aborted) return Promise.reject(signal.reason) + return new Promise((resolve, reject) => { + const onAbort = () => reject(signal.reason) + signal.addEventListener('abort', onAbort, { once: true }) + promise.then(resolve, reject).finally(() => signal.removeEventListener('abort', onAbort)) + }) } function permitsFor(cache: Map, pipeline: PipelineDefinition): PipelinePermits { @@ -498,20 +590,22 @@ function permitsFor(cache: Map, pipeline: PipelineDefin return created } -function resolveEnv(input: ExecutionEnv): ResolvedEnv { +const NEVER_ABORTED = new AbortController().signal + +function resolveEnv(input: ExecutionEnv, deadlineSignal: AbortSignal): ResolvedEnv { const now = input.now ?? (() => new Date()) const maxDurationMs = input.maxDurationMs ?? DEFAULT_MAX_DURATION_MS return { journal: input.journal, destination: input.destination, - signal: input.signal ?? new AbortController().signal, + signal: AbortSignal.any([input.signal ?? NEVER_ABORTED, deadlineSignal]), + hostSignal: input.signal ?? NEVER_ABORTED, maxDurationMs, prefetchBatches: input.prefetchBatches ?? DEFAULT_PREFETCH_BATCHES, now, sleep: input.sleep ?? sleep, random: input.random ?? Math.random, log: input.log ?? (() => undefined), - deadline: now().getTime() + maxDurationMs, } } diff --git a/packages/plugin-ingest/src/ingest.e2e.test.ts b/packages/plugin-ingest/src/ingest.e2e.test.ts index 19e13e22..718998a6 100644 --- a/packages/plugin-ingest/src/ingest.e2e.test.ts +++ b/packages/plugin-ingest/src/ingest.e2e.test.ts @@ -2,7 +2,7 @@ import { afterAll, beforeAll, describe, expect, test } from 'bun:test' import { table, toCreateSQL } from '@chkit/core' import type { ClickHouseExecutor } from '@chkit/clickhouse' -import { createLiveExecutor, createPrefix, getRequiredEnv, quoteIdent, waitForTable } from '@chkit/clickhouse/e2e-testkit' +import { createPrefix, createStatelessLiveExecutor, getRequiredEnv, quoteIdent, waitForTable } from '@chkit/clickhouse/e2e-testkit' import { createClickHouseDestination, ingestionColumns } from './destination.js' import { runIngestion } from './executor.js' @@ -28,7 +28,8 @@ describe('@chkit/plugin-ingest live env e2e', () => { let executor: ClickHouseExecutor beforeAll(async () => { - executor = createLiveExecutor(liveEnv) + // Ingestion fetches, loads and journals concurrently, so it needs a stateless executor. + executor = createStatelessLiveExecutor(liveEnv) await executor.command(toCreateSQL(destinationTable)) await waitForTable(executor, database, destinationTable.name) }) diff --git a/packages/plugin-ingest/src/journal.ts b/packages/plugin-ingest/src/journal.ts index f86f7ba5..01551b2c 100644 --- a/packages/plugin-ingest/src/journal.ts +++ b/packages/plugin-ingest/src/journal.ts @@ -32,6 +32,7 @@ export type JournalRow = { } export interface ClickHouseJournalOptions { + /** Must allow concurrent queries: use a stateless executor, not a session-bound one. */ executor: ClickHouseExecutor database: string targetId: string @@ -70,38 +71,67 @@ export function createClickHouseJournal(options: ClickHouseJournalOptions): Jour }, async readCheckpoint(namespaceId) { - // Physical retry duplicates are allowed; canonicalize by event_id and - // refuse to continue when one deterministic id carries different payloads. - const rows = await options.executor.query<{ - head_seq: string - drifted: string - checkpoint_version: string - checkpoint_json: string - }>( - `SELECT - max(event_seq) AS head_seq, - countIf(distinct_payloads > 1) AS drifted, - argMaxIf(fact_version, (fact_version, event_seq, event_id), fact_kind = 'batch_committed') AS checkpoint_version, - argMaxIf(fact_checkpoint, (fact_version, event_seq, event_id), fact_kind = 'batch_committed') AS checkpoint_json -FROM ( - SELECT + // Physical retry duplicates are allowed, so facts are canonicalized per + // sequence number first. The history is then validated before anything is + // projected from it: a checkpoint read from a damaged journal is worthless. + const facts = `SELECT event_seq, - event_id, + uniqExact(event_id) AS owners, + uniqExact(payload_hash) AS payloads, any(event_kind) AS fact_kind, + any(expected_checkpoint_version) AS fact_expected, any(checkpoint_version) AS fact_version, - any(checkpoint_json) AS fact_checkpoint, - uniqExact(payload_hash) AS distinct_payloads + any(checkpoint_json) AS fact_checkpoint FROM ${qualified} WHERE target_id = ${sqlString(options.targetId)} AND namespace_id = ${sqlString(namespaceId)} - GROUP BY event_seq, event_id + GROUP BY event_seq` + const settings = { select_sequential_consistency: '1' } + const [health, transitions] = await Promise.all([ + options.executor.query<{ + head_seq: string + sequences: string + conflicting_owners: string + drifted: string + checkpoint_version: string + checkpoint_json: string + }>( + `SELECT + max(event_seq) AS head_seq, + count() AS sequences, + countIf(owners > 1) AS conflicting_owners, + countIf(payloads > 1) AS drifted, + argMaxIf(fact_version, event_seq, fact_kind = 'batch_committed') AS checkpoint_version, + argMaxIf(fact_checkpoint, event_seq, fact_kind = 'batch_committed') AS checkpoint_json +FROM (${facts})`, + settings + ), + // Every commit must start from the version the previous commit produced + // and advance it by at most one. + options.executor.query<{ invalid: string }>( + `SELECT countIf(fact_expected != previous_version OR fact_version < fact_expected OR fact_version > fact_expected + 1) AS invalid +FROM ( + SELECT + fact_expected, + fact_version, + lagInFrame(fact_version, 1, toUInt64(0)) OVER (ORDER BY event_seq ASC ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS previous_version + FROM (${facts}) + WHERE fact_kind = 'batch_committed' )`, - { select_sequential_consistency: '1' } - ) - const row = rows[0] - if (!row) return { version: 0, envelope: undefined, headSeq: 0 } - if (Number(row.drifted) > 0) { + settings + ), + ]) + const row = health[0] + if (!row || Number(row.sequences) === 0) return emptyCheckpoint() + + const problems = [ + Number(row.head_seq) !== Number(row.sequences) ? `sequence gap (head ${row.head_seq}, ${row.sequences} facts)` : '', + Number(row.conflicting_owners) > 0 ? `${row.conflicting_owners} sequence number(s) owned by conflicting facts` : '', + Number(row.drifted) > 0 ? `${row.drifted} fact(s) with drifting payloads` : '', + Number(transitions[0]?.invalid ?? 0) > 0 ? `${transitions[0]?.invalid} invalid checkpoint transition(s)` : '', + ].filter((problem) => problem !== '') + if (problems.length > 0) { throw new Error( - `Ingestion journal payload drift detected for namespace "${namespaceId}": a deterministic event id has conflicting payloads. Refusing to continue.` + `Ingestion journal for "${namespaceId}" is not a valid history: ${problems.join('; ')}. Refusing to project a checkpoint from it; more than one executor process may have been active.` ) } return { @@ -116,8 +146,9 @@ FROM ( export function toJournalRow(event: JournalEvent, targetId: string, at: Date): JournalRow { const checkpointJson = event.checkpoint ? canonicalJson(event.checkpoint) : '' const detailJson = canonicalJson(event.detail) - // Identity covers what makes the fact unique; the payload hash covers what a - // replay of that same fact must reproduce. Timestamps are excluded from both. + // Identity covers what makes the fact unique; the payload hash covers every + // authoritative field a retry of that same fact must reproduce. Only the + // physical append time (event_at) is excluded. const eventId = digest([targetId, event.namespaceId, String(event.eventSeq), event.eventKind, event.workId, event.batchId, String(event.attemptNo)]) const payload = digest([ eventId, @@ -127,6 +158,9 @@ export function toJournalRow(event: JournalEvent, targetId: string, at: Date): J event.workState, event.sinkEvidence, event.errorClass, + event.runId, + event.retryAt ? event.retryAt.toISOString() : '', + detailJson, ]) return { target_id: targetId, diff --git a/packages/plugin-ingest/src/loader.ts b/packages/plugin-ingest/src/loader.ts index db08ac74..770b3c19 100644 --- a/packages/plugin-ingest/src/loader.ts +++ b/packages/plugin-ingest/src/loader.ts @@ -1,4 +1,4 @@ -import { BATCH_ID_COLUMN, RUN_ID_COLUMN } from './destination.js' +import { BATCH_ID_COLUMN, INGESTED_AT_COLUMN, RUN_ID_COLUMN } from './destination.js' import type { LoaderFactory } from './types.js' export interface SimpleLoaderOptions { @@ -25,11 +25,11 @@ export function simpleLoader(options: SimpleLoaderOptions = {}): LoaderFactory { async write(batch) { for (let offset = 0, unit = 0; offset < batch.rows.length; offset += maxRows, unit += 1) { ctx.signal.throwIfAborted() - const slice = batch.rows.slice(offset, offset + maxRows).map((row) => ({ - ...row, - [BATCH_ID_COLUMN]: batch.batchId, - [RUN_ID_COLUMN]: ctx.runId, - })) + const slice = batch.rows.slice(offset, offset + maxRows).map((row) => { + // Publication time is destination-owned: never let a mapped row supply it. + const { [INGESTED_AT_COLUMN]: _ignored, ...authored } = row + return { ...authored, [BATCH_ID_COLUMN]: batch.batchId, [RUN_ID_COLUMN]: ctx.runId } + }) await ctx.destination.insert({ table: ctx.table, rows: slice, token: `${batch.batchId}:${unit}` }) rows += slice.length writeUnits += 1 diff --git a/packages/plugin-ingest/src/paginate.ts b/packages/plugin-ingest/src/paginate.ts index d46a1c63..1dfa2880 100644 --- a/packages/plugin-ingest/src/paginate.ts +++ b/packages/plugin-ingest/src/paginate.ts @@ -1,3 +1,4 @@ +import { canonicalJson } from './journal.js' import type { AttemptOptions, ReadContext } from './types.js' export interface Page { @@ -16,8 +17,9 @@ export interface PaginateOptions { /** * Pull-based pagination over one retryable Promise step per page. Every request - * runs through the executor attempt capability, and repeated continuations are - * rejected so a cyclic provider cursor cannot loop forever. + * runs through the executor attempt capability. A continuation that was already + * seen (compared by canonical serialization) is rejected, so a cyclic provider + * cursor cannot loop forever; other non-progress is bounded by execution budgets. */ export async function* paginate( options: PaginateOptions @@ -34,7 +36,7 @@ export async function* paginate( if (page.items.length > 0) yield page.items if (page.next === undefined || page.next === null) return - const key = JSON.stringify(page.next) + const key = canonicalJson(page.next) if (seen.has(key)) { throw new Error(`paginate: provider returned a repeated continuation ${key}; refusing to loop.`) } diff --git a/packages/plugin-ingest/src/plugin.ts b/packages/plugin-ingest/src/plugin.ts index 2b237edc..20c44cb5 100644 --- a/packages/plugin-ingest/src/plugin.ts +++ b/packages/plugin-ingest/src/plugin.ts @@ -1,6 +1,6 @@ import process from 'node:process' -import { createClickHouseExecutor, type ClickHouseExecutor } from '@chkit/clickhouse' +import { createStatelessClickHouseExecutor, type ClickHouseExecutor } from '@chkit/clickhouse' import { createPluginRunner, defineFlags, @@ -236,8 +236,10 @@ function openTarget(context: IngestPluginCommandContext) { const clickhouse = context.config.clickhouse // A direct connection carries per-insert settings (the deduplication token); // fall back to the host-provided executor only when no URL is configured. + // Streams fetch, load and journal concurrently, so the executor must not be + // bound to one ClickHouse HTTP session (a session allows one in-flight query). if (clickhouse) { - const executor = createClickHouseExecutor(clickhouse) + const executor = createStatelessClickHouseExecutor(clickhouse) return { executor, database: clickhouse.database, targetId: targetIdOf(clickhouse.url, clickhouse.database), close: () => executor.close() } } if (context.pluginContext?.hasExecutor) { diff --git a/packages/plugin-ingest/src/types.ts b/packages/plugin-ingest/src/types.ts index 312d58f5..b4ddc7a7 100644 --- a/packages/plugin-ingest/src/types.ts +++ b/packages/plugin-ingest/src/types.ts @@ -18,6 +18,15 @@ export type Row = Record export interface SourceChunk { rows: readonly TRow[] state?: TState + /** + * Stable, non-secret identity of the logical source interval this chunk + * covers (a page cursor, an id range, a day…). Declare it only when a replay + * of the same interval is the same logical write: batch identity then ignores + * row content, so mutable provider fields cannot defeat retry deduplication. + * Without it, identity falls back to a content hash, which prefers a possible + * duplicate over suppressing rows that changed between attempts. + */ + id?: string } export interface AttemptOptions { @@ -89,6 +98,8 @@ export interface RetryOptions { export interface StreamBudget { /** Maximum source chunks pulled in one execution of this stream. */ maxChunks?: number + /** Largest chunk a reader may yield. A bigger chunk fails the stream instead of being buffered. */ + maxChunkRows?: number } export interface StreamDefinition { From 15014f475ff7f0842ab53c1db323dddd1994278a Mon Sep 17 00:00:00 2001 From: KeKs0r Date: Fri, 18 Sep 2026 18:31:34 -0700 Subject: [PATCH 03/11] =?UTF-8?q?=F0=9F=90=9B=20Close=20second-pass=20revi?= =?UTF-8?q?ew=20findings=20in=20ingestion=20runtime?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit One write unit per declared source interval, poisoned append chain after an unconfirmed fact, no progress commits from abandoned work, and envelope-change validation in the journal projection. --- apps/docs/src/content/docs/plugins/ingest.md | 2 +- packages/plugin-ingest/src/executor.ts | 19 +++++++++++++++++-- packages/plugin-ingest/src/journal.ts | 15 +++++++++++---- 3 files changed, 29 insertions(+), 7 deletions(-) diff --git a/apps/docs/src/content/docs/plugins/ingest.md b/apps/docs/src/content/docs/plugins/ingest.md index 23673e78..7e5b8009 100644 --- a/apps/docs/src/content/docs/plugins/ingest.md +++ b/apps/docs/src/content/docs/plugins/ingest.md @@ -89,7 +89,7 @@ definePipeline({ id: 'helpdesk', streams: [ticketStream], maxFetches: 4 }) Spread `ingestionColumns` into every destination table. The loader fills `_chkit_batch_id` and `_chkit_run_id`; `_chkit_ingested_at` is set by ClickHouse at the physical insert. -Rows must be deterministic for a given source page. Batch identity includes a content hash, so a field like `synced_at: new Date()` in a row defeats retry deduplication. +Batch identity decides whether a retry is deduplicated. By default it includes a content hash of the rows, which prefers a possible duplicate over suppressing rows that changed between attempts; a field like `synced_at: new Date()` therefore defeats retry deduplication. When a chunk covers a stable source interval, declare it with `id` (for example `yield { rows, id: \`page:${cursor}\` }`): the chunk then becomes its own write unit and its identity ignores row content. ## Progress and checkpoints diff --git a/packages/plugin-ingest/src/executor.ts b/packages/plugin-ingest/src/executor.ts index 503f7f71..6db840f1 100644 --- a/packages/plugin-ingest/src/executor.ts +++ b/packages/plugin-ingest/src/executor.ts @@ -290,6 +290,9 @@ async function readAndLoad(input: { const queue = createBoundedQueue(env.prefetchBatches) const batchSize = stream.batchSize ?? DEFAULT_BATCH_SIZE let budgetExhausted = false + // Set once this attempt has reported its outcome: detached work that finishes + // later may have written rows (at-least-once) but must never commit progress. + let abandoned = false const produce = async () => { let pending = emptyBatch() @@ -330,13 +333,20 @@ async function readAndLoad(input: { const chunk: unknown = step.value assertValidChunk(stream.id, chunk, maxChunkRows) progress.chunks += 1 + // A chunk that declares its source interval is one logical write unit: + // it is never merged with neighbours, so a replay regroups identically + // even when row counts changed. + if (chunk.id !== undefined && pending.rows.length > 0) { + await queue.push(pending, signal) + pending = emptyBatch() + } pending.rows.push(...chunk.rows) if (pending.intervalIds) { if (chunk.id === undefined) pending.intervalIds = undefined else pending.intervalIds.push(chunk.id) } if (chunk.state !== undefined) pending.state = { value: chunk.state } - if (pending.rows.length >= batchSize) { + if (chunk.id !== undefined || pending.rows.length >= batchSize) { await queue.push(pending, signal) pending = emptyBatch() } @@ -368,6 +378,7 @@ async function readAndLoad(input: { const discriminator = batch.intervalIds ? `interval:${canonicalJson(batch.intervalIds)}` : `content:${canonicalJson(batch.rows)}` const batchId = digest([namespaceId, String(progress.version), String(sinceBoundary), discriminator]).slice(0, 32) const receipt = await loadBatch(input, batchId, batch.rows, signal) + if (abandoned) return const envelope: CheckpointEnvelope | undefined = batch.state ? { strategy: stream.incremental.id, version: stream.incremental.version, state: batch.state.value } : progress.envelope @@ -406,6 +417,7 @@ async function readAndLoad(input: { // After a failure, a source operation that ignores its signal must not pin // the stream (and its permits) forever. await Promise.race([settled, graceAfterAbort(signal, READER_SHUTDOWN_GRACE_MS)]) + abandoned = true if (rootCause) throw rootCause.error if (budgetExhausted) { env.log?.(`${namespaceId}: ${new BudgetExhausted('execution budget exhausted; committed progress is preserved').message}`) @@ -510,7 +522,10 @@ function append( } } }) - progress.appendChain = write.catch(() => undefined) + // A fact that could not be confirmed poisons the chain: a later append must + // not reuse its sequence number, because the unconfirmed write may have landed. + progress.appendChain = write + write.catch(() => undefined) return write } diff --git a/packages/plugin-ingest/src/journal.ts b/packages/plugin-ingest/src/journal.ts index 01551b2c..35d765ed 100644 --- a/packages/plugin-ingest/src/journal.ts +++ b/packages/plugin-ingest/src/journal.ts @@ -105,15 +105,22 @@ export function createClickHouseJournal(options: ClickHouseJournalOptions): Jour FROM (${facts})`, settings ), - // Every commit must start from the version the previous commit produced - // and advance it by at most one. + // Every commit must start from the version the previous commit produced, + // advance it by at most one, and only change the envelope when it advances. + // The two reads are not one snapshot; that is sound because V1 runs a + // single executor process and reads a namespace before appending to it. options.executor.query<{ invalid: string }>( - `SELECT countIf(fact_expected != previous_version OR fact_version < fact_expected OR fact_version > fact_expected + 1) AS invalid + `SELECT countIf( + fact_expected != previous_version OR fact_version < fact_expected OR fact_version > fact_expected + 1 + OR (fact_version = fact_expected AND fact_checkpoint != previous_checkpoint) +) AS invalid FROM ( SELECT fact_expected, fact_version, - lagInFrame(fact_version, 1, toUInt64(0)) OVER (ORDER BY event_seq ASC ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS previous_version + fact_checkpoint, + lagInFrame(fact_version, 1, toUInt64(0)) OVER (ORDER BY event_seq ASC ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS previous_version, + lagInFrame(fact_checkpoint, 1, '') OVER (ORDER BY event_seq ASC ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS previous_checkpoint FROM (${facts}) WHERE fact_kind = 'batch_committed' )`, From d47d8062de1352fae4522fe1bf64a2bf7f8612f8 Mon Sep 17 00:00:00 2001 From: KeKs0r Date: Sat, 19 Sep 2026 08:19:29 -0700 Subject: [PATCH 04/11] =?UTF-8?q?=E2=9C=A8=20Add=20rawTable/rawRows:=20lan?= =?UTF-8?q?d=20provider=20objects=20raw,=20transform=20in=20ClickHouse?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .changeset/ingestion-runtime.md | 2 ++ apps/docs/src/content/docs/plugins/ingest.md | 33 ++++++++++++++++++++ packages/plugin-ingest/src/destination.ts | 32 ++++++++++++++++++- packages/plugin-ingest/src/executor.test.ts | 22 ++++++++++++- packages/plugin-ingest/src/index.ts | 2 +- 5 files changed, 88 insertions(+), 3 deletions(-) diff --git a/.changeset/ingestion-runtime.md b/.changeset/ingestion-runtime.md index bf660798..275ddf46 100644 --- a/.changeset/ingestion-runtime.md +++ b/.changeset/ingestion-runtime.md @@ -9,6 +9,8 @@ Add `@chkit/plugin-ingest`, the first cut of scheduled pull ingestion into Click Progress follows one rule: rows are saved before the bookmark advances. Every batch is written with a stable `insert_deduplication_token`, and only after the ClickHouse acknowledgement does the executor append a `batch_committed` fact to the append-only ingestion journal. Checkpoints are a projection of that journal, so a crashed or lost-acknowledgement run replays from the last durable boundary with the same batch identity instead of skipping rows. Bundled strategies are `timestampWindow`, `cursorState` for provider-owned state, and the full-sync fallback; `--backfill ` runs an explicit range in an isolated checkpoint namespace. +`rawTable` and `rawRows` land provider objects untouched in a native `JSON` column, so typed shapes are derived inside ClickHouse with ordinary views or materialized views instead of being mapped in pipeline code. + Source operations run through `context.attempt`, which owns fetch permits, p-retry-shaped retry policy, `Retry-After`, cancellation, and failure classification (`HttpError.fromResponse` is the canonical boundary for fetch-based readers). Pipelines carry separate `maxStreams`, `maxFetches`, and `maxLoads` ceilings, executions have a duration budget, and the executor emits OpenTelemetry spans. `@chkit/core` gains a singular `entry` config field, mutually exclusive with `schema` globs: the module is imported once, its exported schema definitions are collected, and plugin-domain definitions self-register while it loads. `@chkit/clickhouse` `insert()` accepts per-insert `settings`. diff --git a/apps/docs/src/content/docs/plugins/ingest.md b/apps/docs/src/content/docs/plugins/ingest.md index 7e5b8009..868a9557 100644 --- a/apps/docs/src/content/docs/plugins/ingest.md +++ b/apps/docs/src/content/docs/plugins/ingest.md @@ -91,6 +91,39 @@ Spread `ingestionColumns` into every destination table. The loader fills `_chkit Batch identity decides whether a retry is deduplicated. By default it includes a content hash of the rows, which prefers a possible duplicate over suppressing rows that changed between attempts; a field like `synced_at: new Date()` therefore defeats retry deduplication. When a chunk covers a stable source interval, declare it with `id` (for example `yield { rows, id: \`page:${cursor}\` }`): the chunk then becomes its own write unit and its identity ignores row content. +## Landing raw objects + +Mapping fields in the reader is optional, and usually the wrong place for it. `rawTable` defines a landing table that stores each provider object untouched in a native `JSON` column next to a stable `id`; `rawRows` shapes a page for it. Typed tables are then ordinary chkit views (or materialized views) over the raw layer: + +```ts +import { view } from '@chkit/core' +import { defineStream, rawRows, rawTable } from '@chkit/plugin-ingest' + +export const rawTickets = rawTable({ database: 'crm_raw', name: 'tickets' }) + +export const tickets = view({ + database: 'crm', + name: 'tickets', + as: `SELECT + id AS ticket_id, + raw.subject::String AS subject, + raw.requester.email::String AS requester_email, + arrayMap(t -> t.name::String, raw.tags[]) AS tags, + parseDateTime64BestEffortOrNull(raw.updated_at::String, 3, 'UTC') AS updated_at +FROM crm_raw.tickets FINAL`, +}) + +const ticketStream = defineStream({ + id: 'helpdesk.tickets', + destination: rawTickets, + async *read(context) { + for await (const page of listTickets(context)) yield { rows: rawRows(page, (ticket) => ticket.id) } + }, +}) +``` + +The raw table is a `ReplacingMergeTree`, so overlapping windows and replays collapse to the latest version of each `id`. Because the transform lives in ClickHouse, changing it never requires re-fetching the source: a view picks the change up immediately, and a materialized view can be rebuilt from the raw table. Keep the raw and modelled layers in separate databases so access and retention can differ. + ## Progress and checkpoints | Strategy | Use when | Bookmark advances | diff --git a/packages/plugin-ingest/src/destination.ts b/packages/plugin-ingest/src/destination.ts index 491757fe..bcf66fdc 100644 --- a/packages/plugin-ingest/src/destination.ts +++ b/packages/plugin-ingest/src/destination.ts @@ -1,5 +1,5 @@ import type { ClickHouseExecutor } from '@chkit/clickhouse' -import type { ColumnDefinition } from '@chkit/core' +import { table, type ColumnDefinition, type TableDefinition } from '@chkit/core' import type { DestinationAdapter } from './types.js' @@ -19,6 +19,36 @@ export const ingestionColumns: readonly ColumnDefinition[] = [ { name: INGESTED_AT_COLUMN, type: "DateTime64(6, 'UTC')", default: 'fn:now64(6)' }, ] +// A type alias (not an interface) so it is assignable to the index-signature Row type. +export type RawRow = { + id: string + raw: unknown +} + +/** + * Landing table for provider objects exactly as received: a stable id plus the + * untouched object in a native JSON column. Typed shapes are derived from it + * inside ClickHouse (views or materialized views), so changing a transform + * never requires re-fetching the source. Replays and overlapping windows + * collapse to the latest ingested version of each id. + */ +export function rawTable(input: { database: string; name: string; comment?: string }): TableDefinition { + return table({ + database: input.database, + name: input.name, + comment: input.comment, + columns: [{ name: 'id', type: 'String' }, { name: 'raw', type: 'JSON' }, ...ingestionColumns], + engine: `ReplacingMergeTree(${INGESTED_AT_COLUMN})`, + primaryKey: ['id'], + orderBy: ['id'], + }) +} + +/** Shape provider objects for a {@link rawTable} without mapping their fields. */ +export function rawRows(items: readonly T[], id: (item: T) => string): RawRow[] { + return items.map((item) => ({ id: id(item), raw: item })) +} + /** * A successful synchronous insert response (or an awaited async insert) is the * sink evidence ChKit trusts. A missing response stays ambiguous and is retried diff --git a/packages/plugin-ingest/src/executor.test.ts b/packages/plugin-ingest/src/executor.test.ts index 219dfe5d..aef09d39 100644 --- a/packages/plugin-ingest/src/executor.test.ts +++ b/packages/plugin-ingest/src/executor.test.ts @@ -2,7 +2,7 @@ import { beforeEach, describe, expect, test } from 'bun:test' import { table } from '@chkit/core' -import { ingestionColumns } from './destination.js' +import { ingestionColumns, rawRows, rawTable } from './destination.js' import { HttpError } from './errors.js' import { runIngestion } from './executor.js' import { cursorState, timestampWindow } from './incremental.js' @@ -401,6 +401,26 @@ describe('runtime contracts', () => { }) }) +describe('rawTable', () => { + test('lands provider objects untouched next to a stable id', async () => { + const destination = createMemoryDestination() + const landing = rawTable({ database: 'app_raw', name: 'tickets' }) + const ticket = { key: 'T-1', requester: { email: 'a@example.com' }, tags: [{ name: 'vip' }] } + const stream = defineStream({ + id: 'app.tickets', + destination: landing, + async *read() { + yield { rows: rawRows([ticket], (item) => item.key) } + }, + }) + + await runIngestion({ selected: selectStreams([definePipeline({ id: 'app', streams: [stream] })], []), backfill: undefined }, { journal: createMemoryJournal(), destination }) + + expect(landing.columns.map((column) => `${column.name}:${column.type}`).slice(0, 2)).toEqual(['id:String', 'raw:JSON']) + expect(destination.tables.get('app_raw.tickets')?.[0]).toMatchObject({ id: 'T-1', raw: ticket }) + }) +}) + describe('selectStreams', () => { test('repeated tags use exact AND semantics and an explicit empty selection fails', () => { const hourly = defineStream({ id: 'crm.people', destination: events, tags: ['schedule:1h'], async *read() {} }) diff --git a/packages/plugin-ingest/src/index.ts b/packages/plugin-ingest/src/index.ts index b178043a..2db7f505 100644 --- a/packages/plugin-ingest/src/index.ts +++ b/packages/plugin-ingest/src/index.ts @@ -3,7 +3,7 @@ export { defineStream, definePipeline, listPipelines, selectStreams, type Select export { fullSync, timestampWindow, cursorState, type TimestampRange, type TimestampWindowState } from './incremental.js' export { paginate, type Page } from './paginate.js' export { simpleLoader } from './loader.js' -export { ingestionColumns, createClickHouseDestination } from './destination.js' +export { ingestionColumns, rawTable, rawRows, createClickHouseDestination, type RawRow } from './destination.js' export { createClickHouseJournal } from './journal.js' export { runIngestion, type BackfillRequest, type ExecutionEnv, type ExecutionRequest } from './executor.js' export { HttpError, FetchFailure, IngestConfigError } from './errors.js' From 8283a5caab4cff19339d0fe66b605452cc34de01 Mon Sep 17 00:00:00 2001 From: KeKs0r Date: Sat, 19 Sep 2026 08:42:01 -0700 Subject: [PATCH 05/11] =?UTF-8?q?=F0=9F=93=9D=20Use=20a=20=5Fraw=20table?= =?UTF-8?q?=20suffix=20in=20the=20ingestion=20raw-landing=20docs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/docs/src/content/docs/plugins/ingest.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/docs/src/content/docs/plugins/ingest.md b/apps/docs/src/content/docs/plugins/ingest.md index 868a9557..9e3e8a5f 100644 --- a/apps/docs/src/content/docs/plugins/ingest.md +++ b/apps/docs/src/content/docs/plugins/ingest.md @@ -99,7 +99,7 @@ Mapping fields in the reader is optional, and usually the wrong place for it. `r import { view } from '@chkit/core' import { defineStream, rawRows, rawTable } from '@chkit/plugin-ingest' -export const rawTickets = rawTable({ database: 'crm_raw', name: 'tickets' }) +export const rawTickets = rawTable({ database: 'crm', name: 'tickets_raw' }) export const tickets = view({ database: 'crm', @@ -110,7 +110,7 @@ export const tickets = view({ raw.requester.email::String AS requester_email, arrayMap(t -> t.name::String, raw.tags[]) AS tags, parseDateTime64BestEffortOrNull(raw.updated_at::String, 3, 'UTC') AS updated_at -FROM crm_raw.tickets FINAL`, +FROM crm.tickets_raw FINAL`, }) const ticketStream = defineStream({ @@ -122,7 +122,7 @@ const ticketStream = defineStream({ }) ``` -The raw table is a `ReplacingMergeTree`, so overlapping windows and replays collapse to the latest version of each `id`. Because the transform lives in ClickHouse, changing it never requires re-fetching the source: a view picks the change up immediately, and a materialized view can be rebuilt from the raw table. Keep the raw and modelled layers in separate databases so access and retention can differ. +The raw table is a `ReplacingMergeTree`, so overlapping windows and replays collapse to the latest version of each `id`. Because the transform lives in ClickHouse, changing it never requires re-fetching the source: a view picks the change up immediately, and a materialized view can be rebuilt from the raw table. A `_raw` suffix next to the typed view of the same name keeps the pair easy to find. ## Progress and checkpoints From 592a20ad55ea38c4dce8825862dac6b1af729b2b Mon Sep 17 00:00:00 2001 From: KeKs0r Date: Sat, 19 Sep 2026 15:26:58 -0700 Subject: [PATCH 06/11] =?UTF-8?q?=F0=9F=90=9B=20Finalize=20ingestion=20API?= =?UTF-8?q?=20and=20recovery=20guarantees?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .changeset/ingestion-runtime.md | 10 +- .../content/docs/configuration/overview.md | 2 +- apps/docs/src/content/docs/plugins/ingest.md | 19 ++- packages/cli/src/runtime/config-merge.ts | 4 +- .../cli/src/test/runtime/config-merge.test.ts | 16 ++- packages/core/src/model-types.ts | 4 +- packages/core/src/schema-loader.ts | 19 ++- packages/plugin-ingest/src/api.test.ts | 116 ++++++++++++++++ packages/plugin-ingest/src/executor.test.ts | 126 ++++++++++++++++-- packages/plugin-ingest/src/executor.ts | 81 ++++++----- packages/plugin-ingest/src/incremental.ts | 39 ++++-- packages/plugin-ingest/src/index.ts | 5 +- packages/plugin-ingest/src/ingest.e2e.test.ts | 38 +++++- packages/plugin-ingest/src/journal.ts | 6 +- packages/plugin-ingest/src/paginate.ts | 5 +- packages/plugin-ingest/src/plugin.ts | 24 ++-- packages/plugin-ingest/src/registry.ts | 67 ++++------ packages/plugin-ingest/src/testing.ts | 5 +- packages/plugin-ingest/src/types.ts | 22 +-- 19 files changed, 461 insertions(+), 147 deletions(-) create mode 100644 packages/plugin-ingest/src/api.test.ts diff --git a/.changeset/ingestion-runtime.md b/.changeset/ingestion-runtime.md index 275ddf46..bde0b8aa 100644 --- a/.changeset/ingestion-runtime.md +++ b/.changeset/ingestion-runtime.md @@ -5,12 +5,16 @@ "chkit": patch --- -Add `@chkit/plugin-ingest`, the first cut of scheduled pull ingestion into ClickHouse. Streams are ordinary TypeScript: a `read` async generator fetches, maps, and yields destination-shaped rows, and `definePipeline` registers a tagged, non-durable group of streams from the project entry. `chkit ingest run` executes the selected streams (`--tag` is repeatable with exact AND semantics; an explicit empty selection fails), `chkit ingest list` shows the loaded graph, and `chkit ingest status` prints committed checkpoints. +Add `@chkit/plugin-ingest`, the first cut of scheduled pull ingestion into ClickHouse. Streams are ordinary TypeScript: a `read` async generator fetches, maps, and yields destination-shaped rows, and `definePipeline` returns a tagged, non-durable group of streams; only pipelines exported from the project entry participate, with no global registry. `chkit ingest run` executes the selected streams (`--tag` is repeatable with exact AND semantics; an explicit empty selection fails), `chkit ingest list` shows the loaded graph, and `chkit ingest status` prints committed checkpoints. -Progress follows one rule: rows are saved before the bookmark advances. Every batch is written with a stable `insert_deduplication_token`, and only after the ClickHouse acknowledgement does the executor append a `batch_committed` fact to the append-only ingestion journal. Checkpoints are a projection of that journal, so a crashed or lost-acknowledgement run replays from the last durable boundary with the same batch identity instead of skipping rows. Bundled strategies are `timestampWindow`, `cursorState` for provider-owned state, and the full-sync fallback; `--backfill ` runs an explicit range in an isolated checkpoint namespace. +Progress follows one rule: rows are saved before the bookmark advances. Every batch is written with a stable `insert_deduplication_token`, and only after the ClickHouse acknowledgement does the executor append a `batch_committed` fact to the append-only ingestion journal. Checkpoints are a projection of that journal, so a crashed or lost-acknowledgement run replays from the last durable boundary with the same batch identity instead of skipping rows. Bundled strategies are `timestampWindow({ start, overlapMs })` (with a custom `from` callback alternative), `cursorState` for provider-owned state, and the full-sync fallback; `--backfill ` runs an explicit range in an isolated checkpoint namespace. `rawTable` and `rawRows` land provider objects untouched in a native `JSON` column, so typed shapes are derived inside ClickHouse with ordinary views or materialized views instead of being mapped in pipeline code. +`FetchContext` exposes source request and cancellation capabilities independently of checkpoint types; `ReadContext` extends it. + Source operations run through `context.attempt`, which owns fetch permits, p-retry-shaped retry policy, `Retry-After`, cancellation, and failure classification (`HttpError.fromResponse` is the canonical boundary for fetch-based readers). Pipelines carry separate `maxStreams`, `maxFetches`, and `maxLoads` ceilings, executions have a duration budget, and the executor emits OpenTelemetry spans. -`@chkit/core` gains a singular `entry` config field, mutually exclusive with `schema` globs: the module is imported once, its exported schema definitions are collected, and plugin-domain definitions self-register while it loads. `@chkit/clickhouse` `insert()` accepts per-insert `settings`. +`@chkit/core` gains a singular `entry` config field, mutually exclusive with `schema` globs: the module is imported once, its exported schema definitions are collected, and exported plugin-domain definitions are collected by their plugins. `@chkit/clickhouse` `insert()` accepts per-insert `settings`. + +Successful syncs rotate batch identity using the existing journal, while failed runs retain their replay identity. Execution cancellation also bounds journal I/O, stalled writes cannot report success, and loader construction failures release their permits. Ingestion requires a direct ClickHouse connection; incompatible host executors fail before any writes. Project `entry` and `schema` settings replace the inherited source mode when layering configuration. diff --git a/apps/docs/src/content/docs/configuration/overview.md b/apps/docs/src/content/docs/configuration/overview.md index 66a63aaf..3b2a5d96 100644 --- a/apps/docs/src/content/docs/configuration/overview.md +++ b/apps/docs/src/content/docs/configuration/overview.md @@ -47,7 +47,7 @@ export default defineConfig({ }) ``` -chkit imports the module once. Schema definitions it exports (directly or re-exported from other files) are collected exactly like glob-matched schema files, and plugin-domain definitions such as [ingestion pipelines](/plugins/ingest/) register themselves while it loads. `entry` and `schema` are mutually exclusive. +chkit imports the module once. Schema definitions it exports (directly or re-exported from other files) are collected exactly like glob-matched schema files, and exported plugin-domain definitions such as [ingestion pipelines](/plugins/ingest/) are collected by their plugins. `entry` and `schema` are mutually exclusive. ## Cluster mode (`ON CLUSTER`) diff --git a/apps/docs/src/content/docs/plugins/ingest.md b/apps/docs/src/content/docs/plugins/ingest.md index 9e3e8a5f..09de3962 100644 --- a/apps/docs/src/content/docs/plugins/ingest.md +++ b/apps/docs/src/content/docs/plugins/ingest.md @@ -19,7 +19,9 @@ The plugin never creates or changes destination tables. Your chkit schema stays ## Plugin setup -Ingestion uses the singular `entry` config field instead of `schema` globs. The entry module is imported once: exported tables are collected as schema, and pipelines register themselves while it loads. +Ingestion uses the singular `entry` config field instead of `schema` globs. The entry module is imported once: exported tables are collected as schema, and exported pipelines form the ingestion graph. Importing a pipeline without exporting it does not activate it; remove its export to deactivate it. + +Configure a direct `clickhouse` connection for ingestion. A host-provided executor, including the ObsessionDB workbench executor, cannot currently guarantee JSON row encoding and per-insert deduplication settings. ```ts // clickhouse.config.ts @@ -62,7 +64,8 @@ const ticketStream = defineStream({ tags: ['schedule:1h'], incremental: timestampWindow({ // Re-read one hour of overlap; ReplacingMergeTree reconciles repeats. - from: ({ watermark }) => (watermark ? new Date(watermark.getTime() - 3_600_000) : new Date(0)), + start: new Date(0), + overlapMs: 3_600_000, }), async *read(context) { const pages = paginate({ @@ -84,7 +87,7 @@ const ticketStream = defineStream({ }, }) -definePipeline({ id: 'helpdesk', streams: [ticketStream], maxFetches: 4 }) +export const helpdesk = definePipeline({ id: 'helpdesk', streams: [ticketStream], maxFetches: 4 }) ``` Spread `ingestionColumns` into every destination table. The loader fills `_chkit_batch_id` and `_chkit_run_id`; `_chkit_ingested_at` is set by ClickHouse at the physical insert. @@ -129,9 +132,13 @@ The raw table is a `ReplacingMergeTree`, so overlapping windows and replays coll | Strategy | Use when | Bookmark advances | |---|---|---| | none (full sync) | The source is small or has no change filter | Never; every run reads everything | -| `timestampWindow({ from })` | The API filters by an updated-since timestamp | To the run cutoff, after the whole window loaded | +| `timestampWindow({ start, overlapMs })` | The API filters by an updated-since timestamp | To the run cutoff, after the whole window loaded | | `cursorState({ id, version, parse })` | The provider owns the state: compound cursor, change token, page position | Whenever a yielded chunk carries `state` and its rows have been saved | +`timestampWindow` starts its first sync at `start`. Later runs begin at the committed watermark minus `overlapMs` (zero by default). Explicit backfill bounds take precedence. For custom lower bounds, use `timestampWindow({ from: ({ watermark, cutoff }) => ... })` instead. Both forms retain the same checkpoint format and only advance after the entire window is saved. + +Provider clients can accept the exported `FetchContext` type, containing `attempt` and `signal`. `ReadContext` extends it with the stream selection and checkpoint, so readers can pass their context directly without coupling clients to checkpoint generics. + With `cursorState`, `state` on a chunk must be the complete state that is safe to resume from once every row up to that chunk is saved. Omit it when you cannot make that claim; the run then restarts from the previous checkpoint after a failure. A checkpoint records its strategy id and version. Changing either makes the next run fail rather than reinterpret old state. @@ -157,6 +164,10 @@ A backfill uses its own checkpoint namespace, so it never moves the scheduled bo Ingestion is at-least-once. Each batch is inserted with a stable `insert_deduplication_token`, so a retry after a lost acknowledgement is suppressed while the table's deduplication window covers it. Pick a destination engine that reconciles repeats for your data, for example `ReplacingMergeTree` keyed by the provider id. +Successful syncs start a new batch identity cycle, recorded by the existing journal. Failed or interrupted syncs retain their cycle for replay. This also applies to full syncs, which have no incremental bookmark. + +The duration budget bounds journal operations as well as readers. Shutdown gives unfinished readers or writes up to five seconds to settle; each terminal journal append has a separate five-second limit. Interrupted writes never count as successful ingestion. + Run at most one ingestion process per project and target at a time. Use your scheduler's concurrency control (for example a GitHub Actions concurrency group) to enforce it. ## Options diff --git a/packages/cli/src/runtime/config-merge.ts b/packages/cli/src/runtime/config-merge.ts index 2b7f0aff..1d2e72b1 100644 --- a/packages/cli/src/runtime/config-merge.ts +++ b/packages/cli/src/runtime/config-merge.ts @@ -70,8 +70,8 @@ export function mergeUserConfig( overlay: ChxUserConfig, ): ChxUserConfig { return { - schema: overlay.schema ?? base.schema, - entry: overlay.entry ?? base.entry, + schema: overlay.schema ?? (overlay.entry !== undefined ? undefined : base.schema), + entry: overlay.entry ?? (overlay.schema !== undefined ? undefined : base.entry), outDir: overlay.outDir ?? base.outDir, migrationsDir: overlay.migrationsDir ?? base.migrationsDir, metaDir: overlay.metaDir ?? base.metaDir, diff --git a/packages/cli/src/test/runtime/config-merge.test.ts b/packages/cli/src/test/runtime/config-merge.test.ts index 02983684..1f4d88b8 100644 --- a/packages/cli/src/test/runtime/config-merge.test.ts +++ b/packages/cli/src/test/runtime/config-merge.test.ts @@ -1,6 +1,6 @@ import { describe, expect, test } from 'bun:test' -import type { ChxUserConfig } from '@chkit/core' +import { resolveConfig, type ChxUserConfig } from '@chkit/core' import { mergeUserConfig, pluginNameOf } from '../../runtime/config-merge.js' @@ -20,6 +20,20 @@ const profile: ChxUserConfig = { } describe('mergeUserConfig', () => { + test('project source mode replaces the inherited alternative', () => { + const entry = mergeUserConfig({ schema: './profile/**/*.ts' }, { entry: './src/chkit.ts' }) + expect(resolveConfig(entry).schema).toEqual(['./src/chkit.ts']) + expect(entry.schema).toBeUndefined() + + const globs = mergeUserConfig({ entry: './profile.ts' }, { schema: './src/schema/**/*.ts' }) + expect(resolveConfig(globs).schema).toEqual(['./src/schema/**/*.ts']) + expect(globs.entry).toBeUndefined() + + expect(resolveConfig(mergeUserConfig({ entry: './profile.ts' }, {})).entry).toBe('./profile.ts') + expect(resolveConfig(mergeUserConfig({ entry: './profile.ts' }, { schema: [] })).schema).toEqual([]) + expect(() => resolveConfig(mergeUserConfig({}, { entry: './entry.ts', schema: './schema.ts' }))).toThrow('mutually exclusive') + }) + test('overlay scalar fields beat base', () => { const merged = mergeUserConfig(profile, { schema: ['./schema/*.ts'], diff --git a/packages/core/src/model-types.ts b/packages/core/src/model-types.ts index 86c86078..c4a29ffe 100644 --- a/packages/core/src/model-types.ts +++ b/packages/core/src/model-types.ts @@ -272,8 +272,8 @@ export interface ChxUserConfig { schema?: string | string[] /** * Single project entry module. It is imported once: exported schema - * definitions are collected from it, and plugin-domain definitions (for - * example ingestion pipelines) self-register while it loads. Mutually + * definitions are collected from it, and exported plugin-domain definitions + * (for example ingestion pipelines) are collected by their plugins. Mutually * exclusive with `schema`. */ entry?: string diff --git a/packages/core/src/schema-loader.ts b/packages/core/src/schema-loader.ts index f2fec004..1bac9a0d 100644 --- a/packages/core/src/schema-loader.ts +++ b/packages/core/src/schema-loader.ts @@ -14,6 +14,15 @@ export async function loadSchemaDefinitions( schemaGlobs: string | string[], options: SchemaLoaderOptions = {} ): Promise { + const modules = await loadDefinitionModules(schemaGlobs, options) + return canonicalizeDefinitions(modules.flatMap(collectDefinitionsFromModule)) +} + +/** Load configured entry/schema modules so plugins can inspect their exported definitions. */ +export async function loadDefinitionModules( + schemaGlobs: string | string[], + options: SchemaLoaderOptions = {} +): Promise[]> { const patterns = Array.isArray(schemaGlobs) ? schemaGlobs : [schemaGlobs] const files = await fg(patterns, { cwd: options.cwd ?? process.cwd(), @@ -24,11 +33,7 @@ export async function loadSchemaDefinitions( throw new Error('No schema files matched. Check config.schema patterns.') } - const all: SchemaDefinition[] = [] - for (const file of files) { - const mod = await importModuleFile(file) - all.push(...collectDefinitionsFromModule(mod)) - } - - return canonicalizeDefinitions(all) + const modules: Record[] = [] + for (const file of files) modules.push(await importModuleFile(file)) + return modules } diff --git a/packages/plugin-ingest/src/api.test.ts b/packages/plugin-ingest/src/api.test.ts new file mode 100644 index 00000000..18f7346b --- /dev/null +++ b/packages/plugin-ingest/src/api.test.ts @@ -0,0 +1,116 @@ +import { afterEach, describe, expect, test } from 'bun:test' +import { mkdtemp, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' + +import { resolveConfig } from '@chkit/core' +import type { ClickHouseExecutor } from '@chkit/clickhouse' +import { loadDefinitionModules, loadSchemaDefinitions } from '@chkit/core/schema-loader' + +import { rawTable } from './destination.js' +import { timestampWindow } from './incremental.js' +import { createIngestPlugin } from './plugin.js' +import { collectPipelines, definePipeline, defineStream, selectStreams } from './registry.js' + +const start = new Date('2024-01-01T00:00:00Z') +const cutoff = new Date('2026-09-19T00:00:00Z') +const watermark = '2026-09-18T00:00:00.000Z' + +describe('timestampWindow options', () => { + test('bootstraps at start and applies overlap only to the committed watermark', () => { + const window = timestampWindow({ start, overlapMs: 3_600_000 }) + expect(window.plan({ state: undefined, cutoff, range: undefined })).toEqual({ from: start, to: cutoff }) + const selection = window.plan({ state: window.parseState({ watermark }), cutoff, range: undefined }) + expect(selection.from).toEqual(new Date('2026-09-17T23:00:00Z')) + expect(window.complete?.({ state: { watermark }, selection })).toEqual({ watermark: cutoff.toISOString() }) + expect(timestampWindow({ start }).plan({ state: { watermark }, cutoff, range: undefined }).from).toEqual(new Date(watermark)) + }) + + test('explicit backfill bounds override start, watermark, overlap, and cutoff', () => { + const window = timestampWindow({ start, overlapMs: 3_600_000 }) + const range = { from: new Date('2025-01-01'), to: new Date('2025-02-01') } + const selection = window.plan({ state: { watermark }, cutoff, range }) + expect(selection).toEqual(range) + expect(window.complete?.({ state: { watermark }, selection })).toEqual({ watermark: range.to.toISOString() }) + expect(window.plan({ state: undefined, cutoff, range: { from: undefined, to: range.to } })).toEqual({ from: start, to: range.to }) + }) + + test('retains custom callbacks and the existing checkpoint strategy', () => { + const window = timestampWindow({ from: ({ watermark: saved, cutoff: end }) => saved ?? end }) + expect(window.plan({ state: undefined, cutoff, range: undefined })).toEqual({ from: cutoff, to: cutoff }) + expect(window.plan({ state: { watermark }, cutoff, range: undefined }).from).toEqual(new Date(watermark)) + expect([window.id, window.version]).toEqual(['chkit.timestamp_window', 1]) + }) + + test('rejects invalid bounds and overlap', () => { + expect(() => timestampWindow({ start: new Date('invalid') })).toThrow('valid Date') + for (const overlapMs of [-1, Number.NaN, Number.POSITIVE_INFINITY]) { + expect(() => timestampWindow({ start, overlapMs })).toThrow('non-negative') + } + expect(() => timestampWindow({ start: cutoff }).plan({ state: undefined, cutoff: start, range: undefined })).toThrow('after upper bound') + expect(() => timestampWindow({ from: () => new Date('invalid') }).plan({ state: undefined, cutoff, range: undefined })).toThrow('valid dates') + }) +}) + +const directories: string[] = [] +afterEach(async () => { + await Promise.all(directories.splice(0).map((path) => rm(path, { recursive: true, force: true }))) +}) + +describe('exported pipeline discovery', () => { + test('side-effect imports stay inactive; re-exports activate pipelines even after a cached import', async () => { + const directory = await mkdtemp(join(tmpdir(), 'chkit-exports-')) + directories.push(directory) + const provider = join(directory, 'provider.ts') + const hidden = join(directory, 'hidden.ts') + const active = join(directory, 'active.ts') + await writeFile(provider, ` + import { definePipeline, defineStream } from ${JSON.stringify(new URL('./registry.ts', import.meta.url).pathname)} + import { rawTable } from ${JSON.stringify(new URL('./destination.ts', import.meta.url).pathname)} + export const rows = rawTable({ database: 'test', name: 'rows' }) + const stream = defineStream({ id: 'test.rows', destination: rows, async *read() { yield { rows: [] } } }) + export const pipeline = definePipeline({ id: 'test', streams: [stream] }) + `) + await writeFile(hidden, "import './provider.ts'\nexport { rows } from './provider.ts'\n") + await writeFile(active, "export { rows, pipeline, pipeline as alias } from './provider.ts'\n") + + expect(collectPipelines(await loadDefinitionModules(hidden))).toEqual([]) + expect(collectPipelines(await loadDefinitionModules(active)).map((pipeline) => pipeline.id)).toEqual(['test']) + expect(collectPipelines(await loadDefinitionModules(hidden))).toEqual([]) + expect(await loadSchemaDefinitions(active)).toHaveLength(1) + + const plugin = createIngestPlugin() + const inactiveCheck = await plugin.hooks.onCheck({ config: resolveConfig({ entry: hidden }) }) + expect(inactiveCheck.findings.map((finding) => finding.code)).toEqual(['ingest_no_pipelines']) + const activeCheck = await plugin.hooks.onCheck({ config: resolveConfig({ entry: active }) }) + expect(activeCheck.findings).toEqual([]) + + // A selected host service is insufficient: its insert API may discard JSON/settings. + for (const command of plugin.commands.filter((command) => command.name !== 'list')) { + const output: unknown[] = [] + const code = await command.run({ + args: [], flags: {}, jsonMode: true, + options: { journalTable: 'test_journal', maxDurationSeconds: 1, prefetchBatches: 1 }, + config: resolveConfig({ entry: active }), configPath: active, + pluginContext: { hasExecutor: true, executor: {} as ClickHouseExecutor }, + print: (value) => { output.push(value) }, + }) + expect(code).toBe(2) + expect(output[0]).toMatchObject({ ok: false, error: expect.stringContaining('direct ClickHouse connection') }) + } + }) + + test('validates only the selected graph and rejects duplicate pipeline or stream IDs', () => { + const destination = rawTable({ database: 'test', name: 'rows' }) + const stream = defineStream({ id: 'test.rows', destination, async *read() { yield { rows: [] } } }) + const a = definePipeline({ id: 'a', streams: [stream] }) + const b = definePipeline({ id: 'b', streams: [stream] }) + const duplicate = definePipeline({ id: 'a', streams: [] }) + expect(selectStreams([a], [])).toHaveLength(1) + expect(selectStreams([b], [])).toHaveLength(1) + expect(() => collectPipelines([{ a, duplicate }])).toThrow('Duplicate pipeline') + expect(() => collectPipelines([{ a, b }])).toThrow('globally unique') + expect(() => selectStreams([a, b], ['pipeline:a'])).toThrow('globally unique') + expect(() => definePipeline({ id: 'repeat', streams: [stream, stream] })).toThrow('more than once') + }) +}) diff --git a/packages/plugin-ingest/src/executor.test.ts b/packages/plugin-ingest/src/executor.test.ts index aef09d39..a8a0e150 100644 --- a/packages/plugin-ingest/src/executor.test.ts +++ b/packages/plugin-ingest/src/executor.test.ts @@ -1,4 +1,4 @@ -import { beforeEach, describe, expect, test } from 'bun:test' +import { describe, expect, test } from 'bun:test' import { table } from '@chkit/core' @@ -7,7 +7,7 @@ import { HttpError } from './errors.js' import { runIngestion } from './executor.js' import { cursorState, timestampWindow } from './incremental.js' import { paginate } from './paginate.js' -import { definePipeline, defineStream, resetRegistry, selectStreams } from './registry.js' +import { definePipeline, defineStream, selectStreams } from './registry.js' import { createMemoryDestination, createMemoryJournal } from './testing.js' import type { DestinationAdapter } from './types.js' @@ -28,8 +28,6 @@ function httpError(status: number, headers: Record = {}) { return HttpError.fromResponse(new Response('nope', { status, headers })) } -beforeEach(() => resetRegistry()) - describe('runIngestion', () => { test('loads rows with runtime metadata and journals progress only after sink evidence', async () => { const journal = createMemoryJournal() @@ -126,7 +124,7 @@ describe('runIngestion', () => { const stream = defineStream({ id: 'app.window', destination: events, - incremental: timestampWindow({ from: ({ watermark }) => watermark ?? new Date('2026-01-01T00:00:00Z') }), + incremental: timestampWindow({ start: new Date('2026-01-01T00:00:00Z') }), async *read({ selection }) { selections.push({ from: selection.from.toISOString(), to: selection.to.toISOString() }) yield { rows: [{ id: selections.length }] } @@ -152,7 +150,7 @@ describe('runIngestion', () => { id: 'app.window', destination: events, retry: { retries: 0 }, - incremental: timestampWindow({ from: ({ watermark }) => watermark ?? new Date(0) }), + incremental: timestampWindow({ start: new Date(0) }), async *read() { yield { rows: [{ id: 1 }] } throw new Error('provider exploded') @@ -211,7 +209,6 @@ describe('runIngestion', () => { const journal = createMemoryJournal() const destination = createMemoryDestination() const build = (version: number) => { - resetRegistry() const stream = defineStream({ id: 'app.versioned', destination: events, @@ -258,7 +255,7 @@ describe('runIngestion', () => { const stream = defineStream({ id: 'app.window', destination: events, - incremental: timestampWindow({ from: ({ watermark }) => watermark ?? new Date(0) }), + incremental: timestampWindow({ start: new Date(0) }), async *read() { yield { rows: [{ id: 1 }] } }, @@ -279,9 +276,115 @@ describe('runtime contracts', () => { const run = (pipeline: ReturnType, env: Parameters[1]) => runIngestion({ selected: selectStreams([pipeline], []), backfill: undefined }, { sleep: noSleep, ...env }) + test.each([false, true])('new full syncs preserve changes while failed syncs reuse tokens (declared id: %s)', async (withId) => { + const journal = createMemoryJournal() + const destination = createMemoryDestination() + let label = 'A' + const stream = defineStream({ + id: 'app.full', destination: events, retry: { retries: 0 }, + async *read() { yield { rows: [{ id: 1, label }], ...(withId ? { id: 'page:1' } : {}) } }, + }) + const pipeline = definePipeline({ id: 'app', streams: [stream] }) + expect((await run(pipeline, { journal, destination })).ok).toBe(true) + const firstSuccess = (await journal.readCheckpoint(stream.id)).lastSuccessSeq + expect(firstSuccess).toBeGreaterThan(0) + + label = 'B' + const lossy: DestinationAdapter = { async insert(input) { + await destination.insert(input) + throw new Error('acknowledgement lost') + } } + expect((await run(pipeline, { journal, destination: lossy })).ok).toBe(false) + expect((await journal.readCheckpoint(stream.id)).lastSuccessSeq).toBe(firstSuccess) + expect((await run(pipeline, { journal, destination })).ok).toBe(true) + + label = 'A' + expect((await run(pipeline, { journal, destination })).ok).toBe(true) + expect(destination.tables.get('app.events')?.map((row) => row.label)).toEqual(['A', 'B', 'A']) + expect((await journal.readCheckpoint(stream.id)).envelope).toBeUndefined() + }) + + test.each(['deadline', 'cancel'] as const)('a hung destination never reports success after %s', async (mode) => { + const journal = createMemoryJournal() + const controller = new AbortController() + let finishWrite: () => void = () => undefined + const destination: DestinationAdapter = { insert: () => new Promise((resolve) => { finishWrite = resolve }) } + const stream = defineStream({ + id: 'app.hung_load', destination: events, + async *read() { yield { rows: [{ id: 1 }] } }, + }) + const timer = mode === 'cancel' ? setTimeout(() => controller.abort(), 20) : undefined + try { + const result = await run(definePipeline({ id: 'app', streams: [stream] }), { + journal, destination, signal: controller.signal, maxDurationMs: mode === 'deadline' ? 20 : 60_000, + }) + expect(result.ok).toBe(false) + expect(result.streams[0]?.outcome).toBe(mode === 'deadline' ? 'budget_exhausted' : 'cancelled') + expect(result.streams[0]?.rows).toBe(0) + finishWrite() + await new Promise((resolve) => setTimeout(resolve, 0)) + expect(journal.events.filter((event) => event.eventKind === 'batch_committed')).toEqual([]) + } finally { + clearTimeout(timer) + finishWrite() + } + }, 10_000) + + test.each(['ensure', 'run_read', 'run_started', 'stream_read', 'work_planned', 'batch_committed'])( + 'the execution deadline bounds a hung journal %s', async (stage) => { + const journal = createMemoryJournal() + const append = journal.append.bind(journal) + const read = journal.readCheckpoint.bind(journal) + const hung = () => new Promise(() => undefined) + if (stage === 'ensure') journal.ensure = hung + journal.readCheckpoint = (namespace) => ( + (stage === 'run_read' && namespace === '@run') || (stage === 'stream_read' && namespace !== '@run') + ? hung() : read(namespace) + ) + journal.append = (event) => event.eventKind === stage ? hung() : append(event) + const stream = defineStream({ id: 'app.journal', destination: events, async *read() { yield { rows: [{ id: 1 }] } } }) + const execution = run(definePipeline({ id: 'app', streams: [stream] }), { + journal, destination: createMemoryDestination(), maxDurationMs: 20, + }) + if (['ensure', 'run_read', 'run_started'].includes(stage)) { + await expect(execution).rejects.toThrow('execution budget exhausted') + } else { + const result = await execution + expect(result.ok).toBe(false) + expect(result.streams[0]?.outcome).toBe('budget_exhausted') + } + }, 1000 + ) + + test.each(['work_finished', 'run_finished'])('a hung terminal journal %s has bounded cleanup', async (stage) => { + const journal = createMemoryJournal() + const append = journal.append.bind(journal) + journal.append = (event) => event.eventKind === stage ? new Promise(() => undefined) : append(event) + const stream = defineStream({ id: 'app.terminal', destination: events, async *read() { yield { rows: [{ id: 1 }] } } }) + const execution = run(definePipeline({ id: 'app', streams: [stream] }), { + journal, destination: createMemoryDestination(), maxDurationMs: 100, + }) + if (stage === 'run_finished') await expect(execution).rejects.toThrow() + else expect((await execution).ok).toBe(false) + }, 8000) + + test('a throwing loader factory releases its permit for sibling streams', async () => { + const failed = defineStream({ + id: 'app.factory', destination: events, retry: { retries: 0 }, + loader() { throw new Error('loader construction failed') }, + async *read() { yield { rows: [{ id: 1 }] } }, + }) + const healthy = defineStream({ id: 'app.healthy', destination: events, async *read() { yield { rows: [{ id: 2 }] } } }) + const destination = createMemoryDestination() + const result = await run(definePipeline({ id: 'app', streams: [failed, healthy], maxLoads: 1 }), { + journal: createMemoryJournal(), destination, maxDurationMs: 100, + }) + expect(result.streams.map((stream) => stream.outcome)).toEqual(['failed', 'succeeded']) + expect(destination.tables.get('app.events')?.map((row) => row.id)).toEqual([2]) + }) + test('a declared interval id keeps batch identity stable when replayed rows changed', async () => { const build = (label: string, withId: boolean) => { - resetRegistry() const stream = defineStream({ id: 'app.mutable', destination: events, @@ -435,8 +538,9 @@ describe('selectStreams', () => { test('stream ids are globally unique across pipelines', () => { const stream = defineStream({ id: 'crm.people', destination: events, async *read() {} }) - definePipeline({ id: 'a', streams: [stream] }) - expect(() => definePipeline({ id: 'b', streams: [stream] })).toThrow('globally unique') + const a = definePipeline({ id: 'a', streams: [stream] }) + const b = definePipeline({ id: 'b', streams: [stream] }) + expect(() => selectStreams([a, b], [])).toThrow('globally unique') }) }) diff --git a/packages/plugin-ingest/src/executor.ts b/packages/plugin-ingest/src/executor.ts index 6db840f1..031ba40e 100644 --- a/packages/plugin-ingest/src/executor.ts +++ b/packages/plugin-ingest/src/executor.ts @@ -29,6 +29,8 @@ const DEFAULT_PREFETCH_BATCHES = 1 const DEFAULT_MAX_DURATION_MS = 60 * 60_000 const LOAD_ATTEMPTS = 3 const JOURNAL_APPEND_ATTEMPTS = 4 +// Terminal facts get a bounded chance to land even after the execution is cancelled. +const TERMINAL_JOURNAL_TIMEOUT_MS = 5000 const DEFAULT_MAX_CHUNK_ROWS = 100_000 // How long a failed stream waits for an uncooperative reader before abandoning it. const READER_SHUTDOWN_GRACE_MS = 5000 @@ -87,6 +89,7 @@ interface StreamProgress { /** Serializes appends so a sequence number is only consumed by a confirmed fact. */ appendChain: Promise version: number + lastSuccessSeq: number envelope: CheckpointEnvelope | undefined rows: number batches: number @@ -111,15 +114,15 @@ export async function runIngestion(request: ExecutionRequest, input: ExecutionEn span.setAttribute('chkit.ingest.run_id', runId) span.setAttribute('chkit.ingest.stream_ids', streamIds) try { - await env.journal.ensure() - const runHead = (await env.journal.readCheckpoint(RUN_NAMESPACE)).headSeq - await env.journal.append( + await abortable(() => env.journal.ensure(), env.signal) + const runHead = (await abortable(() => env.journal.readCheckpoint(RUN_NAMESPACE), env.signal)).headSeq + await abortable(() => env.journal.append( runEvent(runHead + 1, 'run_started', runId, '', { streamIds, cutoff: cutoff.toISOString(), backfill: request.backfill?.id, }) - ) + ), env.signal) const permits = new Map() const streams = await Promise.all( @@ -129,11 +132,11 @@ export async function runIngestion(request: ExecutionRequest, input: ExecutionEn ) const ok = streams.every((stream) => stream.outcome === 'succeeded') - await env.journal.append( + await abortable(() => env.journal.append( runEvent(runHead + 2, 'run_finished', runId, ok ? 'succeeded' : 'failed', { outcomes: Object.fromEntries(streams.map((stream) => [stream.namespaceId, stream.outcome])), }) - ) + ), AbortSignal.timeout(TERMINAL_JOURNAL_TIMEOUT_MS)) if (!ok) span.setStatus({ code: SpanStatusCode.ERROR }) return { runId, cutoff: cutoff.toISOString(), streams, ok } } finally { @@ -155,7 +158,7 @@ async function executeSelectedStream( ): Promise { const { stream, pipeline } = entry const namespaceId = backfill ? `${stream.id}#backfill:${backfill.id}` : stream.id - const progress: StreamProgress = { seq: 0, appendChain: Promise.resolve(), version: 0, envelope: undefined, rows: 0, batches: 0, chunks: 0 } + const progress: StreamProgress = { seq: 0, appendChain: Promise.resolve(), version: 0, lastSuccessSeq: 0, envelope: undefined, rows: 0, batches: 0, chunks: 0 } const result = (outcome: StreamOutcome, error: string | undefined): StreamResult => ({ streamId: stream.id, pipelineId: pipeline.id, @@ -172,7 +175,7 @@ async function executeSelectedStream( try { release = await permits.streams.acquire(env.signal) } catch { - return result('cancelled', 'cancelled before start') + return result(env.hostSignal.aborted ? 'cancelled' : 'budget_exhausted', 'execution interrupted before start') } return tracer.startActiveSpan('chkit.ingest.stream', async (span) => { @@ -207,9 +210,10 @@ async function executeStream(input: { env: ResolvedEnv }): Promise { const { stream, pipeline, namespaceId, progress, env } = input - const committed = await env.journal.readCheckpoint(namespaceId) + const committed = await abortable(() => env.journal.readCheckpoint(namespaceId), env.signal) progress.seq = committed.headSeq progress.version = committed.version + progress.lastSuccessSeq = committed.lastSuccessSeq progress.envelope = committed.envelope const retry = mergeRetry(pipeline.retry, stream.retry) @@ -227,7 +231,7 @@ async function executeStream(input: { cutoff: input.cutoff, range: input.backfill ? { from: input.backfill.from, to: input.backfill.to } : undefined, }) - const nextWorkId = digest([namespaceId, String(progress.version), canonicalJson(selection)]).slice(0, 24) + const nextWorkId = digest([namespaceId, String(progress.lastSuccessSeq), String(progress.version), canonicalJson(selection)]).slice(0, 24) if (nextWorkId !== workId) { workId = nextWorkId await append(input, 'work_planned', { workId, workState: 'planned', detail: { selection, strategy: stream.incremental.id, strategyVersion: stream.incremental.version, pipelineId: pipeline.id } }) @@ -328,7 +332,7 @@ async function readAndLoad(input: { const maxChunkRows = stream.budget?.maxChunkRows ?? DEFAULT_MAX_CHUNK_ROWS try { while (!budgetExhausted) { - const step = await abortable(iterator.next(), signal) + const step = await abortable(() => iterator.next(), signal) if (step.done) break const chunk: unknown = step.value assertValidChunk(stream.id, chunk, maxChunkRows) @@ -367,8 +371,8 @@ async function readAndLoad(input: { } const consume = async () => { - // Batch identity is anchored to the last durable boundary: the committed - // checkpoint version plus the batch's position since that version, so a + // Batch identity uses the last successful sync, the committed checkpoint + // version and the batch's position since that version, so a // replay in a fresh process reproduces the same id and deduplication token. // Declared source-interval ids complete the identity; without them a // content hash does, preferring a possible duplicate over suppressing rows @@ -376,7 +380,7 @@ async function readAndLoad(input: { let sinceBoundary = 0 for (let batch = await queue.pop(signal); batch !== undefined; batch = await queue.pop(signal)) { const discriminator = batch.intervalIds ? `interval:${canonicalJson(batch.intervalIds)}` : `content:${canonicalJson(batch.rows)}` - const batchId = digest([namespaceId, String(progress.version), String(sinceBoundary), discriminator]).slice(0, 32) + const batchId = digest([namespaceId, String(progress.lastSuccessSeq), String(progress.version), String(sinceBoundary), discriminator]).slice(0, 32) const receipt = await loadBatch(input, batchId, batch.rows, signal) if (abandoned) return const envelope: CheckpointEnvelope | undefined = batch.state @@ -419,6 +423,7 @@ async function readAndLoad(input: { await Promise.race([settled, graceAfterAbort(signal, READER_SHUTDOWN_GRACE_MS)]) abandoned = true if (rootCause) throw rootCause.error + signal.throwIfAborted() if (budgetExhausted) { env.log?.(`${namespaceId}: ${new BudgetExhausted('execution budget exhausted; committed progress is preserved').message}`) return 'budget_exhausted' @@ -442,22 +447,24 @@ async function loadBatch( for (let attempt = 1; ; attempt += 1) { // The load permit covers only the active write, never the backoff timer. const release = await input.permits.loads.acquire(signal) - const loader = factory({ - streamId: input.stream.id, - runId: input.runId, - table: input.stream.destination, - destination: input.env.destination, - signal, - }) try { - await loader.write({ batchId, rows }) - return await loader.finalize() - } catch (error) { - // A cleanup failure must not replace the write failure that decides the retry. - await loader.abort(error).catch((cleanupError: unknown) => { - span.recordException(cleanupError instanceof Error ? cleanupError : new Error(String(cleanupError))) + const loader = factory({ + streamId: input.stream.id, + runId: input.runId, + table: input.stream.destination, + destination: input.env.destination, + signal, }) - if (signal.aborted || isAbortError(error) || attempt >= LOAD_ATTEMPTS) throw error + try { + await loader.write({ batchId, rows }) + return await loader.finalize() + } catch (error) { + // A cleanup failure must not replace the write failure that decides the retry. + await loader.abort(error).catch((cleanupError: unknown) => { + span.recordException(cleanupError instanceof Error ? cleanupError : new Error(String(cleanupError))) + }) + if (signal.aborted || isAbortError(error) || attempt >= LOAD_ATTEMPTS) throw error + } } finally { release() } @@ -492,7 +499,9 @@ function append( fields: Partial> ): Promise { const { progress, env } = input + const signal = eventKind === 'work_finished' ? AbortSignal.timeout(TERMINAL_JOURNAL_TIMEOUT_MS) : env.signal const write = progress.appendChain.then(async () => { + signal.throwIfAborted() const event: JournalEvent = { namespaceId: input.namespaceId, eventSeq: progress.seq + 1, @@ -512,13 +521,12 @@ function append( } for (let attempt = 1; ; attempt += 1) { try { - await env.journal.append(event) + await abortable(() => env.journal.append(event), signal) progress.seq = event.eventSeq return } catch (error) { - if (attempt >= JOURNAL_APPEND_ATTEMPTS) throw error - // Terminal facts must still land after cancellation, so this wait ignores the run signal. - await env.sleep(250 * 2 ** (attempt - 1), NEVER_ABORTED) + if (signal.aborted || attempt >= JOURNAL_APPEND_ATTEMPTS) throw error + await abortable(() => env.sleep(250 * 2 ** (attempt - 1), signal), signal) } } }) @@ -526,7 +534,7 @@ function append( // not reuse its sequence number, because the unconfirmed write may have landed. progress.appendChain = write write.catch(() => undefined) - return write + return abortable(() => write, signal) } function runEvent(seq: number, eventKind: 'run_started' | 'run_finished', runId: string, workState: JournalEvent['workState'], detail: Record): JournalEvent { @@ -584,12 +592,15 @@ function graceAfterAbort(signal: AbortSignal, graceMs: number): Promise { } /** Settle with the promise, or reject as soon as the signal aborts. */ -function abortable(promise: Promise, signal: AbortSignal): Promise { +function abortable(operation: () => Promise, signal: AbortSignal): Promise { if (signal.aborted) return Promise.reject(signal.reason) return new Promise((resolve, reject) => { const onAbort = () => reject(signal.reason) signal.addEventListener('abort', onAbort, { once: true }) - promise.then(resolve, reject).finally(() => signal.removeEventListener('abort', onAbort)) + Promise.resolve().then(() => { + signal.throwIfAborted() + return operation() + }).then(resolve, reject).finally(() => signal.removeEventListener('abort', onAbort)) }) } diff --git a/packages/plugin-ingest/src/incremental.ts b/packages/plugin-ingest/src/incremental.ts index d10e74b7..3bb75c3d 100644 --- a/packages/plugin-ingest/src/incremental.ts +++ b/packages/plugin-ingest/src/incremental.ts @@ -10,13 +10,20 @@ export interface TimestampRange { to: Date } -export interface TimestampWindowOptions { - /** - * Lower bound for the next read. Bootstrap and overlap are ordinary branches - * here; the result is selection input and is never persisted as the watermark. - */ - from(input: { watermark: Date | undefined; cutoff: Date }): Date -} +export type TimestampWindowOptions = + | { + /** Lower bound of the first sync. Overlap applies only to later runs. */ + start: Date + /** Milliseconds to re-read before the committed watermark. Defaults to zero. */ + overlapMs?: number + from?: never + } + | { + /** Custom lower bound; this selection input is never persisted as the watermark. */ + from(input: { watermark: Date | undefined; cutoff: Date }): Date + start?: never + overlapMs?: never + } export interface CursorStateOptions { /** Stable strategy identifier stored in the checkpoint envelope. */ @@ -44,6 +51,16 @@ export function fullSync(): IncrementalStrategy { export function timestampWindow( options: TimestampWindowOptions ): IncrementalStrategy { + if (options.from === undefined) { + if (!(options.start instanceof Date) || !Number.isFinite(options.start.getTime())) { + throw new IngestConfigError('timestampWindow: start must be a valid Date.') + } + if (!Number.isFinite(options.overlapMs ?? 0) || (options.overlapMs ?? 0) < 0) { + throw new IngestConfigError('timestampWindow: overlapMs must be a finite non-negative number.') + } + } else if (options.start !== undefined || options.overlapMs !== undefined) { + throw new IngestConfigError('timestampWindow: use either start/overlapMs or from, not both.') + } return { id: 'chkit.timestamp_window', version: 1, @@ -58,7 +75,13 @@ export function timestampWindow( }, plan({ state, cutoff, range }) { const to = range?.to ?? cutoff - const from = range?.from ?? options.from({ watermark: state ? new Date(state.watermark) : undefined, cutoff }) + const watermark = state ? new Date(state.watermark) : undefined + const from = range?.from ?? (options.from + ? options.from({ watermark, cutoff }) + : new Date(watermark ? watermark.getTime() - (options.overlapMs ?? 0) : options.start.getTime())) + if (!Number.isFinite(from.getTime()) || !Number.isFinite(to.getTime())) { + throw new IngestConfigError('timestampWindow: planned bounds must be valid dates.') + } if (from.getTime() > to.getTime()) { throw new IngestConfigError( `timestampWindow: planned lower bound ${from.toISOString()} is after upper bound ${to.toISOString()}.` diff --git a/packages/plugin-ingest/src/index.ts b/packages/plugin-ingest/src/index.ts index 2db7f505..88f981ef 100644 --- a/packages/plugin-ingest/src/index.ts +++ b/packages/plugin-ingest/src/index.ts @@ -1,6 +1,6 @@ export { ingest, createIngestPlugin, checkGraph, type IngestPlugin, type IngestPluginOptions } from './plugin.js' -export { defineStream, definePipeline, listPipelines, selectStreams, type SelectedStream } from './registry.js' -export { fullSync, timestampWindow, cursorState, type TimestampRange, type TimestampWindowState } from './incremental.js' +export { defineStream, definePipeline, selectStreams, type SelectedStream } from './registry.js' +export { fullSync, timestampWindow, cursorState, type TimestampRange, type TimestampWindowOptions, type TimestampWindowState } from './incremental.js' export { paginate, type Page } from './paginate.js' export { simpleLoader } from './loader.js' export { ingestionColumns, rawTable, rawRows, createClickHouseDestination, type RawRow } from './destination.js' @@ -12,6 +12,7 @@ export type { ErrorClassifier, ExecutionResult, FailureClass, + FetchContext, IncrementalStrategy, Journal, Loader, diff --git a/packages/plugin-ingest/src/ingest.e2e.test.ts b/packages/plugin-ingest/src/ingest.e2e.test.ts index 718998a6..78379e09 100644 --- a/packages/plugin-ingest/src/ingest.e2e.test.ts +++ b/packages/plugin-ingest/src/ingest.e2e.test.ts @@ -4,11 +4,11 @@ import { table, toCreateSQL } from '@chkit/core' import type { ClickHouseExecutor } from '@chkit/clickhouse' import { createPrefix, createStatelessLiveExecutor, getRequiredEnv, quoteIdent, waitForTable } from '@chkit/clickhouse/e2e-testkit' -import { createClickHouseDestination, ingestionColumns } from './destination.js' +import { createClickHouseDestination, ingestionColumns, rawRows, rawTable } from './destination.js' import { runIngestion } from './executor.js' import { cursorState } from './incremental.js' import { createClickHouseJournal } from './journal.js' -import { definePipeline, defineStream, resetRegistry, selectStreams } from './registry.js' +import { definePipeline, defineStream, selectStreams } from './registry.js' import type { DestinationAdapter } from './types.js' describe('@chkit/plugin-ingest live env e2e', () => { @@ -25,6 +25,10 @@ describe('@chkit/plugin-ingest live env e2e', () => { orderBy: ['id'], settings: { non_replicated_deduplication_window: '100' }, }) + const landing = { + ...rawTable({ database, name: `${prefix}raw` }), + settings: { non_replicated_deduplication_window: '100' }, + } let executor: ClickHouseExecutor beforeAll(async () => { @@ -32,16 +36,18 @@ describe('@chkit/plugin-ingest live env e2e', () => { executor = createStatelessLiveExecutor(liveEnv) await executor.command(toCreateSQL(destinationTable)) await waitForTable(executor, database, destinationTable.name) + await executor.command(toCreateSQL(landing)) + await waitForTable(executor, database, landing.name) }) afterAll(async () => { await executor.command(`DROP TABLE IF EXISTS ${quoteIdent(database)}.${quoteIdent(destinationTable.name)}`) + await executor.command(`DROP TABLE IF EXISTS ${quoteIdent(database)}.${quoteIdent(landing.name)}`) await executor.command(`DROP TABLE IF EXISTS ${quoteIdent(database)}.${quoteIdent(journalTable)}`) await executor.close() }) test('a lost acknowledgement replays from the journaled checkpoint without skipping or duplicating rows', async () => { - resetRegistry() const source = Array.from({ length: 3 }, (_, page) => [0, 1].map((offset) => ({ id: page * 2 + offset, label: `row-${page * 2 + offset}` }))) const stream = defineStream({ id: `${prefix}items`, @@ -91,4 +97,30 @@ describe('@chkit/plugin-ingest live env e2e', () => { ) expect(Number(physical[0]?.rows)).toBe(6) }, 120_000) + + test('successful full syncs land A → B → A instead of suppressing the last update', async () => { + let value = 'A' + const stream = defineStream({ + id: `${prefix}raw`, destination: landing, + async *read() { yield { rows: rawRows([{ id: '1', value }], (item) => item.id) } }, + }) + const selected = selectStreams([definePipeline({ id: `${prefix}raw_pipeline`, streams: [stream] })], []) + let lastSuccessSeq = 0 + for (const next of ['A', 'B', 'A']) { + value = next + const journal = createClickHouseJournal({ executor, database, targetId: `e2e/${prefix}`, table: journalTable }) + const result = await runIngestion({ selected, backfill: undefined }, { + journal, destination: createClickHouseDestination(executor), + }) + expect(result.ok).toBe(true) + const checkpoint = await journal.readCheckpoint(stream.id) + expect(checkpoint.lastSuccessSeq).toBeGreaterThan(lastSuccessSeq) + lastSuccessSeq = checkpoint.lastSuccessSeq + const rows = await executor.query<{ raw: { value: string } }>( + `SELECT raw FROM ${quoteIdent(database)}.${quoteIdent(landing.name)} FINAL`, + { select_sequential_consistency: '1' } + ) + expect(rows.map((row) => row.raw.value)).toEqual([next]) + } + }, 120_000) }) diff --git a/packages/plugin-ingest/src/journal.ts b/packages/plugin-ingest/src/journal.ts index 35d765ed..05c3a515 100644 --- a/packages/plugin-ingest/src/journal.ts +++ b/packages/plugin-ingest/src/journal.ts @@ -79,6 +79,7 @@ export function createClickHouseJournal(options: ClickHouseJournalOptions): Jour uniqExact(event_id) AS owners, uniqExact(payload_hash) AS payloads, any(event_kind) AS fact_kind, + any(work_state) AS fact_work_state, any(expected_checkpoint_version) AS fact_expected, any(checkpoint_version) AS fact_version, any(checkpoint_json) AS fact_checkpoint @@ -89,6 +90,7 @@ export function createClickHouseJournal(options: ClickHouseJournalOptions): Jour const [health, transitions] = await Promise.all([ options.executor.query<{ head_seq: string + last_success_seq: string sequences: string conflicting_owners: string drifted: string @@ -97,6 +99,7 @@ export function createClickHouseJournal(options: ClickHouseJournalOptions): Jour }>( `SELECT max(event_seq) AS head_seq, + maxIf(event_seq, fact_kind = 'work_finished' AND fact_work_state = 'succeeded') AS last_success_seq, count() AS sequences, countIf(owners > 1) AS conflicting_owners, countIf(payloads > 1) AS drifted, @@ -145,6 +148,7 @@ FROM ( version: Number(row.checkpoint_version), envelope: parseEnvelope(row.checkpoint_json), headSeq: Number(row.head_seq), + lastSuccessSeq: Number(row.last_success_seq), } }, } @@ -225,7 +229,7 @@ export function digest(parts: readonly string[]): string { } export function emptyCheckpoint(): CommittedCheckpoint { - return { version: 0, envelope: undefined, headSeq: 0 } + return { version: 0, envelope: undefined, headSeq: 0, lastSuccessSeq: 0 } } function journalTableSql(qualified: string): string { diff --git a/packages/plugin-ingest/src/paginate.ts b/packages/plugin-ingest/src/paginate.ts index 1dfa2880..cfa8ff1f 100644 --- a/packages/plugin-ingest/src/paginate.ts +++ b/packages/plugin-ingest/src/paginate.ts @@ -1,5 +1,5 @@ import { canonicalJson } from './journal.js' -import type { AttemptOptions, ReadContext } from './types.js' +import type { AttemptOptions, FetchContext } from './types.js' export interface Page { items: readonly TItem[] @@ -8,8 +8,7 @@ export interface Page { } export interface PaginateOptions { - // biome-ignore lint/suspicious/noExplicitAny: pagination only needs the attempt capability, not the stream generics - context: Pick, 'attempt' | 'signal'> + context: FetchContext fetchPage(cursor: TCursor | undefined, signal: AbortSignal): Promise> initial?: TCursor label?: AttemptOptions['label'] diff --git a/packages/plugin-ingest/src/plugin.ts b/packages/plugin-ingest/src/plugin.ts index 20c44cb5..f1b2a9c2 100644 --- a/packages/plugin-ingest/src/plugin.ts +++ b/packages/plugin-ingest/src/plugin.ts @@ -8,14 +8,14 @@ import { type ChxInlinePluginRegistration, type ResolvedChxConfig, } from '@chkit/core' -import { loadSchemaDefinitions } from '@chkit/core/schema-loader' +import { loadDefinitionModules } from '@chkit/core/schema-loader' import { z } from 'zod' import { BATCH_ID_COLUMN, createClickHouseDestination, INGESTED_AT_COLUMN, RUN_ID_COLUMN } from './destination.js' import { IngestConfigError } from './errors.js' import { runIngestion, type BackfillRequest } from './executor.js' import { createClickHouseJournal, DEFAULT_JOURNAL_TABLE } from './journal.js' -import { listPipelines, selectStreams, type SelectedStream } from './registry.js' +import { collectPipelines, selectStreams, type SelectedStream } from './registry.js' import type { PipelineDefinition } from './types.js' const REQUIRED_COLUMNS = [BATCH_ID_COLUMN, RUN_ID_COLUMN, INGESTED_AT_COLUMN] @@ -198,7 +198,7 @@ export function ingest(options: IngestPluginOptions = {}): ChxInlinePluginRegist export function checkGraph(pipelines: readonly PipelineDefinition[]) { const findings: Array<{ code: string; message: string; severity: 'info' | 'warn' | 'error' }> = [] if (pipelines.length === 0) { - findings.push({ code: 'ingest_no_pipelines', message: 'No ingestion pipeline is registered by the project entry.', severity: 'warn' }) + findings.push({ code: 'ingest_no_pipelines', message: 'No ingestion pipeline is exported by the project entry.', severity: 'warn' }) } for (const pipeline of pipelines) { for (const stream of pipeline.streams) { @@ -216,17 +216,16 @@ export function checkGraph(pipelines: readonly PipelineDefinition[]) { return findings } -// Importing the entry (or legacy schema files) is what lets definePipeline -// self-register; the module cache guarantees this happens once per process. +// Discover values exported from the configured entry (or legacy schema files). async function loadGraph(config: ResolvedChxConfig): Promise { - if (config.schema.length > 0) await loadSchemaDefinitions(config.schema, { cwd: process.cwd() }) - return listPipelines() + if (config.schema.length === 0) return [] + return collectPipelines(await loadDefinitionModules(config.schema, { cwd: process.cwd() })) } async function loadSelection(context: IngestPluginCommandContext): Promise { const pipelines = await loadGraph(context.config) if (pipelines.length === 0) { - throw new IngestConfigError('No ingestion pipeline is registered. Call definePipeline(...) from the module configured as "entry".') + throw new IngestConfigError('No ingestion pipeline is exported. Export a definePipeline(...) value from the module configured as "entry".') } const tags = context.flags['--tag'] return selectStreams(pipelines, Array.isArray(tags) ? tags : typeof tags === 'string' ? [tags] : []) @@ -234,18 +233,15 @@ async function loadSelection(context: IngestPluginCommandContext): Promise executor.close() } } - if (context.pluginContext?.hasExecutor) { - return { executor: context.pluginContext.executor, database: 'default', targetId: 'host-executor/default', close: async () => undefined } - } - throw new IngestConfigError('Ingestion needs a ClickHouse target. Configure clickhouse in your clickhouse.config.ts.') + throw new IngestConfigError('Ingestion requires a direct ClickHouse connection for JSON rows and deduplication settings. Configure clickhouse in your clickhouse.config.ts; a host-provided executor is not supported.') } function targetIdOf(url: string, database: string): string { diff --git a/packages/plugin-ingest/src/registry.ts b/packages/plugin-ingest/src/registry.ts index 291386fe..ab7a4bf4 100644 --- a/packages/plugin-ingest/src/registry.ts +++ b/packages/plugin-ingest/src/registry.ts @@ -16,16 +16,8 @@ import type { StreamDefinition, } from './types.js' -// The registry lives on globalThis so that definePipeline calls made from a -// project entry reach the plugin even when the package is resolved twice -// (for example `source` vs `default` export conditions). -const REGISTRY_KEY = Symbol.for('chkit.ingest.registry') const STREAM_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9_.:-]*$/ -interface Registry { - pipelines: Map -} - export interface StreamInput { id: string destination: TableDefinition @@ -87,7 +79,7 @@ export function defineStream( } /** - * Register a non-durable named group of streams. Pipeline identity never + * Define a non-durable named group of streams. Pipeline identity never * participates in checkpoint or batch identity, so moving a stream between * pipelines does not reset its state. */ @@ -104,16 +96,6 @@ export function definePipeline(input: PipelineInput): PipelineDefinition { retry: input.retry, } - const registry = getRegistry() - const owners = streamOwners(registry) - for (const stream of pipeline.streams) { - const owner = owners.get(stream.id) - if (owner !== undefined && owner !== pipeline.id) { - throw new IngestConfigError( - `Stream id "${stream.id}" is registered by both pipeline "${owner}" and pipeline "${pipeline.id}". Stream ids must be globally unique.` - ) - } - } const seen = new Set() for (const stream of pipeline.streams) { if (seen.has(stream.id)) { @@ -122,16 +104,35 @@ export function definePipeline(input: PipelineInput): PipelineDefinition { seen.add(stream.id) } - registry.pipelines.set(pipeline.id, pipeline) return pipeline } -export function listPipelines(): PipelineDefinition[] { - return [...getRegistry().pipelines.values()] +/** Validate identities within this graph, without process-wide registration. */ +export function validatePipelines(pipelines: readonly PipelineDefinition[]): void { + const ids = new Set() + const owners = new Map() + for (const pipeline of pipelines) { + if (ids.has(pipeline.id)) throw new IngestConfigError(`Duplicate pipeline id "${pipeline.id}".`) + ids.add(pipeline.id) + for (const stream of pipeline.streams) { + const owner = owners.get(stream.id) + if (owner !== undefined) { + throw new IngestConfigError( + `Stream id "${stream.id}" occurs in pipelines "${owner}" and "${pipeline.id}". Stream ids must be globally unique.` + ) + } + owners.set(stream.id, pipeline.id) + } + } } -export function resetRegistry(): void { - getRegistry().pipelines.clear() +/** Only exported pipeline values participate; aliases of the same value count once. */ +export function collectPipelines(modules: readonly Record[]): PipelineDefinition[] { + const pipelines = [...new Set(modules.flatMap((mod) => Object.values(mod).filter( + (value): value is PipelineDefinition => typeof value === 'object' && value !== null && 'kind' in value && value.kind === 'ingest_pipeline' + )))] + validatePipelines(pipelines) + return pipelines } /** @@ -140,6 +141,7 @@ export function resetRegistry(): void { * No filter selects the complete graph; an explicit empty selection throws. */ export function selectStreams(pipelines: readonly PipelineDefinition[], tags: readonly string[]): SelectedStream[] { + validatePipelines(pipelines) const wanted = dedupe(tags) const all: SelectedStream[] = pipelines.flatMap((pipeline) => pipeline.streams.map((stream) => ({ @@ -159,23 +161,6 @@ export function selectStreams(pipelines: readonly PipelineDefinition[], tags: re return selected } -function getRegistry(): Registry { - const holder = globalThis as { [REGISTRY_KEY]?: Registry } - const existing = holder[REGISTRY_KEY] - if (existing) return existing - const created: Registry = { pipelines: new Map() } - holder[REGISTRY_KEY] = created - return created -} - -function streamOwners(registry: Registry): Map { - const owners = new Map() - for (const pipeline of registry.pipelines.values()) { - for (const stream of pipeline.streams) owners.set(stream.id, pipeline.id) - } - return owners -} - function assertValidId(kind: 'stream' | 'pipeline', id: string): void { if (!STREAM_ID_PATTERN.test(id)) { throw new IngestConfigError( diff --git a/packages/plugin-ingest/src/testing.ts b/packages/plugin-ingest/src/testing.ts index 2eb22fa9..6eb3c8ab 100644 --- a/packages/plugin-ingest/src/testing.ts +++ b/packages/plugin-ingest/src/testing.ts @@ -28,11 +28,14 @@ export function createMemoryJournal(): MemoryJournal { const scoped = events.filter((event) => event.namespaceId === namespaceId) if (scoped.length === 0) return emptyCheckpoint() const headSeq = Math.max(...scoped.map((event) => event.eventSeq)) + const lastSuccessSeq = Math.max(0, ...scoped + .filter((event) => event.eventKind === 'work_finished' && event.workState === 'succeeded') + .map((event) => event.eventSeq)) const committed = scoped .filter((event) => event.eventKind === 'batch_committed') .sort((a, b) => a.checkpointVersion - b.checkpointVersion || a.eventSeq - b.eventSeq) .at(-1) - return { version: committed?.checkpointVersion ?? 0, envelope: committed?.checkpoint, headSeq } + return { version: committed?.checkpointVersion ?? 0, envelope: committed?.checkpoint, headSeq, lastSuccessSeq } }, } } diff --git a/packages/plugin-ingest/src/types.ts b/packages/plugin-ingest/src/types.ts index b4ddc7a7..0e30ca0b 100644 --- a/packages/plugin-ingest/src/types.ts +++ b/packages/plugin-ingest/src/types.ts @@ -34,14 +34,8 @@ export interface AttemptOptions { label?: string } -export interface ReadContext { - streamId: string - /** What to read this execution, planned by the incremental strategy. */ - selection: TSelection - /** Last committed provider state, already validated by the strategy. */ - state: TState | undefined - /** Immutable cutoff shared by every stream selected in this execution. */ - cutoff: Date +/** Source request capabilities, independent of selection and checkpoint types. */ +export interface FetchContext { signal: AbortSignal /** * Run one source operation under executor authority: fetch permit, retry @@ -51,6 +45,16 @@ export interface ReadContext { attempt(operation: (signal: AbortSignal) => Promise, options?: AttemptOptions): Promise } +export interface ReadContext extends FetchContext { + streamId: string + /** What to read this execution, planned by the incremental strategy. */ + selection: TSelection + /** Last committed provider state, already validated by the strategy. */ + state: TState | undefined + /** Immutable cutoff shared by every stream selected in this execution. */ + cutoff: Date +} + /** * Provider-owned incremental strategy inside the ChKit-owned checkpoint * envelope `{ strategy, version, state }`. Core treats `state` as opaque. @@ -227,6 +231,8 @@ export interface CommittedCheckpoint { envelope: CheckpointEnvelope | undefined /** Highest journal sequence observed for the namespace (any event kind). */ headSeq: number + /** Last successful work_finished sequence, or zero. Separates new syncs from replays. */ + lastSuccessSeq: number } /** Authoritative append-only control state. Checkpoints are projections of it. */ From a01f5f64bec50bd70f21cb4e12d93fce78dc8ada Mon Sep 17 00:00:00 2001 From: KeKs0r Date: Sat, 19 Sep 2026 15:59:23 -0700 Subject: [PATCH 07/11] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20Use=20standard=20pro?= =?UTF-8?q?mise=20helpers?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/docs/src/content/docs/plugins/ingest.md | 8 + bun.lock | 13 +- packages/cli/package.json | 5 +- .../cli/src/commands/migrate/async-apply.ts | 5 +- packages/cli/src/runtime/journal-store.ts | 39 ++-- packages/cli/src/test/e2e-testkit.ts | 5 +- .../src/test/runtime/journal-store.test.ts | 19 ++ .../plugin-backfill/src/async-backfill.ts | 6 +- packages/plugin-ingest/package.json | 2 + packages/plugin-ingest/src/executor.test.ts | 123 +++++++++++- packages/plugin-ingest/src/executor.ts | 186 +++++++++--------- packages/plugin-ingest/src/ingest.e2e.test.ts | 5 +- packages/plugin-ingest/src/retry.test.ts | 91 +++++++++ packages/plugin-ingest/src/retry.ts | 117 ++++------- packages/plugin-ingest/src/semaphore.ts | 51 ----- packages/plugin-ingest/src/types.ts | 22 +-- .../plugin-obsessiondb/src/auth/api-client.ts | 3 +- 17 files changed, 412 insertions(+), 288 deletions(-) create mode 100644 packages/plugin-ingest/src/retry.test.ts delete mode 100644 packages/plugin-ingest/src/semaphore.ts diff --git a/apps/docs/src/content/docs/plugins/ingest.md b/apps/docs/src/content/docs/plugins/ingest.md index 5350b8e0..cb4f4c0d 100644 --- a/apps/docs/src/content/docs/plugins/ingest.md +++ b/apps/docs/src/content/docs/plugins/ingest.md @@ -143,6 +143,14 @@ With `cursorState`, `state` on a chunk must be the complete state that is safe t A checkpoint records its strategy id and version. Changing either makes the next run fail rather than reinterpret old state. +## Retry policy + +Pipeline `retry` settings are defaults; a stream can override individual settings. Source attempts and reader restarts use [`p-retry`](https://github.com/sindresorhus/p-retry) for exponential backoff, jitter, retry counts, and `maxRetryTime`. The supported options are `retries`, `factor`, `minTimeout`, `maxTimeout`, `randomize`, `maxRetryTime`, `shouldRetry`, and `shouldConsumeRetry`. + +The policy callbacks receive p-retry's `attemptNumber`, `retriesLeft`, and `retriesConsumed`, plus a `FetchFailure` preserving the original `cause` and normalized `classification`. Returning `false` from `shouldConsumeRetry` follows p-retry's behavior: it skips consuming a retry and skips its backoff. A provider's `Retry-After` still applies, including for those unconsumed retries. Retry waits release fetch and load capacity for other streams. + +Once a source attempt exhausts its policy, it fails the stream; reader recovery does not multiply that retry budget. Opaque reader failures restart from the latest committed checkpoint. + ## Commands ```sh diff --git a/bun.lock b/bun.lock index 4adcbb26..9e563dd8 100644 --- a/bun.lock +++ b/bun.lock @@ -57,6 +57,7 @@ "@chkit/core": "workspace:*", "@logtape/logtape": "^2.0.5", "fast-glob": "^3.3.2", + "p-retry": "^7.1.1", }, "optionalDependencies": { "@chkit/plugin-obsessiondb": "workspace:*", @@ -76,7 +77,7 @@ "name": "@chkit/codegen", "version": "0.1.2-beta.7", "dependencies": { - "@chkit/core": "0.1.0-beta.26", + "@chkit/core": "workspace:*", }, }, "packages/core": { @@ -133,11 +134,13 @@ }, "packages/plugin-ingest": { "name": "@chkit/plugin-ingest", - "version": "0.1.0-beta.0", + "version": "0.1.2-beta.7", "dependencies": { "@chkit/clickhouse": "workspace:*", "@chkit/core": "workspace:*", "@opentelemetry/api": "^1.9.1", + "p-limit": "^7.2.0", + "p-retry": "^7.1.1", "zod": "^4.3.6", }, }, @@ -1167,7 +1170,7 @@ "p-filter": ["p-filter@2.1.0", "", { "dependencies": { "p-map": "^2.0.0" } }, "sha512-ZBxxZ5sL2HghephhpGAQdoskxplTwr7ICaehZwLIlfL6acuVgZPm8yBNuRAFBGEqtD/hmUeq9eqLg2ys9Xr/yw=="], - "p-limit": ["p-limit@2.3.0", "", { "dependencies": { "p-try": "^2.0.0" } }, "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w=="], + "p-limit": ["p-limit@7.3.3", "", { "dependencies": { "yocto-queue": "^1.2.1" } }, "sha512-SKgVvDU5TNIxQ4r30U75BNLC6a+6GOEjngrk/ysM5+Zu3oJuk1SC5ApzIzY/7PjCEjoVPMWJ0zho/4QUiEB7TQ=="], "p-locate": ["p-locate@4.1.0", "", { "dependencies": { "p-limit": "^2.2.0" } }, "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A=="], @@ -1529,6 +1532,8 @@ "@babel/traverse/@babel/types": ["@babel/types@7.29.7", "", { "dependencies": { "@babel/helper-string-parser": "^7.29.7", "@babel/helper-validator-identifier": "^7.29.7" } }, "sha512-4zBIxpPzowiZpusoFkyGVwakdRJUyuH5PxQ/PrqghfdFWWasvnCdPfQXHrenDai+gyLARulZjZowCOj6fjT4pA=="], + "@changesets/cli/p-limit": ["p-limit@2.3.0", "", { "dependencies": { "p-try": "^2.0.0" } }, "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w=="], + "@chkit/plugin-backfill/zod": ["zod@4.3.6", "", {}, "sha512-rftlrkhHZOcjDwkGlnUtZZkvaPHCsDATp4pGpuOOMDaTdDDXF91wuVDJoWoPsKX/3YPQ5fHuF3STjcYyKr+Qhg=="], "@chkit/plugin-codegen/zod": ["zod@4.3.6", "", {}, "sha512-rftlrkhHZOcjDwkGlnUtZZkvaPHCsDATp4pGpuOOMDaTdDDXF91wuVDJoWoPsKX/3YPQ5fHuF3STjcYyKr+Qhg=="], @@ -1601,6 +1606,8 @@ "p-filter/p-map": ["p-map@2.1.0", "", {}, "sha512-y3b8Kpd8OAN444hxfBbFfj1FY/RjtTd8tzYwhUqNYXx0fXx2iX4maP4Qr6qhIKbQXI02wTLAda4fYUbDagTUFw=="], + "p-locate/p-limit": ["p-limit@2.3.0", "", { "dependencies": { "p-try": "^2.0.0" } }, "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w=="], + "parse-entities/@types/unist": ["@types/unist@2.0.11", "", {}, "sha512-CmBKiL6NNo/OqgmMn95Fk9Whlp2mtvIv+KNpQKN2F4SjvrEesubTRWGYSg+BnWZOnlCaSTU1sMpsBOzgbYhnsA=="], "parse5/entities": ["entities@6.0.1", "", {}, "sha512-aN97NXWF6AWBTahfVOIrB/NShkzi5H7F9r1s9mD3cDj4Ko5f2qhhVoYMibXF7GlLveb/D2ioWay8lxI97Ven3g=="], diff --git a/packages/cli/package.json b/packages/cli/package.json index c205648c..e1816efb 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -39,11 +39,12 @@ "clean": "rm -rf dist" }, "dependencies": { - "fast-glob": "^3.3.2", "@chkit/clickhouse": "workspace:*", "@chkit/codegen": "workspace:*", "@chkit/core": "workspace:*", - "@logtape/logtape": "^2.0.5" + "@logtape/logtape": "^2.0.5", + "fast-glob": "^3.3.2", + "p-retry": "^7.1.1" }, "optionalDependencies": { "@chkit/plugin-obsessiondb": "workspace:*" diff --git a/packages/cli/src/commands/migrate/async-apply.ts b/packages/cli/src/commands/migrate/async-apply.ts index 443170c9..6cd038d7 100644 --- a/packages/cli/src/commands/migrate/async-apply.ts +++ b/packages/cli/src/commands/migrate/async-apply.ts @@ -1,4 +1,5 @@ import { createHash } from 'node:crypto' +import { setTimeout as defaultSleep } from 'node:timers/promises' import type { ClickHouseExecutor, QueryStatus } from '@chkit/clickhouse' @@ -408,7 +409,3 @@ export function isoWithoutZone(date: Date): string { function firstLine(value: string): string { return value.split('\n')[0] ?? value } - -function defaultSleep(ms: number): Promise { - return new Promise((resolve) => setTimeout(resolve, ms)) -} diff --git a/packages/cli/src/runtime/journal-store.ts b/packages/cli/src/runtime/journal-store.ts index b8b219cc..3d0af00d 100644 --- a/packages/cli/src/runtime/journal-store.ts +++ b/packages/cli/src/runtime/journal-store.ts @@ -1,5 +1,8 @@ +import { setTimeout as sleep } from 'node:timers/promises' + import { isUnknownDatabaseError, type ClickHouseExecutor } from '@chkit/clickhouse' import { onClusterClause } from '@chkit/core' +import pRetry from 'p-retry' import type { MigrationJournal, MigrationJournalEntry } from './migration-store.js' import { CLI_VERSION } from './version.js' @@ -202,15 +205,10 @@ SETTINGS index_granularity = 1` } throw error } - for (let attempt = 0; attempt < 10; attempt++) { - try { - await db.query(`SELECT name FROM ${journalTable} LIMIT 0`) - debug('journal', `DDL propagation confirmed (attempt ${attempt + 1})`) - break - } catch { - await new Promise((r) => setTimeout(r, 250)) - } - } + await pRetry(async (attempt) => { + await db.query(`SELECT name FROM ${journalTable} LIMIT 0`) + debug('journal', `DDL propagation confirmed (attempt ${attempt})`) + }, { retries: 9, minTimeout: 250, factor: 1 }).catch(() => undefined) bootstrapped = true } @@ -289,19 +287,16 @@ SETTINGS index_granularity = 1` } await ensureTable() const insertSql = `INSERT INTO ${journalTable} (name, applied_at, checksum, chkit_version, migration_completed, operations) VALUES ('${escapeSqlString(state.name)}', '${escapeSqlString(state.appliedAt)}', '${escapeSqlString(state.checksum)}', '${escapeSqlString(state.chkitVersion || CLI_VERSION)}', ${state.migrationCompleted ? 'true' : 'false'}, ${operationsArrayLiteral(state.operations)})` - const maxAttempts = 5 - for (let attempt = 1; attempt <= maxAttempts; attempt += 1) { - try { - await db.command(insertSql) - break - } catch (error) { - if (!isRetryableInsertRace(error) || attempt === maxAttempts) { - throw error - } - debug('journal', `insert race detected — retrying (attempt ${attempt}/${maxAttempts})`) - await new Promise((r) => setTimeout(r, attempt * 150)) - } - } + await pRetry(() => db.command(insertSql), { + retries: 4, + minTimeout: 0, + shouldRetry: async ({ error, attemptNumber }) => { + if (!isRetryableInsertRace(error)) return false + debug('journal', `insert race detected — retrying (attempt ${attemptNumber}/5)`) + await sleep(attemptNumber * 150) + return true + }, + }) await trySyncReplica() }, diff --git a/packages/cli/src/test/e2e-testkit.ts b/packages/cli/src/test/e2e-testkit.ts index 681225be..b793bff2 100644 --- a/packages/cli/src/test/e2e-testkit.ts +++ b/packages/cli/src/test/e2e-testkit.ts @@ -6,6 +6,7 @@ */ import { join, resolve } from 'node:path' +import { setTimeout as sleep } from 'node:timers/promises' // Re-export all shared utilities so CLI tests only need one import export { @@ -79,7 +80,7 @@ export async function runCliWithRetry( const result = runCli(cwd, args, extraEnv) if (result.exitCode === 0 && (!expectJson || isValidJson(result.stdout))) return result if (attempt === maxAttempts) return result - await new Promise((r) => setTimeout(r, delayMs)) + await sleep(delayMs) } return runCli(cwd, args, extraEnv) } @@ -114,7 +115,7 @@ export async function waitForCliJson( formatTestDiagnostic('last attempt', result) ) } - await new Promise((r) => setTimeout(r, delayMs)) + await sleep(delayMs) } throw new Error('waitForCliJson: unreachable') } diff --git a/packages/cli/src/test/runtime/journal-store.test.ts b/packages/cli/src/test/runtime/journal-store.test.ts index e3fa2cfd..40e496fb 100644 --- a/packages/cli/src/test/runtime/journal-store.test.ts +++ b/packages/cli/src/test/runtime/journal-store.test.ts @@ -248,4 +248,23 @@ describe('createJournalStore', () => { expect(insert).toBeDefined() expect(insert).toContain("It\\'s broken: \\'unterminated") }) + + test.each(['recovers', 'exhausts', 'permanent'] as const)('migration insert retry policy %s', async (mode) => { + const { db } = createScriptedExecutor(new Map()) + let attempts = 0 + const failure = new Error(mode === 'permanent' ? 'access denied' : 'Please retry the INSERT') + db.command = async (sql) => { + if (!sql.startsWith('INSERT INTO')) return + attempts += 1 + if (mode !== 'recovers' || attempts < 3) throw failure + } + const write = createJournalStore(db).writeMigrationState({ + name: 'retry.sql', appliedAt: '2026-05-26 12:00:00.000', checksum: 'cs', + chkitVersion: 'v', migrationCompleted: true, operations: [], + }) + if (mode === 'recovers') await write + else await expect(write).rejects.toBe(failure) + expect(attempts).toBe(mode === 'recovers' ? 3 : mode === 'exhausts' ? 5 : 1) + }) + }) diff --git a/packages/plugin-backfill/src/async-backfill.ts b/packages/plugin-backfill/src/async-backfill.ts index bbb7c3b7..6c78543b 100644 --- a/packages/plugin-backfill/src/async-backfill.ts +++ b/packages/plugin-backfill/src/async-backfill.ts @@ -1,3 +1,5 @@ +import { setTimeout as sleep } from 'node:timers/promises' + import type { ClickHouseExecutor, QueryStatus } from '@chkit/clickhouse' import pMap from 'p-map' @@ -47,10 +49,6 @@ export interface BackfillResult { progress: BackfillProgress } -function sleep(ms: number): Promise { - return new Promise((resolve) => setTimeout(resolve, ms)) -} - /** Build the deterministic query ID for a chunk. */ function chunkQueryId(planId: string, chunkId: string): string { return `backfill-${planId}-${chunkId}` diff --git a/packages/plugin-ingest/package.json b/packages/plugin-ingest/package.json index 6e244551..33793881 100644 --- a/packages/plugin-ingest/package.json +++ b/packages/plugin-ingest/package.json @@ -50,6 +50,8 @@ "@chkit/clickhouse": "workspace:*", "@chkit/core": "workspace:*", "@opentelemetry/api": "^1.9.1", + "p-limit": "^7.2.0", + "p-retry": "^7.1.1", "zod": "^4.3.6" } } diff --git a/packages/plugin-ingest/src/executor.test.ts b/packages/plugin-ingest/src/executor.test.ts index a8a0e150..22575235 100644 --- a/packages/plugin-ingest/src/executor.test.ts +++ b/packages/plugin-ingest/src/executor.test.ts @@ -1,4 +1,5 @@ import { describe, expect, test } from 'bun:test' +import { setTimeout as sleep } from 'node:timers/promises' import { table } from '@chkit/core' @@ -20,7 +21,6 @@ const events = table({ orderBy: ['id'], }) -const noSleep = async () => undefined const pages = (count: number, size: number) => Array.from({ length: count }, (_, page) => Array.from({ length: size }, (_, index) => ({ id: page * size + index }))) @@ -106,11 +106,11 @@ describe('runIngestion', () => { }, } - const first = await runIngestion({ selected: selectStreams([pipeline], []), backfill: undefined }, { journal, destination: crashing, sleep: noSleep }) + const first = await runIngestion({ selected: selectStreams([pipeline], []), backfill: undefined }, { journal, destination: crashing }) expect(first.ok).toBe(false) expect((await journal.readCheckpoint('app.cursor')).envelope?.state).toBe(1) - const second = await runIngestion({ selected: selectStreams([pipeline], []), backfill: undefined }, { journal, destination, sleep: noSleep }) + const second = await runIngestion({ selected: selectStreams([pipeline], []), backfill: undefined }, { journal, destination }) expect(second.ok).toBe(true) expect((await journal.readCheckpoint('app.cursor')).envelope?.state).toBe(3) const ids = (destination.tables.get('app.events') ?? []).map((row) => row.id) @@ -158,7 +158,7 @@ describe('runIngestion', () => { }) const pipeline = definePipeline({ id: 'app', streams: [stream] }) - const result = await runIngestion({ selected: selectStreams([pipeline], []), backfill: undefined }, { journal, destination: createMemoryDestination(), sleep: noSleep }) + const result = await runIngestion({ selected: selectStreams([pipeline], []), backfill: undefined }, { journal, destination: createMemoryDestination() }) expect(result.streams[0]?.outcome).toBe('failed') expect(result.streams[0]?.error).toContain('provider exploded') @@ -168,7 +168,6 @@ describe('runIngestion', () => { test('attempt retries rate limits and transient failures but not permanent ones', async () => { const journal = createMemoryJournal() const destination = createMemoryDestination() - const delays: number[] = [] let calls = 0 const flaky = defineStream({ id: 'app.flaky', @@ -177,7 +176,7 @@ describe('runIngestion', () => { async *read(context) { const rows = await context.attempt(async () => { calls += 1 - if (calls === 1) throw await httpError(429, { 'retry-after': '7' }) + if (calls === 1) throw await httpError(429, { 'retry-after': '0.02' }) if (calls === 2) throw await httpError(503) return [{ id: 1 }] }) @@ -195,11 +194,10 @@ describe('runIngestion', () => { const result = await runIngestion( { selected: selectStreams([pipeline], []), backfill: undefined }, - { journal, destination, sleep: async (ms) => { delays.push(ms) } } + { journal, destination } ) expect(calls).toBe(3) - expect(delays).toEqual([7000, 20]) // One failing stream does not stop its sibling. expect(result.streams.map((stream) => stream.outcome)).toEqual(['succeeded', 'failed']) expect(journal.events.filter((event) => event.eventKind === 'retry_scheduled').map((event) => event.errorClass)).toEqual(['rate_limited', 'transient']) @@ -274,7 +272,7 @@ describe('runIngestion', () => { describe('runtime contracts', () => { const run = (pipeline: ReturnType, env: Parameters[1]) => - runIngestion({ selected: selectStreams([pipeline], []), backfill: undefined }, { sleep: noSleep, ...env }) + runIngestion({ selected: selectStreams([pipeline], []), backfill: undefined }, env) test.each([false, true])('new full syncs preserve changes while failed syncs reuse tokens (declared id: %s)', async (withId) => { const journal = createMemoryJournal() @@ -504,6 +502,113 @@ describe('runtime contracts', () => { }) }) +describe('pipeline concurrency and retries', () => { + const run = (pipeline: ReturnType, env: Parameters[1]) => + runIngestion({ selected: selectStreams([pipeline], []), backfill: undefined }, env) + + test.each(['fetches', 'loads'] as const)('%s release capacity during backoff and never exceed the limit', async (mode) => { + const order: number[] = [] + let active = 0 + let peak = 0 + const operation = async (id: number) => { + order.push(id) + active += 1 + peak = Math.max(peak, active) + await sleep(5) + active -= 1 + if (order.length === 1) throw new Error('temporary failure') + return [{ id }] + } + const streams = [1, 2].map((id) => defineStream({ + id: `app.stream${id}`, destination: events, + async *read({ attempt }) { + yield { rows: mode === 'fetches' ? await attempt(() => operation(id)) : [{ id }] } + }, + })) + const destination: DestinationAdapter = { + async insert({ rows }) { if (mode === 'loads') await operation(Number(rows[0]?.id)) }, + } + const result = await run(definePipeline({ + id: 'app', streams, maxStreams: 2, maxFetches: 1, maxLoads: 1, + retry: { retries: 1, minTimeout: 10, randomize: false }, + }), { journal: createMemoryJournal(), destination }) + + expect(result.ok).toBe(true) + expect(order).toEqual([1, 2, 1]) + expect(peak).toBe(1) + }) + + test('stream limits serialize readers and cancellation prevents queued readers from starting', async () => { + for (const cancel of [false, true]) { + const controller = new AbortController() + const order: string[] = [] + const streams = [1, 2].map((id) => defineStream({ + id: `app.stream${id}`, destination: events, + async *read() { + order.push(`start:${id}`) + if (cancel) controller.abort() + await sleep(5) + order.push(`end:${id}`) + yield { rows: [] } + }, + })) + const result = await run(definePipeline({ id: 'app', streams, maxStreams: 1 }), { + journal: createMemoryJournal(), destination: createMemoryDestination(), signal: controller.signal, + }) + if (cancel) { + expect(result.streams.map((stream) => stream.outcome)).toEqual(['cancelled', 'cancelled']) + expect(order).not.toContain('start:2') + } else { + expect(result.ok).toBe(true) + expect(order).toEqual(['start:1', 'end:1', 'start:2', 'end:2']) + } + } + }) + + test('reader retries resume from committed progress and exhausted source retries do not recreate the reader', async () => { + const journal = createMemoryJournal() + const append = journal.append.bind(journal) + let committed: () => void = () => undefined + const saved = new Promise((resolve) => { committed = resolve }) + journal.append = async (event) => { + await append(event) + if (event.eventKind === 'batch_committed') committed() + } + const selections: Array = [] + let sourceCalls = 0 + let readers = 0 + const resumable = defineStream({ + id: 'app.resumable', destination: events, batchSize: 1, + incremental: cursorState({ id: 'page', version: 1, parse: (raw) => Number(raw) }), + async *read({ selection }) { + selections.push(selection) + if (selection === undefined) { + yield { rows: [{ id: 1 }], state: 1 } + await saved + throw new Error('reader disconnected') + } + yield { rows: [{ id: 2 }], state: 2 } + }, + }) + const exhausted = defineStream({ + id: 'app.exhausted', destination: events, + async *read({ attempt }) { + readers += 1 + await attempt(async () => { sourceCalls += 1; throw new Error('source unavailable') }) + }, + }) + const result = await run(definePipeline({ + id: 'app', streams: [resumable, exhausted], retry: { retries: 1, minTimeout: 0 }, + }), { journal, destination: createMemoryDestination() }) + + expect(result.streams.map((stream) => stream.outcome)).toEqual(['succeeded', 'failed']) + expect(selections).toEqual([undefined, 1]) + expect((await journal.readCheckpoint(resumable.id)).envelope?.state).toBe(2) + expect(readers).toBe(1) + expect(sourceCalls).toBe(2) + }) +}) + describe('rawTable', () => { test('lands provider objects untouched next to a stable id', async () => { const destination = createMemoryDestination() diff --git a/packages/plugin-ingest/src/executor.ts b/packages/plugin-ingest/src/executor.ts index 031ba40e..d3306071 100644 --- a/packages/plugin-ingest/src/executor.ts +++ b/packages/plugin-ingest/src/executor.ts @@ -1,14 +1,16 @@ import { randomUUID } from 'node:crypto' +import { setTimeout as sleep } from 'node:timers/promises' import { SpanStatusCode, trace, type Span } from '@opentelemetry/api' +import pLimit, { type LimitFunction } from 'p-limit' +import pRetry, { AbortError } from 'p-retry' import { BudgetExhausted, classifyFailure, FetchFailure, IngestConfigError, isAbortError } from './errors.js' import { canonicalJson, digest } from './journal.js' import { simpleLoader } from './loader.js' import { createBoundedQueue } from './queue.js' import type { SelectedStream } from './registry.js' -import { DEFAULT_RETRY, mergeRetry, runAttempt, sleep } from './retry.js' -import { createSemaphore, type Semaphore } from './semaphore.js' +import { mergeRetry, runAttempt } from './retry.js' import type { AnyStreamDefinition, CheckpointEnvelope, @@ -57,8 +59,6 @@ export interface ExecutionEnv { /** Finite number of mapped batches buffered between fetch and load. */ prefetchBatches?: number now?: () => Date - sleep?: (ms: number, signal: AbortSignal) => Promise - random?: () => number log?: (message: string) => void } @@ -70,9 +70,9 @@ interface ResolvedEnv extends Required> { } interface PipelinePermits { - streams: Semaphore - fetches: Semaphore - loads: Semaphore + streams: LimitFunction + fetches: LimitFunction + loads: LimitFunction } interface PendingBatch { @@ -171,30 +171,28 @@ async function executeSelectedStream( error, }) - let release: (() => void) | undefined - try { - release = await permits.streams.acquire(env.signal) - } catch { - return result(env.hostSignal.aborted ? 'cancelled' : 'budget_exhausted', 'execution interrupted before start') - } - - return tracer.startActiveSpan('chkit.ingest.stream', async (span) => { - span.setAttribute('chkit.ingest.stream_id', stream.id) - span.setAttribute('chkit.ingest.namespace_id', namespaceId) - try { - const outcome = await executeStream({ stream, pipeline, namespaceId, backfill, runId, cutoff, permits, progress, env }) - env.log?.(`${namespaceId}: ${outcome} (${progress.rows} rows, ${progress.batches} batches, checkpoint v${progress.version})`) - return result(outcome, undefined) - } catch (error) { - const message = error instanceof Error ? error.message : String(error) - const outcome: StreamOutcome = env.hostSignal.aborted ? 'cancelled' : env.signal.aborted ? 'budget_exhausted' : 'failed' - recordFailure(span, error) - env.log?.(`${namespaceId}: ${outcome} — ${message}`) - return result(outcome, message) - } finally { - span.end() - release?.() + return permits.streams(async () => { + if (env.signal.aborted) { + return result(env.hostSignal.aborted ? 'cancelled' : 'budget_exhausted', 'execution interrupted before start') } + + return tracer.startActiveSpan('chkit.ingest.stream', async (span) => { + span.setAttribute('chkit.ingest.stream_id', stream.id) + span.setAttribute('chkit.ingest.namespace_id', namespaceId) + try { + const outcome = await executeStream({ stream, pipeline, namespaceId, backfill, runId, cutoff, permits, progress, env }) + env.log?.(`${namespaceId}: ${outcome} (${progress.rows} rows, ${progress.batches} batches, checkpoint v${progress.version})`) + return result(outcome, undefined) + } catch (error) { + const message = error instanceof Error ? error.message : String(error) + const outcome: StreamOutcome = env.hostSignal.aborted ? 'cancelled' : env.signal.aborted ? 'budget_exhausted' : 'failed' + recordFailure(span, error) + env.log?.(`${namespaceId}: ${outcome} — ${message}`) + return result(outcome, message) + } finally { + span.end() + } + }) }) } @@ -217,13 +215,12 @@ async function executeStream(input: { progress.envelope = committed.envelope const retry = mergeRetry(pipeline.retry, stream.retry) - const readerAttempts = (retry.retries ?? DEFAULT_RETRY.retries) + 1 let workId = '' let outcome: StreamOutcome = 'failed' let failure: unknown - for (let attemptNo = 1; attemptNo <= readerAttempts; attemptNo += 1) { - try { + try { + outcome = await runAttempt(async (attemptNo) => { // Re-plan from the latest durable boundary on every reader (re)creation. const state = restoreState(stream, progress.envelope, namespaceId) const selection: unknown = stream.incremental.plan({ @@ -237,25 +234,24 @@ async function executeStream(input: { await append(input, 'work_planned', { workId, workState: 'planned', detail: { selection, strategy: stream.incremental.id, strategyVersion: stream.incremental.version, pipelineId: pipeline.id } }) } await append(input, 'attempt_started', { workId, attemptNo, workState: 'running' }) - outcome = await readAndLoad({ ...input, workId, attemptNo, state, selection, retry }) - failure = undefined - break - } catch (error) { - failure = error - if (error instanceof IngestConfigError) break - const classification = error instanceof FetchFailure ? error.classification : classifyFailure(error, env.signal, stream.classifyError) - if (classification.kind === 'cancelled') { - outcome = env.hostSignal.aborted ? 'cancelled' : 'budget_exhausted' - // Exhausting the budget is an incomplete result, not a failure: committed progress stands. - if (outcome === 'budget_exhausted') failure = undefined - break - } - // A FetchFailure already exhausted its fine-grained retries; an opaque - // iterator failure gets coarse reader recreation from the last checkpoint. - if (error instanceof FetchFailure || classification.kind === 'permanent' || attemptNo === readerAttempts) break - const retryDelay = Math.min(retry.maxTimeout ?? DEFAULT_RETRY.maxTimeout, (retry.minTimeout ?? DEFAULT_RETRY.minTimeout) * 2 ** (attemptNo - 1)) - await append(input, 'retry_scheduled', { workId, attemptNo, retryAt: new Date(env.now().getTime() + retryDelay), errorClass: classification.kind, detail: { error: messageOf(error) } }) - await env.sleep(retryDelay, env.signal) + return readAndLoad({ ...input, workId, attemptNo, state, selection, retry }) + }, retry, { + signal: env.signal, + classifier: stream.classifyError, + onRetry: (context, retryAfterMs) => append(input, 'retry_scheduled', { + workId, + attemptNo: context.attemptNumber, + retryAt: retryAfterMs === undefined ? undefined : new Date(env.now().getTime() + retryAfterMs), + errorClass: context.error.classification.kind, + detail: { error: context.error.message }, + }), + }) + } catch (error) { + failure = error + if (env.signal.aborted || failureKind(error, env.signal, stream) === 'cancelled') { + outcome = env.hostSignal.aborted ? 'cancelled' : 'budget_exhausted' + // Exhausting the budget is incomplete: committed progress stands. + if (outcome === 'budget_exhausted') failure = undefined } } @@ -307,21 +303,20 @@ async function readAndLoad(input: { cutoff: input.cutoff, signal, attempt: (operation, options) => - runAttempt(operation, options?.label ?? 'source', input.retry, { + runAttempt(() => input.permits.fetches(() => { + signal.throwIfAborted() + return operation(signal) + }), input.retry, { signal, - fetchPermits: input.permits.fetches, classifier: stream.classifyError, - sleep: env.sleep, - random: env.random, - now: () => env.now().getTime(), - onAttempt: () => undefined, - onRetry: ({ label, context }) => + onRetry: (context, retryAfterMs) => append(input, 'retry_scheduled', { workId: input.workId, attemptNo: input.attemptNo, - retryAt: new Date(env.now().getTime() + context.retryDelay), + // p-retry owns the randomized backoff; record only a provider's known not-before time. + retryAt: retryAfterMs === undefined ? undefined : new Date(env.now().getTime() + retryAfterMs), errorClass: context.error.classification.kind, - detail: { label, sourceAttempt: context.attemptNumber, error: context.error.message }, + detail: { label: options?.label ?? 'source', sourceAttempt: context.attemptNumber, error: context.error.message }, }), }), }) @@ -444,32 +439,38 @@ async function loadBatch( span.setAttribute('chkit.ingest.batch_id', batchId) span.setAttribute('chkit.ingest.rows', rows.length) try { - for (let attempt = 1; ; attempt += 1) { - // The load permit covers only the active write, never the backoff timer. - const release = await input.permits.loads.acquire(signal) + return await pRetry(() => input.permits.loads(async () => { + signal.throwIfAborted() + // The load limit covers only the active write, never p-retry's backoff. + let loader: ReturnType try { - const loader = factory({ + loader = factory({ streamId: input.stream.id, runId: input.runId, table: input.stream.destination, destination: input.env.destination, signal, }) - try { - await loader.write({ batchId, rows }) - return await loader.finalize() - } catch (error) { - // A cleanup failure must not replace the write failure that decides the retry. - await loader.abort(error).catch((cleanupError: unknown) => { - span.recordException(cleanupError instanceof Error ? cleanupError : new Error(String(cleanupError))) - }) - if (signal.aborted || isAbortError(error) || attempt >= LOAD_ATTEMPTS) throw error - } - } finally { - release() + } catch (error) { + // Construction errors are not ambiguous sink acknowledgements. + throw new AbortError(error instanceof Error ? error : new Error(String(error))) } - await input.env.sleep(1000 * 2 ** (attempt - 1), signal) - } + try { + await loader.write({ batchId, rows }) + return await loader.finalize() + } catch (error) { + // Cleanup must not replace the write failure that decides the retry. + await loader.abort(error).catch((cleanupError: unknown) => { + span.recordException(cleanupError instanceof Error ? cleanupError : new Error(String(cleanupError))) + }) + throw error + } + }), { + retries: LOAD_ATTEMPTS - 1, + minTimeout: 1000, + signal, + shouldRetry: ({ error }) => !isAbortError(error), + }) } catch (error) { recordFailure(span, error) throw error @@ -519,16 +520,12 @@ function append( errorClass: fields.errorClass ?? '', detail: fields.detail ?? {}, } - for (let attempt = 1; ; attempt += 1) { - try { - await abortable(() => env.journal.append(event), signal) - progress.seq = event.eventSeq - return - } catch (error) { - if (signal.aborted || attempt >= JOURNAL_APPEND_ATTEMPTS) throw error - await abortable(() => env.sleep(250 * 2 ** (attempt - 1), signal), signal) - } - } + await pRetry(() => abortable(() => env.journal.append(event), signal), { + retries: JOURNAL_APPEND_ATTEMPTS - 1, + minTimeout: 250, + signal, + }) + progress.seq = event.eventSeq }) // A fact that could not be confirmed poisons the chain: a later append must // not reuse its sequence number, because the unconfirmed write may have landed. @@ -583,8 +580,7 @@ function emptyBatch(): PendingBatch { function graceAfterAbort(signal: AbortSignal, graceMs: number): Promise { return new Promise((resolve) => { const start = () => { - const timer = setTimeout(resolve, graceMs) - if (typeof timer === 'object' && 'unref' in timer) timer.unref() + void sleep(graceMs, undefined, { ref: false }).then(resolve) } if (signal.aborted) start() else signal.addEventListener('abort', start, { once: true }) @@ -608,9 +604,9 @@ function permitsFor(cache: Map, pipeline: PipelineDefin const existing = cache.get(pipeline.id) if (existing) return existing const created: PipelinePermits = { - streams: createSemaphore(pipeline.maxStreams), - fetches: createSemaphore(pipeline.maxFetches), - loads: createSemaphore(pipeline.maxLoads), + streams: pLimit(pipeline.maxStreams), + fetches: pLimit(pipeline.maxFetches), + loads: pLimit(pipeline.maxLoads), } cache.set(pipeline.id, created) return created @@ -629,8 +625,6 @@ function resolveEnv(input: ExecutionEnv, deadlineSignal: AbortSignal): ResolvedE maxDurationMs, prefetchBatches: input.prefetchBatches ?? DEFAULT_PREFETCH_BATCHES, now, - sleep: input.sleep ?? sleep, - random: input.random ?? Math.random, log: input.log ?? (() => undefined), } } diff --git a/packages/plugin-ingest/src/ingest.e2e.test.ts b/packages/plugin-ingest/src/ingest.e2e.test.ts index 78379e09..054540d8 100644 --- a/packages/plugin-ingest/src/ingest.e2e.test.ts +++ b/packages/plugin-ingest/src/ingest.e2e.test.ts @@ -73,14 +73,13 @@ describe('@chkit/plugin-ingest live env e2e', () => { await destination.insert(input) }, } - const noSleep = async () => undefined - const first = await runIngestion({ selected: selectStreams([pipeline], []), backfill: undefined }, { journal: journal(), destination: lossy, sleep: noSleep }) + const first = await runIngestion({ selected: selectStreams([pipeline], []), backfill: undefined }, { journal: journal(), destination: lossy }) expect(first.ok).toBe(false) expect((await journal().readCheckpoint(stream.id)).envelope?.state).toBe(1) // A fresh executor process reconstructs everything from the journal. - const second = await runIngestion({ selected: selectStreams([pipeline], []), backfill: undefined }, { journal: journal(), destination, sleep: noSleep }) + const second = await runIngestion({ selected: selectStreams([pipeline], []), backfill: undefined }, { journal: journal(), destination }) expect(second.ok).toBe(true) const checkpoint = await journal().readCheckpoint(stream.id) expect(checkpoint.envelope).toEqual({ strategy: 'e2e.page', version: 1, state: 3 }) diff --git a/packages/plugin-ingest/src/retry.test.ts b/packages/plugin-ingest/src/retry.test.ts new file mode 100644 index 00000000..35a962e8 --- /dev/null +++ b/packages/plugin-ingest/src/retry.test.ts @@ -0,0 +1,91 @@ +import { describe, expect, test } from 'bun:test' + +import { FetchFailure, HttpError, IngestConfigError } from './errors.js' +import { runAttempt } from './retry.js' +import type { RetryContext } from './types.js' + +const signal = new AbortController().signal +const env = { signal, classifier: undefined, onRetry: () => undefined } +const immediate = { minTimeout: 0, randomize: false } +const httpError = (status: number, retryAfter = '0') => + HttpError.fromResponse(new Response('', { status, headers: { 'retry-after': retryAfter } })) + +describe('provider retry policy', () => { + test('honors Retry-After even when rate limits do not consume the retry budget', async () => { + const limited = await httpError(429, '0.03') + const transient = await httpError(503) + const retries: RetryContext[] = [] + const times: number[] = [] + const value = await runAttempt(async (attempt) => { + times.push(performance.now()) + if (attempt === 1) throw limited + if (attempt === 2) throw transient + return 'ok' + }, { + ...immediate, + retries: 1, + shouldConsumeRetry: ({ error }) => error.classification.kind !== 'rate_limited', + }, { ...env, onRetry: (context) => { retries.push(context) } }) + + expect(value).toBe('ok') + expect(times).toHaveLength(3) + expect((times[1] ?? 0) - (times[0] ?? 0)).toBeGreaterThanOrEqual(25) + expect(retries.map(({ attemptNumber, retriesLeft, retriesConsumed }) => ({ attemptNumber, retriesLeft, retriesConsumed }))).toEqual([ + { attemptNumber: 1, retriesLeft: 1, retriesConsumed: 0 }, + { attemptNumber: 2, retriesLeft: 1, retriesConsumed: 0 }, + ]) + expect(retries[0]?.error.cause).toBe(limited) + }) + + test('does not schedule a retry when Retry-After exceeds maxRetryTime', async () => { + let calls = 0 + let scheduled = 0 + const failure = await httpError(429, '60') + await expect(runAttempt(async () => { + calls += 1 + throw failure + }, { ...immediate, maxRetryTime: 100 }, { + ...env, onRetry: () => { scheduled += 1 }, + })).rejects.toMatchObject({ cause: failure }) + expect(calls).toBe(1) + expect(scheduled).toBe(0) + }) + + test.each(['backoff', 'retry-after'] as const)('cancellation interrupts %s without calling the provider again', async (mode) => { + const controller = new AbortController() + const failure = await httpError(mode === 'backoff' ? 503 : 429, '60') + let calls = 0 + const timer = setTimeout(() => controller.abort(new Error('cancelled')), 20) + try { + await expect(runAttempt(async () => { + calls += 1 + throw failure + }, { minTimeout: mode === 'backoff' ? 60_000 : 0 }, { ...env, signal: controller.signal })).rejects.toThrow() + expect(calls).toBe(1) + } finally { + clearTimeout(timer) + } + }) + + test('preserves the thrown value and allows the retry veto', async () => { + const cause = { providerCode: 'NO_ACCESS' } + let calls = 0 + await expect(runAttempt(async () => { + calls += 1 + throw cause + }, { ...immediate, shouldRetry: ({ error }) => error.cause !== cause }, env)).rejects.toMatchObject({ cause }) + expect(calls).toBe(1) + }) + + test('does not retry permanent, configuration, or already-exhausted source failures', async () => { + const exhausted = new FetchFailure(new Error('already retried'), { kind: 'transient' }) + for (const cause of [await httpError(401), new IngestConfigError('invalid'), exhausted]) { + let calls = 0 + await expect(runAttempt(async () => { + calls += 1 + throw cause + }, immediate, env)).rejects.toThrow() + expect(calls).toBe(1) + } + }) +}) diff --git a/packages/plugin-ingest/src/retry.ts b/packages/plugin-ingest/src/retry.ts index 152beefd..a3c4dd5e 100644 --- a/packages/plugin-ingest/src/retry.ts +++ b/packages/plugin-ingest/src/retry.ts @@ -1,8 +1,11 @@ -import { classifyFailure, FetchFailure } from './errors.js' -import type { Semaphore } from './semaphore.js' +import { setTimeout as sleep } from 'node:timers/promises' + +import pRetry, { AbortError } from 'p-retry' + +import { classifyFailure, FetchFailure, IngestConfigError } from './errors.js' import type { ErrorClassifier, RetryContext, RetryOptions } from './types.js' -export const DEFAULT_RETRY: Required> = { +export const DEFAULT_RETRY = { retries: 5, factor: 2, minTimeout: 1000, @@ -11,15 +14,10 @@ export const DEFAULT_RETRY: Required Promise - random: () => number - now: () => number - onAttempt: (input: { label: string; attemptNumber: number }) => Promise | void - onRetry: (input: { label: string; context: RetryContext }) => Promise | void + onRetry: (context: RetryContext, retryAfterMs: number | undefined) => Promise | void } /** Pipeline default, partially overridden by the stream. */ @@ -34,79 +32,46 @@ export function mergeRetry(...layers: Array): RetryOpt return merged } -/** - * Run one source operation. A fetch permit is acquired per attempt and released - * while waiting for a retry timer, so backoff never occupies fetch capacity. - */ -export async function runAttempt( - operation: (signal: AbortSignal) => Promise, - label: string, +/** p-retry owns retry budgets and backoff; this adapter adds provider classification and Retry-After. */ +export function runAttempt( + operation: (attemptNumber: number) => Promise, policy: RetryOptions, env: AttemptEnv ): Promise { - const retries = policy.retries ?? DEFAULT_RETRY.retries - const maxRetryTime = policy.maxRetryTime ?? DEFAULT_RETRY.maxRetryTime - const startedAt = env.now() - let attemptNumber = 0 - let retriesConsumed = 0 + const options = { ...DEFAULT_RETRY, ...mergeRetry(policy) } + const deadline = performance.now() + options.maxRetryTime + let notBefore = 0 - while (true) { - attemptNumber += 1 + return pRetry(async (attemptNumber) => { + // Backoff and Retry-After overlap: only wait out the remaining provider embargo. + const remaining = notBefore - performance.now() + if (remaining > 0) await sleep(remaining, undefined, { signal: env.signal }) env.signal.throwIfAborted() - await env.onAttempt({ label, attemptNumber }) - - const release = await env.fetchPermits.acquire(env.signal) try { - return await operation(env.signal) + return await operation(attemptNumber) } catch (cause) { - const classification = classifyFailure(cause, env.signal, env.classifier) - const failure = new FetchFailure(cause, classification) - if (classification.kind === 'cancelled' || classification.kind === 'permanent') throw failure - - const retriesLeft = retries - retriesConsumed - const backoff = backoffDelay(policy, retriesConsumed, env.random) - const hinted = classification.kind === 'rate_limited' ? classification.retryAfterMs : undefined - const retryDelay = hinted === undefined ? backoff : Math.max(hinted, backoff) - const context: RetryContext = { error: failure, attemptNumber, retriesLeft, retriesConsumed, retryDelay } - - if (retriesLeft <= 0) throw failure - if (env.now() - startedAt + retryDelay > maxRetryTime) throw failure - if (policy.shouldRetry && !(await policy.shouldRetry(context))) throw failure - const consume = policy.shouldConsumeRetry ? await policy.shouldConsumeRetry(context) : true - if (consume) retriesConsumed += 1 - - await env.onRetry({ label, context }) - release() - await env.sleep(retryDelay, env.signal) - } finally { - release() + // A nested source attempt already exhausted its retries. Never replay it + // through the reader's coarser retry boundary. + if (cause instanceof FetchFailure || cause instanceof IngestConfigError) throw new AbortError(cause) + const failure = new FetchFailure(cause, classifyFailure(cause, env.signal, env.classifier)) + if (failure.classification.kind === 'cancelled' || failure.classification.kind === 'permanent') throw new AbortError(failure) + throw failure } - } -} - -export function sleep(ms: number, signal: AbortSignal): Promise { - return new Promise((resolve, reject) => { - if (signal.aborted) { - reject(signal.reason) - return - } - const onAbort = () => { - clearTimeout(timer) - reject(signal.reason) - } - const timer = setTimeout(() => { - signal.removeEventListener('abort', onAbort) - resolve() - }, ms) - signal.addEventListener('abort', onAbort, { once: true }) + }, { + ...options, + signal: env.signal, + shouldConsumeRetry: ({ error, ...context }) => + error instanceof FetchFailure ? (policy.shouldConsumeRetry?.({ ...context, error }) ?? true) : true, + shouldRetry: async ({ error, ...context }) => { + env.signal.throwIfAborted() + if (!(error instanceof FetchFailure)) return false + const retryContext = { ...context, error } + if (policy.shouldRetry && !(await policy.shouldRetry(retryContext))) return false + const retryAfterMs = error.classification.kind === 'rate_limited' ? error.classification.retryAfterMs : undefined + notBefore = performance.now() + (retryAfterMs ?? 0) + if (notBefore > deadline) return false + await env.onRetry(retryContext, retryAfterMs) + return true + }, }) } - -function backoffDelay(policy: RetryOptions, retriesConsumed: number, random: () => number): number { - const factor = policy.factor ?? DEFAULT_RETRY.factor - const minTimeout = policy.minTimeout ?? DEFAULT_RETRY.minTimeout - const maxTimeout = policy.maxTimeout ?? DEFAULT_RETRY.maxTimeout - const randomize = policy.randomize ?? DEFAULT_RETRY.randomize - const jitter = randomize ? 1 + random() : 1 - return Math.min(maxTimeout, Math.round(jitter * minTimeout * factor ** retriesConsumed)) -} diff --git a/packages/plugin-ingest/src/semaphore.ts b/packages/plugin-ingest/src/semaphore.ts deleted file mode 100644 index dcf7e15e..00000000 --- a/packages/plugin-ingest/src/semaphore.ts +++ /dev/null @@ -1,51 +0,0 @@ -export interface Semaphore { - /** Resolves with an idempotent release function. */ - acquire(signal: AbortSignal): Promise<() => void> -} - -export function createSemaphore(limit: number): Semaphore { - let active = 0 - const waiters: Array<{ grant: () => void; cancel: (reason: unknown) => void }> = [] - - const release = () => { - active -= 1 - const next = waiters.shift() - if (next) next.grant() - } - - const onceRelease = () => { - let released = false - return () => { - if (released) return - released = true - release() - } - } - - return { - acquire(signal) { - if (signal.aborted) return Promise.reject(signal.reason) - if (active < limit) { - active += 1 - return Promise.resolve(onceRelease()) - } - return new Promise((resolve, reject) => { - const waiter = { - grant: () => { - signal.removeEventListener('abort', onAbort) - active += 1 - resolve(onceRelease()) - }, - cancel: reject, - } - const onAbort = () => { - const index = waiters.indexOf(waiter) - if (index >= 0) waiters.splice(index, 1) - waiter.cancel(signal.reason) - } - signal.addEventListener('abort', onAbort, { once: true }) - waiters.push(waiter) - }) - }, - } -} diff --git a/packages/plugin-ingest/src/types.ts b/packages/plugin-ingest/src/types.ts index 0e30ca0b..810742fc 100644 --- a/packages/plugin-ingest/src/types.ts +++ b/packages/plugin-ingest/src/types.ts @@ -1,3 +1,5 @@ +import type { Options as PRetryOptions, RetryContext as PRetryContext } from 'p-retry' + import type { TableDefinition } from '@chkit/core' import type { FetchFailure } from './errors.js' @@ -79,22 +81,12 @@ export interface PlanInput { range: { from: Date | undefined; to: Date | undefined } | undefined } -export interface RetryContext { - error: FetchFailure - attemptNumber: number - retriesLeft: number - retriesConsumed: number - retryDelay: number -} +/** p-retry context with the original provider failure and its classification. */ +export type RetryContext = Omit & { error: FetchFailure } -/** Portable p-retry-shaped subset. */ -export interface RetryOptions { - retries?: number - factor?: number - minTimeout?: number - maxTimeout?: number - randomize?: boolean - maxRetryTime?: number +export type RetryOptions = Pick & { shouldRetry?: (context: RetryContext) => boolean | Promise shouldConsumeRetry?: (context: RetryContext) => boolean | Promise } diff --git a/packages/plugin-obsessiondb/src/auth/api-client.ts b/packages/plugin-obsessiondb/src/auth/api-client.ts index 3c427f49..c98649b4 100644 --- a/packages/plugin-obsessiondb/src/auth/api-client.ts +++ b/packages/plugin-obsessiondb/src/auth/api-client.ts @@ -1,4 +1,5 @@ import { createRequire } from 'node:module' +import { setTimeout as sleep } from 'node:timers/promises' interface DeviceCodeResponse { device_code: string @@ -81,7 +82,7 @@ export async function pollDeviceToken( let pollInterval = interval * 1000 while (Date.now() < deadline) { - await new Promise((r) => setTimeout(r, pollInterval)) + await sleep(pollInterval) const res = await fetch(`${baseUrl}/api/auth/device/token`, { method: 'POST', From 78e8fe6b970efb84fc71e97a38c5c04cd900076e Mon Sep 17 00:00:00 2001 From: KeKs0r Date: Sat, 19 Sep 2026 16:55:03 -0700 Subject: [PATCH 08/11] =?UTF-8?q?=F0=9F=90=9B=20Fix=20ingestion=20unused-e?= =?UTF-8?q?xport=20checks?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/plugin-ingest/src/errors.ts | 2 +- packages/plugin-ingest/src/journal.ts | 2 +- packages/plugin-ingest/src/plugin.ts | 2 +- packages/plugin-ingest/src/registry.ts | 2 +- packages/plugin-ingest/src/retry.ts | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/plugin-ingest/src/errors.ts b/packages/plugin-ingest/src/errors.ts index fbdb0682..dd9915b5 100644 --- a/packages/plugin-ingest/src/errors.ts +++ b/packages/plugin-ingest/src/errors.ts @@ -85,7 +85,7 @@ export function classifyFailure( return classifier?.(cause, fallback) ?? fallback } -export function classifyCommon(cause: unknown): FailureClass { +function classifyCommon(cause: unknown): FailureClass { if (cause instanceof HttpError) { if (cause.status === 429) return { kind: 'rate_limited', retryAfterMs: cause.retryAfterMs } if (cause.status === 408 || cause.status === 425 || cause.status >= 500) return { kind: 'transient' } diff --git a/packages/plugin-ingest/src/journal.ts b/packages/plugin-ingest/src/journal.ts index 05c3a515..5afe24ba 100644 --- a/packages/plugin-ingest/src/journal.ts +++ b/packages/plugin-ingest/src/journal.ts @@ -196,7 +196,7 @@ export function toJournalRow(event: JournalEvent, targetId: string, at: Date): J } } -export function parseEnvelope(json: string): CheckpointEnvelope | undefined { +function parseEnvelope(json: string): CheckpointEnvelope | undefined { if (json === '') return undefined const parsed: unknown = JSON.parse(json) if ( diff --git a/packages/plugin-ingest/src/plugin.ts b/packages/plugin-ingest/src/plugin.ts index f1b2a9c2..97a904b2 100644 --- a/packages/plugin-ingest/src/plugin.ts +++ b/packages/plugin-ingest/src/plugin.ts @@ -42,7 +42,7 @@ const RUN_FLAGS = defineFlags([ { name: '--max-duration', type: 'string', description: 'Execution budget in seconds', placeholder: '' }, ] as const) -export interface IngestPluginCommandContext { +interface IngestPluginCommandContext { args: string[] flags: Record jsonMode: boolean diff --git a/packages/plugin-ingest/src/registry.ts b/packages/plugin-ingest/src/registry.ts index ab7a4bf4..d5419a22 100644 --- a/packages/plugin-ingest/src/registry.ts +++ b/packages/plugin-ingest/src/registry.ts @@ -108,7 +108,7 @@ export function definePipeline(input: PipelineInput): PipelineDefinition { } /** Validate identities within this graph, without process-wide registration. */ -export function validatePipelines(pipelines: readonly PipelineDefinition[]): void { +function validatePipelines(pipelines: readonly PipelineDefinition[]): void { const ids = new Set() const owners = new Map() for (const pipeline of pipelines) { diff --git a/packages/plugin-ingest/src/retry.ts b/packages/plugin-ingest/src/retry.ts index a3c4dd5e..4517eb7c 100644 --- a/packages/plugin-ingest/src/retry.ts +++ b/packages/plugin-ingest/src/retry.ts @@ -5,7 +5,7 @@ import pRetry, { AbortError } from 'p-retry' import { classifyFailure, FetchFailure, IngestConfigError } from './errors.js' import type { ErrorClassifier, RetryContext, RetryOptions } from './types.js' -export const DEFAULT_RETRY = { +const DEFAULT_RETRY = { retries: 5, factor: 2, minTimeout: 1000, From 4c62a84f95248d60b892f2586aece986883ce570 Mon Sep 17 00:00:00 2001 From: KeKs0r Date: Sat, 19 Sep 2026 17:00:04 -0700 Subject: [PATCH 09/11] =?UTF-8?q?=E2=9C=85=20Target=20replay=20failures=20?= =?UTF-8?q?by=20batch=20identity?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/plugin-ingest/src/executor.test.ts | 18 +++++++++++++----- packages/plugin-ingest/src/ingest.e2e.test.ts | 12 ++++++++---- 2 files changed, 21 insertions(+), 9 deletions(-) diff --git a/packages/plugin-ingest/src/executor.test.ts b/packages/plugin-ingest/src/executor.test.ts index 22575235..c807f784 100644 --- a/packages/plugin-ingest/src/executor.test.ts +++ b/packages/plugin-ingest/src/executor.test.ts @@ -92,22 +92,30 @@ describe('runIngestion', () => { }, }) const pipeline = definePipeline({ id: 'app', streams: [stream], retry: { retries: 0 } }) - let inserts = 0 + let firstWrite = true + let acknowledgementLost = false const crashing: DestinationAdapter = { insert: async (input) => { - inserts += 1 + // A transient error on the first batch must not shift the simulated + // acknowledgement loss onto that batch's retry. + if (firstWrite) { + firstWrite = false + throw new Error('temporary insert failure') + } // Second batch: the write lands but the acknowledgement is lost. - if (inserts === 2) { + if (input.rows[0]?.id === 2) { + if (acknowledgementLost) throw new Error('still down') await destination.insert(input) + acknowledgementLost = true throw new Error('socket hang up') } - if (inserts > 2 && inserts <= 4) throw new Error('still down') await destination.insert(input) }, } const first = await runIngestion({ selected: selectStreams([pipeline], []), backfill: undefined }, { journal, destination: crashing }) expect(first.ok).toBe(false) + expect(first.streams[0]).toMatchObject({ outcome: 'failed', rows: 2, batches: 1, error: 'still down' }) expect((await journal.readCheckpoint('app.cursor')).envelope?.state).toBe(1) const second = await runIngestion({ selected: selectStreams([pipeline], []), backfill: undefined }, { journal, destination }) @@ -115,7 +123,7 @@ describe('runIngestion', () => { expect((await journal.readCheckpoint('app.cursor')).envelope?.state).toBe(3) const ids = (destination.tables.get('app.events') ?? []).map((row) => row.id) expect(ids).toEqual([0, 1, 2, 3, 4, 5]) - }) + }, 10_000) test('timestampWindow commits the cutoff as watermark only after the whole window loaded', async () => { const journal = createMemoryJournal() diff --git a/packages/plugin-ingest/src/ingest.e2e.test.ts b/packages/plugin-ingest/src/ingest.e2e.test.ts index 054540d8..94d061a8 100644 --- a/packages/plugin-ingest/src/ingest.e2e.test.ts +++ b/packages/plugin-ingest/src/ingest.e2e.test.ts @@ -61,25 +61,29 @@ describe('@chkit/plugin-ingest live env e2e', () => { const pipeline = definePipeline({ id: `${prefix}pipeline`, streams: [stream], retry: { retries: 0 } }) const journal = () => createClickHouseJournal({ executor, database, targetId: `e2e/${prefix}`, table: journalTable }) const destination = createClickHouseDestination(executor) - let inserts = 0 + let acknowledgementLost = false const lossy: DestinationAdapter = { insert: async (input) => { - inserts += 1 - if (inserts === 2) { + // Target the second batch, not the second call: the live destination + // can itself need retries before the first batch is acknowledged. + if (input.rows[0]?.id === 2) { + if (acknowledgementLost) throw new Error('target unavailable') await destination.insert(input) + acknowledgementLost = true throw new Error('acknowledgement lost') } - if (inserts > 2) throw new Error('target unavailable') await destination.insert(input) }, } const first = await runIngestion({ selected: selectStreams([pipeline], []), backfill: undefined }, { journal: journal(), destination: lossy }) expect(first.ok).toBe(false) + expect(first.streams[0]).toMatchObject({ outcome: 'failed', rows: 2, batches: 1, error: 'target unavailable' }) expect((await journal().readCheckpoint(stream.id)).envelope?.state).toBe(1) // A fresh executor process reconstructs everything from the journal. const second = await runIngestion({ selected: selectStreams([pipeline], []), backfill: undefined }, { journal: journal(), destination }) + expect(second.streams[0]).toMatchObject({ outcome: 'succeeded', error: undefined }) expect(second.ok).toBe(true) const checkpoint = await journal().readCheckpoint(stream.id) expect(checkpoint.envelope).toEqual({ strategy: 'e2e.page', version: 1, state: 3 }) From 63407b585d42b397a451fb77c1b648d631ff0c03 Mon Sep 17 00:00:00 2001 From: KeKs0r Date: Sat, 19 Sep 2026 18:00:35 -0700 Subject: [PATCH 10/11] =?UTF-8?q?=F0=9F=90=9B=20Fix=20ingestion=20serializ?= =?UTF-8?q?ation=20and=20retry=20bounds?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/plugin-ingest/src/destination.ts | 9 +- packages/plugin-ingest/src/errors.ts | 9 +- packages/plugin-ingest/src/executor.test.ts | 119 ++++++++++++++++++ packages/plugin-ingest/src/executor.ts | 19 ++- packages/plugin-ingest/src/ingest.e2e.test.ts | 37 ++++++ packages/plugin-ingest/src/retry.test.ts | 16 +++ packages/plugin-ingest/src/types.ts | 1 + 7 files changed, 196 insertions(+), 14 deletions(-) diff --git a/packages/plugin-ingest/src/destination.ts b/packages/plugin-ingest/src/destination.ts index bcf66fdc..b91be89c 100644 --- a/packages/plugin-ingest/src/destination.ts +++ b/packages/plugin-ingest/src/destination.ts @@ -1,7 +1,7 @@ import type { ClickHouseExecutor } from '@chkit/clickhouse' import { table, type ColumnDefinition, type TableDefinition } from '@chkit/core' -import type { DestinationAdapter } from './types.js' +import type { DestinationAdapter, Row } from './types.js' export const BATCH_ID_COLUMN = '_chkit_batch_id' export const RUN_ID_COLUMN = '_chkit_run_id' @@ -60,7 +60,7 @@ export function createClickHouseDestination(executor: ClickHouseExecutor): Desti if (rows.length === 0) return await executor.insert({ table: `${table.database}.${table.name}`, - values: [...rows], + values: toJsonRows(rows), settings: { insert_deduplication_token: token, wait_for_async_insert: 1, @@ -69,3 +69,8 @@ export function createClickHouseDestination(executor: ClickHouseExecutor): Desti }, } } + +/** JSONEachRow encodes big integers as exact decimal strings, including nested values. */ +export function toJsonRows(rows: readonly Row[]): Row[] { + return JSON.parse(JSON.stringify(rows, (_key, value: unknown) => typeof value === 'bigint' ? value.toString() : value)) as Row[] +} diff --git a/packages/plugin-ingest/src/errors.ts b/packages/plugin-ingest/src/errors.ts index dd9915b5..402022f8 100644 --- a/packages/plugin-ingest/src/errors.ts +++ b/packages/plugin-ingest/src/errors.ts @@ -80,7 +80,7 @@ export function classifyFailure( signal: AbortSignal, classifier: ErrorClassifier | undefined ): FailureClass { - if (signal.aborted || isAbortError(cause)) return { kind: 'cancelled' } + if (signal.aborted) return { kind: 'cancelled' } const fallback = classifyCommon(cause) return classifier?.(cause, fallback) ?? fallback } @@ -95,14 +95,11 @@ function classifyCommon(cause: unknown): FailureClass { const code = errorCode(cause) if (code !== undefined && TRANSIENT_NETWORK_CODES.has(code)) return { kind: 'transient' } if (cause instanceof TypeError && /fetch failed|network|socket/i.test(cause.message)) return { kind: 'transient' } - if (cause instanceof Error && cause.name === 'TimeoutError') return { kind: 'transient' } + // A request's own timeout/abort does not cancel the ingestion execution. + if (cause instanceof Error && (cause.name === 'TimeoutError' || cause.name === 'AbortError')) return { kind: 'transient' } return { kind: 'unknown' } } -export function isAbortError(value: unknown): boolean { - return value instanceof Error && value.name === 'AbortError' -} - function errorCode(value: unknown): string | undefined { if (typeof value !== 'object' || value === null) return undefined if ('code' in value && typeof value.code === 'string') return value.code diff --git a/packages/plugin-ingest/src/executor.test.ts b/packages/plugin-ingest/src/executor.test.ts index c807f784..84ad1d1d 100644 --- a/packages/plugin-ingest/src/executor.test.ts +++ b/packages/plugin-ingest/src/executor.test.ts @@ -282,6 +282,125 @@ describe('runtime contracts', () => { const run = (pipeline: ReturnType, env: Parameters[1]) => runIngestion({ selected: selectStreams([pipeline], []), backfill: undefined }, env) + test('content-based batch identity supports BigInt rows and stays stable on replay', async () => { + const journal = createMemoryJournal() + const destination = createMemoryDestination() + const row = { id: 9007199254740993n, nested: { values: [-9223372036854775808n] } } + const tokens: string[] = [] + let loseAcknowledgement = true + const stream = defineStream({ + id: 'app.bigint', destination: events, retry: { retries: 0 }, + async *read() { yield { rows: [row] } }, + }) + const pipeline = definePipeline({ id: 'app', streams: [stream] }) + const observed: DestinationAdapter = { async insert(input) { + tokens.push(input.token) + // Custom destinations continue receiving the original values. + expect(input.rows[0]?.id).toBe(row.id) + await destination.insert(input) + if (loseAcknowledgement) throw new TypeError('acknowledgement lost') + } } + + expect((await run(pipeline, { journal, destination: observed })).streams[0]?.error).toBe('acknowledgement lost') + loseAcknowledgement = false + expect((await run(pipeline, { journal, destination: observed })).ok).toBe(true) + expect(tokens).toHaveLength(2) + expect(tokens[0]).toBe(tokens[1]) + expect(destination.tables.get('app.events')).toHaveLength(1) + }) + + test.each(['source', 'reader', 'sink'] as const)('a request-local AbortError in the %s is retried without cancelling the run', async (boundary) => { + const journal = createMemoryJournal() + const destination = createMemoryDestination() + let calls = 0 + let classifications = 0 + const operation = async () => { + if (++calls === 1) throw new DOMException('request timed out', 'AbortError') + return [{ id: 1 }] + } + const stream = defineStream({ + id: 'app.local_abort', destination: events, + retry: { retries: 1, minTimeout: 0, randomize: false }, + classifyError: () => { classifications += 1; return { kind: 'transient' } }, + async *read({ attempt }) { + yield { rows: boundary === 'sink' ? [{ id: 1 }] : await (boundary === 'source' ? attempt(operation) : operation()) } + }, + }) + const result = await run(definePipeline({ id: 'app', streams: [stream] }), { + journal, + destination: boundary === 'sink' ? { async insert(input) { await operation(); await destination.insert(input) } } : destination, + }) + + expect(result.streams[0]).toMatchObject({ outcome: 'succeeded', rows: 1, error: undefined }) + expect(calls).toBe(2) + expect(classifications).toBe(boundary === 'sink' ? 0 : 1) + }) + + test('exhausted request-local abort retries report failure, not budget exhaustion', async () => { + let calls = 0 + const stream = defineStream({ + id: 'app.local_abort', destination: events, + retry: { retries: 1, minTimeout: 0, randomize: false }, + async *read({ attempt }) { + yield { rows: await attempt(async () => { calls += 1; throw new DOMException('request timed out', 'AbortError') }) } + }, + }) + const result = await run(definePipeline({ id: 'app', streams: [stream] }), { + journal: createMemoryJournal(), destination: createMemoryDestination(), + }) + expect(result.streams[0]).toMatchObject({ outcome: 'failed', error: 'request timed out' }) + expect(calls).toBe(2) + }) + + test('a sink failure at the chunk limit cannot recreate the reader or hide the failure', async () => { + const journal = createMemoryJournal() + let readers = 0 + let pulled = 0 + const stream = defineStream({ + id: 'app.bounded', destination: events, batchSize: 1, budget: { maxChunks: 1 }, + retry: { retries: 2, minTimeout: 0, randomize: false }, + incremental: timestampWindow({ start: new Date(0) }), + async *read() { + readers += 1 + for (let id = 0; id < 3; id += 1) { pulled += 1; yield { rows: [{ id }] } } + }, + }) + const result = await run(definePipeline({ id: 'app', streams: [stream] }), { + journal, destination: { async insert() { throw new TypeError('sink serialization failed') } }, + }) + expect(result.streams[0]).toMatchObject({ outcome: 'failed', chunks: 1, error: 'sink serialization failed' }) + expect(readers).toBe(1) + expect(pulled).toBe(1) + expect((await journal.readCheckpoint(stream.id)).envelope).toBeUndefined() + expect(journal.events.filter((event) => event.eventKind === 'retry_scheduled')).toHaveLength(0) + }) + + test('reader retries share the chunk budget and never complete a partial window', async () => { + const journal = createMemoryJournal() + let readers = 0 + let pulled = 0 + const stream = defineStream({ + id: 'app.bounded', destination: events, budget: { maxChunks: 2 }, + retry: { retries: 2, minTimeout: 0, randomize: false }, + incremental: timestampWindow({ start: new Date(0) }), + async *read() { + readers += 1 + for (let id = 0; id < 3; id += 1) { + pulled += 1 + yield { rows: [{ id }] } + if (readers === 1) throw new Error('reader disconnected') + } + }, + }) + const result = await run(definePipeline({ id: 'app', streams: [stream] }), { + journal, destination: createMemoryDestination(), + }) + expect(result.streams[0]).toMatchObject({ outcome: 'budget_exhausted', chunks: 2, rows: 1 }) + expect(readers).toBe(2) + expect(pulled).toBe(2) + expect((await journal.readCheckpoint(stream.id)).envelope).toBeUndefined() + }) + test.each([false, true])('new full syncs preserve changes while failed syncs reuse tokens (declared id: %s)', async (withId) => { const journal = createMemoryJournal() const destination = createMemoryDestination() diff --git a/packages/plugin-ingest/src/executor.ts b/packages/plugin-ingest/src/executor.ts index d3306071..cc3b9bbe 100644 --- a/packages/plugin-ingest/src/executor.ts +++ b/packages/plugin-ingest/src/executor.ts @@ -5,7 +5,8 @@ import { SpanStatusCode, trace, type Span } from '@opentelemetry/api' import pLimit, { type LimitFunction } from 'p-limit' import pRetry, { AbortError } from 'p-retry' -import { BudgetExhausted, classifyFailure, FetchFailure, IngestConfigError, isAbortError } from './errors.js' +import { toJsonRows } from './destination.js' +import { BudgetExhausted, classifyFailure, FetchFailure, IngestConfigError } from './errors.js' import { canonicalJson, digest } from './journal.js' import { simpleLoader } from './loader.js' import { createBoundedQueue } from './queue.js' @@ -235,7 +236,12 @@ async function executeStream(input: { } await append(input, 'attempt_started', { workId, attemptNo, workState: 'running' }) return readAndLoad({ ...input, workId, attemptNo, state, selection, retry }) - }, retry, { + }, { + ...retry, + // Reader recreation consumes the same execution-wide chunk budget. + // Preserve the original failure when no further chunks may be pulled. + shouldRetry: (context) => progress.chunks < (stream.budget?.maxChunks ?? Infinity) && (retry.shouldRetry?.(context) ?? true), + }, { signal: env.signal, classifier: stream.classifyError, onRetry: (context, retryAfterMs) => append(input, 'retry_scheduled', { @@ -248,7 +254,7 @@ async function executeStream(input: { }) } catch (error) { failure = error - if (env.signal.aborted || failureKind(error, env.signal, stream) === 'cancelled') { + if (env.signal.aborted) { outcome = env.hostSignal.aborted ? 'cancelled' : 'budget_exhausted' // Exhausting the budget is incomplete: committed progress stands. if (outcome === 'budget_exhausted') failure = undefined @@ -285,6 +291,8 @@ async function readAndLoad(input: { retry: ReturnType }): Promise { const { stream, namespaceId, progress, env } = input + const maxChunks = stream.budget?.maxChunks ?? Infinity + if (progress.chunks >= maxChunks) return 'budget_exhausted' const local = new AbortController() const signal = AbortSignal.any([env.signal, local.signal]) const queue = createBoundedQueue(env.prefetchBatches) @@ -349,7 +357,7 @@ async function readAndLoad(input: { await queue.push(pending, signal) pending = emptyBatch() } - if (stream.budget?.maxChunks !== undefined && progress.chunks >= stream.budget.maxChunks) budgetExhausted = true + if (progress.chunks >= maxChunks) budgetExhausted = true } } finally { // Never await an uncooperative reader: cleanup is best effort once we leave. @@ -374,7 +382,7 @@ async function readAndLoad(input: { // that changed between attempts. let sinceBoundary = 0 for (let batch = await queue.pop(signal); batch !== undefined; batch = await queue.pop(signal)) { - const discriminator = batch.intervalIds ? `interval:${canonicalJson(batch.intervalIds)}` : `content:${canonicalJson(batch.rows)}` + const discriminator = batch.intervalIds ? `interval:${canonicalJson(batch.intervalIds)}` : `content:${canonicalJson(toJsonRows(batch.rows))}` const batchId = digest([namespaceId, String(progress.lastSuccessSeq), String(progress.version), String(sinceBoundary), discriminator]).slice(0, 32) const receipt = await loadBatch(input, batchId, batch.rows, signal) if (abandoned) return @@ -469,7 +477,6 @@ async function loadBatch( retries: LOAD_ATTEMPTS - 1, minTimeout: 1000, signal, - shouldRetry: ({ error }) => !isAbortError(error), }) } catch (error) { recordFailure(span, error) diff --git a/packages/plugin-ingest/src/ingest.e2e.test.ts b/packages/plugin-ingest/src/ingest.e2e.test.ts index 94d061a8..051a429f 100644 --- a/packages/plugin-ingest/src/ingest.e2e.test.ts +++ b/packages/plugin-ingest/src/ingest.e2e.test.ts @@ -29,6 +29,18 @@ describe('@chkit/plugin-ingest live env e2e', () => { ...rawTable({ database, name: `${prefix}raw` }), settings: { non_replicated_deduplication_window: '100' }, } + const integers = table({ + database, + name: `${prefix}integers`, + columns: [ + { name: 'id', type: 'UInt64' }, + { name: 'signed', type: 'Int64' }, + { name: 'values', type: 'Array(UInt64)' }, + ...ingestionColumns, + ], + engine: 'MergeTree()', + orderBy: ['id'], + }) let executor: ClickHouseExecutor beforeAll(async () => { @@ -38,11 +50,14 @@ describe('@chkit/plugin-ingest live env e2e', () => { await waitForTable(executor, database, destinationTable.name) await executor.command(toCreateSQL(landing)) await waitForTable(executor, database, landing.name) + await executor.command(toCreateSQL(integers)) + await waitForTable(executor, database, integers.name) }) afterAll(async () => { await executor.command(`DROP TABLE IF EXISTS ${quoteIdent(database)}.${quoteIdent(destinationTable.name)}`) await executor.command(`DROP TABLE IF EXISTS ${quoteIdent(database)}.${quoteIdent(landing.name)}`) + await executor.command(`DROP TABLE IF EXISTS ${quoteIdent(database)}.${quoteIdent(integers.name)}`) await executor.command(`DROP TABLE IF EXISTS ${quoteIdent(database)}.${quoteIdent(journalTable)}`) await executor.close() }) @@ -126,4 +141,26 @@ describe('@chkit/plugin-ingest live env e2e', () => { expect(rows.map((row) => row.raw.value)).toEqual([next]) } }, 120_000) + + test('BigInt rows round-trip through content hashing and JSONEachRow without losing precision', async () => { + const row = { id: 18446744073709551615n, signed: -9223372036854775808n, values: [9007199254740993n, 18446744073709551615n] } + const stream = defineStream({ + id: `${prefix}integers`, destination: integers, + async *read() { yield { rows: [row] } }, + }) + const result = await runIngestion({ + selected: selectStreams([definePipeline({ id: `${prefix}integers_pipeline`, streams: [stream] })], []), + backfill: undefined, + }, { + journal: createClickHouseJournal({ executor, database, targetId: `e2e/${prefix}`, table: journalTable }), + destination: createClickHouseDestination(executor), + }) + expect(result.streams[0]).toMatchObject({ outcome: 'succeeded', rows: 1, error: undefined }) + const rows = await executor.query<{ id: string; signed: string; values: string[] }>( + `SELECT id, signed, values FROM ${quoteIdent(database)}.${quoteIdent(integers.name)}`, + { select_sequential_consistency: '1', output_format_json_quote_64bit_integers: '1' } + ) + expect(rows).toEqual([{ id: row.id.toString(), signed: row.signed.toString(), values: row.values.map(String) }]) + expect(typeof row.id).toBe('bigint') + }, 120_000) }) diff --git a/packages/plugin-ingest/src/retry.test.ts b/packages/plugin-ingest/src/retry.test.ts index 35a962e8..e7a3c5b2 100644 --- a/packages/plugin-ingest/src/retry.test.ts +++ b/packages/plugin-ingest/src/retry.test.ts @@ -77,6 +77,22 @@ describe('provider retry policy', () => { expect(calls).toBe(1) }) + test('execution cancellation stays authoritative over the provider classifier', async () => { + const controller = new AbortController() + let classifications = 0 + let calls = 0 + await expect(runAttempt(async () => { + calls += 1 + controller.abort(new Error('host cancelled')) + throw new DOMException('request aborted', 'AbortError') + }, immediate, { + ...env, signal: controller.signal, + classifier: () => { classifications += 1; return { kind: 'transient' } }, + })).rejects.toThrow() + expect(calls).toBe(1) + expect(classifications).toBe(0) + }) + test('does not retry permanent, configuration, or already-exhausted source failures', async () => { const exhausted = new FetchFailure(new Error('already retried'), { kind: 'transient' }) for (const cause of [await httpError(401), new IngestConfigError('invalid'), exhausted]) { diff --git a/packages/plugin-ingest/src/types.ts b/packages/plugin-ingest/src/types.ts index 810742fc..6526170f 100644 --- a/packages/plugin-ingest/src/types.ts +++ b/packages/plugin-ingest/src/types.ts @@ -6,6 +6,7 @@ import type { FetchFailure } from './errors.js' // ───── Authoring: streams and pipelines ───── +/** JSON-compatible column values; the ClickHouse destination encodes bigint as decimal strings. */ export type Row = Record /** From 8fc6fdbcc39b2e59d49238aaf5c5491b67a7d3d1 Mon Sep 17 00:00:00 2001 From: KeKs0r Date: Sat, 19 Sep 2026 18:04:25 -0700 Subject: [PATCH 11/11] =?UTF-8?q?=E2=9C=85=20Expose=20live=20ingestion=20f?= =?UTF-8?q?ailure=20details?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/plugin-ingest/src/ingest.e2e.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/plugin-ingest/src/ingest.e2e.test.ts b/packages/plugin-ingest/src/ingest.e2e.test.ts index 051a429f..292148ec 100644 --- a/packages/plugin-ingest/src/ingest.e2e.test.ts +++ b/packages/plugin-ingest/src/ingest.e2e.test.ts @@ -130,6 +130,7 @@ describe('@chkit/plugin-ingest live env e2e', () => { const result = await runIngestion({ selected, backfill: undefined }, { journal, destination: createClickHouseDestination(executor), }) + expect(result.streams[0]).toMatchObject({ outcome: 'succeeded', error: undefined }) expect(result.ok).toBe(true) const checkpoint = await journal.readCheckpoint(stream.id) expect(checkpoint.lastSuccessSeq).toBeGreaterThan(lastSuccessSeq)