diff --git a/.changeset/config.json b/.changeset/config.json index 488b07f8..efd130ea 100644 --- a/.changeset/config.json +++ b/.changeset/config.json @@ -12,6 +12,7 @@ "@chkit/plugin-codegen", "@chkit/plugin-pull", "@chkit/plugin-backfill", + "@chkit/plugin-ingest", "@chkit/plugin-obsessiondb" ] ], diff --git a/.changeset/ingestion-runtime.md b/.changeset/ingestion-runtime.md new file mode 100644 index 00000000..bde0b8aa --- /dev/null +++ b/.changeset/ingestion-runtime.md @@ -0,0 +1,20 @@ +--- +"@chkit/plugin-ingest": patch +"@chkit/clickhouse": patch +"@chkit/core": patch +"chkit": patch +--- + +Add `@chkit/plugin-ingest`, the first cut of scheduled pull ingestion into ClickHouse. Streams are ordinary TypeScript: a `read` async generator fetches, maps, and yields destination-shaped rows, and `definePipeline` returns a tagged, non-durable group of streams; only pipelines exported from the project entry participate, with no global registry. `chkit ingest run` executes the selected streams (`--tag` is repeatable with exact AND semantics; an explicit empty selection fails), `chkit ingest list` shows the loaded graph, and `chkit ingest status` prints committed checkpoints. + +Progress follows one rule: rows are saved before the bookmark advances. Every batch is written with a stable `insert_deduplication_token`, and only after the ClickHouse acknowledgement does the executor append a `batch_committed` fact to the append-only ingestion journal. Checkpoints are a projection of that journal, so a crashed or lost-acknowledgement run replays from the last durable boundary with the same batch identity instead of skipping rows. Bundled strategies are `timestampWindow({ start, overlapMs })` (with a custom `from` callback alternative), `cursorState` for provider-owned state, and the full-sync fallback; `--backfill ` runs an explicit range in an isolated checkpoint namespace. + +`rawTable` and `rawRows` land provider objects untouched in a native `JSON` column, so typed shapes are derived inside ClickHouse with ordinary views or materialized views instead of being mapped in pipeline code. + +`FetchContext` exposes source request and cancellation capabilities independently of checkpoint types; `ReadContext` extends it. + +Source operations run through `context.attempt`, which owns fetch permits, p-retry-shaped retry policy, `Retry-After`, cancellation, and failure classification (`HttpError.fromResponse` is the canonical boundary for fetch-based readers). Pipelines carry separate `maxStreams`, `maxFetches`, and `maxLoads` ceilings, executions have a duration budget, and the executor emits OpenTelemetry spans. + +`@chkit/core` gains a singular `entry` config field, mutually exclusive with `schema` globs: the module is imported once, its exported schema definitions are collected, and exported plugin-domain definitions are collected by their plugins. `@chkit/clickhouse` `insert()` accepts per-insert `settings`. + +Successful syncs rotate batch identity using the existing journal, while failed runs retain their replay identity. Execution cancellation also bounds journal I/O, stalled writes cannot report success, and loader construction failures release their permits. Ingestion requires a direct ClickHouse connection; incompatible host executors fail before any writes. Project `entry` and `schema` settings replace the inherited source mode when layering configuration. diff --git a/CLAUDE.md b/CLAUDE.md index d2adff38..93dbac37 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -21,6 +21,7 @@ This is a monorepo managed with Bun workspaces and Turborepo. | `packages/plugin-codegen` | `@chkit/plugin-codegen` | Plugin: TypeScript type + Zod schema generation | | `packages/plugin-pull` | `@chkit/plugin-pull` | Plugin: introspect live ClickHouse into schema files | | `packages/plugin-backfill` | `@chkit/plugin-backfill` | Plugin: time-windowed data backfill with checkpoints | +| `packages/plugin-ingest` | `@chkit/plugin-ingest` | Plugin: scheduled pull ingestion with journaled checkpoints | | `packages/plugin-obsessiondb` | `@chkit/plugin-obsessiondb` | Plugin: ObsessionDB integration; auto-rewrites `Shared` engines for non-ObsessionDB targets | ### Documentation diff --git a/README.md b/README.md index 839c06aa..2ddae244 100644 --- a/README.md +++ b/README.md @@ -114,6 +114,7 @@ See the [configuration docs](https://chkit.obsessiondb.com/configuration/overvie | [`@chkit/plugin-pull`](packages/plugin-pull) | Pull live schema into local files | | [`@chkit/plugin-codegen`](packages/plugin-codegen) | Codegen plugin for the CLI | | [`@chkit/plugin-backfill`](packages/plugin-backfill) | Backfill plugin for data migrations | +| [`@chkit/plugin-ingest`](packages/plugin-ingest) | Ingestion plugin: scheduled API pulls with journaled checkpoints | | [`@chkit/plugin-obsessiondb`](packages/plugin-obsessiondb) | ObsessionDB integration: auto-rewrite `Shared` engines for ClickHouse targets | ## Python diff --git a/apps/docs/src/content/docs/configuration/overview.mdx b/apps/docs/src/content/docs/configuration/overview.mdx index b5a58509..a1eea3a9 100644 --- a/apps/docs/src/content/docs/configuration/overview.mdx +++ b/apps/docs/src/content/docs/configuration/overview.mdx @@ -5,11 +5,12 @@ description: "clickhouse.config.ts / clickhouse.config.py structure and defaults import { Tabs, TabItem } from '@astrojs/starlight/components'; -`chkit` is configured through `clickhouse.config.ts` (TypeScript) or `clickhouse.config.py` (Python). The option keys and defaults are identical. +`chkit` is configured through `clickhouse.config.ts` (TypeScript) or `clickhouse.config.py` (Python). Shared configuration options use the same keys and defaults in both languages. ## Core Fields - `schema`: glob path to [schema files](/schema/dsl-reference/) +- `entry`: optional project entry module in place of `schema` globs (TypeScript only) - `outDir`: root folder for generated artifacts - `migrationsDir`: SQL migration file folder - `metaDir`: state folder (`snapshot.json`) @@ -65,6 +66,18 @@ Migration state (the journal of applied migrations) is not stored in `metaDir`. +## Project entry (`entry`, TypeScript only) + +Instead of `schema` globs you can point chkit at one entry module: + +```ts +export default defineConfig({ + entry: './src/chkit.ts', +}) +``` + +chkit imports the module once. Schema definitions it exports (directly or re-exported from other files) are collected exactly like glob-matched schema files, and exported plugin-domain definitions such as [ingestion pipelines](/plugins/ingest/) are collected by their plugins. `entry` and `schema` are mutually exclusive. + ## Cluster mode (`ON CLUSTER`) For self-managed multi-node ClickHouse clusters, set `clickhouse.cluster` to the cluster name from your server's `remote_servers` config: diff --git a/apps/docs/src/content/docs/plugins/ingest.md b/apps/docs/src/content/docs/plugins/ingest.md new file mode 100644 index 00000000..cb4f4c0d --- /dev/null +++ b/apps/docs/src/content/docs/plugins/ingest.md @@ -0,0 +1,187 @@ +--- +title: Ingest Plugin +description: Scheduled pull ingestion from application APIs into ClickHouse with journaled checkpoints. +sidebar: + order: 5 +--- + +This document covers practical usage of the optional TypeScript `ingest` plugin. + +## What it does + +- Runs finite, scheduled pulls from application APIs into chkit-managed tables. +- Keeps progress in an append-only ingestion journal inside the target database. Checkpoints are a projection of that journal. +- Saves rows before advancing the bookmark. A crash may cause rereading; it never causes unsaved rows to be skipped. +- Retries source requests with backoff, `Retry-After`, and failure classification. +- Selects streams by exact tags so any external scheduler (cron, CI, Kubernetes) can drive it. + +The plugin never creates or changes destination tables. Your chkit schema stays the only DDL authority. + +## Plugin setup + +Ingestion uses the singular `entry` config field instead of `schema` globs. The entry module is imported once: exported tables are collected as schema, and exported pipelines form the ingestion graph. Importing a pipeline without exporting it does not activate it; remove its export to deactivate it. + +Configure a direct `clickhouse` connection for ingestion. A host-provided executor, including the ObsessionDB workbench executor, cannot currently guarantee JSON row encoding and per-insert deduplication settings. + +```ts +// clickhouse.config.ts +import { defineConfig } from '@chkit/core' +import { ingest } from '@chkit/plugin-ingest' + +export default defineConfig({ + entry: './src/chkit.ts', + plugins: [ingest()], + clickhouse: { url: process.env.CLICKHOUSE_URL ?? '' }, +}) +``` + +## Writing a stream + +A stream is a destination table plus a `read` generator. Fetching and mapping are ordinary code inside `read`; each yielded chunk carries rows already shaped for the table. + +```ts +// src/chkit.ts +import { table } from '@chkit/core' +import { HttpError, definePipeline, defineStream, ingestionColumns, paginate, timestampWindow } from '@chkit/plugin-ingest' + +export const tickets = table({ + database: 'crm', + name: 'tickets', + columns: [ + { name: 'id', type: 'String' }, + { name: 'updated_at', type: "DateTime64(3, 'UTC')" }, + { name: 'raw', type: 'String' }, + ...ingestionColumns, + ], + engine: 'ReplacingMergeTree(updated_at)', + primaryKey: ['id'], + orderBy: ['id'], +}) + +const ticketStream = defineStream({ + id: 'helpdesk.tickets', + destination: tickets, + tags: ['schedule:1h'], + incremental: timestampWindow({ + // Re-read one hour of overlap; ReplacingMergeTree reconciles repeats. + start: new Date(0), + overlapMs: 3_600_000, + }), + async *read(context) { + const pages = paginate({ + context, + label: 'GET /tickets', + fetchPage: async (cursor: string | undefined, signal) => { + const url = new URL('https://api.example.com/tickets') + url.searchParams.set('updated_since', context.selection.from.toISOString()) + if (cursor) url.searchParams.set('cursor', cursor) + const response = await fetch(url, { signal, headers: { Authorization: `Bearer ${process.env.HELPDESK_TOKEN}` } }) + if (!response.ok) throw await HttpError.fromResponse(response) + const body = await response.json() + return { items: body.data, next: body.next_cursor ?? undefined } + }, + }) + for await (const items of pages) { + yield { rows: items.map((item) => ({ id: item.id, updated_at: item.updated_at, raw: JSON.stringify(item) })) } + } + }, +}) + +export const helpdesk = definePipeline({ id: 'helpdesk', streams: [ticketStream], maxFetches: 4 }) +``` + +Spread `ingestionColumns` into every destination table. The loader fills `_chkit_batch_id` and `_chkit_run_id`; `_chkit_ingested_at` is set by ClickHouse at the physical insert. + +Batch identity decides whether a retry is deduplicated. By default it includes a content hash of the rows, which prefers a possible duplicate over suppressing rows that changed between attempts; a field like `synced_at: new Date()` therefore defeats retry deduplication. When a chunk covers a stable source interval, declare it with `id` (for example `yield { rows, id: \`page:${cursor}\` }`): the chunk then becomes its own write unit and its identity ignores row content. + +## Landing raw objects + +Mapping fields in the reader is optional, and usually the wrong place for it. `rawTable` defines a landing table that stores each provider object untouched in a native `JSON` column next to a stable `id`; `rawRows` shapes a page for it. Typed tables are then ordinary chkit views (or materialized views) over the raw layer: + +```ts +import { view } from '@chkit/core' +import { defineStream, rawRows, rawTable } from '@chkit/plugin-ingest' + +export const rawTickets = rawTable({ database: 'crm', name: 'tickets_raw' }) + +export const tickets = view({ + database: 'crm', + name: 'tickets', + as: `SELECT + id AS ticket_id, + raw.subject::String AS subject, + raw.requester.email::String AS requester_email, + arrayMap(t -> t.name::String, raw.tags[]) AS tags, + parseDateTime64BestEffortOrNull(raw.updated_at::String, 3, 'UTC') AS updated_at +FROM crm.tickets_raw FINAL`, +}) + +const ticketStream = defineStream({ + id: 'helpdesk.tickets', + destination: rawTickets, + async *read(context) { + for await (const page of listTickets(context)) yield { rows: rawRows(page, (ticket) => ticket.id) } + }, +}) +``` + +The raw table is a `ReplacingMergeTree`, so overlapping windows and replays collapse to the latest version of each `id`. Because the transform lives in ClickHouse, changing it never requires re-fetching the source: a view picks the change up immediately, and a materialized view can be rebuilt from the raw table. A `_raw` suffix next to the typed view of the same name keeps the pair easy to find. + +## Progress and checkpoints + +| Strategy | Use when | Bookmark advances | +|---|---|---| +| none (full sync) | The source is small or has no change filter | Never; every run reads everything | +| `timestampWindow({ start, overlapMs })` | The API filters by an updated-since timestamp | To the run cutoff, after the whole window loaded | +| `cursorState({ id, version, parse })` | The provider owns the state: compound cursor, change token, page position | Whenever a yielded chunk carries `state` and its rows have been saved | + +`timestampWindow` starts its first sync at `start`. Later runs begin at the committed watermark minus `overlapMs` (zero by default). Explicit backfill bounds take precedence. For custom lower bounds, use `timestampWindow({ from: ({ watermark, cutoff }) => ... })` instead. Both forms retain the same checkpoint format and only advance after the entire window is saved. + +Provider clients can accept the exported `FetchContext` type, containing `attempt` and `signal`. `ReadContext` extends it with the stream selection and checkpoint, so readers can pass their context directly without coupling clients to checkpoint generics. + +With `cursorState`, `state` on a chunk must be the complete state that is safe to resume from once every row up to that chunk is saved. Omit it when you cannot make that claim; the run then restarts from the previous checkpoint after a failure. + +A checkpoint records its strategy id and version. Changing either makes the next run fail rather than reinterpret old state. + +## Retry policy + +Pipeline `retry` settings are defaults; a stream can override individual settings. Source attempts and reader restarts use [`p-retry`](https://github.com/sindresorhus/p-retry) for exponential backoff, jitter, retry counts, and `maxRetryTime`. The supported options are `retries`, `factor`, `minTimeout`, `maxTimeout`, `randomize`, `maxRetryTime`, `shouldRetry`, and `shouldConsumeRetry`. + +The policy callbacks receive p-retry's `attemptNumber`, `retriesLeft`, and `retriesConsumed`, plus a `FetchFailure` preserving the original `cause` and normalized `classification`. Returning `false` from `shouldConsumeRetry` follows p-retry's behavior: it skips consuming a retry and skips its backoff. A provider's `Retry-After` still applies, including for those unconsumed retries. Retry waits release fetch and load capacity for other streams. + +Once a source attempt exhausts its policy, it fails the stream; reader recovery does not multiply that retry budget. Opaque reader failures restart from the latest committed checkpoint. + +## Commands + +```sh +chkit ingest list # streams in the loaded graph +chkit ingest run # run every stream +chkit ingest run --tag schedule:1h # exact tag match; repeat --tag for AND +chkit ingest run --tag stream:helpdesk.tickets +chkit ingest status # committed checkpoint per stream +chkit ingest run --backfill jan --from 2026-01-01 --to 2026-02-01 +``` + +Every stream also carries the derived tags `pipeline:` and `stream:`. `schedule:` is a convention only: chkit never interprets it. A `--tag` filter that matches nothing fails before any work runs. + +A backfill uses its own checkpoint namespace, so it never moves the scheduled bookmark. Rerunning the same `--backfill` id resumes it. + +`chkit check` verifies that every stream destination carries the ingestion metadata columns. + +## Delivery guarantee + +Ingestion is at-least-once. Each batch is inserted with a stable `insert_deduplication_token`, so a retry after a lost acknowledgement is suppressed while the table's deduplication window covers it. Pick a destination engine that reconciles repeats for your data, for example `ReplacingMergeTree` keyed by the provider id. + +Successful syncs start a new batch identity cycle, recorded by the existing journal. Failed or interrupted syncs retain their cycle for replay. This also applies to full syncs, which have no incremental bookmark. + +The duration budget bounds journal operations as well as readers. Shutdown gives unfinished readers or writes up to five seconds to settle; each terminal journal append has a separate five-second limit. Interrupted writes never count as successful ingestion. + +Run at most one ingestion process per project and target at a time. Use your scheduler's concurrency control (for example a GitHub Actions concurrency group) to enforce it. + +## Options + +| Option | Default | Description | +|---|---|---| +| `journalTable` | `_chkit_ingestion_journal` | Journal table name in the configured database | +| `maxDurationSeconds` | `3600` | Execution budget. Exhausting it ends the run as incomplete and keeps committed progress | +| `prefetchBatches` | `1` | Mapped batches buffered between fetching and loading | diff --git a/apps/docs/src/content/docs/plugins/overview.mdx b/apps/docs/src/content/docs/plugins/overview.mdx index dcabb115..93ccc149 100644 --- a/apps/docs/src/content/docs/plugins/overview.mdx +++ b/apps/docs/src/content/docs/plugins/overview.mdx @@ -62,3 +62,4 @@ If you deploy to [ObsessionDB](https://obsessiondb.com), start at the dedicated - [`@chkit/plugin-codegen`](/plugins/codegen/) — TypeScript row types and optional Zod schemas (Python: Pydantic models), generated from your schema files. - [`@chkit/plugin-pull`](/plugins/pull/) — introspect a live ClickHouse database into local schema files. Useful for adopting chkit on an existing database. Built into the Python CLI as `chkit pull`. - [`@chkit/plugin-backfill`](/plugins/backfill/) — time-windowed data backfill with checkpoints, for materialized views and historical data loads. +- [`@chkit/plugin-ingest`](/plugins/ingest/) — scheduled pull ingestion from application APIs with journaled checkpoints (TypeScript only). diff --git a/bun.lock b/bun.lock index 4cc04496..9e563dd8 100644 --- a/bun.lock +++ b/bun.lock @@ -47,7 +47,7 @@ }, "packages/cli": { "name": "chkit", - "version": "0.1.2-beta.4", + "version": "0.1.2-beta.7", "bin": { "chkit": "./dist/bin/chkit.js", }, @@ -57,6 +57,7 @@ "@chkit/core": "workspace:*", "@logtape/logtape": "^2.0.5", "fast-glob": "^3.3.2", + "p-retry": "^7.1.1", }, "optionalDependencies": { "@chkit/plugin-obsessiondb": "workspace:*", @@ -64,7 +65,7 @@ }, "packages/clickhouse": { "name": "@chkit/clickhouse", - "version": "0.1.2-beta.4", + "version": "0.1.2-beta.7", "dependencies": { "@chkit/core": "workspace:*", "@clickhouse/client": "^1.18.0", @@ -74,14 +75,14 @@ }, "packages/codegen": { "name": "@chkit/codegen", - "version": "0.1.2-beta.4", + "version": "0.1.2-beta.7", "dependencies": { - "@chkit/core": "0.1.0-beta.26", + "@chkit/core": "workspace:*", }, }, "packages/core": { "name": "@chkit/core", - "version": "0.1.2-beta.4", + "version": "0.1.2-beta.7", "dependencies": { "fast-glob": "^3.3.2", "jiti": "^2.7.0", @@ -92,7 +93,7 @@ }, "packages/create-chkit": { "name": "create-chkit", - "version": "0.1.2-beta.4", + "version": "0.1.2-beta.7", "bin": { "create-chkit": "./dist/bin/create-chkit.js", }, @@ -109,7 +110,7 @@ }, "packages/plugin-backfill": { "name": "@chkit/plugin-backfill", - "version": "0.1.2-beta.4", + "version": "0.1.2-beta.7", "dependencies": { "@chkit/clickhouse": "workspace:*", "@chkit/core": "workspace:*", @@ -120,7 +121,7 @@ }, "packages/plugin-codegen": { "name": "@chkit/plugin-codegen", - "version": "0.1.2-beta.4", + "version": "0.1.2-beta.7", "dependencies": { "@chkit/core": "workspace:*", }, @@ -131,9 +132,21 @@ "zod": "^4.0.0", }, }, + "packages/plugin-ingest": { + "name": "@chkit/plugin-ingest", + "version": "0.1.2-beta.7", + "dependencies": { + "@chkit/clickhouse": "workspace:*", + "@chkit/core": "workspace:*", + "@opentelemetry/api": "^1.9.1", + "p-limit": "^7.2.0", + "p-retry": "^7.1.1", + "zod": "^4.3.6", + }, + }, "packages/plugin-obsessiondb": { "name": "@chkit/plugin-obsessiondb", - "version": "0.1.2-beta.4", + "version": "0.1.2-beta.7", "dependencies": { "@chkit/clickhouse": "workspace:*", "@chkit/core": "workspace:*", @@ -146,7 +159,7 @@ }, "packages/plugin-pull": { "name": "@chkit/plugin-pull", - "version": "0.1.2-beta.4", + "version": "0.1.2-beta.7", "dependencies": { "@chkit/clickhouse": "workspace:*", "@chkit/core": "workspace:*", @@ -281,6 +294,8 @@ "@chkit/plugin-codegen": ["@chkit/plugin-codegen@workspace:packages/plugin-codegen"], + "@chkit/plugin-ingest": ["@chkit/plugin-ingest@workspace:packages/plugin-ingest"], + "@chkit/plugin-obsessiondb": ["@chkit/plugin-obsessiondb@workspace:packages/plugin-obsessiondb"], "@chkit/plugin-pull": ["@chkit/plugin-pull@workspace:packages/plugin-pull"], @@ -451,6 +466,8 @@ "@nodelib/fs.walk": ["@nodelib/fs.walk@1.2.8", "", { "dependencies": { "@nodelib/fs.scandir": "2.1.5", "fastq": "^1.6.0" } }, "sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg=="], + "@opentelemetry/api": ["@opentelemetry/api@1.9.1", "", {}, "sha512-gLyJlPHPZYdAk1JENA9LeHejZe1Ti77/pTeFm/nMXmQH/HFZlcS/O2XJB+L8fkbrNSqhdtlvjBVjxwUYanNH5Q=="], + "@orpc/client": ["@orpc/client@1.13.4", "", { "dependencies": { "@orpc/shared": "1.13.4", "@orpc/standard-server": "1.13.4", "@orpc/standard-server-fetch": "1.13.4", "@orpc/standard-server-peer": "1.13.4" } }, "sha512-s13GPMeoooJc5Th2EaYT5HMFtWG8S03DUVytYfJv8pIhP87RYKl94w52A36denH6r/B4LaAgBeC9nTAOslK+Og=="], "@orpc/contract": ["@orpc/contract@1.13.4", "", { "dependencies": { "@orpc/client": "1.13.4", "@orpc/shared": "1.13.4", "@standard-schema/spec": "^1.1.0", "openapi-types": "^12.1.3" } }, "sha512-TIxyaF67uOlihCRcasjHZxguZpbqfNK7aMrDLnhoufmQBE4OKvguNzmrOFHgsuM0OXoopX0Nuhun1ccaxKP10A=="], @@ -1153,7 +1170,7 @@ "p-filter": ["p-filter@2.1.0", "", { "dependencies": { "p-map": "^2.0.0" } }, "sha512-ZBxxZ5sL2HghephhpGAQdoskxplTwr7ICaehZwLIlfL6acuVgZPm8yBNuRAFBGEqtD/hmUeq9eqLg2ys9Xr/yw=="], - "p-limit": ["p-limit@2.3.0", "", { "dependencies": { "p-try": "^2.0.0" } }, "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w=="], + "p-limit": ["p-limit@7.3.3", "", { "dependencies": { "yocto-queue": "^1.2.1" } }, "sha512-SKgVvDU5TNIxQ4r30U75BNLC6a+6GOEjngrk/ysM5+Zu3oJuk1SC5ApzIzY/7PjCEjoVPMWJ0zho/4QUiEB7TQ=="], "p-locate": ["p-locate@4.1.0", "", { "dependencies": { "p-limit": "^2.2.0" } }, "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A=="], @@ -1515,10 +1532,14 @@ "@babel/traverse/@babel/types": ["@babel/types@7.29.7", "", { "dependencies": { "@babel/helper-string-parser": "^7.29.7", "@babel/helper-validator-identifier": "^7.29.7" } }, "sha512-4zBIxpPzowiZpusoFkyGVwakdRJUyuH5PxQ/PrqghfdFWWasvnCdPfQXHrenDai+gyLARulZjZowCOj6fjT4pA=="], + "@changesets/cli/p-limit": ["p-limit@2.3.0", "", { "dependencies": { "p-try": "^2.0.0" } }, "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w=="], + "@chkit/plugin-backfill/zod": ["zod@4.3.6", "", {}, "sha512-rftlrkhHZOcjDwkGlnUtZZkvaPHCsDATp4pGpuOOMDaTdDDXF91wuVDJoWoPsKX/3YPQ5fHuF3STjcYyKr+Qhg=="], "@chkit/plugin-codegen/zod": ["zod@4.3.6", "", {}, "sha512-rftlrkhHZOcjDwkGlnUtZZkvaPHCsDATp4pGpuOOMDaTdDDXF91wuVDJoWoPsKX/3YPQ5fHuF3STjcYyKr+Qhg=="], + "@chkit/plugin-ingest/zod": ["zod@4.3.6", "", {}, "sha512-rftlrkhHZOcjDwkGlnUtZZkvaPHCsDATp4pGpuOOMDaTdDDXF91wuVDJoWoPsKX/3YPQ5fHuF3STjcYyKr+Qhg=="], + "@chkit/plugin-obsessiondb/zod": ["zod@4.3.6", "", {}, "sha512-rftlrkhHZOcjDwkGlnUtZZkvaPHCsDATp4pGpuOOMDaTdDDXF91wuVDJoWoPsKX/3YPQ5fHuF3STjcYyKr+Qhg=="], "@chkit/plugin-pull/zod": ["zod@4.3.6", "", {}, "sha512-rftlrkhHZOcjDwkGlnUtZZkvaPHCsDATp4pGpuOOMDaTdDDXF91wuVDJoWoPsKX/3YPQ5fHuF3STjcYyKr+Qhg=="], @@ -1585,6 +1606,8 @@ "p-filter/p-map": ["p-map@2.1.0", "", {}, "sha512-y3b8Kpd8OAN444hxfBbFfj1FY/RjtTd8tzYwhUqNYXx0fXx2iX4maP4Qr6qhIKbQXI02wTLAda4fYUbDagTUFw=="], + "p-locate/p-limit": ["p-limit@2.3.0", "", { "dependencies": { "p-try": "^2.0.0" } }, "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w=="], + "parse-entities/@types/unist": ["@types/unist@2.0.11", "", {}, "sha512-CmBKiL6NNo/OqgmMn95Fk9Whlp2mtvIv+KNpQKN2F4SjvrEesubTRWGYSg+BnWZOnlCaSTU1sMpsBOzgbYhnsA=="], "parse5/entities": ["entities@6.0.1", "", {}, "sha512-aN97NXWF6AWBTahfVOIrB/NShkzi5H7F9r1s9mD3cDj4Ko5f2qhhVoYMibXF7GlLveb/D2ioWay8lxI97Ven3g=="], diff --git a/packages/cli/package.json b/packages/cli/package.json index c205648c..e1816efb 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -39,11 +39,12 @@ "clean": "rm -rf dist" }, "dependencies": { - "fast-glob": "^3.3.2", "@chkit/clickhouse": "workspace:*", "@chkit/codegen": "workspace:*", "@chkit/core": "workspace:*", - "@logtape/logtape": "^2.0.5" + "@logtape/logtape": "^2.0.5", + "fast-glob": "^3.3.2", + "p-retry": "^7.1.1" }, "optionalDependencies": { "@chkit/plugin-obsessiondb": "workspace:*" diff --git a/packages/cli/src/commands/migrate/async-apply.ts b/packages/cli/src/commands/migrate/async-apply.ts index 443170c9..6cd038d7 100644 --- a/packages/cli/src/commands/migrate/async-apply.ts +++ b/packages/cli/src/commands/migrate/async-apply.ts @@ -1,4 +1,5 @@ import { createHash } from 'node:crypto' +import { setTimeout as defaultSleep } from 'node:timers/promises' import type { ClickHouseExecutor, QueryStatus } from '@chkit/clickhouse' @@ -408,7 +409,3 @@ export function isoWithoutZone(date: Date): string { function firstLine(value: string): string { return value.split('\n')[0] ?? value } - -function defaultSleep(ms: number): Promise { - return new Promise((resolve) => setTimeout(resolve, ms)) -} diff --git a/packages/cli/src/runtime/config-merge.ts b/packages/cli/src/runtime/config-merge.ts index 20b9705a..1d2e72b1 100644 --- a/packages/cli/src/runtime/config-merge.ts +++ b/packages/cli/src/runtime/config-merge.ts @@ -70,7 +70,8 @@ export function mergeUserConfig( overlay: ChxUserConfig, ): ChxUserConfig { return { - schema: overlay.schema ?? base.schema, + schema: overlay.schema ?? (overlay.entry !== undefined ? undefined : base.schema), + entry: overlay.entry ?? (overlay.schema !== undefined ? undefined : base.entry), outDir: overlay.outDir ?? base.outDir, migrationsDir: overlay.migrationsDir ?? base.migrationsDir, metaDir: overlay.metaDir ?? base.metaDir, diff --git a/packages/cli/src/runtime/journal-store.ts b/packages/cli/src/runtime/journal-store.ts index b8b219cc..3d0af00d 100644 --- a/packages/cli/src/runtime/journal-store.ts +++ b/packages/cli/src/runtime/journal-store.ts @@ -1,5 +1,8 @@ +import { setTimeout as sleep } from 'node:timers/promises' + import { isUnknownDatabaseError, type ClickHouseExecutor } from '@chkit/clickhouse' import { onClusterClause } from '@chkit/core' +import pRetry from 'p-retry' import type { MigrationJournal, MigrationJournalEntry } from './migration-store.js' import { CLI_VERSION } from './version.js' @@ -202,15 +205,10 @@ SETTINGS index_granularity = 1` } throw error } - for (let attempt = 0; attempt < 10; attempt++) { - try { - await db.query(`SELECT name FROM ${journalTable} LIMIT 0`) - debug('journal', `DDL propagation confirmed (attempt ${attempt + 1})`) - break - } catch { - await new Promise((r) => setTimeout(r, 250)) - } - } + await pRetry(async (attempt) => { + await db.query(`SELECT name FROM ${journalTable} LIMIT 0`) + debug('journal', `DDL propagation confirmed (attempt ${attempt})`) + }, { retries: 9, minTimeout: 250, factor: 1 }).catch(() => undefined) bootstrapped = true } @@ -289,19 +287,16 @@ SETTINGS index_granularity = 1` } await ensureTable() const insertSql = `INSERT INTO ${journalTable} (name, applied_at, checksum, chkit_version, migration_completed, operations) VALUES ('${escapeSqlString(state.name)}', '${escapeSqlString(state.appliedAt)}', '${escapeSqlString(state.checksum)}', '${escapeSqlString(state.chkitVersion || CLI_VERSION)}', ${state.migrationCompleted ? 'true' : 'false'}, ${operationsArrayLiteral(state.operations)})` - const maxAttempts = 5 - for (let attempt = 1; attempt <= maxAttempts; attempt += 1) { - try { - await db.command(insertSql) - break - } catch (error) { - if (!isRetryableInsertRace(error) || attempt === maxAttempts) { - throw error - } - debug('journal', `insert race detected — retrying (attempt ${attempt}/${maxAttempts})`) - await new Promise((r) => setTimeout(r, attempt * 150)) - } - } + await pRetry(() => db.command(insertSql), { + retries: 4, + minTimeout: 0, + shouldRetry: async ({ error, attemptNumber }) => { + if (!isRetryableInsertRace(error)) return false + debug('journal', `insert race detected — retrying (attempt ${attemptNumber}/5)`) + await sleep(attemptNumber * 150) + return true + }, + }) await trySyncReplica() }, diff --git a/packages/cli/src/test/e2e-testkit.ts b/packages/cli/src/test/e2e-testkit.ts index 681225be..b793bff2 100644 --- a/packages/cli/src/test/e2e-testkit.ts +++ b/packages/cli/src/test/e2e-testkit.ts @@ -6,6 +6,7 @@ */ import { join, resolve } from 'node:path' +import { setTimeout as sleep } from 'node:timers/promises' // Re-export all shared utilities so CLI tests only need one import export { @@ -79,7 +80,7 @@ export async function runCliWithRetry( const result = runCli(cwd, args, extraEnv) if (result.exitCode === 0 && (!expectJson || isValidJson(result.stdout))) return result if (attempt === maxAttempts) return result - await new Promise((r) => setTimeout(r, delayMs)) + await sleep(delayMs) } return runCli(cwd, args, extraEnv) } @@ -114,7 +115,7 @@ export async function waitForCliJson( formatTestDiagnostic('last attempt', result) ) } - await new Promise((r) => setTimeout(r, delayMs)) + await sleep(delayMs) } throw new Error('waitForCliJson: unreachable') } diff --git a/packages/cli/src/test/runtime/config-merge.test.ts b/packages/cli/src/test/runtime/config-merge.test.ts index 02983684..1f4d88b8 100644 --- a/packages/cli/src/test/runtime/config-merge.test.ts +++ b/packages/cli/src/test/runtime/config-merge.test.ts @@ -1,6 +1,6 @@ import { describe, expect, test } from 'bun:test' -import type { ChxUserConfig } from '@chkit/core' +import { resolveConfig, type ChxUserConfig } from '@chkit/core' import { mergeUserConfig, pluginNameOf } from '../../runtime/config-merge.js' @@ -20,6 +20,20 @@ const profile: ChxUserConfig = { } describe('mergeUserConfig', () => { + test('project source mode replaces the inherited alternative', () => { + const entry = mergeUserConfig({ schema: './profile/**/*.ts' }, { entry: './src/chkit.ts' }) + expect(resolveConfig(entry).schema).toEqual(['./src/chkit.ts']) + expect(entry.schema).toBeUndefined() + + const globs = mergeUserConfig({ entry: './profile.ts' }, { schema: './src/schema/**/*.ts' }) + expect(resolveConfig(globs).schema).toEqual(['./src/schema/**/*.ts']) + expect(globs.entry).toBeUndefined() + + expect(resolveConfig(mergeUserConfig({ entry: './profile.ts' }, {})).entry).toBe('./profile.ts') + expect(resolveConfig(mergeUserConfig({ entry: './profile.ts' }, { schema: [] })).schema).toEqual([]) + expect(() => resolveConfig(mergeUserConfig({}, { entry: './entry.ts', schema: './schema.ts' }))).toThrow('mutually exclusive') + }) + test('overlay scalar fields beat base', () => { const merged = mergeUserConfig(profile, { schema: ['./schema/*.ts'], diff --git a/packages/cli/src/test/runtime/journal-store.test.ts b/packages/cli/src/test/runtime/journal-store.test.ts index e3fa2cfd..40e496fb 100644 --- a/packages/cli/src/test/runtime/journal-store.test.ts +++ b/packages/cli/src/test/runtime/journal-store.test.ts @@ -248,4 +248,23 @@ describe('createJournalStore', () => { expect(insert).toBeDefined() expect(insert).toContain("It\\'s broken: \\'unterminated") }) + + test.each(['recovers', 'exhausts', 'permanent'] as const)('migration insert retry policy %s', async (mode) => { + const { db } = createScriptedExecutor(new Map()) + let attempts = 0 + const failure = new Error(mode === 'permanent' ? 'access denied' : 'Please retry the INSERT') + db.command = async (sql) => { + if (!sql.startsWith('INSERT INTO')) return + attempts += 1 + if (mode !== 'recovers' || attempts < 3) throw failure + } + const write = createJournalStore(db).writeMigrationState({ + name: 'retry.sql', appliedAt: '2026-05-26 12:00:00.000', checksum: 'cs', + chkitVersion: 'v', migrationCompleted: true, operations: [], + }) + if (mode === 'recovers') await write + else await expect(write).rejects.toBe(failure) + expect(attempts).toBe(mode === 'recovers' ? 3 : mode === 'exhausts' ? 5 : 1) + }) + }) diff --git a/packages/clickhouse/src/index.ts b/packages/clickhouse/src/index.ts index ca3d6270..8a20c013 100644 --- a/packages/clickhouse/src/index.ts +++ b/packages/clickhouse/src/index.ts @@ -42,6 +42,8 @@ export interface ClickHouseInsertParams> { table: string values: T[] compressed?: boolean + /** Per-insert settings, e.g. a stable `insert_deduplication_token`. */ + settings?: ClickHouseSettings } export interface ClickHouseJsonQueryResult< @@ -735,6 +737,7 @@ export function createExecutorWithClient( table: params.table, values: params.values, format: 'JSONEachRow', + ...(params.settings ? { clickhouse_settings: params.settings } : {}), }) assertStreamedQuerySucceeded({ response_headers: result.response_headers, diff --git a/packages/core/src/model-types.ts b/packages/core/src/model-types.ts index 8ecf54c7..c4a29ffe 100644 --- a/packages/core/src/model-types.ts +++ b/packages/core/src/model-types.ts @@ -266,7 +266,17 @@ export interface ChxInlinePluginRegistration< export type ChxPluginRegistration = ChxInlinePluginRegistration export interface ChxUserConfig { - schema: string | string[] + /** + * Glob patterns for schema files. Mutually exclusive with `entry`. + */ + schema?: string | string[] + /** + * Single project entry module. It is imported once: exported schema + * definitions are collected from it, and exported plugin-domain definitions + * (for example ingestion pipelines) are collected by their plugins. Mutually + * exclusive with `schema`. + */ + entry?: string outDir?: string migrationsDir?: string metaDir?: string @@ -278,6 +288,7 @@ export interface ChxUserConfig { export interface ChxResolvedConfig { schema: string[] + entry?: string outDir: string migrationsDir: string metaDir: string diff --git a/packages/core/src/model.ts b/packages/core/src/model.ts index 7d46afb0..6fb4cb20 100644 --- a/packages/core/src/model.ts +++ b/packages/core/src/model.ts @@ -53,8 +53,16 @@ export function resolveConfig(config: ChxUserConfig): ChxResolvedConfig { const migrationsDir = config.migrationsDir ?? join(outDir, 'migrations') const metaDir = config.metaDir ?? join(outDir, 'meta') + const schemaGlobs = config.schema === undefined ? [] : Array.isArray(config.schema) ? config.schema : [config.schema] + if (config.entry !== undefined && schemaGlobs.length > 0) { + throw new Error('Config fields "entry" and "schema" are mutually exclusive. Use one project entry module or schema globs, not both.') + } + return { - schema: Array.isArray(config.schema) ? config.schema : [config.schema], + // In entry mode the entry module is the only schema source: its exported + // definitions are collected exactly like any other schema file. + schema: config.entry !== undefined ? [config.entry] : schemaGlobs, + entry: config.entry, outDir, migrationsDir, metaDir, diff --git a/packages/core/src/on-cluster.test.ts b/packages/core/src/on-cluster.test.ts index 163771b1..788b43f1 100644 --- a/packages/core/src/on-cluster.test.ts +++ b/packages/core/src/on-cluster.test.ts @@ -209,6 +209,18 @@ describe('applyOnClusterToPlan', () => { }) }) +describe('resolveConfig project entry', () => { + test('entry mode makes the entry module the only schema source', () => { + const resolved = resolveConfig({ entry: './src/chkit.ts' }) + expect(resolved.entry).toBe('./src/chkit.ts') + expect(resolved.schema).toEqual(['./src/chkit.ts']) + }) + + test('entry and schema globs are mutually exclusive', () => { + expect(() => resolveConfig({ entry: './src/chkit.ts', schema: './src/schema/**/*.ts' })).toThrow('mutually exclusive') + }) +}) + describe('resolveConfig cluster validation', () => { test('passes through an identifier and a macro', () => { expect(resolveConfig({ schema: 's', clickhouse: { url: 'u', cluster: 'my_cluster' } }).clickhouse?.cluster).toBe( diff --git a/packages/core/src/schema-loader.ts b/packages/core/src/schema-loader.ts index f2fec004..1bac9a0d 100644 --- a/packages/core/src/schema-loader.ts +++ b/packages/core/src/schema-loader.ts @@ -14,6 +14,15 @@ export async function loadSchemaDefinitions( schemaGlobs: string | string[], options: SchemaLoaderOptions = {} ): Promise { + const modules = await loadDefinitionModules(schemaGlobs, options) + return canonicalizeDefinitions(modules.flatMap(collectDefinitionsFromModule)) +} + +/** Load configured entry/schema modules so plugins can inspect their exported definitions. */ +export async function loadDefinitionModules( + schemaGlobs: string | string[], + options: SchemaLoaderOptions = {} +): Promise[]> { const patterns = Array.isArray(schemaGlobs) ? schemaGlobs : [schemaGlobs] const files = await fg(patterns, { cwd: options.cwd ?? process.cwd(), @@ -24,11 +33,7 @@ export async function loadSchemaDefinitions( throw new Error('No schema files matched. Check config.schema patterns.') } - const all: SchemaDefinition[] = [] - for (const file of files) { - const mod = await importModuleFile(file) - all.push(...collectDefinitionsFromModule(mod)) - } - - return canonicalizeDefinitions(all) + const modules: Record[] = [] + for (const file of files) modules.push(await importModuleFile(file)) + return modules } diff --git a/packages/plugin-backfill/src/async-backfill.ts b/packages/plugin-backfill/src/async-backfill.ts index bbb7c3b7..6c78543b 100644 --- a/packages/plugin-backfill/src/async-backfill.ts +++ b/packages/plugin-backfill/src/async-backfill.ts @@ -1,3 +1,5 @@ +import { setTimeout as sleep } from 'node:timers/promises' + import type { ClickHouseExecutor, QueryStatus } from '@chkit/clickhouse' import pMap from 'p-map' @@ -47,10 +49,6 @@ export interface BackfillResult { progress: BackfillProgress } -function sleep(ms: number): Promise { - return new Promise((resolve) => setTimeout(resolve, ms)) -} - /** Build the deterministic query ID for a chunk. */ function chunkQueryId(planId: string, chunkId: string): string { return `backfill-${planId}-${chunkId}` diff --git a/packages/plugin-ingest/README.md b/packages/plugin-ingest/README.md new file mode 100644 index 00000000..001c968d --- /dev/null +++ b/packages/plugin-ingest/README.md @@ -0,0 +1,22 @@ +# @chkit/plugin-ingest + +Scheduled pull ingestion from application APIs into ClickHouse for [chkit](https://www.npmjs.com/package/chkit), with journaled checkpoints. + +Rows are saved before the bookmark advances: a crash may cause rereading, never skipped rows. + +```ts +import { defineConfig } from '@chkit/core' +import { ingest } from '@chkit/plugin-ingest' + +export default defineConfig({ + entry: './src/chkit.ts', + plugins: [ingest()], + clickhouse: { url: process.env.CLICKHOUSE_URL ?? '' }, +}) +``` + +```sh +chkit ingest run --tag schedule:1h +``` + +Documentation: https://chkit.obsessiondb.com/plugins/ingest/ diff --git a/packages/plugin-ingest/package.json b/packages/plugin-ingest/package.json new file mode 100644 index 00000000..33793881 --- /dev/null +++ b/packages/plugin-ingest/package.json @@ -0,0 +1,57 @@ +{ + "name": "@chkit/plugin-ingest", + "version": "0.1.2-beta.7", + "description": "Scheduled pull ingestion into ClickHouse with journaled checkpoints for chkit", + "license": "MIT", + "author": "ObsessionDB", + "keywords": [ + "clickhouse", + "ingestion", + "etl", + "chkit-plugin" + ], + "homepage": "https://chkit.obsessiondb.com", + "bugs": { + "url": "https://github.com/obsessiondb/chkit/issues" + }, + "repository": { + "type": "git", + "url": "https://github.com/obsessiondb/chkit.git", + "directory": "packages/plugin-ingest" + }, + "type": "module", + "main": "./dist/index.js", + "types": "./dist/index.d.ts", + "exports": { + ".": { + "source": "./src/index.ts", + "types": "./dist/index.d.ts", + "default": "./dist/index.js" + }, + "./testing": { + "source": "./src/testing.ts", + "types": "./dist/testing.d.ts", + "default": "./dist/testing.js" + } + }, + "files": [ + "dist" + ], + "scripts": { + "build": "tsc -p tsconfig.json", + "dev": "tsc -p tsconfig.json --watch", + "typecheck": "tsc -p tsconfig.json --noEmit", + "lint": "biome lint src", + "test": "bun test src", + "test:env": "doppler run --project chkit --config ci -- bun test src", + "clean": "rm -rf dist" + }, + "dependencies": { + "@chkit/clickhouse": "workspace:*", + "@chkit/core": "workspace:*", + "@opentelemetry/api": "^1.9.1", + "p-limit": "^7.2.0", + "p-retry": "^7.1.1", + "zod": "^4.3.6" + } +} diff --git a/packages/plugin-ingest/src/api.test.ts b/packages/plugin-ingest/src/api.test.ts new file mode 100644 index 00000000..18f7346b --- /dev/null +++ b/packages/plugin-ingest/src/api.test.ts @@ -0,0 +1,116 @@ +import { afterEach, describe, expect, test } from 'bun:test' +import { mkdtemp, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' + +import { resolveConfig } from '@chkit/core' +import type { ClickHouseExecutor } from '@chkit/clickhouse' +import { loadDefinitionModules, loadSchemaDefinitions } from '@chkit/core/schema-loader' + +import { rawTable } from './destination.js' +import { timestampWindow } from './incremental.js' +import { createIngestPlugin } from './plugin.js' +import { collectPipelines, definePipeline, defineStream, selectStreams } from './registry.js' + +const start = new Date('2024-01-01T00:00:00Z') +const cutoff = new Date('2026-09-19T00:00:00Z') +const watermark = '2026-09-18T00:00:00.000Z' + +describe('timestampWindow options', () => { + test('bootstraps at start and applies overlap only to the committed watermark', () => { + const window = timestampWindow({ start, overlapMs: 3_600_000 }) + expect(window.plan({ state: undefined, cutoff, range: undefined })).toEqual({ from: start, to: cutoff }) + const selection = window.plan({ state: window.parseState({ watermark }), cutoff, range: undefined }) + expect(selection.from).toEqual(new Date('2026-09-17T23:00:00Z')) + expect(window.complete?.({ state: { watermark }, selection })).toEqual({ watermark: cutoff.toISOString() }) + expect(timestampWindow({ start }).plan({ state: { watermark }, cutoff, range: undefined }).from).toEqual(new Date(watermark)) + }) + + test('explicit backfill bounds override start, watermark, overlap, and cutoff', () => { + const window = timestampWindow({ start, overlapMs: 3_600_000 }) + const range = { from: new Date('2025-01-01'), to: new Date('2025-02-01') } + const selection = window.plan({ state: { watermark }, cutoff, range }) + expect(selection).toEqual(range) + expect(window.complete?.({ state: { watermark }, selection })).toEqual({ watermark: range.to.toISOString() }) + expect(window.plan({ state: undefined, cutoff, range: { from: undefined, to: range.to } })).toEqual({ from: start, to: range.to }) + }) + + test('retains custom callbacks and the existing checkpoint strategy', () => { + const window = timestampWindow({ from: ({ watermark: saved, cutoff: end }) => saved ?? end }) + expect(window.plan({ state: undefined, cutoff, range: undefined })).toEqual({ from: cutoff, to: cutoff }) + expect(window.plan({ state: { watermark }, cutoff, range: undefined }).from).toEqual(new Date(watermark)) + expect([window.id, window.version]).toEqual(['chkit.timestamp_window', 1]) + }) + + test('rejects invalid bounds and overlap', () => { + expect(() => timestampWindow({ start: new Date('invalid') })).toThrow('valid Date') + for (const overlapMs of [-1, Number.NaN, Number.POSITIVE_INFINITY]) { + expect(() => timestampWindow({ start, overlapMs })).toThrow('non-negative') + } + expect(() => timestampWindow({ start: cutoff }).plan({ state: undefined, cutoff: start, range: undefined })).toThrow('after upper bound') + expect(() => timestampWindow({ from: () => new Date('invalid') }).plan({ state: undefined, cutoff, range: undefined })).toThrow('valid dates') + }) +}) + +const directories: string[] = [] +afterEach(async () => { + await Promise.all(directories.splice(0).map((path) => rm(path, { recursive: true, force: true }))) +}) + +describe('exported pipeline discovery', () => { + test('side-effect imports stay inactive; re-exports activate pipelines even after a cached import', async () => { + const directory = await mkdtemp(join(tmpdir(), 'chkit-exports-')) + directories.push(directory) + const provider = join(directory, 'provider.ts') + const hidden = join(directory, 'hidden.ts') + const active = join(directory, 'active.ts') + await writeFile(provider, ` + import { definePipeline, defineStream } from ${JSON.stringify(new URL('./registry.ts', import.meta.url).pathname)} + import { rawTable } from ${JSON.stringify(new URL('./destination.ts', import.meta.url).pathname)} + export const rows = rawTable({ database: 'test', name: 'rows' }) + const stream = defineStream({ id: 'test.rows', destination: rows, async *read() { yield { rows: [] } } }) + export const pipeline = definePipeline({ id: 'test', streams: [stream] }) + `) + await writeFile(hidden, "import './provider.ts'\nexport { rows } from './provider.ts'\n") + await writeFile(active, "export { rows, pipeline, pipeline as alias } from './provider.ts'\n") + + expect(collectPipelines(await loadDefinitionModules(hidden))).toEqual([]) + expect(collectPipelines(await loadDefinitionModules(active)).map((pipeline) => pipeline.id)).toEqual(['test']) + expect(collectPipelines(await loadDefinitionModules(hidden))).toEqual([]) + expect(await loadSchemaDefinitions(active)).toHaveLength(1) + + const plugin = createIngestPlugin() + const inactiveCheck = await plugin.hooks.onCheck({ config: resolveConfig({ entry: hidden }) }) + expect(inactiveCheck.findings.map((finding) => finding.code)).toEqual(['ingest_no_pipelines']) + const activeCheck = await plugin.hooks.onCheck({ config: resolveConfig({ entry: active }) }) + expect(activeCheck.findings).toEqual([]) + + // A selected host service is insufficient: its insert API may discard JSON/settings. + for (const command of plugin.commands.filter((command) => command.name !== 'list')) { + const output: unknown[] = [] + const code = await command.run({ + args: [], flags: {}, jsonMode: true, + options: { journalTable: 'test_journal', maxDurationSeconds: 1, prefetchBatches: 1 }, + config: resolveConfig({ entry: active }), configPath: active, + pluginContext: { hasExecutor: true, executor: {} as ClickHouseExecutor }, + print: (value) => { output.push(value) }, + }) + expect(code).toBe(2) + expect(output[0]).toMatchObject({ ok: false, error: expect.stringContaining('direct ClickHouse connection') }) + } + }) + + test('validates only the selected graph and rejects duplicate pipeline or stream IDs', () => { + const destination = rawTable({ database: 'test', name: 'rows' }) + const stream = defineStream({ id: 'test.rows', destination, async *read() { yield { rows: [] } } }) + const a = definePipeline({ id: 'a', streams: [stream] }) + const b = definePipeline({ id: 'b', streams: [stream] }) + const duplicate = definePipeline({ id: 'a', streams: [] }) + expect(selectStreams([a], [])).toHaveLength(1) + expect(selectStreams([b], [])).toHaveLength(1) + expect(() => collectPipelines([{ a, duplicate }])).toThrow('Duplicate pipeline') + expect(() => collectPipelines([{ a, b }])).toThrow('globally unique') + expect(() => selectStreams([a, b], ['pipeline:a'])).toThrow('globally unique') + expect(() => definePipeline({ id: 'repeat', streams: [stream, stream] })).toThrow('more than once') + }) +}) diff --git a/packages/plugin-ingest/src/destination.ts b/packages/plugin-ingest/src/destination.ts new file mode 100644 index 00000000..b91be89c --- /dev/null +++ b/packages/plugin-ingest/src/destination.ts @@ -0,0 +1,76 @@ +import type { ClickHouseExecutor } from '@chkit/clickhouse' +import { table, type ColumnDefinition, type TableDefinition } from '@chkit/core' + +import type { DestinationAdapter, Row } from './types.js' + +export const BATCH_ID_COLUMN = '_chkit_batch_id' +export const RUN_ID_COLUMN = '_chkit_run_id' +export const INGESTED_AT_COLUMN = '_chkit_ingested_at' + +/** + * Runtime-owned metadata every ingestion destination table carries. Spread it + * into the table's `columns`. `_chkit_ingested_at` is destination-owned: it is + * never sent by the loader, so it records the physical publication time even + * when a retry recomputes it under the same deduplication token. + */ +export const ingestionColumns: readonly ColumnDefinition[] = [ + { name: BATCH_ID_COLUMN, type: 'String' }, + { name: RUN_ID_COLUMN, type: 'String' }, + { name: INGESTED_AT_COLUMN, type: "DateTime64(6, 'UTC')", default: 'fn:now64(6)' }, +] + +// A type alias (not an interface) so it is assignable to the index-signature Row type. +export type RawRow = { + id: string + raw: unknown +} + +/** + * Landing table for provider objects exactly as received: a stable id plus the + * untouched object in a native JSON column. Typed shapes are derived from it + * inside ClickHouse (views or materialized views), so changing a transform + * never requires re-fetching the source. Replays and overlapping windows + * collapse to the latest ingested version of each id. + */ +export function rawTable(input: { database: string; name: string; comment?: string }): TableDefinition { + return table({ + database: input.database, + name: input.name, + comment: input.comment, + columns: [{ name: 'id', type: 'String' }, { name: 'raw', type: 'JSON' }, ...ingestionColumns], + engine: `ReplacingMergeTree(${INGESTED_AT_COLUMN})`, + primaryKey: ['id'], + orderBy: ['id'], + }) +} + +/** Shape provider objects for a {@link rawTable} without mapping their fields. */ +export function rawRows(items: readonly T[], id: (item: T) => string): RawRow[] { + return items.map((item) => ({ id: id(item), raw: item })) +} + +/** + * A successful synchronous insert response (or an awaited async insert) is the + * sink evidence ChKit trusts. A missing response stays ambiguous and is retried + * with the same token. + */ +export function createClickHouseDestination(executor: ClickHouseExecutor): DestinationAdapter { + return { + async insert({ table, rows, token }) { + if (rows.length === 0) return + await executor.insert({ + table: `${table.database}.${table.name}`, + values: toJsonRows(rows), + settings: { + insert_deduplication_token: token, + wait_for_async_insert: 1, + }, + }) + }, + } +} + +/** JSONEachRow encodes big integers as exact decimal strings, including nested values. */ +export function toJsonRows(rows: readonly Row[]): Row[] { + return JSON.parse(JSON.stringify(rows, (_key, value: unknown) => typeof value === 'bigint' ? value.toString() : value)) as Row[] +} diff --git a/packages/plugin-ingest/src/errors.ts b/packages/plugin-ingest/src/errors.ts new file mode 100644 index 00000000..402022f8 --- /dev/null +++ b/packages/plugin-ingest/src/errors.ts @@ -0,0 +1,136 @@ +import type { ErrorClassifier, FailureClass } from './types.js' + +const DIAGNOSTIC_BODY_LIMIT = 2048 +const TRANSIENT_NETWORK_CODES = new Set([ + 'ECONNRESET', + 'ECONNREFUSED', + 'ECONNABORTED', + 'ETIMEDOUT', + 'EPIPE', + 'EAI_AGAIN', + 'ENOTFOUND', + 'UND_ERR_SOCKET', + 'UND_ERR_CONNECT_TIMEOUT', + 'UND_ERR_HEADERS_TIMEOUT', + 'UND_ERR_BODY_TIMEOUT', +]) + +export class IngestConfigError extends Error { + constructor(message: string) { + super(message) + this.name = 'IngestConfigError' + } +} + +/** + * Canonical boundary for fetch-based readers when `response.ok` is false. + * Keeps normalized HTTP facts plus the original Response for raw inspection. + */ +export class HttpError extends Error { + readonly status: number + readonly statusText: string + readonly url: string + readonly retryAfterMs: number | undefined + readonly body: string + readonly response: Response + + private constructor(response: Response, body: string) { + super(`HTTP ${response.status} ${response.statusText} for ${redactUrl(response.url)}${body ? `: ${body}` : ''}`) + this.name = 'HttpError' + this.status = response.status + this.statusText = response.statusText + this.url = redactUrl(response.url) + this.retryAfterMs = parseRetryAfter(response.headers.get('retry-after')) + this.body = body + this.response = response + } + + static async fromResponse(response: Response): Promise { + const body = await readDiagnosticBody(response) + return new HttpError(response, body) + } +} + +/** Executor-owned failure that always preserves the exact thrown value. */ +export class FetchFailure extends Error { + readonly classification: FailureClass + override readonly cause: unknown + + constructor(cause: unknown, classification: FailureClass) { + super(cause instanceof Error ? cause.message : String(cause)) + this.name = 'FetchFailure' + this.cause = cause + this.classification = classification + } +} + +export class BudgetExhausted extends Error { + constructor(message: string) { + super(message) + this.name = 'BudgetExhausted' + } +} + +/** + * Cancellation is authoritative, then the common HTTP/network fallback is + * derived, then an optional provider classifier may enrich or override it. + */ +export function classifyFailure( + cause: unknown, + signal: AbortSignal, + classifier: ErrorClassifier | undefined +): FailureClass { + if (signal.aborted) return { kind: 'cancelled' } + const fallback = classifyCommon(cause) + return classifier?.(cause, fallback) ?? fallback +} + +function classifyCommon(cause: unknown): FailureClass { + if (cause instanceof HttpError) { + if (cause.status === 429) return { kind: 'rate_limited', retryAfterMs: cause.retryAfterMs } + if (cause.status === 408 || cause.status === 425 || cause.status >= 500) return { kind: 'transient' } + if (cause.status >= 400) return { kind: 'permanent' } + return { kind: 'unknown' } + } + const code = errorCode(cause) + if (code !== undefined && TRANSIENT_NETWORK_CODES.has(code)) return { kind: 'transient' } + if (cause instanceof TypeError && /fetch failed|network|socket/i.test(cause.message)) return { kind: 'transient' } + // A request's own timeout/abort does not cancel the ingestion execution. + if (cause instanceof Error && (cause.name === 'TimeoutError' || cause.name === 'AbortError')) return { kind: 'transient' } + return { kind: 'unknown' } +} + +function errorCode(value: unknown): string | undefined { + if (typeof value !== 'object' || value === null) return undefined + if ('code' in value && typeof value.code === 'string') return value.code + if ('cause' in value) return errorCode(value.cause) + return undefined +} + +function parseRetryAfter(header: string | null): number | undefined { + if (header === null) return undefined + const seconds = Number(header) + if (Number.isFinite(seconds)) return Math.max(0, seconds * 1000) + const date = Date.parse(header) + if (Number.isNaN(date)) return undefined + return Math.max(0, date - Date.now()) +} + +async function readDiagnosticBody(response: Response): Promise { + try { + const text = await response.clone().text() + return text.length > DIAGNOSTIC_BODY_LIMIT ? `${text.slice(0, DIAGNOSTIC_BODY_LIMIT)}…` : text + } catch { + return '' + } +} + +// Query strings may carry API keys; keep only origin and path in diagnostics. +function redactUrl(url: string): string { + try { + const parsed = new URL(url) + return `${parsed.origin}${parsed.pathname}` + } catch { + return url + } +} diff --git a/packages/plugin-ingest/src/executor.test.ts b/packages/plugin-ingest/src/executor.test.ts new file mode 100644 index 00000000..84ad1d1d --- /dev/null +++ b/packages/plugin-ingest/src/executor.test.ts @@ -0,0 +1,789 @@ +import { describe, expect, test } from 'bun:test' +import { setTimeout as sleep } from 'node:timers/promises' + +import { table } from '@chkit/core' + +import { ingestionColumns, rawRows, rawTable } from './destination.js' +import { HttpError } from './errors.js' +import { runIngestion } from './executor.js' +import { cursorState, timestampWindow } from './incremental.js' +import { paginate } from './paginate.js' +import { definePipeline, defineStream, selectStreams } from './registry.js' +import { createMemoryDestination, createMemoryJournal } from './testing.js' +import type { DestinationAdapter } from './types.js' + +const events = table({ + database: 'app', + name: 'events', + columns: [{ name: 'id', type: 'UInt64' }, ...ingestionColumns], + engine: 'MergeTree()', + primaryKey: ['id'], + orderBy: ['id'], +}) + +const pages = (count: number, size: number) => + Array.from({ length: count }, (_, page) => Array.from({ length: size }, (_, index) => ({ id: page * size + index }))) + +function httpError(status: number, headers: Record = {}) { + return HttpError.fromResponse(new Response('nope', { status, headers })) +} + +describe('runIngestion', () => { + test('loads rows with runtime metadata and journals progress only after sink evidence', async () => { + const journal = createMemoryJournal() + const destination = createMemoryDestination() + const order: string[] = [] + const observed: DestinationAdapter = { + insert: async (input) => { + await destination.insert(input) + order.push('insert') + }, + } + const stream = defineStream({ + id: 'app.events', + destination: events, + async *read() { + yield { rows: [{ id: 1 }, { id: 2 }] } + }, + }) + const pipeline = definePipeline({ id: 'app', streams: [stream] }) + const append = journal.append.bind(journal) + journal.append = async (event) => { + if (event.eventKind === 'batch_committed') order.push('commit') + await append(event) + } + + const result = await runIngestion({ selected: selectStreams([pipeline], []), backfill: undefined }, { journal, destination: observed }) + + expect(result.ok).toBe(true) + expect(order).toEqual(['insert', 'commit']) + const rows = destination.tables.get('app.events') ?? [] + expect(rows.map((row) => row.id)).toEqual([1, 2]) + expect(rows[0]?._chkit_run_id).toBe(result.runId) + expect(typeof rows[0]?._chkit_batch_id).toBe('string') + expect(rows[0]).not.toHaveProperty('_chkit_ingested_at') + expect(journal.events.map((event) => event.eventKind)).toEqual([ + 'run_started', + 'work_planned', + 'attempt_started', + 'batch_committed', + 'work_finished', + 'run_finished', + ]) + }) + + test('a crash before sink evidence never advances the bookmark, and the restart neither skips nor duplicates rows', async () => { + const journal = createMemoryJournal() + const destination = createMemoryDestination() + const source = pages(3, 2) + const stream = defineStream({ + id: 'app.cursor', + destination: events, + batchSize: 2, + incremental: cursorState({ + id: 'test.page', + version: 1, + parse: (raw) => Number(raw), + }), + async *read({ selection }) { + for (let page = selection ?? 0; page < source.length; page += 1) { + yield { rows: source[page] ?? [], state: page + 1 } + } + }, + }) + const pipeline = definePipeline({ id: 'app', streams: [stream], retry: { retries: 0 } }) + let firstWrite = true + let acknowledgementLost = false + const crashing: DestinationAdapter = { + insert: async (input) => { + // A transient error on the first batch must not shift the simulated + // acknowledgement loss onto that batch's retry. + if (firstWrite) { + firstWrite = false + throw new Error('temporary insert failure') + } + // Second batch: the write lands but the acknowledgement is lost. + if (input.rows[0]?.id === 2) { + if (acknowledgementLost) throw new Error('still down') + await destination.insert(input) + acknowledgementLost = true + throw new Error('socket hang up') + } + await destination.insert(input) + }, + } + + const first = await runIngestion({ selected: selectStreams([pipeline], []), backfill: undefined }, { journal, destination: crashing }) + expect(first.ok).toBe(false) + expect(first.streams[0]).toMatchObject({ outcome: 'failed', rows: 2, batches: 1, error: 'still down' }) + expect((await journal.readCheckpoint('app.cursor')).envelope?.state).toBe(1) + + const second = await runIngestion({ selected: selectStreams([pipeline], []), backfill: undefined }, { journal, destination }) + expect(second.ok).toBe(true) + expect((await journal.readCheckpoint('app.cursor')).envelope?.state).toBe(3) + const ids = (destination.tables.get('app.events') ?? []).map((row) => row.id) + expect(ids).toEqual([0, 1, 2, 3, 4, 5]) + }, 10_000) + + test('timestampWindow commits the cutoff as watermark only after the whole window loaded', async () => { + const journal = createMemoryJournal() + const destination = createMemoryDestination() + const selections: Array<{ from: string; to: string }> = [] + const stream = defineStream({ + id: 'app.window', + destination: events, + incremental: timestampWindow({ start: new Date('2026-01-01T00:00:00Z') }), + async *read({ selection }) { + selections.push({ from: selection.from.toISOString(), to: selection.to.toISOString() }) + yield { rows: [{ id: selections.length }] } + }, + }) + const pipeline = definePipeline({ id: 'app', streams: [stream] }) + const times = [new Date('2026-02-01T00:00:00Z'), new Date('2026-03-01T00:00:00Z')] + + for (const time of times) { + await runIngestion({ selected: selectStreams([pipeline], []), backfill: undefined }, { journal, destination, now: () => time }) + } + + expect(selections).toEqual([ + { from: '2026-01-01T00:00:00.000Z', to: '2026-02-01T00:00:00.000Z' }, + { from: '2026-02-01T00:00:00.000Z', to: '2026-03-01T00:00:00.000Z' }, + ]) + expect((await journal.readCheckpoint('app.window')).envelope?.state).toEqual({ watermark: '2026-03-01T00:00:00.000Z' }) + }) + + test('a failed window leaves the watermark untouched', async () => { + const journal = createMemoryJournal() + const stream = defineStream({ + id: 'app.window', + destination: events, + retry: { retries: 0 }, + incremental: timestampWindow({ start: new Date(0) }), + async *read() { + yield { rows: [{ id: 1 }] } + throw new Error('provider exploded') + }, + }) + const pipeline = definePipeline({ id: 'app', streams: [stream] }) + + const result = await runIngestion({ selected: selectStreams([pipeline], []), backfill: undefined }, { journal, destination: createMemoryDestination() }) + + expect(result.streams[0]?.outcome).toBe('failed') + expect(result.streams[0]?.error).toContain('provider exploded') + expect((await journal.readCheckpoint('app.window')).envelope).toBeUndefined() + }) + + test('attempt retries rate limits and transient failures but not permanent ones', async () => { + const journal = createMemoryJournal() + const destination = createMemoryDestination() + let calls = 0 + const flaky = defineStream({ + id: 'app.flaky', + destination: events, + retry: { randomize: false, minTimeout: 10 }, + async *read(context) { + const rows = await context.attempt(async () => { + calls += 1 + if (calls === 1) throw await httpError(429, { 'retry-after': '0.02' }) + if (calls === 2) throw await httpError(503) + return [{ id: 1 }] + }) + yield { rows } + }, + }) + const denied = defineStream({ + id: 'app.denied', + destination: events, + async *read(context) { + yield { rows: await context.attempt(async () => { throw await httpError(401) }) } + }, + }) + const pipeline = definePipeline({ id: 'app', streams: [flaky, denied] }) + + const result = await runIngestion( + { selected: selectStreams([pipeline], []), backfill: undefined }, + { journal, destination } + ) + + expect(calls).toBe(3) + // One failing stream does not stop its sibling. + expect(result.streams.map((stream) => stream.outcome)).toEqual(['succeeded', 'failed']) + expect(journal.events.filter((event) => event.eventKind === 'retry_scheduled').map((event) => event.errorClass)).toEqual(['rate_limited', 'transient']) + }) + + test('refuses to reinterpret a checkpoint written by another strategy version', async () => { + const journal = createMemoryJournal() + const destination = createMemoryDestination() + const build = (version: number) => { + const stream = defineStream({ + id: 'app.versioned', + destination: events, + incremental: cursorState({ id: 'test.cursor', version, parse: (raw) => String(raw) }), + async *read() { + yield { rows: [{ id: 1 }], state: 'c1' } + }, + }) + return definePipeline({ id: 'app', streams: [stream] }) + } + + await runIngestion({ selected: selectStreams([build(1)], []), backfill: undefined }, { journal, destination }) + const result = await runIngestion({ selected: selectStreams([build(2)], []), backfill: undefined }, { journal, destination }) + + expect(result.streams[0]?.outcome).toBe('failed') + expect(result.streams[0]?.error).toContain('never silently reinterprets') + }) + + test('budget exhaustion is incomplete but preserves committed progress', async () => { + const journal = createMemoryJournal() + const destination = createMemoryDestination() + const stream = defineStream({ + id: 'app.budget', + destination: events, + batchSize: 1, + budget: { maxChunks: 2 }, + incremental: cursorState({ id: 'test.page', version: 1, parse: (raw) => Number(raw) }), + async *read({ selection }) { + for (let page = selection ?? 0; page < 5; page += 1) yield { rows: [{ id: page }], state: page + 1 } + }, + }) + const pipeline = definePipeline({ id: 'app', streams: [stream] }) + + const result = await runIngestion({ selected: selectStreams([pipeline], []), backfill: undefined }, { journal, destination }) + + expect(result.ok).toBe(false) + expect(result.streams[0]?.outcome).toBe('budget_exhausted') + expect((await journal.readCheckpoint('app.budget')).envelope?.state).toBe(2) + }) + + test('a backfill uses an isolated checkpoint namespace', async () => { + const journal = createMemoryJournal() + const destination = createMemoryDestination() + const stream = defineStream({ + id: 'app.window', + destination: events, + incremental: timestampWindow({ start: new Date(0) }), + async *read() { + yield { rows: [{ id: 1 }] } + }, + }) + const pipeline = definePipeline({ id: 'app', streams: [stream] }) + + await runIngestion( + { selected: selectStreams([pipeline], []), backfill: { id: 'jan', from: new Date('2026-01-01'), to: new Date('2026-02-01') } }, + { journal, destination } + ) + + expect((await journal.readCheckpoint('app.window')).envelope).toBeUndefined() + expect((await journal.readCheckpoint('app.window#backfill:jan')).envelope?.state).toEqual({ watermark: '2026-02-01T00:00:00.000Z' }) + }) +}) + +describe('runtime contracts', () => { + const run = (pipeline: ReturnType, env: Parameters[1]) => + runIngestion({ selected: selectStreams([pipeline], []), backfill: undefined }, env) + + test('content-based batch identity supports BigInt rows and stays stable on replay', async () => { + const journal = createMemoryJournal() + const destination = createMemoryDestination() + const row = { id: 9007199254740993n, nested: { values: [-9223372036854775808n] } } + const tokens: string[] = [] + let loseAcknowledgement = true + const stream = defineStream({ + id: 'app.bigint', destination: events, retry: { retries: 0 }, + async *read() { yield { rows: [row] } }, + }) + const pipeline = definePipeline({ id: 'app', streams: [stream] }) + const observed: DestinationAdapter = { async insert(input) { + tokens.push(input.token) + // Custom destinations continue receiving the original values. + expect(input.rows[0]?.id).toBe(row.id) + await destination.insert(input) + if (loseAcknowledgement) throw new TypeError('acknowledgement lost') + } } + + expect((await run(pipeline, { journal, destination: observed })).streams[0]?.error).toBe('acknowledgement lost') + loseAcknowledgement = false + expect((await run(pipeline, { journal, destination: observed })).ok).toBe(true) + expect(tokens).toHaveLength(2) + expect(tokens[0]).toBe(tokens[1]) + expect(destination.tables.get('app.events')).toHaveLength(1) + }) + + test.each(['source', 'reader', 'sink'] as const)('a request-local AbortError in the %s is retried without cancelling the run', async (boundary) => { + const journal = createMemoryJournal() + const destination = createMemoryDestination() + let calls = 0 + let classifications = 0 + const operation = async () => { + if (++calls === 1) throw new DOMException('request timed out', 'AbortError') + return [{ id: 1 }] + } + const stream = defineStream({ + id: 'app.local_abort', destination: events, + retry: { retries: 1, minTimeout: 0, randomize: false }, + classifyError: () => { classifications += 1; return { kind: 'transient' } }, + async *read({ attempt }) { + yield { rows: boundary === 'sink' ? [{ id: 1 }] : await (boundary === 'source' ? attempt(operation) : operation()) } + }, + }) + const result = await run(definePipeline({ id: 'app', streams: [stream] }), { + journal, + destination: boundary === 'sink' ? { async insert(input) { await operation(); await destination.insert(input) } } : destination, + }) + + expect(result.streams[0]).toMatchObject({ outcome: 'succeeded', rows: 1, error: undefined }) + expect(calls).toBe(2) + expect(classifications).toBe(boundary === 'sink' ? 0 : 1) + }) + + test('exhausted request-local abort retries report failure, not budget exhaustion', async () => { + let calls = 0 + const stream = defineStream({ + id: 'app.local_abort', destination: events, + retry: { retries: 1, minTimeout: 0, randomize: false }, + async *read({ attempt }) { + yield { rows: await attempt(async () => { calls += 1; throw new DOMException('request timed out', 'AbortError') }) } + }, + }) + const result = await run(definePipeline({ id: 'app', streams: [stream] }), { + journal: createMemoryJournal(), destination: createMemoryDestination(), + }) + expect(result.streams[0]).toMatchObject({ outcome: 'failed', error: 'request timed out' }) + expect(calls).toBe(2) + }) + + test('a sink failure at the chunk limit cannot recreate the reader or hide the failure', async () => { + const journal = createMemoryJournal() + let readers = 0 + let pulled = 0 + const stream = defineStream({ + id: 'app.bounded', destination: events, batchSize: 1, budget: { maxChunks: 1 }, + retry: { retries: 2, minTimeout: 0, randomize: false }, + incremental: timestampWindow({ start: new Date(0) }), + async *read() { + readers += 1 + for (let id = 0; id < 3; id += 1) { pulled += 1; yield { rows: [{ id }] } } + }, + }) + const result = await run(definePipeline({ id: 'app', streams: [stream] }), { + journal, destination: { async insert() { throw new TypeError('sink serialization failed') } }, + }) + expect(result.streams[0]).toMatchObject({ outcome: 'failed', chunks: 1, error: 'sink serialization failed' }) + expect(readers).toBe(1) + expect(pulled).toBe(1) + expect((await journal.readCheckpoint(stream.id)).envelope).toBeUndefined() + expect(journal.events.filter((event) => event.eventKind === 'retry_scheduled')).toHaveLength(0) + }) + + test('reader retries share the chunk budget and never complete a partial window', async () => { + const journal = createMemoryJournal() + let readers = 0 + let pulled = 0 + const stream = defineStream({ + id: 'app.bounded', destination: events, budget: { maxChunks: 2 }, + retry: { retries: 2, minTimeout: 0, randomize: false }, + incremental: timestampWindow({ start: new Date(0) }), + async *read() { + readers += 1 + for (let id = 0; id < 3; id += 1) { + pulled += 1 + yield { rows: [{ id }] } + if (readers === 1) throw new Error('reader disconnected') + } + }, + }) + const result = await run(definePipeline({ id: 'app', streams: [stream] }), { + journal, destination: createMemoryDestination(), + }) + expect(result.streams[0]).toMatchObject({ outcome: 'budget_exhausted', chunks: 2, rows: 1 }) + expect(readers).toBe(2) + expect(pulled).toBe(2) + expect((await journal.readCheckpoint(stream.id)).envelope).toBeUndefined() + }) + + test.each([false, true])('new full syncs preserve changes while failed syncs reuse tokens (declared id: %s)', async (withId) => { + const journal = createMemoryJournal() + const destination = createMemoryDestination() + let label = 'A' + const stream = defineStream({ + id: 'app.full', destination: events, retry: { retries: 0 }, + async *read() { yield { rows: [{ id: 1, label }], ...(withId ? { id: 'page:1' } : {}) } }, + }) + const pipeline = definePipeline({ id: 'app', streams: [stream] }) + expect((await run(pipeline, { journal, destination })).ok).toBe(true) + const firstSuccess = (await journal.readCheckpoint(stream.id)).lastSuccessSeq + expect(firstSuccess).toBeGreaterThan(0) + + label = 'B' + const lossy: DestinationAdapter = { async insert(input) { + await destination.insert(input) + throw new Error('acknowledgement lost') + } } + expect((await run(pipeline, { journal, destination: lossy })).ok).toBe(false) + expect((await journal.readCheckpoint(stream.id)).lastSuccessSeq).toBe(firstSuccess) + expect((await run(pipeline, { journal, destination })).ok).toBe(true) + + label = 'A' + expect((await run(pipeline, { journal, destination })).ok).toBe(true) + expect(destination.tables.get('app.events')?.map((row) => row.label)).toEqual(['A', 'B', 'A']) + expect((await journal.readCheckpoint(stream.id)).envelope).toBeUndefined() + }) + + test.each(['deadline', 'cancel'] as const)('a hung destination never reports success after %s', async (mode) => { + const journal = createMemoryJournal() + const controller = new AbortController() + let finishWrite: () => void = () => undefined + const destination: DestinationAdapter = { insert: () => new Promise((resolve) => { finishWrite = resolve }) } + const stream = defineStream({ + id: 'app.hung_load', destination: events, + async *read() { yield { rows: [{ id: 1 }] } }, + }) + const timer = mode === 'cancel' ? setTimeout(() => controller.abort(), 20) : undefined + try { + const result = await run(definePipeline({ id: 'app', streams: [stream] }), { + journal, destination, signal: controller.signal, maxDurationMs: mode === 'deadline' ? 20 : 60_000, + }) + expect(result.ok).toBe(false) + expect(result.streams[0]?.outcome).toBe(mode === 'deadline' ? 'budget_exhausted' : 'cancelled') + expect(result.streams[0]?.rows).toBe(0) + finishWrite() + await new Promise((resolve) => setTimeout(resolve, 0)) + expect(journal.events.filter((event) => event.eventKind === 'batch_committed')).toEqual([]) + } finally { + clearTimeout(timer) + finishWrite() + } + }, 10_000) + + test.each(['ensure', 'run_read', 'run_started', 'stream_read', 'work_planned', 'batch_committed'])( + 'the execution deadline bounds a hung journal %s', async (stage) => { + const journal = createMemoryJournal() + const append = journal.append.bind(journal) + const read = journal.readCheckpoint.bind(journal) + const hung = () => new Promise(() => undefined) + if (stage === 'ensure') journal.ensure = hung + journal.readCheckpoint = (namespace) => ( + (stage === 'run_read' && namespace === '@run') || (stage === 'stream_read' && namespace !== '@run') + ? hung() : read(namespace) + ) + journal.append = (event) => event.eventKind === stage ? hung() : append(event) + const stream = defineStream({ id: 'app.journal', destination: events, async *read() { yield { rows: [{ id: 1 }] } } }) + const execution = run(definePipeline({ id: 'app', streams: [stream] }), { + journal, destination: createMemoryDestination(), maxDurationMs: 20, + }) + if (['ensure', 'run_read', 'run_started'].includes(stage)) { + await expect(execution).rejects.toThrow('execution budget exhausted') + } else { + const result = await execution + expect(result.ok).toBe(false) + expect(result.streams[0]?.outcome).toBe('budget_exhausted') + } + }, 1000 + ) + + test.each(['work_finished', 'run_finished'])('a hung terminal journal %s has bounded cleanup', async (stage) => { + const journal = createMemoryJournal() + const append = journal.append.bind(journal) + journal.append = (event) => event.eventKind === stage ? new Promise(() => undefined) : append(event) + const stream = defineStream({ id: 'app.terminal', destination: events, async *read() { yield { rows: [{ id: 1 }] } } }) + const execution = run(definePipeline({ id: 'app', streams: [stream] }), { + journal, destination: createMemoryDestination(), maxDurationMs: 100, + }) + if (stage === 'run_finished') await expect(execution).rejects.toThrow() + else expect((await execution).ok).toBe(false) + }, 8000) + + test('a throwing loader factory releases its permit for sibling streams', async () => { + const failed = defineStream({ + id: 'app.factory', destination: events, retry: { retries: 0 }, + loader() { throw new Error('loader construction failed') }, + async *read() { yield { rows: [{ id: 1 }] } }, + }) + const healthy = defineStream({ id: 'app.healthy', destination: events, async *read() { yield { rows: [{ id: 2 }] } } }) + const destination = createMemoryDestination() + const result = await run(definePipeline({ id: 'app', streams: [failed, healthy], maxLoads: 1 }), { + journal: createMemoryJournal(), destination, maxDurationMs: 100, + }) + expect(result.streams.map((stream) => stream.outcome)).toEqual(['failed', 'succeeded']) + expect(destination.tables.get('app.events')?.map((row) => row.id)).toEqual([2]) + }) + + test('a declared interval id keeps batch identity stable when replayed rows changed', async () => { + const build = (label: string, withId: boolean) => { + const stream = defineStream({ + id: 'app.mutable', + destination: events, + async *read() { + yield { rows: [{ id: 1, label }], ...(withId ? { id: 'page:1' } : {}) } + }, + }) + return definePipeline({ id: 'app', streams: [stream] }) + } + + const declared = createMemoryDestination() + await run(build('before', true), { journal: createMemoryJournal(), destination: declared }) + await run(build('after', true), { journal: createMemoryJournal(), destination: declared }) + expect(declared.tables.get('app.events')).toHaveLength(1) + + // Without a declared interval the content hash wins: a duplicate over a suppressed change. + const undeclared = createMemoryDestination() + await run(build('before', false), { journal: createMemoryJournal(), destination: undeclared }) + await run(build('after', false), { journal: createMemoryJournal(), destination: undeclared }) + expect(undeclared.tables.get('app.events')).toHaveLength(2) + }) + + test('the execution budget interrupts a hung reader and preserves committed progress', async () => { + const journal = createMemoryJournal() + const stream = defineStream({ + id: 'app.hung', + destination: events, + batchSize: 1, + incremental: cursorState({ id: 'test.page', version: 1, parse: (raw) => Number(raw) }), + async *read() { + yield { rows: [{ id: 1 }], state: 1 } + await new Promise(() => undefined) + }, + }) + const pipeline = definePipeline({ id: 'app', streams: [stream] }) + + const result = await run(pipeline, { journal, destination: createMemoryDestination(), maxDurationMs: 50 }) + + expect(result.streams[0]?.outcome).toBe('budget_exhausted') + expect((await journal.readCheckpoint('app.hung')).envelope?.state).toBe(1) + expect(journal.events.at(-1)?.eventKind).toBe('run_finished') + }) + + test('an oversized chunk fails the stream instead of being buffered', async () => { + const stream = defineStream({ + id: 'app.big', + destination: events, + budget: { maxChunkRows: 2 }, + async *read() { + yield { rows: [{ id: 1 }, { id: 2 }, { id: 3 }] } + }, + }) + const result = await run(definePipeline({ id: 'app', streams: [stream] }), { journal: createMemoryJournal(), destination: createMemoryDestination() }) + expect(result.streams[0]?.error).toContain('above the 2-row bound') + }) + + test('a mapped row cannot supply the destination-owned publication time', async () => { + const destination = createMemoryDestination() + const stream = defineStream({ + id: 'app.stamped', + destination: events, + async *read() { + yield { rows: [{ id: 1, _chkit_ingested_at: '2000-01-01 00:00:00' }] } + }, + }) + await run(definePipeline({ id: 'app', streams: [stream] }), { journal: createMemoryJournal(), destination }) + expect(destination.tables.get('app.events')?.[0]).not.toHaveProperty('_chkit_ingested_at') + }) + + test('an ambiguous journal append retries the same fact without consuming a sequence number', async () => { + const journal = createMemoryJournal() + const append = journal.append.bind(journal) + let failures = 0 + journal.append = async (event) => { + if (event.eventKind === 'batch_committed' && failures === 0) { + failures += 1 + throw new Error('acknowledgement lost') + } + await append(event) + } + const stream = defineStream({ id: 'app.events', destination: events, async *read() { yield { rows: [{ id: 1 }] } } }) + + const result = await run(definePipeline({ id: 'app', streams: [stream] }), { journal, destination: createMemoryDestination() }) + + expect(result.ok).toBe(true) + const sequences = journal.events.filter((event) => event.namespaceId === 'app.events').map((event) => event.eventSeq) + expect(sequences).toEqual([1, 2, 3, 4]) + }) + + test('a failing loader cleanup does not mask the write failure or stop the retry', async () => { + const destination = createMemoryDestination() + let writes = 0 + const stream = defineStream({ + id: 'app.cleanup', + destination: events, + loader: (ctx) => ({ + ctx, + async write(batch) { + writes += 1 + if (writes === 1) throw new Error('write failed') + await ctx.destination.insert({ table: ctx.table, rows: batch.rows, token: batch.batchId }) + }, + finalize: async () => ({ evidence: 'clickhouse_ack', rows: 1, writeUnits: 1 }), + abort: async () => { + throw new Error('cleanup failed') + }, + }), + async *read() { + yield { rows: [{ id: 1 }] } + }, + }) + + const result = await run(definePipeline({ id: 'app', streams: [stream] }), { journal: createMemoryJournal(), destination }) + + expect(result.ok).toBe(true) + expect(writes).toBe(2) + }) +}) + +describe('pipeline concurrency and retries', () => { + const run = (pipeline: ReturnType, env: Parameters[1]) => + runIngestion({ selected: selectStreams([pipeline], []), backfill: undefined }, env) + + test.each(['fetches', 'loads'] as const)('%s release capacity during backoff and never exceed the limit', async (mode) => { + const order: number[] = [] + let active = 0 + let peak = 0 + const operation = async (id: number) => { + order.push(id) + active += 1 + peak = Math.max(peak, active) + await sleep(5) + active -= 1 + if (order.length === 1) throw new Error('temporary failure') + return [{ id }] + } + const streams = [1, 2].map((id) => defineStream({ + id: `app.stream${id}`, destination: events, + async *read({ attempt }) { + yield { rows: mode === 'fetches' ? await attempt(() => operation(id)) : [{ id }] } + }, + })) + const destination: DestinationAdapter = { + async insert({ rows }) { if (mode === 'loads') await operation(Number(rows[0]?.id)) }, + } + const result = await run(definePipeline({ + id: 'app', streams, maxStreams: 2, maxFetches: 1, maxLoads: 1, + retry: { retries: 1, minTimeout: 10, randomize: false }, + }), { journal: createMemoryJournal(), destination }) + + expect(result.ok).toBe(true) + expect(order).toEqual([1, 2, 1]) + expect(peak).toBe(1) + }) + + test('stream limits serialize readers and cancellation prevents queued readers from starting', async () => { + for (const cancel of [false, true]) { + const controller = new AbortController() + const order: string[] = [] + const streams = [1, 2].map((id) => defineStream({ + id: `app.stream${id}`, destination: events, + async *read() { + order.push(`start:${id}`) + if (cancel) controller.abort() + await sleep(5) + order.push(`end:${id}`) + yield { rows: [] } + }, + })) + const result = await run(definePipeline({ id: 'app', streams, maxStreams: 1 }), { + journal: createMemoryJournal(), destination: createMemoryDestination(), signal: controller.signal, + }) + if (cancel) { + expect(result.streams.map((stream) => stream.outcome)).toEqual(['cancelled', 'cancelled']) + expect(order).not.toContain('start:2') + } else { + expect(result.ok).toBe(true) + expect(order).toEqual(['start:1', 'end:1', 'start:2', 'end:2']) + } + } + }) + + test('reader retries resume from committed progress and exhausted source retries do not recreate the reader', async () => { + const journal = createMemoryJournal() + const append = journal.append.bind(journal) + let committed: () => void = () => undefined + const saved = new Promise((resolve) => { committed = resolve }) + journal.append = async (event) => { + await append(event) + if (event.eventKind === 'batch_committed') committed() + } + const selections: Array = [] + let sourceCalls = 0 + let readers = 0 + const resumable = defineStream({ + id: 'app.resumable', destination: events, batchSize: 1, + incremental: cursorState({ id: 'page', version: 1, parse: (raw) => Number(raw) }), + async *read({ selection }) { + selections.push(selection) + if (selection === undefined) { + yield { rows: [{ id: 1 }], state: 1 } + await saved + throw new Error('reader disconnected') + } + yield { rows: [{ id: 2 }], state: 2 } + }, + }) + const exhausted = defineStream({ + id: 'app.exhausted', destination: events, + async *read({ attempt }) { + readers += 1 + await attempt(async () => { sourceCalls += 1; throw new Error('source unavailable') }) + }, + }) + const result = await run(definePipeline({ + id: 'app', streams: [resumable, exhausted], retry: { retries: 1, minTimeout: 0 }, + }), { journal, destination: createMemoryDestination() }) + + expect(result.streams.map((stream) => stream.outcome)).toEqual(['succeeded', 'failed']) + expect(selections).toEqual([undefined, 1]) + expect((await journal.readCheckpoint(resumable.id)).envelope?.state).toBe(2) + expect(readers).toBe(1) + expect(sourceCalls).toBe(2) + }) +}) + +describe('rawTable', () => { + test('lands provider objects untouched next to a stable id', async () => { + const destination = createMemoryDestination() + const landing = rawTable({ database: 'app_raw', name: 'tickets' }) + const ticket = { key: 'T-1', requester: { email: 'a@example.com' }, tags: [{ name: 'vip' }] } + const stream = defineStream({ + id: 'app.tickets', + destination: landing, + async *read() { + yield { rows: rawRows([ticket], (item) => item.key) } + }, + }) + + await runIngestion({ selected: selectStreams([definePipeline({ id: 'app', streams: [stream] })], []), backfill: undefined }, { journal: createMemoryJournal(), destination }) + + expect(landing.columns.map((column) => `${column.name}:${column.type}`).slice(0, 2)).toEqual(['id:String', 'raw:JSON']) + expect(destination.tables.get('app_raw.tickets')?.[0]).toMatchObject({ id: 'T-1', raw: ticket }) + }) +}) + +describe('selectStreams', () => { + test('repeated tags use exact AND semantics and an explicit empty selection fails', () => { + const hourly = defineStream({ id: 'crm.people', destination: events, tags: ['schedule:1h'], async *read() {} }) + const daily = defineStream({ id: 'crm.deals', destination: events, tags: ['schedule:1d'], async *read() {} }) + const pipeline = definePipeline({ id: 'crm', tags: ['crm'], streams: [hourly, daily] }) + + expect(selectStreams([pipeline], []).map((entry) => entry.stream.id)).toEqual(['crm.people', 'crm.deals']) + expect(selectStreams([pipeline], ['crm', 'schedule:1h']).map((entry) => entry.stream.id)).toEqual(['crm.people']) + expect(selectStreams([pipeline], ['stream:crm.deals']).map((entry) => entry.stream.id)).toEqual(['crm.deals']) + expect(() => selectStreams([pipeline], ['schedule'])).toThrow('No stream matches every requested tag') + }) + + test('stream ids are globally unique across pipelines', () => { + const stream = defineStream({ id: 'crm.people', destination: events, async *read() {} }) + const a = definePipeline({ id: 'a', streams: [stream] }) + const b = definePipeline({ id: 'b', streams: [stream] }) + expect(() => selectStreams([a, b], [])).toThrow('globally unique') + }) +}) + +describe('paginate', () => { + test('rejects a repeated continuation instead of looping forever', async () => { + const context = { signal: new AbortController().signal, attempt: (operation: (signal: AbortSignal) => Promise) => operation(new AbortController().signal) } + const iterate = async () => { + for await (const _ of paginate({ context, fetchPage: async () => ({ items: [1], next: 'same' }) })) { + // drain + } + } + await expect(iterate()).rejects.toThrow('repeated continuation') + }) +}) diff --git a/packages/plugin-ingest/src/executor.ts b/packages/plugin-ingest/src/executor.ts new file mode 100644 index 00000000..cc3b9bbe --- /dev/null +++ b/packages/plugin-ingest/src/executor.ts @@ -0,0 +1,652 @@ +import { randomUUID } from 'node:crypto' +import { setTimeout as sleep } from 'node:timers/promises' + +import { SpanStatusCode, trace, type Span } from '@opentelemetry/api' +import pLimit, { type LimitFunction } from 'p-limit' +import pRetry, { AbortError } from 'p-retry' + +import { toJsonRows } from './destination.js' +import { BudgetExhausted, classifyFailure, FetchFailure, IngestConfigError } from './errors.js' +import { canonicalJson, digest } from './journal.js' +import { simpleLoader } from './loader.js' +import { createBoundedQueue } from './queue.js' +import type { SelectedStream } from './registry.js' +import { mergeRetry, runAttempt } from './retry.js' +import type { + AnyStreamDefinition, + CheckpointEnvelope, + DestinationAdapter, + ExecutionResult, + Journal, + JournalEvent, + PipelineDefinition, + Row, + SinkReceipt, + StreamOutcome, + StreamResult, +} from './types.js' + +const RUN_NAMESPACE = '@run' +const DEFAULT_BATCH_SIZE = 10_000 +const DEFAULT_PREFETCH_BATCHES = 1 +const DEFAULT_MAX_DURATION_MS = 60 * 60_000 +const LOAD_ATTEMPTS = 3 +const JOURNAL_APPEND_ATTEMPTS = 4 +// Terminal facts get a bounded chance to land even after the execution is cancelled. +const TERMINAL_JOURNAL_TIMEOUT_MS = 5000 +const DEFAULT_MAX_CHUNK_ROWS = 100_000 +// How long a failed stream waits for an uncooperative reader before abandoning it. +const READER_SHUTDOWN_GRACE_MS = 5000 +const tracer = trace.getTracer('@chkit/plugin-ingest') + +export interface BackfillRequest { + /** Stable identity: resuming the same backfill reuses its isolated checkpoint namespace. */ + id: string + from: Date | undefined + to: Date | undefined +} + +export interface ExecutionRequest { + selected: readonly SelectedStream[] + backfill: BackfillRequest | undefined +} + +export interface ExecutionEnv { + journal: Journal + destination: DestinationAdapter + signal?: AbortSignal + /** Host-default execution budget. Exhaustion preserves committed progress. */ + maxDurationMs?: number + /** Finite number of mapped batches buffered between fetch and load. */ + prefetchBatches?: number + now?: () => Date + log?: (message: string) => void +} + +interface ResolvedEnv extends Required> { + /** Host cancellation combined with the execution deadline. */ + signal: AbortSignal + /** Host cancellation only: distinguishes a cancelled run from an exhausted budget. */ + hostSignal: AbortSignal +} + +interface PipelinePermits { + streams: LimitFunction + fetches: LimitFunction + loads: LimitFunction +} + +interface PendingBatch { + rows: Row[] + /** Declared source-interval ids, or `undefined` once any chunk omitted its id. */ + intervalIds: string[] | undefined + /** Candidate provider state that becomes safe once these rows have sink evidence. */ + state: { value: unknown } | undefined +} + +interface StreamProgress { + /** Last journal sequence confirmed for this namespace. */ + seq: number + /** Serializes appends so a sequence number is only consumed by a confirmed fact. */ + appendChain: Promise + version: number + lastSuccessSeq: number + envelope: CheckpointEnvelope | undefined + rows: number + batches: number + chunks: number +} + +/** + * One transient execution context for one exact stream selection. The run id + * correlates journal and telemetry evidence only: every stream plans and + * recovers independently from its own journal-backed checkpoint. + */ +export async function runIngestion(request: ExecutionRequest, input: ExecutionEnv): Promise { + const deadline = new AbortController() + const env = resolveEnv(input, deadline.signal) + // The budget is a real bound: it interrupts hung readers, retry timers and waits. + const deadlineTimer = setTimeout(() => deadline.abort(new BudgetExhausted('execution budget exhausted')), env.maxDurationMs) + const runId = randomUUID() + const cutoff = env.now() + const streamIds = request.selected.map((entry) => entry.stream.id) + + return tracer.startActiveSpan('chkit.ingest.execution', async (span) => { + span.setAttribute('chkit.ingest.run_id', runId) + span.setAttribute('chkit.ingest.stream_ids', streamIds) + try { + await abortable(() => env.journal.ensure(), env.signal) + const runHead = (await abortable(() => env.journal.readCheckpoint(RUN_NAMESPACE), env.signal)).headSeq + await abortable(() => env.journal.append( + runEvent(runHead + 1, 'run_started', runId, '', { + streamIds, + cutoff: cutoff.toISOString(), + backfill: request.backfill?.id, + }) + ), env.signal) + + const permits = new Map() + const streams = await Promise.all( + request.selected.map((entry) => + executeSelectedStream(entry, permitsFor(permits, entry.pipeline), request.backfill, runId, cutoff, env) + ) + ) + + const ok = streams.every((stream) => stream.outcome === 'succeeded') + await abortable(() => env.journal.append( + runEvent(runHead + 2, 'run_finished', runId, ok ? 'succeeded' : 'failed', { + outcomes: Object.fromEntries(streams.map((stream) => [stream.namespaceId, stream.outcome])), + }) + ), AbortSignal.timeout(TERMINAL_JOURNAL_TIMEOUT_MS)) + if (!ok) span.setStatus({ code: SpanStatusCode.ERROR }) + return { runId, cutoff: cutoff.toISOString(), streams, ok } + } finally { + clearTimeout(deadlineTimer) + span.end() + } + }) +} + +// An ordinary failure in one stream never prevents unrelated selected streams +// from being attempted: every failure is folded into that stream's result. +async function executeSelectedStream( + entry: SelectedStream, + permits: PipelinePermits, + backfill: BackfillRequest | undefined, + runId: string, + cutoff: Date, + env: ResolvedEnv +): Promise { + const { stream, pipeline } = entry + const namespaceId = backfill ? `${stream.id}#backfill:${backfill.id}` : stream.id + const progress: StreamProgress = { seq: 0, appendChain: Promise.resolve(), version: 0, lastSuccessSeq: 0, envelope: undefined, rows: 0, batches: 0, chunks: 0 } + const result = (outcome: StreamOutcome, error: string | undefined): StreamResult => ({ + streamId: stream.id, + pipelineId: pipeline.id, + namespaceId, + outcome, + rows: progress.rows, + batches: progress.batches, + chunks: progress.chunks, + checkpointVersion: progress.version, + error, + }) + + return permits.streams(async () => { + if (env.signal.aborted) { + return result(env.hostSignal.aborted ? 'cancelled' : 'budget_exhausted', 'execution interrupted before start') + } + + return tracer.startActiveSpan('chkit.ingest.stream', async (span) => { + span.setAttribute('chkit.ingest.stream_id', stream.id) + span.setAttribute('chkit.ingest.namespace_id', namespaceId) + try { + const outcome = await executeStream({ stream, pipeline, namespaceId, backfill, runId, cutoff, permits, progress, env }) + env.log?.(`${namespaceId}: ${outcome} (${progress.rows} rows, ${progress.batches} batches, checkpoint v${progress.version})`) + return result(outcome, undefined) + } catch (error) { + const message = error instanceof Error ? error.message : String(error) + const outcome: StreamOutcome = env.hostSignal.aborted ? 'cancelled' : env.signal.aborted ? 'budget_exhausted' : 'failed' + recordFailure(span, error) + env.log?.(`${namespaceId}: ${outcome} — ${message}`) + return result(outcome, message) + } finally { + span.end() + } + }) + }) +} + +async function executeStream(input: { + stream: AnyStreamDefinition + pipeline: PipelineDefinition + namespaceId: string + backfill: BackfillRequest | undefined + runId: string + cutoff: Date + permits: PipelinePermits + progress: StreamProgress + env: ResolvedEnv +}): Promise { + const { stream, pipeline, namespaceId, progress, env } = input + const committed = await abortable(() => env.journal.readCheckpoint(namespaceId), env.signal) + progress.seq = committed.headSeq + progress.version = committed.version + progress.lastSuccessSeq = committed.lastSuccessSeq + progress.envelope = committed.envelope + + const retry = mergeRetry(pipeline.retry, stream.retry) + let workId = '' + let outcome: StreamOutcome = 'failed' + let failure: unknown + + try { + outcome = await runAttempt(async (attemptNo) => { + // Re-plan from the latest durable boundary on every reader (re)creation. + const state = restoreState(stream, progress.envelope, namespaceId) + const selection: unknown = stream.incremental.plan({ + state, + cutoff: input.cutoff, + range: input.backfill ? { from: input.backfill.from, to: input.backfill.to } : undefined, + }) + const nextWorkId = digest([namespaceId, String(progress.lastSuccessSeq), String(progress.version), canonicalJson(selection)]).slice(0, 24) + if (nextWorkId !== workId) { + workId = nextWorkId + await append(input, 'work_planned', { workId, workState: 'planned', detail: { selection, strategy: stream.incremental.id, strategyVersion: stream.incremental.version, pipelineId: pipeline.id } }) + } + await append(input, 'attempt_started', { workId, attemptNo, workState: 'running' }) + return readAndLoad({ ...input, workId, attemptNo, state, selection, retry }) + }, { + ...retry, + // Reader recreation consumes the same execution-wide chunk budget. + // Preserve the original failure when no further chunks may be pulled. + shouldRetry: (context) => progress.chunks < (stream.budget?.maxChunks ?? Infinity) && (retry.shouldRetry?.(context) ?? true), + }, { + signal: env.signal, + classifier: stream.classifyError, + onRetry: (context, retryAfterMs) => append(input, 'retry_scheduled', { + workId, + attemptNo: context.attemptNumber, + retryAt: retryAfterMs === undefined ? undefined : new Date(env.now().getTime() + retryAfterMs), + errorClass: context.error.classification.kind, + detail: { error: context.error.message }, + }), + }) + } catch (error) { + failure = error + if (env.signal.aborted) { + outcome = env.hostSignal.aborted ? 'cancelled' : 'budget_exhausted' + // Exhausting the budget is incomplete: committed progress stands. + if (outcome === 'budget_exhausted') failure = undefined + } + } + + await append(input, 'work_finished', { + workId, + workState: failure === undefined ? outcome : outcome === 'cancelled' ? 'cancelled' : 'failed', + errorClass: failure === undefined ? '' : failureKind(failure, env.signal, stream), + detail: { rows: progress.rows, batches: progress.batches, chunks: progress.chunks, error: failure === undefined ? undefined : messageOf(failure) }, + }) + if (failure !== undefined) throw failure + return outcome +} + +/** + * Pull the reader through a finite buffer into the loader. Exactly one next() + * is pending at a time, and source progress is appended to the journal only + * after the covering rows have sink evidence. + */ +async function readAndLoad(input: { + stream: AnyStreamDefinition + namespaceId: string + runId: string + cutoff: Date + permits: PipelinePermits + progress: StreamProgress + env: ResolvedEnv + workId: string + attemptNo: number + state: unknown + selection: unknown + retry: ReturnType +}): Promise { + const { stream, namespaceId, progress, env } = input + const maxChunks = stream.budget?.maxChunks ?? Infinity + if (progress.chunks >= maxChunks) return 'budget_exhausted' + const local = new AbortController() + const signal = AbortSignal.any([env.signal, local.signal]) + const queue = createBoundedQueue(env.prefetchBatches) + const batchSize = stream.batchSize ?? DEFAULT_BATCH_SIZE + let budgetExhausted = false + // Set once this attempt has reported its outcome: detached work that finishes + // later may have written rows (at-least-once) but must never commit progress. + let abandoned = false + + const produce = async () => { + let pending = emptyBatch() + const reader = stream.read({ + streamId: stream.id, + selection: input.selection, + state: input.state, + cutoff: input.cutoff, + signal, + attempt: (operation, options) => + runAttempt(() => input.permits.fetches(() => { + signal.throwIfAborted() + return operation(signal) + }), input.retry, { + signal, + classifier: stream.classifyError, + onRetry: (context, retryAfterMs) => + append(input, 'retry_scheduled', { + workId: input.workId, + attemptNo: input.attemptNo, + // p-retry owns the randomized backoff; record only a provider's known not-before time. + retryAt: retryAfterMs === undefined ? undefined : new Date(env.now().getTime() + retryAfterMs), + errorClass: context.error.classification.kind, + detail: { label: options?.label ?? 'source', sourceAttempt: context.attemptNumber, error: context.error.message }, + }), + }), + }) + + // Iterate by hand so a hung next() can be abandoned on abort; return() still + // runs on every exit so the reader's finally blocks own cursor cleanup. + const iterator = reader[Symbol.asyncIterator]() + const maxChunkRows = stream.budget?.maxChunkRows ?? DEFAULT_MAX_CHUNK_ROWS + try { + while (!budgetExhausted) { + const step = await abortable(() => iterator.next(), signal) + if (step.done) break + const chunk: unknown = step.value + assertValidChunk(stream.id, chunk, maxChunkRows) + progress.chunks += 1 + // A chunk that declares its source interval is one logical write unit: + // it is never merged with neighbours, so a replay regroups identically + // even when row counts changed. + if (chunk.id !== undefined && pending.rows.length > 0) { + await queue.push(pending, signal) + pending = emptyBatch() + } + pending.rows.push(...chunk.rows) + if (pending.intervalIds) { + if (chunk.id === undefined) pending.intervalIds = undefined + else pending.intervalIds.push(chunk.id) + } + if (chunk.state !== undefined) pending.state = { value: chunk.state } + if (chunk.id !== undefined || pending.rows.length >= batchSize) { + await queue.push(pending, signal) + pending = emptyBatch() + } + if (progress.chunks >= maxChunks) budgetExhausted = true + } + } finally { + // Never await an uncooperative reader: cleanup is best effort once we leave. + void Promise.resolve(iterator.return?.()).catch(() => undefined) + } + + // Only a fully consumed selection may claim the strategy's completion state. + if (!budgetExhausted) { + const completed: unknown = stream.incremental.complete?.({ state: input.state, selection: input.selection }) + if (completed !== undefined) pending.state = { value: completed } + } + if (pending.rows.length > 0 || pending.state !== undefined) await queue.push(pending, signal) + queue.close() + } + + const consume = async () => { + // Batch identity uses the last successful sync, the committed checkpoint + // version and the batch's position since that version, so a + // replay in a fresh process reproduces the same id and deduplication token. + // Declared source-interval ids complete the identity; without them a + // content hash does, preferring a possible duplicate over suppressing rows + // that changed between attempts. + let sinceBoundary = 0 + for (let batch = await queue.pop(signal); batch !== undefined; batch = await queue.pop(signal)) { + const discriminator = batch.intervalIds ? `interval:${canonicalJson(batch.intervalIds)}` : `content:${canonicalJson(toJsonRows(batch.rows))}` + const batchId = digest([namespaceId, String(progress.lastSuccessSeq), String(progress.version), String(sinceBoundary), discriminator]).slice(0, 32) + const receipt = await loadBatch(input, batchId, batch.rows, signal) + if (abandoned) return + const envelope: CheckpointEnvelope | undefined = batch.state + ? { strategy: stream.incremental.id, version: stream.incremental.version, state: batch.state.value } + : progress.envelope + const advanced = canonicalJson(envelope ?? null) !== canonicalJson(progress.envelope ?? null) + const expected = progress.version + const version = advanced ? expected + 1 : expected + await append(input, 'batch_committed', { + workId: input.workId, + attemptNo: input.attemptNo, + batchId, + expectedCheckpointVersion: expected, + checkpointVersion: version, + checkpoint: envelope, + sinkEvidence: receipt.evidence, + detail: { rows: receipt.rows, writeUnits: receipt.writeUnits }, + }) + sinceBoundary = advanced ? 0 : sinceBoundary + 1 + progress.version = version + progress.envelope = envelope + progress.rows += receipt.rows + progress.batches += 1 + } + } + + // The first failure is the root cause; the sibling only fails because of the + // induced abort, so it must not mask what actually went wrong. + let rootCause: { error: unknown } | undefined + const settled = Promise.allSettled( + [produce(), consume()].map((task) => + task.catch((error: unknown) => { + rootCause ??= { error } + local.abort(error) + }) + ) + ) + // After a failure, a source operation that ignores its signal must not pin + // the stream (and its permits) forever. + await Promise.race([settled, graceAfterAbort(signal, READER_SHUTDOWN_GRACE_MS)]) + abandoned = true + if (rootCause) throw rootCause.error + signal.throwIfAborted() + if (budgetExhausted) { + env.log?.(`${namespaceId}: ${new BudgetExhausted('execution budget exhausted; committed progress is preserved').message}`) + return 'budget_exhausted' + } + return 'succeeded' +} + +// A fresh Loader instance per attempt. An ambiguous acknowledgement replays the +// same stable batch identity and accepts a possible duplicate over data loss. +async function loadBatch( + input: { stream: AnyStreamDefinition; runId: string; permits: PipelinePermits; env: ResolvedEnv }, + batchId: string, + rows: readonly Row[], + signal: AbortSignal +): Promise { + const factory = input.stream.loader ?? simpleLoader() + return tracer.startActiveSpan('chkit.ingest.load', async (span) => { + span.setAttribute('chkit.ingest.batch_id', batchId) + span.setAttribute('chkit.ingest.rows', rows.length) + try { + return await pRetry(() => input.permits.loads(async () => { + signal.throwIfAborted() + // The load limit covers only the active write, never p-retry's backoff. + let loader: ReturnType + try { + loader = factory({ + streamId: input.stream.id, + runId: input.runId, + table: input.stream.destination, + destination: input.env.destination, + signal, + }) + } catch (error) { + // Construction errors are not ambiguous sink acknowledgements. + throw new AbortError(error instanceof Error ? error : new Error(String(error))) + } + try { + await loader.write({ batchId, rows }) + return await loader.finalize() + } catch (error) { + // Cleanup must not replace the write failure that decides the retry. + await loader.abort(error).catch((cleanupError: unknown) => { + span.recordException(cleanupError instanceof Error ? cleanupError : new Error(String(cleanupError))) + }) + throw error + } + }), { + retries: LOAD_ATTEMPTS - 1, + minTimeout: 1000, + signal, + }) + } catch (error) { + recordFailure(span, error) + throw error + } finally { + span.end() + } + }) +} + +function restoreState(stream: AnyStreamDefinition, envelope: CheckpointEnvelope | undefined, namespaceId: string): unknown { + if (!envelope) return undefined + if (envelope.strategy !== stream.incremental.id || envelope.version !== stream.incremental.version) { + throw new IngestConfigError( + `Checkpoint for "${namespaceId}" was written by strategy ${envelope.strategy}@${envelope.version}, but the stream now declares ${stream.incremental.id}@${stream.incremental.version}. ` + + 'ChKit never silently reinterprets a checkpoint: migrate the state explicitly, use a new stream id, or run a separately namespaced backfill.' + ) + } + return stream.incremental.parseState(envelope.state) +} + +// Appends for one namespace are serialized and the sequence number is taken +// only when the write starts, so a fact that never lands cannot leave a gap. +// An ambiguous failure retries the exact same deterministic fact. +function append( + input: { namespaceId: string; runId: string; progress: StreamProgress; env: ResolvedEnv }, + eventKind: JournalEvent['eventKind'], + fields: Partial> +): Promise { + const { progress, env } = input + const signal = eventKind === 'work_finished' ? AbortSignal.timeout(TERMINAL_JOURNAL_TIMEOUT_MS) : env.signal + const write = progress.appendChain.then(async () => { + signal.throwIfAborted() + const event: JournalEvent = { + namespaceId: input.namespaceId, + eventSeq: progress.seq + 1, + eventKind, + runId: input.runId, + workId: fields.workId ?? '', + attemptNo: fields.attemptNo ?? 0, + batchId: fields.batchId ?? '', + expectedCheckpointVersion: fields.expectedCheckpointVersion ?? progress.version, + checkpointVersion: fields.checkpointVersion ?? progress.version, + checkpoint: fields.checkpoint, + workState: fields.workState ?? '', + sinkEvidence: fields.sinkEvidence ?? '', + retryAt: fields.retryAt, + errorClass: fields.errorClass ?? '', + detail: fields.detail ?? {}, + } + await pRetry(() => abortable(() => env.journal.append(event), signal), { + retries: JOURNAL_APPEND_ATTEMPTS - 1, + minTimeout: 250, + signal, + }) + progress.seq = event.eventSeq + }) + // A fact that could not be confirmed poisons the chain: a later append must + // not reuse its sequence number, because the unconfirmed write may have landed. + progress.appendChain = write + write.catch(() => undefined) + return abortable(() => write, signal) +} + +function runEvent(seq: number, eventKind: 'run_started' | 'run_finished', runId: string, workState: JournalEvent['workState'], detail: Record): JournalEvent { + return { + namespaceId: RUN_NAMESPACE, + eventSeq: seq, + eventKind, + runId, + workId: runId, + attemptNo: 0, + batchId: '', + expectedCheckpointVersion: 0, + checkpointVersion: 0, + checkpoint: undefined, + workState, + sinkEvidence: '', + retryAt: undefined, + errorClass: '', + detail, + } +} + +function assertValidChunk( + streamId: string, + chunk: unknown, + maxChunkRows: number +): asserts chunk is { rows: readonly Row[]; state?: unknown; id?: string } { + if (typeof chunk !== 'object' || chunk === null || !('rows' in chunk) || !Array.isArray(chunk.rows)) { + throw new IngestConfigError(`Stream "${streamId}" yielded a chunk without a "rows" array.`) + } + if ('id' in chunk && chunk.id !== undefined && typeof chunk.id !== 'string') { + throw new IngestConfigError(`Stream "${streamId}" yielded a chunk whose "id" is not a string.`) + } + if (chunk.rows.length > maxChunkRows) { + throw new IngestConfigError( + `Stream "${streamId}" yielded a chunk of ${chunk.rows.length} rows, above the ${maxChunkRows}-row bound. Yield smaller chunks or raise budget.maxChunkRows.` + ) + } +} + +function emptyBatch(): PendingBatch { + return { rows: [], intervalIds: [], state: undefined } +} + +/** Resolves a fixed grace period after the signal aborts; never resolves otherwise. */ +function graceAfterAbort(signal: AbortSignal, graceMs: number): Promise { + return new Promise((resolve) => { + const start = () => { + void sleep(graceMs, undefined, { ref: false }).then(resolve) + } + if (signal.aborted) start() + else signal.addEventListener('abort', start, { once: true }) + }) +} + +/** Settle with the promise, or reject as soon as the signal aborts. */ +function abortable(operation: () => Promise, signal: AbortSignal): Promise { + if (signal.aborted) return Promise.reject(signal.reason) + return new Promise((resolve, reject) => { + const onAbort = () => reject(signal.reason) + signal.addEventListener('abort', onAbort, { once: true }) + Promise.resolve().then(() => { + signal.throwIfAborted() + return operation() + }).then(resolve, reject).finally(() => signal.removeEventListener('abort', onAbort)) + }) +} + +function permitsFor(cache: Map, pipeline: PipelineDefinition): PipelinePermits { + const existing = cache.get(pipeline.id) + if (existing) return existing + const created: PipelinePermits = { + streams: pLimit(pipeline.maxStreams), + fetches: pLimit(pipeline.maxFetches), + loads: pLimit(pipeline.maxLoads), + } + cache.set(pipeline.id, created) + return created +} + +const NEVER_ABORTED = new AbortController().signal + +function resolveEnv(input: ExecutionEnv, deadlineSignal: AbortSignal): ResolvedEnv { + const now = input.now ?? (() => new Date()) + const maxDurationMs = input.maxDurationMs ?? DEFAULT_MAX_DURATION_MS + return { + journal: input.journal, + destination: input.destination, + signal: AbortSignal.any([input.signal ?? NEVER_ABORTED, deadlineSignal]), + hostSignal: input.signal ?? NEVER_ABORTED, + maxDurationMs, + prefetchBatches: input.prefetchBatches ?? DEFAULT_PREFETCH_BATCHES, + now, + log: input.log ?? (() => undefined), + } +} + +function failureKind(error: unknown, signal: AbortSignal, stream: AnyStreamDefinition): string { + if (error instanceof IngestConfigError) return 'config' + if (error instanceof FetchFailure) return error.classification.kind + return classifyFailure(error, signal, stream.classifyError).kind +} + +function recordFailure(span: Span, error: unknown): void { + span.setStatus({ code: SpanStatusCode.ERROR, message: messageOf(error) }) + if (error instanceof Error) span.recordException(error) +} + +function messageOf(error: unknown): string { + return error instanceof Error ? error.message : String(error) +} diff --git a/packages/plugin-ingest/src/incremental.ts b/packages/plugin-ingest/src/incremental.ts new file mode 100644 index 00000000..3bb75c3d --- /dev/null +++ b/packages/plugin-ingest/src/incremental.ts @@ -0,0 +1,110 @@ +import { IngestConfigError } from './errors.js' +import type { IncrementalStrategy } from './types.js' + +export interface TimestampWindowState { + watermark: string +} + +export interface TimestampRange { + from: Date + to: Date +} + +export type TimestampWindowOptions = + | { + /** Lower bound of the first sync. Overlap applies only to later runs. */ + start: Date + /** Milliseconds to re-read before the committed watermark. Defaults to zero. */ + overlapMs?: number + from?: never + } + | { + /** Custom lower bound; this selection input is never persisted as the watermark. */ + from(input: { watermark: Date | undefined; cutoff: Date }): Date + start?: never + overlapMs?: never + } + +export interface CursorStateOptions { + /** Stable strategy identifier stored in the checkpoint envelope. */ + id: string + version: number + /** Validate committed state. Throw when it must not be reinterpreted. */ + parse(raw: unknown): TState +} + +/** No incremental selection: every execution reads the whole source. */ +export function fullSync(): IncrementalStrategy { + return { + id: 'chkit.full_sync', + version: 1, + parseState: () => undefined, + plan: () => undefined, + } +} + +/** + * Bundled timestamp strategy. The upper bound is fixed to the execution cutoff + * and `range.to` becomes the committed watermark only after the whole window + * has sink evidence. + */ +export function timestampWindow( + options: TimestampWindowOptions +): IncrementalStrategy { + if (options.from === undefined) { + if (!(options.start instanceof Date) || !Number.isFinite(options.start.getTime())) { + throw new IngestConfigError('timestampWindow: start must be a valid Date.') + } + if (!Number.isFinite(options.overlapMs ?? 0) || (options.overlapMs ?? 0) < 0) { + throw new IngestConfigError('timestampWindow: overlapMs must be a finite non-negative number.') + } + } else if (options.start !== undefined || options.overlapMs !== undefined) { + throw new IngestConfigError('timestampWindow: use either start/overlapMs or from, not both.') + } + return { + id: 'chkit.timestamp_window', + version: 1, + parseState(raw) { + if (typeof raw !== 'object' || raw === null || !('watermark' in raw) || typeof raw.watermark !== 'string') { + throw new IngestConfigError('timestampWindow: committed state has no string "watermark".') + } + if (Number.isNaN(Date.parse(raw.watermark))) { + throw new IngestConfigError(`timestampWindow: committed watermark "${raw.watermark}" is not a timestamp.`) + } + return { watermark: raw.watermark } + }, + plan({ state, cutoff, range }) { + const to = range?.to ?? cutoff + const watermark = state ? new Date(state.watermark) : undefined + const from = range?.from ?? (options.from + ? options.from({ watermark, cutoff }) + : new Date(watermark ? watermark.getTime() - (options.overlapMs ?? 0) : options.start.getTime())) + if (!Number.isFinite(from.getTime()) || !Number.isFinite(to.getTime())) { + throw new IngestConfigError('timestampWindow: planned bounds must be valid dates.') + } + if (from.getTime() > to.getTime()) { + throw new IngestConfigError( + `timestampWindow: planned lower bound ${from.toISOString()} is after upper bound ${to.toISOString()}.` + ) + } + return { from, to } + }, + complete({ selection }) { + return { watermark: selection.to.toISOString() } + }, + } +} + +/** + * Generic provider-owned state (compound cursor, change token, snapshot id…). + * The reader receives the committed state as its selection and advances it by + * yielding chunks that carry complete candidate state. + */ +export function cursorState(options: CursorStateOptions): IncrementalStrategy { + return { + id: options.id, + version: options.version, + parseState: options.parse, + plan: ({ state }) => state, + } +} diff --git a/packages/plugin-ingest/src/index.ts b/packages/plugin-ingest/src/index.ts new file mode 100644 index 00000000..88f981ef --- /dev/null +++ b/packages/plugin-ingest/src/index.ts @@ -0,0 +1,29 @@ +export { ingest, createIngestPlugin, checkGraph, type IngestPlugin, type IngestPluginOptions } from './plugin.js' +export { defineStream, definePipeline, selectStreams, type SelectedStream } from './registry.js' +export { fullSync, timestampWindow, cursorState, type TimestampRange, type TimestampWindowOptions, type TimestampWindowState } from './incremental.js' +export { paginate, type Page } from './paginate.js' +export { simpleLoader } from './loader.js' +export { ingestionColumns, rawTable, rawRows, createClickHouseDestination, type RawRow } from './destination.js' +export { createClickHouseJournal } from './journal.js' +export { runIngestion, type BackfillRequest, type ExecutionEnv, type ExecutionRequest } from './executor.js' +export { HttpError, FetchFailure, IngestConfigError } from './errors.js' +export type { + DestinationAdapter, + ErrorClassifier, + ExecutionResult, + FailureClass, + FetchContext, + IncrementalStrategy, + Journal, + Loader, + LoaderContext, + LoaderFactory, + PipelineDefinition, + ReadContext, + RetryOptions, + Row, + SinkReceipt, + SourceChunk, + StreamDefinition, + StreamResult, +} from './types.js' diff --git a/packages/plugin-ingest/src/ingest.e2e.test.ts b/packages/plugin-ingest/src/ingest.e2e.test.ts new file mode 100644 index 00000000..292148ec --- /dev/null +++ b/packages/plugin-ingest/src/ingest.e2e.test.ts @@ -0,0 +1,167 @@ +import { afterAll, beforeAll, describe, expect, test } from 'bun:test' + +import { table, toCreateSQL } from '@chkit/core' +import type { ClickHouseExecutor } from '@chkit/clickhouse' +import { createPrefix, createStatelessLiveExecutor, getRequiredEnv, quoteIdent, waitForTable } from '@chkit/clickhouse/e2e-testkit' + +import { createClickHouseDestination, ingestionColumns, rawRows, rawTable } from './destination.js' +import { runIngestion } from './executor.js' +import { cursorState } from './incremental.js' +import { createClickHouseJournal } from './journal.js' +import { definePipeline, defineStream, selectStreams } from './registry.js' +import type { DestinationAdapter } from './types.js' + +describe('@chkit/plugin-ingest live env e2e', () => { + const liveEnv = getRequiredEnv() + const prefix = createPrefix('ingest') + const database = liveEnv.clickhouseDatabase + const journalTable = `${prefix}journal` + const destinationTable = table({ + database, + name: `${prefix}items`, + columns: [{ name: 'id', type: 'UInt64' }, { name: 'label', type: 'String' }, ...ingestionColumns], + engine: 'MergeTree()', + primaryKey: ['id'], + orderBy: ['id'], + settings: { non_replicated_deduplication_window: '100' }, + }) + const landing = { + ...rawTable({ database, name: `${prefix}raw` }), + settings: { non_replicated_deduplication_window: '100' }, + } + const integers = table({ + database, + name: `${prefix}integers`, + columns: [ + { name: 'id', type: 'UInt64' }, + { name: 'signed', type: 'Int64' }, + { name: 'values', type: 'Array(UInt64)' }, + ...ingestionColumns, + ], + engine: 'MergeTree()', + orderBy: ['id'], + }) + let executor: ClickHouseExecutor + + beforeAll(async () => { + // Ingestion fetches, loads and journals concurrently, so it needs a stateless executor. + executor = createStatelessLiveExecutor(liveEnv) + await executor.command(toCreateSQL(destinationTable)) + await waitForTable(executor, database, destinationTable.name) + await executor.command(toCreateSQL(landing)) + await waitForTable(executor, database, landing.name) + await executor.command(toCreateSQL(integers)) + await waitForTable(executor, database, integers.name) + }) + + afterAll(async () => { + await executor.command(`DROP TABLE IF EXISTS ${quoteIdent(database)}.${quoteIdent(destinationTable.name)}`) + await executor.command(`DROP TABLE IF EXISTS ${quoteIdent(database)}.${quoteIdent(landing.name)}`) + await executor.command(`DROP TABLE IF EXISTS ${quoteIdent(database)}.${quoteIdent(integers.name)}`) + await executor.command(`DROP TABLE IF EXISTS ${quoteIdent(database)}.${quoteIdent(journalTable)}`) + await executor.close() + }) + + test('a lost acknowledgement replays from the journaled checkpoint without skipping or duplicating rows', async () => { + const source = Array.from({ length: 3 }, (_, page) => [0, 1].map((offset) => ({ id: page * 2 + offset, label: `row-${page * 2 + offset}` }))) + const stream = defineStream({ + id: `${prefix}items`, + destination: destinationTable, + batchSize: 2, + incremental: cursorState({ id: 'e2e.page', version: 1, parse: (raw) => Number(raw) }), + async *read({ selection }) { + for (let page = selection ?? 0; page < source.length; page += 1) yield { rows: source[page] ?? [], state: page + 1 } + }, + }) + const pipeline = definePipeline({ id: `${prefix}pipeline`, streams: [stream], retry: { retries: 0 } }) + const journal = () => createClickHouseJournal({ executor, database, targetId: `e2e/${prefix}`, table: journalTable }) + const destination = createClickHouseDestination(executor) + let acknowledgementLost = false + const lossy: DestinationAdapter = { + insert: async (input) => { + // Target the second batch, not the second call: the live destination + // can itself need retries before the first batch is acknowledged. + if (input.rows[0]?.id === 2) { + if (acknowledgementLost) throw new Error('target unavailable') + await destination.insert(input) + acknowledgementLost = true + throw new Error('acknowledgement lost') + } + await destination.insert(input) + }, + } + + const first = await runIngestion({ selected: selectStreams([pipeline], []), backfill: undefined }, { journal: journal(), destination: lossy }) + expect(first.ok).toBe(false) + expect(first.streams[0]).toMatchObject({ outcome: 'failed', rows: 2, batches: 1, error: 'target unavailable' }) + expect((await journal().readCheckpoint(stream.id)).envelope?.state).toBe(1) + + // A fresh executor process reconstructs everything from the journal. + const second = await runIngestion({ selected: selectStreams([pipeline], []), backfill: undefined }, { journal: journal(), destination }) + expect(second.streams[0]).toMatchObject({ outcome: 'succeeded', error: undefined }) + expect(second.ok).toBe(true) + const checkpoint = await journal().readCheckpoint(stream.id) + expect(checkpoint.envelope).toEqual({ strategy: 'e2e.page', version: 1, state: 3 }) + expect(checkpoint.version).toBe(3) + + const rows = await executor.query<{ id: string; run_ids: string }>( + `SELECT id, uniqExact(_chkit_run_id) AS run_ids FROM ${quoteIdent(database)}.${quoteIdent(destinationTable.name)} GROUP BY id ORDER BY id`, + { select_sequential_consistency: '1' } + ) + expect(rows.map((row) => Number(row.id))).toEqual([0, 1, 2, 3, 4, 5]) + const physical = await executor.query<{ rows: string }>( + `SELECT count() AS rows FROM ${quoteIdent(database)}.${quoteIdent(destinationTable.name)}`, + { select_sequential_consistency: '1' } + ) + expect(Number(physical[0]?.rows)).toBe(6) + }, 120_000) + + test('successful full syncs land A → B → A instead of suppressing the last update', async () => { + let value = 'A' + const stream = defineStream({ + id: `${prefix}raw`, destination: landing, + async *read() { yield { rows: rawRows([{ id: '1', value }], (item) => item.id) } }, + }) + const selected = selectStreams([definePipeline({ id: `${prefix}raw_pipeline`, streams: [stream] })], []) + let lastSuccessSeq = 0 + for (const next of ['A', 'B', 'A']) { + value = next + const journal = createClickHouseJournal({ executor, database, targetId: `e2e/${prefix}`, table: journalTable }) + const result = await runIngestion({ selected, backfill: undefined }, { + journal, destination: createClickHouseDestination(executor), + }) + expect(result.streams[0]).toMatchObject({ outcome: 'succeeded', error: undefined }) + expect(result.ok).toBe(true) + const checkpoint = await journal.readCheckpoint(stream.id) + expect(checkpoint.lastSuccessSeq).toBeGreaterThan(lastSuccessSeq) + lastSuccessSeq = checkpoint.lastSuccessSeq + const rows = await executor.query<{ raw: { value: string } }>( + `SELECT raw FROM ${quoteIdent(database)}.${quoteIdent(landing.name)} FINAL`, + { select_sequential_consistency: '1' } + ) + expect(rows.map((row) => row.raw.value)).toEqual([next]) + } + }, 120_000) + + test('BigInt rows round-trip through content hashing and JSONEachRow without losing precision', async () => { + const row = { id: 18446744073709551615n, signed: -9223372036854775808n, values: [9007199254740993n, 18446744073709551615n] } + const stream = defineStream({ + id: `${prefix}integers`, destination: integers, + async *read() { yield { rows: [row] } }, + }) + const result = await runIngestion({ + selected: selectStreams([definePipeline({ id: `${prefix}integers_pipeline`, streams: [stream] })], []), + backfill: undefined, + }, { + journal: createClickHouseJournal({ executor, database, targetId: `e2e/${prefix}`, table: journalTable }), + destination: createClickHouseDestination(executor), + }) + expect(result.streams[0]).toMatchObject({ outcome: 'succeeded', rows: 1, error: undefined }) + const rows = await executor.query<{ id: string; signed: string; values: string[] }>( + `SELECT id, signed, values FROM ${quoteIdent(database)}.${quoteIdent(integers.name)}`, + { select_sequential_consistency: '1', output_format_json_quote_64bit_integers: '1' } + ) + expect(rows).toEqual([{ id: row.id.toString(), signed: row.signed.toString(), values: row.values.map(String) }]) + expect(typeof row.id).toBe('bigint') + }, 120_000) +}) diff --git a/packages/plugin-ingest/src/journal.ts b/packages/plugin-ingest/src/journal.ts new file mode 100644 index 00000000..5afe24ba --- /dev/null +++ b/packages/plugin-ingest/src/journal.ts @@ -0,0 +1,283 @@ +import { createHash } from 'node:crypto' + +import type { ClickHouseExecutor } from '@chkit/clickhouse' + +import { IngestConfigError } from './errors.js' +import type { CheckpointEnvelope, CommittedCheckpoint, Journal, JournalEvent } from './types.js' + +export const DEFAULT_JOURNAL_TABLE = '_chkit_ingestion_journal' +const TABLE_NAME_PATTERN = /^[A-Za-z_][A-Za-z0-9_]*$/ + +// A type alias (not an interface) so it is assignable to the executor's Record-based insert values. +export type JournalRow = { + target_id: string + namespace_id: string + event_seq: string + event_id: string + payload_hash: string + event_at: string + event_kind: string + run_id: string + work_id: string + attempt_no: number + batch_id: string + expected_checkpoint_version: string + checkpoint_version: string + checkpoint_json: string + work_state: string + sink_evidence: string + retry_at: string | null + error_class: string + detail_json: string +} + +export interface ClickHouseJournalOptions { + /** Must allow concurrent queries: use a stateless executor, not a session-bound one. */ + executor: ClickHouseExecutor + database: string + targetId: string + table?: string + now?: () => Date +} + +/** + * Append-only, target-linked journal. It is the sole authority for durable + * ingestion control state: checkpoints are read as a projection over + * `batch_committed` facts and never stored anywhere else. + */ +export function createClickHouseJournal(options: ClickHouseJournalOptions): Journal { + const table = options.table ?? DEFAULT_JOURNAL_TABLE + if (!TABLE_NAME_PATTERN.test(table)) throw new IngestConfigError(`Invalid journal table name "${table}".`) + if (!TABLE_NAME_PATTERN.test(options.database)) { + throw new IngestConfigError(`Invalid journal database name "${options.database}".`) + } + const qualified = `\`${options.database}\`.\`${table}\`` + const now = options.now ?? (() => new Date()) + + return { + async ensure() { + await options.executor.command(journalTableSql(qualified)) + }, + + async append(event) { + const row = toJournalRow(event, options.targetId, now()) + await options.executor.insert({ + table: `${options.database}.${table}`, + values: [row], + // A retried append of the same deterministic fact is suppressed while + // the deduplication window lasts; readers canonicalize by event_id anyway. + settings: { insert_deduplication_token: row.event_id, async_insert: 0 }, + }) + }, + + async readCheckpoint(namespaceId) { + // Physical retry duplicates are allowed, so facts are canonicalized per + // sequence number first. The history is then validated before anything is + // projected from it: a checkpoint read from a damaged journal is worthless. + const facts = `SELECT + event_seq, + uniqExact(event_id) AS owners, + uniqExact(payload_hash) AS payloads, + any(event_kind) AS fact_kind, + any(work_state) AS fact_work_state, + any(expected_checkpoint_version) AS fact_expected, + any(checkpoint_version) AS fact_version, + any(checkpoint_json) AS fact_checkpoint + FROM ${qualified} + WHERE target_id = ${sqlString(options.targetId)} AND namespace_id = ${sqlString(namespaceId)} + GROUP BY event_seq` + const settings = { select_sequential_consistency: '1' } + const [health, transitions] = await Promise.all([ + options.executor.query<{ + head_seq: string + last_success_seq: string + sequences: string + conflicting_owners: string + drifted: string + checkpoint_version: string + checkpoint_json: string + }>( + `SELECT + max(event_seq) AS head_seq, + maxIf(event_seq, fact_kind = 'work_finished' AND fact_work_state = 'succeeded') AS last_success_seq, + count() AS sequences, + countIf(owners > 1) AS conflicting_owners, + countIf(payloads > 1) AS drifted, + argMaxIf(fact_version, event_seq, fact_kind = 'batch_committed') AS checkpoint_version, + argMaxIf(fact_checkpoint, event_seq, fact_kind = 'batch_committed') AS checkpoint_json +FROM (${facts})`, + settings + ), + // Every commit must start from the version the previous commit produced, + // advance it by at most one, and only change the envelope when it advances. + // The two reads are not one snapshot; that is sound because V1 runs a + // single executor process and reads a namespace before appending to it. + options.executor.query<{ invalid: string }>( + `SELECT countIf( + fact_expected != previous_version OR fact_version < fact_expected OR fact_version > fact_expected + 1 + OR (fact_version = fact_expected AND fact_checkpoint != previous_checkpoint) +) AS invalid +FROM ( + SELECT + fact_expected, + fact_version, + fact_checkpoint, + lagInFrame(fact_version, 1, toUInt64(0)) OVER (ORDER BY event_seq ASC ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS previous_version, + lagInFrame(fact_checkpoint, 1, '') OVER (ORDER BY event_seq ASC ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS previous_checkpoint + FROM (${facts}) + WHERE fact_kind = 'batch_committed' +)`, + settings + ), + ]) + const row = health[0] + if (!row || Number(row.sequences) === 0) return emptyCheckpoint() + + const problems = [ + Number(row.head_seq) !== Number(row.sequences) ? `sequence gap (head ${row.head_seq}, ${row.sequences} facts)` : '', + Number(row.conflicting_owners) > 0 ? `${row.conflicting_owners} sequence number(s) owned by conflicting facts` : '', + Number(row.drifted) > 0 ? `${row.drifted} fact(s) with drifting payloads` : '', + Number(transitions[0]?.invalid ?? 0) > 0 ? `${transitions[0]?.invalid} invalid checkpoint transition(s)` : '', + ].filter((problem) => problem !== '') + if (problems.length > 0) { + throw new Error( + `Ingestion journal for "${namespaceId}" is not a valid history: ${problems.join('; ')}. Refusing to project a checkpoint from it; more than one executor process may have been active.` + ) + } + return { + version: Number(row.checkpoint_version), + envelope: parseEnvelope(row.checkpoint_json), + headSeq: Number(row.head_seq), + lastSuccessSeq: Number(row.last_success_seq), + } + }, + } +} + +export function toJournalRow(event: JournalEvent, targetId: string, at: Date): JournalRow { + const checkpointJson = event.checkpoint ? canonicalJson(event.checkpoint) : '' + const detailJson = canonicalJson(event.detail) + // Identity covers what makes the fact unique; the payload hash covers every + // authoritative field a retry of that same fact must reproduce. Only the + // physical append time (event_at) is excluded. + const eventId = digest([targetId, event.namespaceId, String(event.eventSeq), event.eventKind, event.workId, event.batchId, String(event.attemptNo)]) + const payload = digest([ + eventId, + String(event.expectedCheckpointVersion), + String(event.checkpointVersion), + checkpointJson, + event.workState, + event.sinkEvidence, + event.errorClass, + event.runId, + event.retryAt ? event.retryAt.toISOString() : '', + detailJson, + ]) + return { + target_id: targetId, + namespace_id: event.namespaceId, + event_seq: String(event.eventSeq), + event_id: eventId, + payload_hash: BigInt(`0x${payload.slice(0, 16)}`).toString(), + event_at: toClickHouseDateTime(at), + event_kind: event.eventKind, + run_id: event.runId, + work_id: event.workId, + attempt_no: event.attemptNo, + batch_id: event.batchId, + expected_checkpoint_version: String(event.expectedCheckpointVersion), + checkpoint_version: String(event.checkpointVersion), + checkpoint_json: checkpointJson, + work_state: event.workState, + sink_evidence: event.sinkEvidence, + retry_at: event.retryAt ? toClickHouseDateTime(event.retryAt) : null, + error_class: event.errorClass, + detail_json: detailJson, + } +} + +function parseEnvelope(json: string): CheckpointEnvelope | undefined { + if (json === '') return undefined + const parsed: unknown = JSON.parse(json) + if ( + typeof parsed !== 'object' || + parsed === null || + !('strategy' in parsed) || + typeof parsed.strategy !== 'string' || + !('version' in parsed) || + typeof parsed.version !== 'number' + ) { + throw new Error('Committed checkpoint is not a valid ChKit checkpoint envelope.') + } + return { strategy: parsed.strategy, version: parsed.version, state: 'state' in parsed ? parsed.state : undefined } +} + +/** Stable key order so equal values always serialize identically. */ +export function canonicalJson(value: unknown): string { + // JSON.stringify(undefined) is undefined, not a string. + return JSON.stringify(sortKeys(value)) ?? 'null' +} + +export function digest(parts: readonly string[]): string { + const hash = createHash('sha256') + for (const part of parts) { + hash.update(String(part.length)) + hash.update(':') + hash.update(part) + } + return hash.digest('hex') +} + +export function emptyCheckpoint(): CommittedCheckpoint { + return { version: 0, envelope: undefined, headSeq: 0, lastSuccessSeq: 0 } +} + +function journalTableSql(qualified: string): string { + return `CREATE TABLE IF NOT EXISTS ${qualified} +( + target_id LowCardinality(String), + namespace_id String, + event_seq UInt64, + event_id String, + payload_hash UInt64, + event_at DateTime64(6, 'UTC'), + event_kind LowCardinality(String), + run_id String, + work_id String, + attempt_no UInt32, + batch_id String, + expected_checkpoint_version UInt64, + checkpoint_version UInt64, + checkpoint_json String, + work_state LowCardinality(String), + sink_evidence LowCardinality(String), + retry_at Nullable(DateTime64(6, 'UTC')), + error_class LowCardinality(String), + detail_json String +) +ENGINE = MergeTree +PARTITION BY toYYYYMM(event_at) +ORDER BY (target_id, namespace_id, event_seq, event_id)` +} + +function sortKeys(value: unknown): unknown { + if (Array.isArray(value)) return value.map(sortKeys) + if (value instanceof Date) return value.toISOString() + if (typeof value === 'object' && value !== null) { + return Object.fromEntries( + Object.entries(value) + .filter(([, entry]) => entry !== undefined) + .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)) + .map(([key, entry]) => [key, sortKeys(entry)]) + ) + } + return value +} + +function toClickHouseDateTime(date: Date): string { + return date.toISOString().replace('T', ' ').replace('Z', '') +} + +function sqlString(value: string): string { + return `'${value.replace(/\\/g, '\\\\').replace(/'/g, "\\'")}'` +} diff --git a/packages/plugin-ingest/src/loader.ts b/packages/plugin-ingest/src/loader.ts new file mode 100644 index 00000000..770b3c19 --- /dev/null +++ b/packages/plugin-ingest/src/loader.ts @@ -0,0 +1,46 @@ +import { BATCH_ID_COLUMN, INGESTED_AT_COLUMN, RUN_ID_COLUMN } from './destination.js' +import type { LoaderFactory } from './types.js' + +export interface SimpleLoaderOptions { + /** Rows per physical INSERT. A larger batch is split into deterministic chunks. */ + maxRowsPerInsert?: number +} + +const DEFAULT_MAX_ROWS_PER_INSERT = 50_000 + +/** + * Direct at-least-once destination writes. Every write unit reuses one stable + * token across retries: a batch written by one INSERT has one token, a + * deterministically chunked batch has one distinct token per chunk. + */ +export function simpleLoader(options: SimpleLoaderOptions = {}): LoaderFactory { + const maxRows = options.maxRowsPerInsert ?? DEFAULT_MAX_ROWS_PER_INSERT + + return (ctx) => { + let rows = 0 + let writeUnits = 0 + + return { + ctx, + async write(batch) { + for (let offset = 0, unit = 0; offset < batch.rows.length; offset += maxRows, unit += 1) { + ctx.signal.throwIfAborted() + const slice = batch.rows.slice(offset, offset + maxRows).map((row) => { + // Publication time is destination-owned: never let a mapped row supply it. + const { [INGESTED_AT_COLUMN]: _ignored, ...authored } = row + return { ...authored, [BATCH_ID_COLUMN]: batch.batchId, [RUN_ID_COLUMN]: ctx.runId } + }) + await ctx.destination.insert({ table: ctx.table, rows: slice, token: `${batch.batchId}:${unit}` }) + rows += slice.length + writeUnits += 1 + } + }, + async finalize() { + return { evidence: writeUnits === 0 ? 'none_required' : 'clickhouse_ack', rows, writeUnits } + }, + async abort() { + // Direct writes cannot be withdrawn; replay reuses the same tokens. + }, + } + } +} diff --git a/packages/plugin-ingest/src/paginate.ts b/packages/plugin-ingest/src/paginate.ts new file mode 100644 index 00000000..cfa8ff1f --- /dev/null +++ b/packages/plugin-ingest/src/paginate.ts @@ -0,0 +1,45 @@ +import { canonicalJson } from './journal.js' +import type { AttemptOptions, FetchContext } from './types.js' + +export interface Page { + items: readonly TItem[] + /** Continuation for the next request; `undefined` ends the sequence. */ + next: TCursor | undefined +} + +export interface PaginateOptions { + context: FetchContext + fetchPage(cursor: TCursor | undefined, signal: AbortSignal): Promise> + initial?: TCursor + label?: AttemptOptions['label'] +} + +/** + * Pull-based pagination over one retryable Promise step per page. Every request + * runs through the executor attempt capability. A continuation that was already + * seen (compared by canonical serialization) is rejected, so a cyclic provider + * cursor cannot loop forever; other non-progress is bounded by execution budgets. + */ +export async function* paginate( + options: PaginateOptions +): AsyncGenerator { + const seen = new Set() + let cursor = options.initial + + while (true) { + options.context.signal.throwIfAborted() + const current = cursor + const page = await options.context.attempt((signal) => options.fetchPage(current, signal), { + label: options.label, + }) + if (page.items.length > 0) yield page.items + if (page.next === undefined || page.next === null) return + + const key = canonicalJson(page.next) + if (seen.has(key)) { + throw new Error(`paginate: provider returned a repeated continuation ${key}; refusing to loop.`) + } + seen.add(key) + cursor = page.next + } +} diff --git a/packages/plugin-ingest/src/plugin.ts b/packages/plugin-ingest/src/plugin.ts new file mode 100644 index 00000000..97a904b2 --- /dev/null +++ b/packages/plugin-ingest/src/plugin.ts @@ -0,0 +1,289 @@ +import process from 'node:process' + +import { createStatelessClickHouseExecutor, type ClickHouseExecutor } from '@chkit/clickhouse' +import { + createPluginRunner, + defineFlags, + withFactoryDefaults, + type ChxInlinePluginRegistration, + type ResolvedChxConfig, +} from '@chkit/core' +import { loadDefinitionModules } from '@chkit/core/schema-loader' +import { z } from 'zod' + +import { BATCH_ID_COLUMN, createClickHouseDestination, INGESTED_AT_COLUMN, RUN_ID_COLUMN } from './destination.js' +import { IngestConfigError } from './errors.js' +import { runIngestion, type BackfillRequest } from './executor.js' +import { createClickHouseJournal, DEFAULT_JOURNAL_TABLE } from './journal.js' +import { collectPipelines, selectStreams, type SelectedStream } from './registry.js' +import type { PipelineDefinition } from './types.js' + +const REQUIRED_COLUMNS = [BATCH_ID_COLUMN, RUN_ID_COLUMN, INGESTED_AT_COLUMN] + +const IngestOptionsSchema = z.object({ + journalTable: z.string().min(1).default(DEFAULT_JOURNAL_TABLE), + /** Host-default execution budget in seconds. */ + maxDurationSeconds: z.number().positive().default(3600), + prefetchBatches: z.number().int().positive().default(1), +}) +type IngestOptions = z.infer + +export type IngestPluginOptions = Partial + +const SELECTION_FLAGS = defineFlags([ + { name: '--tag', type: 'string[]', description: 'Exact tag every selected stream must carry (repeatable, AND)', placeholder: '' }, +] as const) + +const RUN_FLAGS = defineFlags([ + ...SELECTION_FLAGS, + { name: '--backfill', type: 'string', description: 'Stable backfill id; uses an isolated checkpoint namespace', placeholder: '' }, + { name: '--from', type: 'string', description: 'Backfill range lower bound (ISO timestamp)', placeholder: '' }, + { name: '--to', type: 'string', description: 'Backfill range upper bound (ISO timestamp)', placeholder: '' }, + { name: '--max-duration', type: 'string', description: 'Execution budget in seconds', placeholder: '' }, +] as const) + +interface IngestPluginCommandContext { + args: string[] + flags: Record + jsonMode: boolean + options: IngestOptions + config: ResolvedChxConfig + configPath: string + print: (value: unknown) => void + pluginContext?: { executor: ClickHouseExecutor; hasExecutor: boolean } +} + +const runCommand = createPluginRunner({ configErrorClass: IngestConfigError }) + +export function createIngestPlugin(options: IngestPluginOptions = {}) { + const optionsSchema = withFactoryDefaults(IngestOptionsSchema, options) + + return { + manifest: { name: 'ingest' as const, apiVersion: 1 as const }, + optionsSchema, + commands: [ + { + name: 'run', + description: 'Execute the selected ingestion streams from their journaled checkpoints', + flags: RUN_FLAGS, + optionsSchema, + run: runCommand({ + command: 'run', + label: 'Ingest run', + fn: async (context) => { + const selected = await loadSelection(context) + const backfill = parseBackfill(context.flags) + const maxDurationSeconds = parseMaxDuration(context.flags['--max-duration']) ?? context.options.maxDurationSeconds + const target = openTarget(context) + const abort = new AbortController() + const onSignal = () => abort.abort(new DOMException('Interrupted', 'AbortError')) + process.once('SIGINT', onSignal) + process.once('SIGTERM', onSignal) + + try { + const result = await runIngestion( + { selected, backfill }, + { + journal: createClickHouseJournal({ + executor: target.executor, + database: target.database, + targetId: target.targetId, + table: context.options.journalTable, + }), + destination: createClickHouseDestination(target.executor), + signal: abort.signal, + maxDurationMs: maxDurationSeconds * 1000, + prefetchBatches: context.options.prefetchBatches, + log: context.jsonMode ? undefined : (message) => context.print(message), + } + ) + if (context.jsonMode) { + context.print({ command: 'run', ...result }) + } else { + const rows = result.streams.reduce((sum, stream) => sum + stream.rows, 0) + context.print(`Ingest run ${result.runId}: ${result.ok ? 'ok' : 'incomplete'} (${result.streams.length} streams, ${rows} rows)`) + } + return result.ok ? 0 : 1 + } finally { + process.off('SIGINT', onSignal) + process.off('SIGTERM', onSignal) + await target.close() + } + }, + }), + }, + { + name: 'list', + description: 'List the ingestion streams in the loaded definition graph', + flags: SELECTION_FLAGS, + optionsSchema, + run: runCommand({ + command: 'list', + label: 'Ingest list', + fn: async (context) => { + const selected = await loadSelection(context) + const streams = selected.map(describeStream) + if (context.jsonMode) { + context.print({ ok: true, command: 'list', streams }) + } else { + for (const stream of streams) { + context.print(`${stream.streamId} -> ${stream.destination} [${stream.tags.join(', ')}]`) + } + } + return 0 + }, + }), + }, + { + name: 'status', + description: 'Show the committed checkpoint of each selected stream', + flags: SELECTION_FLAGS, + optionsSchema, + run: runCommand({ + command: 'status', + label: 'Ingest status', + fn: async (context) => { + const selected = await loadSelection(context) + const target = openTarget(context) + try { + const journal = createClickHouseJournal({ + executor: target.executor, + database: target.database, + targetId: target.targetId, + table: context.options.journalTable, + }) + await journal.ensure() + const streams = await Promise.all( + selected.map(async (entry) => { + const checkpoint = await journal.readCheckpoint(entry.stream.id) + return { ...describeStream(entry), checkpointVersion: checkpoint.version, checkpoint: checkpoint.envelope ?? null } + }) + ) + if (context.jsonMode) { + context.print({ ok: true, command: 'status', targetId: target.targetId, streams }) + } else { + for (const stream of streams) { + context.print(`${stream.streamId}: v${stream.checkpointVersion} ${stream.checkpoint ? JSON.stringify(stream.checkpoint.state) : '(no checkpoint)'}`) + } + } + return 0 + } finally { + await target.close() + } + }, + }), + }, + ], + hooks: { + async onCheck(context: { config: ResolvedChxConfig }) { + const findings = checkGraph(await loadGraph(context.config)) + return { + plugin: 'ingest', + evaluated: true, + ok: findings.every((finding) => finding.severity !== 'error'), + findings, + } + }, + }, + } +} + +export type IngestPlugin = ReturnType + +export function ingest(options: IngestPluginOptions = {}): ChxInlinePluginRegistration { + return { plugin: createIngestPlugin(options), name: 'ingest', enabled: true, options } +} + +/** Local, zero-network graph checks: they hold in `--offline` mode too. */ +export function checkGraph(pipelines: readonly PipelineDefinition[]) { + const findings: Array<{ code: string; message: string; severity: 'info' | 'warn' | 'error' }> = [] + if (pipelines.length === 0) { + findings.push({ code: 'ingest_no_pipelines', message: 'No ingestion pipeline is exported by the project entry.', severity: 'warn' }) + } + for (const pipeline of pipelines) { + for (const stream of pipeline.streams) { + const columns = new Set(stream.destination.columns.map((column) => column.name)) + const missing = REQUIRED_COLUMNS.filter((name) => !columns.has(name)) + if (missing.length > 0) { + findings.push({ + code: 'ingest_missing_metadata_columns', + message: `Stream "${stream.id}" destination ${stream.destination.database}.${stream.destination.name} is missing ${missing.join(', ')}. Spread ingestionColumns into its columns.`, + severity: 'error', + }) + } + } + } + return findings +} + +// Discover values exported from the configured entry (or legacy schema files). +async function loadGraph(config: ResolvedChxConfig): Promise { + if (config.schema.length === 0) return [] + return collectPipelines(await loadDefinitionModules(config.schema, { cwd: process.cwd() })) +} + +async function loadSelection(context: IngestPluginCommandContext): Promise { + const pipelines = await loadGraph(context.config) + if (pipelines.length === 0) { + throw new IngestConfigError('No ingestion pipeline is exported. Export a definePipeline(...) value from the module configured as "entry".') + } + const tags = context.flags['--tag'] + return selectStreams(pipelines, Array.isArray(tags) ? tags : typeof tags === 'string' ? [tags] : []) +} + +function openTarget(context: IngestPluginCommandContext) { + const clickhouse = context.config.clickhouse + // Ingestion requires JSONEachRow and per-insert deduplication settings. + // The host executor contract does not guarantee either capability. + // Streams fetch, load and journal concurrently, so the executor must not be + // bound to one ClickHouse HTTP session (a session allows one in-flight query). + if (clickhouse) { + const executor = createStatelessClickHouseExecutor(clickhouse) + return { executor, database: clickhouse.database, targetId: targetIdOf(clickhouse.url, clickhouse.database), close: () => executor.close() } + } + throw new IngestConfigError('Ingestion requires a direct ClickHouse connection for JSON rows and deduplication settings. Configure clickhouse in your clickhouse.config.ts; a host-provided executor is not supported.') +} + +function targetIdOf(url: string, database: string): string { + try { + return `${new URL(url).host}/${database}` + } catch { + return `${url}/${database}` + } +} + +function parseBackfill(flags: IngestPluginCommandContext['flags']): BackfillRequest | undefined { + const id = flags['--backfill'] + const from = parseTimestamp(flags['--from'], '--from') + const to = parseTimestamp(flags['--to'], '--to') + if (typeof id !== 'string') { + if (from || to) throw new IngestConfigError('--from and --to require --backfill : explicit ranges never touch the scheduled checkpoint.') + return undefined + } + if (!/^[A-Za-z0-9][A-Za-z0-9_.-]*$/.test(id)) throw new IngestConfigError(`Invalid --backfill id "${id}".`) + return { id, from, to } +} + +function parseTimestamp(raw: string | string[] | boolean | undefined, flag: string): Date | undefined { + if (typeof raw !== 'string') return undefined + const date = new Date(raw) + if (Number.isNaN(date.getTime())) throw new IngestConfigError(`Invalid timestamp for ${flag}: ${raw}`) + return date +} + +function parseMaxDuration(raw: string | string[] | boolean | undefined): number | undefined { + if (typeof raw !== 'string') return undefined + const seconds = Number(raw) + if (!Number.isFinite(seconds) || seconds <= 0) throw new IngestConfigError(`Invalid value for --max-duration: ${raw}`) + return seconds +} + +function describeStream(entry: SelectedStream) { + return { + streamId: entry.stream.id, + pipelineId: entry.pipeline.id, + destination: `${entry.stream.destination.database}.${entry.stream.destination.name}`, + strategy: `${entry.stream.incremental.id}@${entry.stream.incremental.version}`, + tags: [...entry.effectiveTags], + } +} diff --git a/packages/plugin-ingest/src/queue.ts b/packages/plugin-ingest/src/queue.ts new file mode 100644 index 00000000..0b1bb0a6 --- /dev/null +++ b/packages/plugin-ingest/src/queue.ts @@ -0,0 +1,56 @@ +export interface BoundedQueue { + /** Waits while the queue is full, which is what applies backpressure to the reader. */ + push(item: T, signal: AbortSignal): Promise + /** Resolves `undefined` once the queue is closed and drained. */ + pop(signal: AbortSignal): Promise + close(): void +} + +export function createBoundedQueue(capacity: number): BoundedQueue { + const limit = Math.max(1, capacity) + const items: T[] = [] + let closed = false + let wake: Array<() => void> = [] + + const notify = () => { + const waiting = wake + wake = [] + for (const resolve of waiting) resolve() + } + + const wait = (signal: AbortSignal) => + new Promise((resolve, reject) => { + const onAbort = () => reject(signal.reason) + signal.addEventListener('abort', onAbort, { once: true }) + wake.push(() => { + signal.removeEventListener('abort', onAbort) + resolve() + }) + }) + + return { + async push(item, signal) { + while (items.length >= limit) { + signal.throwIfAborted() + await wait(signal) + } + signal.throwIfAborted() + items.push(item) + notify() + }, + async pop(signal) { + while (items.length === 0) { + if (closed) return undefined + signal.throwIfAborted() + await wait(signal) + } + const item = items.shift() + notify() + return item + }, + close() { + closed = true + notify() + }, + } +} diff --git a/packages/plugin-ingest/src/registry.ts b/packages/plugin-ingest/src/registry.ts new file mode 100644 index 00000000..d5419a22 --- /dev/null +++ b/packages/plugin-ingest/src/registry.ts @@ -0,0 +1,182 @@ +import type { TableDefinition } from '@chkit/core' + +import { IngestConfigError } from './errors.js' +import { fullSync } from './incremental.js' +import type { + AnyStreamDefinition, + ErrorClassifier, + IncrementalStrategy, + LoaderFactory, + PipelineDefinition, + ReadContext, + RetryOptions, + Row, + SourceChunk, + StreamBudget, + StreamDefinition, +} from './types.js' + +const STREAM_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9_.:-]*$/ + +export interface StreamInput { + id: string + destination: TableDefinition + tags?: readonly string[] + incremental: IncrementalStrategy + read: (context: ReadContext) => AsyncIterable> + loader?: LoaderFactory + retry?: RetryOptions + batchSize?: number + budget?: StreamBudget + classifyError?: ErrorClassifier +} + +export interface FullSyncStreamInput + extends Omit, 'incremental'> { + incremental?: undefined +} + +export interface PipelineInput { + id: string + tags?: readonly string[] + streams: readonly AnyStreamDefinition[] + maxStreams?: number + maxFetches?: number + maxLoads?: number + retry?: RetryOptions +} + +export interface SelectedStream { + stream: AnyStreamDefinition + pipeline: PipelineDefinition + effectiveTags: readonly string[] +} + +export function defineStream(input: FullSyncStreamInput): StreamDefinition +export function defineStream( + input: StreamInput +): StreamDefinition +export function defineStream( + input: StreamInput | FullSyncStreamInput +): AnyStreamDefinition { + assertValidId('stream', input.id) + if (input.batchSize !== undefined && (!Number.isInteger(input.batchSize) || input.batchSize <= 0)) { + throw new IngestConfigError(`Stream "${input.id}": batchSize must be a positive integer.`) + } + return { + kind: 'ingest_stream', + id: input.id, + destination: input.destination, + tags: dedupe(input.tags ?? []), + incremental: input.incremental ?? fullSync(), + read: input.read, + loader: input.loader, + retry: input.retry, + batchSize: input.batchSize, + budget: input.budget, + classifyError: input.classifyError, + } +} + +/** + * Define a non-durable named group of streams. Pipeline identity never + * participates in checkpoint or batch identity, so moving a stream between + * pipelines does not reset its state. + */ +export function definePipeline(input: PipelineInput): PipelineDefinition { + assertValidId('pipeline', input.id) + const pipeline: PipelineDefinition = { + kind: 'ingest_pipeline', + id: input.id, + tags: dedupe(input.tags ?? []), + streams: [...input.streams], + maxStreams: positiveCeiling(input.id, 'maxStreams', input.maxStreams, 4), + maxFetches: positiveCeiling(input.id, 'maxFetches', input.maxFetches, 4), + maxLoads: positiveCeiling(input.id, 'maxLoads', input.maxLoads, 2), + retry: input.retry, + } + + const seen = new Set() + for (const stream of pipeline.streams) { + if (seen.has(stream.id)) { + throw new IngestConfigError(`Pipeline "${pipeline.id}" lists stream "${stream.id}" more than once.`) + } + seen.add(stream.id) + } + + return pipeline +} + +/** Validate identities within this graph, without process-wide registration. */ +function validatePipelines(pipelines: readonly PipelineDefinition[]): void { + const ids = new Set() + const owners = new Map() + for (const pipeline of pipelines) { + if (ids.has(pipeline.id)) throw new IngestConfigError(`Duplicate pipeline id "${pipeline.id}".`) + ids.add(pipeline.id) + for (const stream of pipeline.streams) { + const owner = owners.get(stream.id) + if (owner !== undefined) { + throw new IngestConfigError( + `Stream id "${stream.id}" occurs in pipelines "${owner}" and "${pipeline.id}". Stream ids must be globally unique.` + ) + } + owners.set(stream.id, pipeline.id) + } + } +} + +/** Only exported pipeline values participate; aliases of the same value count once. */ +export function collectPipelines(modules: readonly Record[]): PipelineDefinition[] { + const pipelines = [...new Set(modules.flatMap((mod) => Object.values(mod).filter( + (value): value is PipelineDefinition => typeof value === 'object' && value !== null && 'kind' in value && value.kind === 'ingest_pipeline' + )))] + validatePipelines(pipelines) + return pipelines +} + +/** + * Repeated exact case-sensitive tags with AND semantics over the union of + * pipeline and stream tags plus derived `pipeline:` and `stream:`. + * No filter selects the complete graph; an explicit empty selection throws. + */ +export function selectStreams(pipelines: readonly PipelineDefinition[], tags: readonly string[]): SelectedStream[] { + validatePipelines(pipelines) + const wanted = dedupe(tags) + const all: SelectedStream[] = pipelines.flatMap((pipeline) => + pipeline.streams.map((stream) => ({ + stream, + pipeline, + effectiveTags: dedupe([...pipeline.tags, ...stream.tags, `pipeline:${pipeline.id}`, `stream:${stream.id}`]), + })) + ) + if (wanted.length === 0) return all + + const selected = all.filter((entry) => wanted.every((tag) => entry.effectiveTags.includes(tag))) + if (selected.length === 0) { + throw new IngestConfigError( + `No stream matches every requested tag: ${wanted.map((tag) => `--tag ${tag}`).join(' ')}. Nothing was executed.` + ) + } + return selected +} + +function assertValidId(kind: 'stream' | 'pipeline', id: string): void { + if (!STREAM_ID_PATTERN.test(id)) { + throw new IngestConfigError( + `Invalid ${kind} id "${id}". Use letters, digits, "_", ".", ":" or "-", starting with a letter or digit.` + ) + } +} + +function positiveCeiling(pipelineId: string, name: string, value: number | undefined, fallback: number): number { + if (value === undefined) return fallback + if (!Number.isInteger(value) || value <= 0) { + throw new IngestConfigError(`Pipeline "${pipelineId}": ${name} must be a positive integer.`) + } + return value +} + +function dedupe(values: readonly string[]): string[] { + return [...new Set(values)] +} diff --git a/packages/plugin-ingest/src/retry.test.ts b/packages/plugin-ingest/src/retry.test.ts new file mode 100644 index 00000000..e7a3c5b2 --- /dev/null +++ b/packages/plugin-ingest/src/retry.test.ts @@ -0,0 +1,107 @@ +import { describe, expect, test } from 'bun:test' + +import { FetchFailure, HttpError, IngestConfigError } from './errors.js' +import { runAttempt } from './retry.js' +import type { RetryContext } from './types.js' + +const signal = new AbortController().signal +const env = { signal, classifier: undefined, onRetry: () => undefined } +const immediate = { minTimeout: 0, randomize: false } +const httpError = (status: number, retryAfter = '0') => + HttpError.fromResponse(new Response('', { status, headers: { 'retry-after': retryAfter } })) + +describe('provider retry policy', () => { + test('honors Retry-After even when rate limits do not consume the retry budget', async () => { + const limited = await httpError(429, '0.03') + const transient = await httpError(503) + const retries: RetryContext[] = [] + const times: number[] = [] + const value = await runAttempt(async (attempt) => { + times.push(performance.now()) + if (attempt === 1) throw limited + if (attempt === 2) throw transient + return 'ok' + }, { + ...immediate, + retries: 1, + shouldConsumeRetry: ({ error }) => error.classification.kind !== 'rate_limited', + }, { ...env, onRetry: (context) => { retries.push(context) } }) + + expect(value).toBe('ok') + expect(times).toHaveLength(3) + expect((times[1] ?? 0) - (times[0] ?? 0)).toBeGreaterThanOrEqual(25) + expect(retries.map(({ attemptNumber, retriesLeft, retriesConsumed }) => ({ attemptNumber, retriesLeft, retriesConsumed }))).toEqual([ + { attemptNumber: 1, retriesLeft: 1, retriesConsumed: 0 }, + { attemptNumber: 2, retriesLeft: 1, retriesConsumed: 0 }, + ]) + expect(retries[0]?.error.cause).toBe(limited) + }) + + test('does not schedule a retry when Retry-After exceeds maxRetryTime', async () => { + let calls = 0 + let scheduled = 0 + const failure = await httpError(429, '60') + await expect(runAttempt(async () => { + calls += 1 + throw failure + }, { ...immediate, maxRetryTime: 100 }, { + ...env, onRetry: () => { scheduled += 1 }, + })).rejects.toMatchObject({ cause: failure }) + expect(calls).toBe(1) + expect(scheduled).toBe(0) + }) + + test.each(['backoff', 'retry-after'] as const)('cancellation interrupts %s without calling the provider again', async (mode) => { + const controller = new AbortController() + const failure = await httpError(mode === 'backoff' ? 503 : 429, '60') + let calls = 0 + const timer = setTimeout(() => controller.abort(new Error('cancelled')), 20) + try { + await expect(runAttempt(async () => { + calls += 1 + throw failure + }, { minTimeout: mode === 'backoff' ? 60_000 : 0 }, { ...env, signal: controller.signal })).rejects.toThrow() + expect(calls).toBe(1) + } finally { + clearTimeout(timer) + } + }) + + test('preserves the thrown value and allows the retry veto', async () => { + const cause = { providerCode: 'NO_ACCESS' } + let calls = 0 + await expect(runAttempt(async () => { + calls += 1 + throw cause + }, { ...immediate, shouldRetry: ({ error }) => error.cause !== cause }, env)).rejects.toMatchObject({ cause }) + expect(calls).toBe(1) + }) + + test('execution cancellation stays authoritative over the provider classifier', async () => { + const controller = new AbortController() + let classifications = 0 + let calls = 0 + await expect(runAttempt(async () => { + calls += 1 + controller.abort(new Error('host cancelled')) + throw new DOMException('request aborted', 'AbortError') + }, immediate, { + ...env, signal: controller.signal, + classifier: () => { classifications += 1; return { kind: 'transient' } }, + })).rejects.toThrow() + expect(calls).toBe(1) + expect(classifications).toBe(0) + }) + + test('does not retry permanent, configuration, or already-exhausted source failures', async () => { + const exhausted = new FetchFailure(new Error('already retried'), { kind: 'transient' }) + for (const cause of [await httpError(401), new IngestConfigError('invalid'), exhausted]) { + let calls = 0 + await expect(runAttempt(async () => { + calls += 1 + throw cause + }, immediate, env)).rejects.toThrow() + expect(calls).toBe(1) + } + }) +}) diff --git a/packages/plugin-ingest/src/retry.ts b/packages/plugin-ingest/src/retry.ts new file mode 100644 index 00000000..4517eb7c --- /dev/null +++ b/packages/plugin-ingest/src/retry.ts @@ -0,0 +1,77 @@ +import { setTimeout as sleep } from 'node:timers/promises' + +import pRetry, { AbortError } from 'p-retry' + +import { classifyFailure, FetchFailure, IngestConfigError } from './errors.js' +import type { ErrorClassifier, RetryContext, RetryOptions } from './types.js' + +const DEFAULT_RETRY = { + retries: 5, + factor: 2, + minTimeout: 1000, + maxTimeout: 60_000, + randomize: true, + maxRetryTime: 10 * 60_000, +} + +interface AttemptEnv { + signal: AbortSignal + classifier: ErrorClassifier | undefined + onRetry: (context: RetryContext, retryAfterMs: number | undefined) => Promise | void +} + +/** Pipeline default, partially overridden by the stream. */ +export function mergeRetry(...layers: Array): RetryOptions { + const merged: RetryOptions = {} + for (const layer of layers) { + if (!layer) continue + for (const [key, value] of Object.entries(layer)) { + if (value !== undefined) Object.assign(merged, { [key]: value }) + } + } + return merged +} + +/** p-retry owns retry budgets and backoff; this adapter adds provider classification and Retry-After. */ +export function runAttempt( + operation: (attemptNumber: number) => Promise, + policy: RetryOptions, + env: AttemptEnv +): Promise { + const options = { ...DEFAULT_RETRY, ...mergeRetry(policy) } + const deadline = performance.now() + options.maxRetryTime + let notBefore = 0 + + return pRetry(async (attemptNumber) => { + // Backoff and Retry-After overlap: only wait out the remaining provider embargo. + const remaining = notBefore - performance.now() + if (remaining > 0) await sleep(remaining, undefined, { signal: env.signal }) + env.signal.throwIfAborted() + try { + return await operation(attemptNumber) + } catch (cause) { + // A nested source attempt already exhausted its retries. Never replay it + // through the reader's coarser retry boundary. + if (cause instanceof FetchFailure || cause instanceof IngestConfigError) throw new AbortError(cause) + const failure = new FetchFailure(cause, classifyFailure(cause, env.signal, env.classifier)) + if (failure.classification.kind === 'cancelled' || failure.classification.kind === 'permanent') throw new AbortError(failure) + throw failure + } + }, { + ...options, + signal: env.signal, + shouldConsumeRetry: ({ error, ...context }) => + error instanceof FetchFailure ? (policy.shouldConsumeRetry?.({ ...context, error }) ?? true) : true, + shouldRetry: async ({ error, ...context }) => { + env.signal.throwIfAborted() + if (!(error instanceof FetchFailure)) return false + const retryContext = { ...context, error } + if (policy.shouldRetry && !(await policy.shouldRetry(retryContext))) return false + const retryAfterMs = error.classification.kind === 'rate_limited' ? error.classification.retryAfterMs : undefined + notBefore = performance.now() + (retryAfterMs ?? 0) + if (notBefore > deadline) return false + await env.onRetry(retryContext, retryAfterMs) + return true + }, + }) +} diff --git a/packages/plugin-ingest/src/testing.ts b/packages/plugin-ingest/src/testing.ts new file mode 100644 index 00000000..6eb3c8ab --- /dev/null +++ b/packages/plugin-ingest/src/testing.ts @@ -0,0 +1,57 @@ +import { emptyCheckpoint, toJournalRow, type JournalRow } from './journal.js' +import type { CommittedCheckpoint, DestinationAdapter, Journal, JournalEvent, Row } from './types.js' + +export interface MemoryJournal extends Journal { + readonly events: JournalEvent[] + readonly rows: JournalRow[] +} + +export interface MemoryDestination extends DestinationAdapter { + /** Physical rows per `database.table`, after token deduplication. */ + readonly tables: Map + readonly tokens: string[] +} + +/** In-memory journal with the same projection semantics as the ClickHouse one. */ +export function createMemoryJournal(): MemoryJournal { + const events: JournalEvent[] = [] + const rows: JournalRow[] = [] + return { + events, + rows, + async ensure() {}, + async append(event) { + events.push(event) + rows.push(toJournalRow(event, 'memory', new Date(0))) + }, + async readCheckpoint(namespaceId): Promise { + const scoped = events.filter((event) => event.namespaceId === namespaceId) + if (scoped.length === 0) return emptyCheckpoint() + const headSeq = Math.max(...scoped.map((event) => event.eventSeq)) + const lastSuccessSeq = Math.max(0, ...scoped + .filter((event) => event.eventKind === 'work_finished' && event.workState === 'succeeded') + .map((event) => event.eventSeq)) + const committed = scoped + .filter((event) => event.eventKind === 'batch_committed') + .sort((a, b) => a.checkpointVersion - b.checkpointVersion || a.eventSeq - b.eventSeq) + .at(-1) + return { version: committed?.checkpointVersion ?? 0, envelope: committed?.checkpoint, headSeq, lastSuccessSeq } + }, + } +} + +/** In-memory destination that honours `insert_deduplication_token` like ClickHouse. */ +export function createMemoryDestination(): MemoryDestination { + const tables = new Map() + const tokens: string[] = [] + return { + tables, + tokens, + async insert({ table, rows, token }) { + const key = `${table.database}.${table.name}` + if (tokens.includes(`${key}:${token}`)) return + tokens.push(`${key}:${token}`) + tables.set(key, [...(tables.get(key) ?? []), ...rows]) + }, + } +} diff --git a/packages/plugin-ingest/src/types.ts b/packages/plugin-ingest/src/types.ts new file mode 100644 index 00000000..6526170f --- /dev/null +++ b/packages/plugin-ingest/src/types.ts @@ -0,0 +1,259 @@ +import type { Options as PRetryOptions, RetryContext as PRetryContext } from 'p-retry' + +import type { TableDefinition } from '@chkit/core' + +import type { FetchFailure } from './errors.js' + +// ───── Authoring: streams and pipelines ───── + +/** JSON-compatible column values; the ClickHouse destination encodes bigint as decimal strings. */ +export type Row = Record + +/** + * One bounded unit produced by a stream reader. Rows are already shaped for the + * destination table: mapping is ordinary code inside the reader. + * + * `state` is optional and, when present, must be the COMPLETE candidate + * provider state that is safe to commit once every row up to and including + * this chunk has sink evidence. The executor never infers ordering: a reader + * that cannot claim a safe frontier simply omits `state`. + */ +export interface SourceChunk { + rows: readonly TRow[] + state?: TState + /** + * Stable, non-secret identity of the logical source interval this chunk + * covers (a page cursor, an id range, a day…). Declare it only when a replay + * of the same interval is the same logical write: batch identity then ignores + * row content, so mutable provider fields cannot defeat retry deduplication. + * Without it, identity falls back to a content hash, which prefers a possible + * duplicate over suppressing rows that changed between attempts. + */ + id?: string +} + +export interface AttemptOptions { + /** Short non-secret label used in journal and telemetry, e.g. `GET /v2/objects`. */ + label?: string +} + +/** Source request capabilities, independent of selection and checkpoint types. */ +export interface FetchContext { + signal: AbortSignal + /** + * Run one source operation under executor authority: fetch permit, retry + * policy, failure classification and cancellation. Operations that bypass it + * only get coarse recovery (reader recreation from the last checkpoint). + */ + attempt(operation: (signal: AbortSignal) => Promise, options?: AttemptOptions): Promise +} + +export interface ReadContext extends FetchContext { + streamId: string + /** What to read this execution, planned by the incremental strategy. */ + selection: TSelection + /** Last committed provider state, already validated by the strategy. */ + state: TState | undefined + /** Immutable cutoff shared by every stream selected in this execution. */ + cutoff: Date +} + +/** + * Provider-owned incremental strategy inside the ChKit-owned checkpoint + * envelope `{ strategy, version, state }`. Core treats `state` as opaque. + */ +export interface IncrementalStrategy { + readonly id: string + readonly version: number + /** Validate previously committed state. Throw when it cannot be trusted. */ + parseState(raw: unknown): TState + plan(input: PlanInput): TSelection + /** + * Candidate state once the WHOLE selection has sink evidence. Return + * `undefined` to leave the checkpoint untouched. + */ + complete?(input: { state: TState | undefined; selection: TSelection }): TState | undefined +} + +export interface PlanInput { + state: TState | undefined + cutoff: Date + /** Explicit historical range from a backfill invocation. */ + range: { from: Date | undefined; to: Date | undefined } | undefined +} + +/** p-retry context with the original provider failure and its classification. */ +export type RetryContext = Omit & { error: FetchFailure } + +export type RetryOptions = Pick & { + shouldRetry?: (context: RetryContext) => boolean | Promise + shouldConsumeRetry?: (context: RetryContext) => boolean | Promise +} + +export interface StreamBudget { + /** Maximum source chunks pulled in one execution of this stream. */ + maxChunks?: number + /** Largest chunk a reader may yield. A bigger chunk fails the stream instead of being buffered. */ + maxChunkRows?: number +} + +export interface StreamDefinition { + readonly kind: 'ingest_stream' + /** Globally stable identity. Owns the checkpoint; never derived from the pipeline. */ + readonly id: string + readonly destination: TableDefinition + readonly tags: readonly string[] + readonly incremental: IncrementalStrategy + readonly read: (context: ReadContext) => AsyncIterable> + readonly loader: LoaderFactory | undefined + readonly retry: RetryOptions | undefined + readonly batchSize: number | undefined + readonly budget: StreamBudget | undefined + readonly classifyError: ErrorClassifier | undefined +} + +// biome-ignore lint/suspicious/noExplicitAny: a heterogeneous stream list cannot share row/state generics +export type AnyStreamDefinition = StreamDefinition + +export interface PipelineDefinition { + readonly kind: 'ingest_pipeline' + readonly id: string + readonly tags: readonly string[] + readonly streams: readonly AnyStreamDefinition[] + readonly maxStreams: number + readonly maxFetches: number + readonly maxLoads: number + readonly retry: RetryOptions | undefined +} + +// ───── Failure classification ───── + +export type FailureClass = + | { kind: 'cancelled' } + | { kind: 'rate_limited'; retryAfterMs: number | undefined } + | { kind: 'transient' } + | { kind: 'permanent' } + | { kind: 'unknown' } + +export type ErrorClassifier = (cause: unknown, fallback: FailureClass) => FailureClass | undefined + +// ───── Loading ───── + +export type SinkEvidenceKind = 'clickhouse_ack' | 'none_required' + +export interface SinkReceipt { + evidence: SinkEvidenceKind + rows: number + writeUnits: number +} + +export interface LoadBatch { + /** Stable across every retry and crash replay of the same logical batch. */ + batchId: string + rows: readonly Row[] +} + +export interface LoaderContext { + streamId: string + runId: string + table: TableDefinition + destination: DestinationAdapter + signal: AbortSignal +} + +export interface Loader { + readonly ctx: LoaderContext + write(batch: LoadBatch): Promise + finalize(): Promise + abort(reason: unknown): Promise +} + +export type LoaderFactory = (ctx: LoaderContext) => Loader + +/** Low-level destination writes performed on behalf of a portable Loader. */ +export interface DestinationAdapter { + insert(input: { + table: TableDefinition + rows: readonly Row[] + /** Deterministic `insert_deduplication_token` for this write unit. */ + token: string + }): Promise +} + +// ───── Journal ───── + +export type JournalEventKind = + | 'run_started' + | 'work_planned' + | 'attempt_started' + | 'retry_scheduled' + | 'batch_committed' + | 'work_finished' + | 'run_finished' + +export type WorkState = '' | 'planned' | 'running' | 'succeeded' | 'failed' | 'budget_exhausted' | 'cancelled' + +export interface CheckpointEnvelope { + strategy: string + version: number + state: unknown +} + +export interface JournalEvent { + namespaceId: string + eventSeq: number + eventKind: JournalEventKind + runId: string + workId: string + attemptNo: number + batchId: string + expectedCheckpointVersion: number + checkpointVersion: number + checkpoint: CheckpointEnvelope | undefined + workState: WorkState + sinkEvidence: SinkEvidenceKind | '' + retryAt: Date | undefined + errorClass: string + detail: Record +} + +export interface CommittedCheckpoint { + version: number + envelope: CheckpointEnvelope | undefined + /** Highest journal sequence observed for the namespace (any event kind). */ + headSeq: number + /** Last successful work_finished sequence, or zero. Separates new syncs from replays. */ + lastSuccessSeq: number +} + +/** Authoritative append-only control state. Checkpoints are projections of it. */ +export interface Journal { + ensure(): Promise + append(event: JournalEvent): Promise + readCheckpoint(namespaceId: string): Promise +} + +// ───── Execution ───── + +export type StreamOutcome = 'succeeded' | 'failed' | 'budget_exhausted' | 'cancelled' + +export interface StreamResult { + streamId: string + pipelineId: string + namespaceId: string + outcome: StreamOutcome + rows: number + batches: number + chunks: number + checkpointVersion: number + error: string | undefined +} + +export interface ExecutionResult { + runId: string + cutoff: string + streams: StreamResult[] + ok: boolean +} diff --git a/packages/plugin-ingest/tsconfig.json b/packages/plugin-ingest/tsconfig.json new file mode 100644 index 00000000..d38ac708 --- /dev/null +++ b/packages/plugin-ingest/tsconfig.json @@ -0,0 +1,9 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "rootDir": "src", + "outDir": "dist" + }, + "include": ["src/**/*.ts"], + "exclude": ["src/**/*.test.ts"] +} diff --git a/packages/plugin-obsessiondb/src/auth/api-client.ts b/packages/plugin-obsessiondb/src/auth/api-client.ts index 3c427f49..c98649b4 100644 --- a/packages/plugin-obsessiondb/src/auth/api-client.ts +++ b/packages/plugin-obsessiondb/src/auth/api-client.ts @@ -1,4 +1,5 @@ import { createRequire } from 'node:module' +import { setTimeout as sleep } from 'node:timers/promises' interface DeviceCodeResponse { device_code: string @@ -81,7 +82,7 @@ export async function pollDeviceToken( let pollInterval = interval * 1000 while (Date.now() < deadline) { - await new Promise((r) => setTimeout(r, pollInterval)) + await sleep(pollInterval) const res = await fetch(`${baseUrl}/api/auth/device/token`, { method: 'POST',