From 461c3bac849005708f8b5853aae8af5317a7b3a6 Mon Sep 17 00:00:00 2001 From: vesperships Date: Mon, 28 Sep 2026 17:42:45 -0700 Subject: [PATCH] chore: version packages --- .changeset/pre.json | 20 +++++- bun.lock | 20 +++--- packages/cli/CHANGELOG.md | 80 ++++++++++++++++++++++++ packages/cli/package.json | 2 +- packages/clickhouse/CHANGELOG.md | 62 ++++++++++++++++++ packages/clickhouse/package.json | 2 +- packages/codegen/CHANGELOG.md | 14 +++++ packages/codegen/package.json | 2 +- packages/core/CHANGELOG.md | 55 ++++++++++++++++ packages/core/package.json | 2 +- packages/create-chkit/CHANGELOG.md | 13 ++++ packages/create-chkit/package.json | 2 +- packages/plugin-backfill/CHANGELOG.md | 20 ++++++ packages/plugin-backfill/package.json | 2 +- packages/plugin-codegen/CHANGELOG.md | 23 +++++++ packages/plugin-codegen/package.json | 2 +- packages/plugin-ingest/CHANGELOG.md | 34 ++++++++++ packages/plugin-ingest/package.json | 2 +- packages/plugin-obsessiondb/CHANGELOG.md | 26 ++++++++ packages/plugin-obsessiondb/package.json | 2 +- packages/plugin-pull/CHANGELOG.md | 55 ++++++++++++++++ packages/plugin-pull/package.json | 2 +- 22 files changed, 420 insertions(+), 22 deletions(-) create mode 100644 packages/plugin-ingest/CHANGELOG.md diff --git a/.changeset/pre.json b/.changeset/pre.json index 3244c220..d4f01121 100644 --- a/.changeset/pre.json +++ b/.changeset/pre.json @@ -11,17 +11,33 @@ "@chkit/plugin-backfill": "0.1.1", "@chkit/plugin-codegen": "0.1.1", "@chkit/plugin-obsessiondb": "0.1.1", - "@chkit/plugin-pull": "0.1.1" + "@chkit/plugin-pull": "0.1.1", + "@chkit/plugin-ingest": "0.1.2-beta.7" }, "changesets": [ + "api-sync-docs-rename", "backfill-mv-replay-empty-target", "backfill-replay-all-mvs", "backfill-sql-scanner", + "cli-skills-proxy", + "create-chkit-next-steps-dedup", "dictionary-schema-primitive", + "drift-index-arguments", "generate-empty-migration", "index-only-projections", + "ingest-journal-replica-resilience", + "ingest-run-namespace", + "ingestion-runtime", + "init-connect-runbook-consistency", + "kafka-table-engine", "key-clause-function-expressions", + "migrate-async-poll-resilience", + "obsessiondb-json-no-bare-string", + "obsessiondb-logout-no-session", + "orpc-deserialization-security", "pull-clause-and-derived-primary-key", - "shared-plugin-command-runner" + "pull-route-through-executor", + "shared-plugin-command-runner", + "text-index-round-trips" ] } diff --git a/bun.lock b/bun.lock index 918afc4c..e4c1e8c9 100644 --- a/bun.lock +++ b/bun.lock @@ -47,7 +47,7 @@ }, "packages/cli": { "name": "chkit", - "version": "0.1.2-beta.7", + "version": "0.2.0-beta.8", "bin": { "chkit": "./dist/bin/chkit.js", }, @@ -65,7 +65,7 @@ }, "packages/clickhouse": { "name": "@chkit/clickhouse", - "version": "0.1.2-beta.7", + "version": "0.2.0-beta.8", "dependencies": { "@chkit/core": "workspace:*", "@clickhouse/client": "^1.18.0", @@ -75,14 +75,14 @@ }, "packages/codegen": { "name": "@chkit/codegen", - "version": "0.1.2-beta.7", + "version": "0.2.0-beta.8", "dependencies": { "@chkit/core": "workspace:*", }, }, "packages/core": { "name": "@chkit/core", - "version": "0.1.2-beta.7", + "version": "0.2.0-beta.8", "dependencies": { "fast-glob": "^3.3.2", "jiti": "^2.7.0", @@ -93,7 +93,7 @@ }, "packages/create-chkit": { "name": "create-chkit", - "version": "0.1.2-beta.7", + "version": "0.2.0-beta.8", "bin": { "create-chkit": "./dist/bin/create-chkit.js", }, @@ -110,7 +110,7 @@ }, "packages/plugin-backfill": { "name": "@chkit/plugin-backfill", - "version": "0.1.2-beta.7", + "version": "0.2.0-beta.8", "dependencies": { "@chkit/clickhouse": "workspace:*", "@chkit/core": "workspace:*", @@ -121,7 +121,7 @@ }, "packages/plugin-codegen": { "name": "@chkit/plugin-codegen", - "version": "0.1.2-beta.7", + "version": "0.2.0-beta.8", "dependencies": { "@chkit/core": "workspace:*", }, @@ -134,7 +134,7 @@ }, "packages/plugin-ingest": { "name": "@chkit/plugin-ingest", - "version": "0.1.2-beta.7", + "version": "0.2.0-beta.8", "dependencies": { "@chkit/clickhouse": "workspace:*", "@chkit/core": "workspace:*", @@ -146,7 +146,7 @@ }, "packages/plugin-obsessiondb": { "name": "@chkit/plugin-obsessiondb", - "version": "0.1.2-beta.7", + "version": "0.2.0-beta.8", "dependencies": { "@chkit/clickhouse": "workspace:*", "@chkit/core": "workspace:*", @@ -159,7 +159,7 @@ }, "packages/plugin-pull": { "name": "@chkit/plugin-pull", - "version": "0.1.2-beta.7", + "version": "0.2.0-beta.8", "dependencies": { "@chkit/clickhouse": "workspace:*", "@chkit/core": "workspace:*", diff --git a/packages/cli/CHANGELOG.md b/packages/cli/CHANGELOG.md index b8720131..06d44053 100644 --- a/packages/cli/CHANGELOG.md +++ b/packages/cli/CHANGELOG.md @@ -1,5 +1,85 @@ # chkit +## 0.2.0-beta.8 + +### Minor Changes + +- fedbf56: Support Kafka engine tables without MergeTree key clauses, with escaped literal + settings, pull round trips, and normalized drift/check comparisons. Preserve the + existing setting-string contract for other engines. Reject unsupported Kafka + changes before writing migration artifacts and document an explicit, destructive- + gated replacement workflow. Handle quoted delimiters and escaped trailing + backslashes in introspection and migration statement splitting. +- 256ec62: Support ClickHouse `text` indexes in schemas, migrations, pull, and drift. Text + indexes require a tokenizer and support preprocessing, postprocessing, phrase + search, and dictionary/posting-list options when supported by the server. + Granularity is automatic. Preserve whitespace and escapes inside SQL literals, + compare parameter order and SQL formatting consistently, and reject unsupported + or malformed metadata instead of silently losing settings. Includes Python parity + and live adversarial round-trip tests on ClickHouse 26.3 and 26.8. + +### Patch Changes + +- 3042c56: Point README documentation links at the renamed API Sync section (`/api-sync/`). +- 4ded781: Add a `chkit skills` command that proxies to the external `skills` CLI (e.g. `chkit skills add obsessiondb/chkit` runs `npx skills add obsessiondb/chkit`). The agent skill is installed by the separate `skills` tool, not a chkit subcommand, so users who reached for `chkit skills add …` previously hit "Unknown command: skills". The command forwards its arguments and passes through the underlying exit code, and is handled before config loading so it works without a project. +- 65c90d6: Add `dictionary()` as a first-class ClickHouse schema primitive, mirroring `materializedView()` across the full lifecycle: DSL authoring, validation, canonicalization, SQL rendering, migration planning/diff, drift, `check`, destructive-op safety, `pull` introspection, and `codegen` typed interfaces. + + - `dictionary({ database, name, attributes, primaryKey, source, layout, lifetime, range?, settings?, comment? })` — attributes support `default`/`expression` (mutually exclusive), `hierarchical`, `bidirectional` (requires `hierarchical`), `injective`, and `isObjectId`. `range: { min, max }` renders `RANGE(MIN ... MAX ...)` for `RANGE_HASHED`/`COMPLEX_KEY_RANGE_HASHED` layouts, and `settings` renders `SETTINGS(...)`. + - ClickHouse has no `ALTER DICTIONARY`, so any structural change plans a single atomic `CREATE OR REPLACE DICTIONARY`. Dropping a dictionary is treated as destructive and blocked without `--allow-destructive`. + - Set `renamedFrom` on a dictionary (or pass `--rename-dictionary old_db.old=new_db.new` to `chkit generate`) to rename a dictionary via `RENAME DICTIONARY IF EXISTS ... TO ...` instead of a destructive drop + create. + - `chkit pull` introspects live dictionaries (including `RANGE`/`SETTINGS` and all attribute modifiers) into typed schema files, preserving ClickHouse's `[HIDDEN]` password redaction on `SOURCE(...)` credentials. A `SOURCE(...)` password change diffs and migrates like any other field change; `chkit generate` warns when a literal password is about to be written into migration SQL as plain text. `chkit pull` warns in two cases: when an introspected password comes back as `[HIDDEN]` (chkit can't recover the real value, so that dictionary's `source` is excluded from future diffs until it's replaced), and when ClickHouse is configured to reveal real passwords on introspection (`display_secrets_in_show_and_select` + `displaySecretsInShowAndSelect`), since that writes a plain-text credential into the generated schema file with no other indication. All warnings print to the console and are included as a `warnings` array in `--json` output. + - `codegen` generates a typed interface (and optional Zod schema) for each dictionary from its `attributes`, always included regardless of `includeViews`. + - `ON CLUSTER` mode stamps `ON CLUSTER ` onto every dictionary DDL statement, including `CREATE OR REPLACE DICTIONARY` and `RENAME DICTIONARY`. + +- 75d15e9: Stop `chkit drift` from reporting `index_mismatch` for skip indexes it just created. Introspection read `system.data_skipping_indices.type`, which holds only the index name (`ngrambf_v1`), so every argument parsed as 0; it now reads `type_full` (`ngrambf_v1(3, 4096, 2, 0)`). chkit renders `INDEX name (expr)` and ClickHouse keeps those parentheses in `expr`, so the comparison now drops one pair when it encloses the whole expression. chkit-py introspection reads `type_full` as well. +- 278e4fa: Add `chkit generate --empty` to scaffold a blank manual migration. Unlike a normal `generate`, empty mode skips the schema diff, plugin pipeline, and table scoping entirely and writes a timestamped `.sql` stub with the standard migration header (`operation-count: 0`) plus a placeholder comment. The snapshot is left untouched, so an empty migration never absorbs pending schema drift. Use it for DDL that chkit does not model — backfills, `OPTIMIZE`, dictionary reloads, or one-off data fixes. The `--name` (default `manual`) and `--migration-id` flags apply; `migrate` picks the file up like any generated one. New `generateEmptyMigration` helper exported from `@chkit/codegen`. +- 3f1db03: Support index-only projections (`PROJECTION p INDEX (a, b) TYPE basic`) end to end. `ProjectionDefinition` is now a union of the existing `{ name, query }` SELECT form and a new `{ name, index, type }` index-only form, which renders without the wrapping parens that made the SELECT form invalid for it. `chkit pull` previously parsed only the SELECT form and dropped index-only projections on the floor, so a pulled schema silently recreated the table without them; they now round-trip through pull, generate, migrate, and drift. + + Index expressions are normalized to the exact form ClickHouse stores — a single expression bare (`INDEX a`), several as a tuple (`INDEX (a, b)`), redundant parens peeled at every level, and a space after each argument separator — so `'(a)'` and `'a'`, or `'concat(x,y)'` and `'concat(x, y)'`, describe the same table and no longer read as drift. + + Two new validation errors guard the new form: `projection_ambiguous_kind` when an entry sets both `query` and `index` (which would otherwise silently discard the SELECT body), and `projection_empty_index` when the index expression is empty (which would otherwise emit invalid DDL). + +- f8238db: Add `@chkit/plugin-ingest`, the first cut of scheduled pull ingestion into ClickHouse. Streams are ordinary TypeScript: a `read` async generator fetches, maps, and yields destination-shaped rows, and `definePipeline` returns a tagged, non-durable group of streams; only pipelines exported from the project entry participate, with no global registry. `chkit ingest run` executes the selected streams (`--tag` is repeatable with exact AND semantics; an explicit empty selection fails), `chkit ingest list` shows the loaded graph, and `chkit ingest status` prints committed checkpoints. + + Progress follows one rule: rows are saved before the bookmark advances. Every batch is written with a stable `insert_deduplication_token`, and only after the ClickHouse acknowledgement does the executor append a `batch_committed` fact to the append-only ingestion journal. Checkpoints are a projection of that journal, so a crashed or lost-acknowledgement run replays from the last durable boundary with the same batch identity instead of skipping rows. Bundled strategies are `timestampWindow({ start, overlapMs })` (with a custom `from` callback alternative), `cursorState` for provider-owned state, and the full-sync fallback; `--backfill ` runs an explicit range in an isolated checkpoint namespace. + + `rawTable` and `rawRows` land provider objects untouched in a native `JSON` column, so typed shapes are derived inside ClickHouse with ordinary views or materialized views instead of being mapped in pipeline code. + + `FetchContext` exposes source request and cancellation capabilities independently of checkpoint types; `ReadContext` extends it. + + Source operations run through `context.attempt`, which owns fetch permits, p-retry-shaped retry policy, `Retry-After`, cancellation, and failure classification (`HttpError.fromResponse` is the canonical boundary for fetch-based readers). Pipelines carry separate `maxStreams`, `maxFetches`, and `maxLoads` ceilings, executions have a duration budget, and the executor emits OpenTelemetry spans. + + `@chkit/core` gains a singular `entry` config field, mutually exclusive with `schema` globs: the module is imported once, its exported schema definitions are collected, and exported plugin-domain definitions are collected by their plugins. `@chkit/clickhouse` `insert()` accepts per-insert `settings`. + + Successful syncs rotate batch identity using the existing journal, while failed runs retain their replay identity. Execution cancellation also bounds journal I/O, stalled writes cannot report success, and loader construction failures release their permits. Ingestion requires a direct ClickHouse connection; incompatible host executors fail before any writes. Project `entry` and `schema` settings replace the inherited source mode when layering configuration. + +- 4ded781: Make `chkit init` consistent with `create-chkit` for connecting a database, and stop hiding plugin import failures. In a non-TTY shell `init` now prints the same connect runbook `create-chkit` does (when the obsessiondb plugin is installed) instead of silently skipping it; `--yes` still keeps init a silent file-writer for CI. The dynamic plugin import now only degrades silently when the plugin is genuinely not installed — any other load failure propagates instead of a false silent pass. The static next-steps also use `npx` rather than a hardcoded `bunx`. +- 5a8d805: Support ClickHouse function expressions in `primaryKey`/`orderBy` (e.g. `toDate(ts)`, `toStartOfHour(session_end)`). Validation no longer reports `primary_key_missing_column`/`order_by_missing_column` for expression entries, and generated DDL emits them verbatim instead of quoting the whole expression as a column name. Plain column references are still validated and backtick-quoted as before. + + Migration planning now compares key clauses independent of insignificant whitespace and identifier backtick-quoting, so an expression written as `toStartOfHour( ts )` or a column written bare as `user-id` no longer diffs against ClickHouse's normalized `toStartOfHour(ts)` / `` `user-id` `` and triggers a phantom table recreate on `migrate`/`drift`/`check`. + +- 4ded781: Keep polling an async data-load migration through transient gateway errors instead of aborting. A single HTTP 524 (or other transient failure) on a status-poll request no longer cancels the migration: the server-side query keeps running, so chkit tolerates a bounded number of poll errors and only gives up after the budget, with an explicit message that the load may still be running and that re-running re-attaches via the deterministic `query_id`. Only a real query exception, or a submit-time failure, is fatal. This affects only operations marked `mode=async` (data loads); ordinary schema DDL is synchronous and unaffected. +- 4ded781: Make `--json` always emit a JSON object, never a bare JSON-encoded string. `printOutput` now wraps any plain string printed under `--json` in `{ schemaVersion, message }`, closing the whole class of bug at the serializer so no command can leak a bare string. `chkit obsessiondb whoami` gains a structured envelope (`status: logged_in | not_logged_in | session_expired`), and `chkit obsessiondb service list` emits a single object with a `services[]` array instead of one JSON line per service (which was not valid single-JSON). Previously these commands `JSON.stringify`'d a prose string (e.g. `"Not logged in…"`), breaking any pipe to `jq`. Text-mode output is unchanged. Note: this changes the `--json` output shape of `whoami` and `service list` from a string to an object. +- 8296b8a: Fix two related pull/drift bugs around tables whose `ORDER BY` is declared alongside a projection or a derived primary key. + + `chkit pull` parsed table-level clauses (`ENGINE`, `ORDER BY`, `PRIMARY KEY`, `PARTITION BY`, `TTL`, `SETTINGS`) by matching the first keyword anywhere in `SHOW CREATE TABLE`. A projection whose `SELECT` body contains `ORDER BY` — or a column-level `TTL` — sits in the column list before those clauses, so the parser matched the inner keyword and swallowed the engine into `orderBy`/`primaryKey`, producing an invalid pulled schema (#190). Table-level clauses are now parsed only from the portion after the column list. + + `chkit drift` always reported `primary_key_mismatch` for any table whose `PRIMARY KEY` is derived from `ORDER BY`. ClickHouse omits the derived key from `SHOW CREATE TABLE`, but the schema carries it, so the two never matched. Drift now applies the same derivation to the live side, so a derived primary key reads clean while a genuine primary-key difference is still reported (#194). + +- Updated dependencies [65c90d6] +- Updated dependencies [75d15e9] +- Updated dependencies [278e4fa] +- Updated dependencies [3f1db03] +- Updated dependencies [f8238db] +- Updated dependencies [fedbf56] +- Updated dependencies [5a8d805] +- Updated dependencies [8296b8a] +- Updated dependencies [b501f5d] +- Updated dependencies [256ec62] + - @chkit/core@0.2.0-beta.8 + - @chkit/clickhouse@0.2.0-beta.8 + - @chkit/codegen@0.2.0-beta.8 + ## 0.1.2-beta.7 ### Patch Changes diff --git a/packages/cli/package.json b/packages/cli/package.json index e1816efb..75e9b0ae 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "chkit", - "version": "0.1.2-beta.7", + "version": "0.2.0-beta.8", "description": "ClickHouse schema and migration CLI for TypeScript projects", "license": "MIT", "author": "ObsessionDB", diff --git a/packages/clickhouse/CHANGELOG.md b/packages/clickhouse/CHANGELOG.md index 2ae8ae0a..2b5adc24 100644 --- a/packages/clickhouse/CHANGELOG.md +++ b/packages/clickhouse/CHANGELOG.md @@ -1,5 +1,67 @@ # @chkit/clickhouse +## 0.2.0-beta.8 + +### Patch Changes + +- 65c90d6: Add `dictionary()` as a first-class ClickHouse schema primitive, mirroring `materializedView()` across the full lifecycle: DSL authoring, validation, canonicalization, SQL rendering, migration planning/diff, drift, `check`, destructive-op safety, `pull` introspection, and `codegen` typed interfaces. + + - `dictionary({ database, name, attributes, primaryKey, source, layout, lifetime, range?, settings?, comment? })` — attributes support `default`/`expression` (mutually exclusive), `hierarchical`, `bidirectional` (requires `hierarchical`), `injective`, and `isObjectId`. `range: { min, max }` renders `RANGE(MIN ... MAX ...)` for `RANGE_HASHED`/`COMPLEX_KEY_RANGE_HASHED` layouts, and `settings` renders `SETTINGS(...)`. + - ClickHouse has no `ALTER DICTIONARY`, so any structural change plans a single atomic `CREATE OR REPLACE DICTIONARY`. Dropping a dictionary is treated as destructive and blocked without `--allow-destructive`. + - Set `renamedFrom` on a dictionary (or pass `--rename-dictionary old_db.old=new_db.new` to `chkit generate`) to rename a dictionary via `RENAME DICTIONARY IF EXISTS ... TO ...` instead of a destructive drop + create. + - `chkit pull` introspects live dictionaries (including `RANGE`/`SETTINGS` and all attribute modifiers) into typed schema files, preserving ClickHouse's `[HIDDEN]` password redaction on `SOURCE(...)` credentials. A `SOURCE(...)` password change diffs and migrates like any other field change; `chkit generate` warns when a literal password is about to be written into migration SQL as plain text. `chkit pull` warns in two cases: when an introspected password comes back as `[HIDDEN]` (chkit can't recover the real value, so that dictionary's `source` is excluded from future diffs until it's replaced), and when ClickHouse is configured to reveal real passwords on introspection (`display_secrets_in_show_and_select` + `displaySecretsInShowAndSelect`), since that writes a plain-text credential into the generated schema file with no other indication. All warnings print to the console and are included as a `warnings` array in `--json` output. + - `codegen` generates a typed interface (and optional Zod schema) for each dictionary from its `attributes`, always included regardless of `includeViews`. + - `ON CLUSTER` mode stamps `ON CLUSTER ` onto every dictionary DDL statement, including `CREATE OR REPLACE DICTIONARY` and `RENAME DICTIONARY`. + +- 75d15e9: Stop `chkit drift` from reporting `index_mismatch` for skip indexes it just created. Introspection read `system.data_skipping_indices.type`, which holds only the index name (`ngrambf_v1`), so every argument parsed as 0; it now reads `type_full` (`ngrambf_v1(3, 4096, 2, 0)`). chkit renders `INDEX name (expr)` and ClickHouse keeps those parentheses in `expr`, so the comparison now drops one pair when it encloses the whole expression. chkit-py introspection reads `type_full` as well. +- 3f1db03: Support index-only projections (`PROJECTION p INDEX (a, b) TYPE basic`) end to end. `ProjectionDefinition` is now a union of the existing `{ name, query }` SELECT form and a new `{ name, index, type }` index-only form, which renders without the wrapping parens that made the SELECT form invalid for it. `chkit pull` previously parsed only the SELECT form and dropped index-only projections on the floor, so a pulled schema silently recreated the table without them; they now round-trip through pull, generate, migrate, and drift. + + Index expressions are normalized to the exact form ClickHouse stores — a single expression bare (`INDEX a`), several as a tuple (`INDEX (a, b)`), redundant parens peeled at every level, and a space after each argument separator — so `'(a)'` and `'a'`, or `'concat(x,y)'` and `'concat(x, y)'`, describe the same table and no longer read as drift. + + Two new validation errors guard the new form: `projection_ambiguous_kind` when an entry sets both `query` and `index` (which would otherwise silently discard the SELECT body), and `projection_empty_index` when the index expression is empty (which would otherwise emit invalid DDL). + +- f8238db: Add `@chkit/plugin-ingest`, the first cut of scheduled pull ingestion into ClickHouse. Streams are ordinary TypeScript: a `read` async generator fetches, maps, and yields destination-shaped rows, and `definePipeline` returns a tagged, non-durable group of streams; only pipelines exported from the project entry participate, with no global registry. `chkit ingest run` executes the selected streams (`--tag` is repeatable with exact AND semantics; an explicit empty selection fails), `chkit ingest list` shows the loaded graph, and `chkit ingest status` prints committed checkpoints. + + Progress follows one rule: rows are saved before the bookmark advances. Every batch is written with a stable `insert_deduplication_token`, and only after the ClickHouse acknowledgement does the executor append a `batch_committed` fact to the append-only ingestion journal. Checkpoints are a projection of that journal, so a crashed or lost-acknowledgement run replays from the last durable boundary with the same batch identity instead of skipping rows. Bundled strategies are `timestampWindow({ start, overlapMs })` (with a custom `from` callback alternative), `cursorState` for provider-owned state, and the full-sync fallback; `--backfill ` runs an explicit range in an isolated checkpoint namespace. + + `rawTable` and `rawRows` land provider objects untouched in a native `JSON` column, so typed shapes are derived inside ClickHouse with ordinary views or materialized views instead of being mapped in pipeline code. + + `FetchContext` exposes source request and cancellation capabilities independently of checkpoint types; `ReadContext` extends it. + + Source operations run through `context.attempt`, which owns fetch permits, p-retry-shaped retry policy, `Retry-After`, cancellation, and failure classification (`HttpError.fromResponse` is the canonical boundary for fetch-based readers). Pipelines carry separate `maxStreams`, `maxFetches`, and `maxLoads` ceilings, executions have a duration budget, and the executor emits OpenTelemetry spans. + + `@chkit/core` gains a singular `entry` config field, mutually exclusive with `schema` globs: the module is imported once, its exported schema definitions are collected, and exported plugin-domain definitions are collected by their plugins. `@chkit/clickhouse` `insert()` accepts per-insert `settings`. + + Successful syncs rotate batch identity using the existing journal, while failed runs retain their replay identity. Execution cancellation also bounds journal I/O, stalled writes cannot report success, and loader construction failures release their permits. Ingestion requires a direct ClickHouse connection; incompatible host executors fail before any writes. Project `entry` and `schema` settings replace the inherited source mode when layering configuration. + +- fedbf56: Support Kafka engine tables without MergeTree key clauses, with escaped literal + settings, pull round trips, and normalized drift/check comparisons. Preserve the + existing setting-string contract for other engines. Reject unsupported Kafka + changes before writing migration artifacts and document an explicit, destructive- + gated replacement workflow. Handle quoted delimiters and escaped trailing + backslashes in introspection and migration statement splitting. +- 8296b8a: Fix two related pull/drift bugs around tables whose `ORDER BY` is declared alongside a projection or a derived primary key. + + `chkit pull` parsed table-level clauses (`ENGINE`, `ORDER BY`, `PRIMARY KEY`, `PARTITION BY`, `TTL`, `SETTINGS`) by matching the first keyword anywhere in `SHOW CREATE TABLE`. A projection whose `SELECT` body contains `ORDER BY` — or a column-level `TTL` — sits in the column list before those clauses, so the parser matched the inner keyword and swallowed the engine into `orderBy`/`primaryKey`, producing an invalid pulled schema (#190). Table-level clauses are now parsed only from the portion after the column list. + + `chkit drift` always reported `primary_key_mismatch` for any table whose `PRIMARY KEY` is derived from `ORDER BY`. ClickHouse omits the derived key from `SHOW CREATE TABLE`, but the schema carries it, so the two never matched. Drift now applies the same derivation to the live side, so a derived primary key reads clean while a genuine primary-key difference is still reported (#194). + +- 256ec62: Support ClickHouse `text` indexes in schemas, migrations, pull, and drift. Text + indexes require a tokenizer and support preprocessing, postprocessing, phrase + search, and dictionary/posting-list options when supported by the server. + Granularity is automatic. Preserve whitespace and escapes inside SQL literals, + compare parameter order and SQL formatting consistently, and reject unsupported + or malformed metadata instead of silently losing settings. Includes Python parity + and live adversarial round-trip tests on ClickHouse 26.3 and 26.8. +- Updated dependencies [65c90d6] +- Updated dependencies [3f1db03] +- Updated dependencies [f8238db] +- Updated dependencies [fedbf56] +- Updated dependencies [5a8d805] +- Updated dependencies [b501f5d] +- Updated dependencies [256ec62] + - @chkit/core@0.2.0-beta.8 + ## 0.1.2-beta.7 ### Patch Changes diff --git a/packages/clickhouse/package.json b/packages/clickhouse/package.json index 079320d9..1735e4da 100644 --- a/packages/clickhouse/package.json +++ b/packages/clickhouse/package.json @@ -1,6 +1,6 @@ { "name": "@chkit/clickhouse", - "version": "0.1.2-beta.7", + "version": "0.2.0-beta.8", "description": "ClickHouse client wrapper for chkit", "license": "MIT", "author": "ObsessionDB", diff --git a/packages/codegen/CHANGELOG.md b/packages/codegen/CHANGELOG.md index 71b43173..16fdfc62 100644 --- a/packages/codegen/CHANGELOG.md +++ b/packages/codegen/CHANGELOG.md @@ -1,5 +1,19 @@ # @chkit/codegen +## 0.2.0-beta.8 + +### Patch Changes + +- 278e4fa: Add `chkit generate --empty` to scaffold a blank manual migration. Unlike a normal `generate`, empty mode skips the schema diff, plugin pipeline, and table scoping entirely and writes a timestamped `.sql` stub with the standard migration header (`operation-count: 0`) plus a placeholder comment. The snapshot is left untouched, so an empty migration never absorbs pending schema drift. Use it for DDL that chkit does not model — backfills, `OPTIMIZE`, dictionary reloads, or one-off data fixes. The `--name` (default `manual`) and `--migration-id` flags apply; `migrate` picks the file up like any generated one. New `generateEmptyMigration` helper exported from `@chkit/codegen`. +- Updated dependencies [65c90d6] +- Updated dependencies [3f1db03] +- Updated dependencies [f8238db] +- Updated dependencies [fedbf56] +- Updated dependencies [5a8d805] +- Updated dependencies [b501f5d] +- Updated dependencies [256ec62] + - @chkit/core@0.2.0-beta.8 + ## 0.1.2-beta.7 ### Patch Changes diff --git a/packages/codegen/package.json b/packages/codegen/package.json index 72fae612..568a2967 100644 --- a/packages/codegen/package.json +++ b/packages/codegen/package.json @@ -1,6 +1,6 @@ { "name": "@chkit/codegen", - "version": "0.1.2-beta.7", + "version": "0.2.0-beta.8", "description": "Migration artifact generator for chkit", "license": "MIT", "author": "ObsessionDB", diff --git a/packages/core/CHANGELOG.md b/packages/core/CHANGELOG.md index da26b73a..c6ba068c 100644 --- a/packages/core/CHANGELOG.md +++ b/packages/core/CHANGELOG.md @@ -1,5 +1,60 @@ # @chkit/core +## 0.2.0-beta.8 + +### Minor Changes + +- fedbf56: Support Kafka engine tables without MergeTree key clauses, with escaped literal + settings, pull round trips, and normalized drift/check comparisons. Preserve the + existing setting-string contract for other engines. Reject unsupported Kafka + changes before writing migration artifacts and document an explicit, destructive- + gated replacement workflow. Handle quoted delimiters and escaped trailing + backslashes in introspection and migration statement splitting. +- 256ec62: Support ClickHouse `text` indexes in schemas, migrations, pull, and drift. Text + indexes require a tokenizer and support preprocessing, postprocessing, phrase + search, and dictionary/posting-list options when supported by the server. + Granularity is automatic. Preserve whitespace and escapes inside SQL literals, + compare parameter order and SQL formatting consistently, and reject unsupported + or malformed metadata instead of silently losing settings. Includes Python parity + and live adversarial round-trip tests on ClickHouse 26.3 and 26.8. + +### Patch Changes + +- 65c90d6: Add `dictionary()` as a first-class ClickHouse schema primitive, mirroring `materializedView()` across the full lifecycle: DSL authoring, validation, canonicalization, SQL rendering, migration planning/diff, drift, `check`, destructive-op safety, `pull` introspection, and `codegen` typed interfaces. + + - `dictionary({ database, name, attributes, primaryKey, source, layout, lifetime, range?, settings?, comment? })` — attributes support `default`/`expression` (mutually exclusive), `hierarchical`, `bidirectional` (requires `hierarchical`), `injective`, and `isObjectId`. `range: { min, max }` renders `RANGE(MIN ... MAX ...)` for `RANGE_HASHED`/`COMPLEX_KEY_RANGE_HASHED` layouts, and `settings` renders `SETTINGS(...)`. + - ClickHouse has no `ALTER DICTIONARY`, so any structural change plans a single atomic `CREATE OR REPLACE DICTIONARY`. Dropping a dictionary is treated as destructive and blocked without `--allow-destructive`. + - Set `renamedFrom` on a dictionary (or pass `--rename-dictionary old_db.old=new_db.new` to `chkit generate`) to rename a dictionary via `RENAME DICTIONARY IF EXISTS ... TO ...` instead of a destructive drop + create. + - `chkit pull` introspects live dictionaries (including `RANGE`/`SETTINGS` and all attribute modifiers) into typed schema files, preserving ClickHouse's `[HIDDEN]` password redaction on `SOURCE(...)` credentials. A `SOURCE(...)` password change diffs and migrates like any other field change; `chkit generate` warns when a literal password is about to be written into migration SQL as plain text. `chkit pull` warns in two cases: when an introspected password comes back as `[HIDDEN]` (chkit can't recover the real value, so that dictionary's `source` is excluded from future diffs until it's replaced), and when ClickHouse is configured to reveal real passwords on introspection (`display_secrets_in_show_and_select` + `displaySecretsInShowAndSelect`), since that writes a plain-text credential into the generated schema file with no other indication. All warnings print to the console and are included as a `warnings` array in `--json` output. + - `codegen` generates a typed interface (and optional Zod schema) for each dictionary from its `attributes`, always included regardless of `includeViews`. + - `ON CLUSTER` mode stamps `ON CLUSTER ` onto every dictionary DDL statement, including `CREATE OR REPLACE DICTIONARY` and `RENAME DICTIONARY`. + +- 3f1db03: Support index-only projections (`PROJECTION p INDEX (a, b) TYPE basic`) end to end. `ProjectionDefinition` is now a union of the existing `{ name, query }` SELECT form and a new `{ name, index, type }` index-only form, which renders without the wrapping parens that made the SELECT form invalid for it. `chkit pull` previously parsed only the SELECT form and dropped index-only projections on the floor, so a pulled schema silently recreated the table without them; they now round-trip through pull, generate, migrate, and drift. + + Index expressions are normalized to the exact form ClickHouse stores — a single expression bare (`INDEX a`), several as a tuple (`INDEX (a, b)`), redundant parens peeled at every level, and a space after each argument separator — so `'(a)'` and `'a'`, or `'concat(x,y)'` and `'concat(x, y)'`, describe the same table and no longer read as drift. + + Two new validation errors guard the new form: `projection_ambiguous_kind` when an entry sets both `query` and `index` (which would otherwise silently discard the SELECT body), and `projection_empty_index` when the index expression is empty (which would otherwise emit invalid DDL). + +- f8238db: Add `@chkit/plugin-ingest`, the first cut of scheduled pull ingestion into ClickHouse. Streams are ordinary TypeScript: a `read` async generator fetches, maps, and yields destination-shaped rows, and `definePipeline` returns a tagged, non-durable group of streams; only pipelines exported from the project entry participate, with no global registry. `chkit ingest run` executes the selected streams (`--tag` is repeatable with exact AND semantics; an explicit empty selection fails), `chkit ingest list` shows the loaded graph, and `chkit ingest status` prints committed checkpoints. + + Progress follows one rule: rows are saved before the bookmark advances. Every batch is written with a stable `insert_deduplication_token`, and only after the ClickHouse acknowledgement does the executor append a `batch_committed` fact to the append-only ingestion journal. Checkpoints are a projection of that journal, so a crashed or lost-acknowledgement run replays from the last durable boundary with the same batch identity instead of skipping rows. Bundled strategies are `timestampWindow({ start, overlapMs })` (with a custom `from` callback alternative), `cursorState` for provider-owned state, and the full-sync fallback; `--backfill ` runs an explicit range in an isolated checkpoint namespace. + + `rawTable` and `rawRows` land provider objects untouched in a native `JSON` column, so typed shapes are derived inside ClickHouse with ordinary views or materialized views instead of being mapped in pipeline code. + + `FetchContext` exposes source request and cancellation capabilities independently of checkpoint types; `ReadContext` extends it. + + Source operations run through `context.attempt`, which owns fetch permits, p-retry-shaped retry policy, `Retry-After`, cancellation, and failure classification (`HttpError.fromResponse` is the canonical boundary for fetch-based readers). Pipelines carry separate `maxStreams`, `maxFetches`, and `maxLoads` ceilings, executions have a duration budget, and the executor emits OpenTelemetry spans. + + `@chkit/core` gains a singular `entry` config field, mutually exclusive with `schema` globs: the module is imported once, its exported schema definitions are collected, and exported plugin-domain definitions are collected by their plugins. `@chkit/clickhouse` `insert()` accepts per-insert `settings`. + + Successful syncs rotate batch identity using the existing journal, while failed runs retain their replay identity. Execution cancellation also bounds journal I/O, stalled writes cannot report success, and loader construction failures release their permits. Ingestion requires a direct ClickHouse connection; incompatible host executors fail before any writes. Project `entry` and `schema` settings replace the inherited source mode when layering configuration. + +- 5a8d805: Support ClickHouse function expressions in `primaryKey`/`orderBy` (e.g. `toDate(ts)`, `toStartOfHour(session_end)`). Validation no longer reports `primary_key_missing_column`/`order_by_missing_column` for expression entries, and generated DDL emits them verbatim instead of quoting the whole expression as a column name. Plain column references are still validated and backtick-quoted as before. + + Migration planning now compares key clauses independent of insignificant whitespace and identifier backtick-quoting, so an expression written as `toStartOfHour( ts )` or a column written bare as `user-id` no longer diffs against ClickHouse's normalized `toStartOfHour(ts)` / `` `user-id` `` and triggers a phantom table recreate on `migrate`/`drift`/`check`. + +- b501f5d: Extract shared plugin command scaffolding into `@chkit/core`: new `createPluginRunner` (binds a plugin's config-error class once and wraps command `run` handlers in the shared error-to-exit-code envelope) and `withFactoryDefaults` (layers plugin-factory options under parsed data). The backfill, codegen, and pull plugins now use these helpers instead of private copies — no behavior change, but the plugins require the matching `@chkit/core` version. + ## 0.1.2-beta.7 ### Patch Changes diff --git a/packages/core/package.json b/packages/core/package.json index f40cdcbe..4d5f3ec8 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -1,6 +1,6 @@ { "name": "@chkit/core", - "version": "0.1.2-beta.7", + "version": "0.2.0-beta.8", "description": "Schema DSL, config, diff engine, and migration planner for chkit", "license": "MIT", "author": "ObsessionDB", diff --git a/packages/create-chkit/CHANGELOG.md b/packages/create-chkit/CHANGELOG.md index b0847ae9..48f0e0b9 100644 --- a/packages/create-chkit/CHANGELOG.md +++ b/packages/create-chkit/CHANGELOG.md @@ -1,5 +1,18 @@ # create-chkit +## 0.2.0-beta.8 + +### Patch Changes + +- 4ded781: Print the "Next steps" block once and with the correct runner for the selected package manager. `create-chkit` previously printed it twice — once package-manager-aware and once from onboarding with a hardcoded `bunx` — so `--package-manager npm` users were told to run `bunx chkit …`. Onboarding now derives the runner (`npx` / `pnpm dlx` / `yarn dlx` / `bunx`) from the package manager, and `create-chkit` only prints its own next-steps when onboarding is skipped, removing the duplicate. +- Updated dependencies [3f9a246] +- Updated dependencies [4ded781] +- Updated dependencies [75d15e9] +- Updated dependencies [4ded781] +- Updated dependencies [4ded781] +- Updated dependencies [3cc768d] + - @chkit/plugin-obsessiondb@0.2.0-beta.8 + ## 0.1.2-beta.7 ### Patch Changes diff --git a/packages/create-chkit/package.json b/packages/create-chkit/package.json index 9eb17208..585cab53 100644 --- a/packages/create-chkit/package.json +++ b/packages/create-chkit/package.json @@ -1,6 +1,6 @@ { "name": "create-chkit", - "version": "0.1.2-beta.7", + "version": "0.2.0-beta.8", "description": "Scaffold a new chkit project from an example", "license": "MIT", "author": "ObsessionDB", diff --git a/packages/plugin-backfill/CHANGELOG.md b/packages/plugin-backfill/CHANGELOG.md index bae8f3d7..244e9c8c 100644 --- a/packages/plugin-backfill/CHANGELOG.md +++ b/packages/plugin-backfill/CHANGELOG.md @@ -1,5 +1,25 @@ # @chkit/plugin-backfill +## 0.2.0-beta.8 + +### Patch Changes + +- f85f568: Fix `mv_replay` backfill of a from-scratch empty aggregate target. Chunk planning now sizes chunks against the materialized view's source table (the one it reads `FROM`) instead of the target, so bootstrapping an empty rollup no longer fails with "No partitions found for <target>". The empty-check still guards the source, and multi-view fan-in from different sources keeps its existing behaviour. +- 3f9a246: Fix `backfill` mv_replay so it rebuilds **every** materialized view feeding the target table, not just the first. ClickHouse allows several MVs to share one destination table; previously only the first-declared MV was replayed and the rest were silently dropped, leaving the backfill incomplete. Each chunk now runs one `INSERT INTO target … SELECT … UNION ALL SELECT …` covering all matching MVs, so a single query id and idempotency token still cover the chunk. Single-MV plans are unchanged. +- 9ad23f9: Refactor the backfill chunk-SQL rewriter (`chunking/sql.ts`): fold the duplicated quote/paren-aware scan loops into one shared `scanSqlTokens` primitive (with `findTopLevelKeywords`/`splitTopLevel` on top) and split the oversized `rewriteSelectColumns` into focused helpers. Behavior is unchanged — the same customer SQL rewriting is now covered by direct unit tests for quoted-string, escaped-quote, nested-subquery, and missing-FROM edge cases. +- b501f5d: Extract shared plugin command scaffolding into `@chkit/core`: new `createPluginRunner` (binds a plugin's config-error class once and wraps command `run` handlers in the shared error-to-exit-code envelope) and `withFactoryDefaults` (layers plugin-factory options under parsed data). The backfill, codegen, and pull plugins now use these helpers instead of private copies — no behavior change, but the plugins require the matching `@chkit/core` version. +- Updated dependencies [65c90d6] +- Updated dependencies [75d15e9] +- Updated dependencies [3f1db03] +- Updated dependencies [f8238db] +- Updated dependencies [fedbf56] +- Updated dependencies [5a8d805] +- Updated dependencies [8296b8a] +- Updated dependencies [b501f5d] +- Updated dependencies [256ec62] + - @chkit/core@0.2.0-beta.8 + - @chkit/clickhouse@0.2.0-beta.8 + ## 0.1.2-beta.7 ### Patch Changes diff --git a/packages/plugin-backfill/package.json b/packages/plugin-backfill/package.json index 579eaa44..83387ea3 100644 --- a/packages/plugin-backfill/package.json +++ b/packages/plugin-backfill/package.json @@ -1,6 +1,6 @@ { "name": "@chkit/plugin-backfill", - "version": "0.1.2-beta.7", + "version": "0.2.0-beta.8", "description": "Time-windowed data backfill with checkpoints for chkit", "license": "MIT", "author": "ObsessionDB", diff --git a/packages/plugin-codegen/CHANGELOG.md b/packages/plugin-codegen/CHANGELOG.md index 304121e5..33351506 100644 --- a/packages/plugin-codegen/CHANGELOG.md +++ b/packages/plugin-codegen/CHANGELOG.md @@ -1,5 +1,28 @@ # @chkit/plugin-codegen +## 0.2.0-beta.8 + +### Patch Changes + +- 65c90d6: Add `dictionary()` as a first-class ClickHouse schema primitive, mirroring `materializedView()` across the full lifecycle: DSL authoring, validation, canonicalization, SQL rendering, migration planning/diff, drift, `check`, destructive-op safety, `pull` introspection, and `codegen` typed interfaces. + + - `dictionary({ database, name, attributes, primaryKey, source, layout, lifetime, range?, settings?, comment? })` — attributes support `default`/`expression` (mutually exclusive), `hierarchical`, `bidirectional` (requires `hierarchical`), `injective`, and `isObjectId`. `range: { min, max }` renders `RANGE(MIN ... MAX ...)` for `RANGE_HASHED`/`COMPLEX_KEY_RANGE_HASHED` layouts, and `settings` renders `SETTINGS(...)`. + - ClickHouse has no `ALTER DICTIONARY`, so any structural change plans a single atomic `CREATE OR REPLACE DICTIONARY`. Dropping a dictionary is treated as destructive and blocked without `--allow-destructive`. + - Set `renamedFrom` on a dictionary (or pass `--rename-dictionary old_db.old=new_db.new` to `chkit generate`) to rename a dictionary via `RENAME DICTIONARY IF EXISTS ... TO ...` instead of a destructive drop + create. + - `chkit pull` introspects live dictionaries (including `RANGE`/`SETTINGS` and all attribute modifiers) into typed schema files, preserving ClickHouse's `[HIDDEN]` password redaction on `SOURCE(...)` credentials. A `SOURCE(...)` password change diffs and migrates like any other field change; `chkit generate` warns when a literal password is about to be written into migration SQL as plain text. `chkit pull` warns in two cases: when an introspected password comes back as `[HIDDEN]` (chkit can't recover the real value, so that dictionary's `source` is excluded from future diffs until it's replaced), and when ClickHouse is configured to reveal real passwords on introspection (`display_secrets_in_show_and_select` + `displaySecretsInShowAndSelect`), since that writes a plain-text credential into the generated schema file with no other indication. All warnings print to the console and are included as a `warnings` array in `--json` output. + - `codegen` generates a typed interface (and optional Zod schema) for each dictionary from its `attributes`, always included regardless of `includeViews`. + - `ON CLUSTER` mode stamps `ON CLUSTER ` onto every dictionary DDL statement, including `CREATE OR REPLACE DICTIONARY` and `RENAME DICTIONARY`. + +- b501f5d: Extract shared plugin command scaffolding into `@chkit/core`: new `createPluginRunner` (binds a plugin's config-error class once and wraps command `run` handlers in the shared error-to-exit-code envelope) and `withFactoryDefaults` (layers plugin-factory options under parsed data). The backfill, codegen, and pull plugins now use these helpers instead of private copies — no behavior change, but the plugins require the matching `@chkit/core` version. +- Updated dependencies [65c90d6] +- Updated dependencies [3f1db03] +- Updated dependencies [f8238db] +- Updated dependencies [fedbf56] +- Updated dependencies [5a8d805] +- Updated dependencies [b501f5d] +- Updated dependencies [256ec62] + - @chkit/core@0.2.0-beta.8 + ## 0.1.2-beta.7 ### Patch Changes diff --git a/packages/plugin-codegen/package.json b/packages/plugin-codegen/package.json index 3e0f7640..6bea3edf 100644 --- a/packages/plugin-codegen/package.json +++ b/packages/plugin-codegen/package.json @@ -1,6 +1,6 @@ { "name": "@chkit/plugin-codegen", - "version": "0.1.2-beta.7", + "version": "0.2.0-beta.8", "description": "TypeScript type and Zod schema generation plugin for chkit", "license": "MIT", "author": "ObsessionDB", diff --git a/packages/plugin-ingest/CHANGELOG.md b/packages/plugin-ingest/CHANGELOG.md new file mode 100644 index 00000000..651092a9 --- /dev/null +++ b/packages/plugin-ingest/CHANGELOG.md @@ -0,0 +1,34 @@ +# @chkit/plugin-ingest + +## 0.2.0-beta.8 + +### Patch Changes + +- 3042c56: Point README documentation links at the renamed API Sync section (`/api-sync/`). +- cffcbd9: Harden the ingestion journal on replicated and managed ClickHouse (such as ObsessionDB). `ensure()` now waits until the journal table is visible after creating it. The run-level `run_started` and `run_finished` facts get the same bounded retry as stream facts, so one transient error, such as a lagging replica that doesn't know the journal table yet, no longer fails the whole run. Terminal facts (`work_finished`, `run_finished`) are no longer cut off after a fixed 5 seconds while the run is still live, so a slow but healthy journal write no longer turns a successful run into a `TimeoutError`. The 5-second grace still applies once a run is cancelled or its execution budget runs out. +- 97949a3: Each ingestion run now journals its `run_started` and `run_finished` facts under its own `@run:` namespace. Previously every run shared one `@run` namespace, so two executor processes that overlapped once (for example a local run during a scheduled one) left conflicting facts at the same sequence number and every later run refused to start. Stream checkpoints were never affected; per-stream namespaces still detect overlap. +- f8238db: Add `@chkit/plugin-ingest`, the first cut of scheduled pull ingestion into ClickHouse. Streams are ordinary TypeScript: a `read` async generator fetches, maps, and yields destination-shaped rows, and `definePipeline` returns a tagged, non-durable group of streams; only pipelines exported from the project entry participate, with no global registry. `chkit ingest run` executes the selected streams (`--tag` is repeatable with exact AND semantics; an explicit empty selection fails), `chkit ingest list` shows the loaded graph, and `chkit ingest status` prints committed checkpoints. + + Progress follows one rule: rows are saved before the bookmark advances. Every batch is written with a stable `insert_deduplication_token`, and only after the ClickHouse acknowledgement does the executor append a `batch_committed` fact to the append-only ingestion journal. Checkpoints are a projection of that journal, so a crashed or lost-acknowledgement run replays from the last durable boundary with the same batch identity instead of skipping rows. Bundled strategies are `timestampWindow({ start, overlapMs })` (with a custom `from` callback alternative), `cursorState` for provider-owned state, and the full-sync fallback; `--backfill ` runs an explicit range in an isolated checkpoint namespace. + + `rawTable` and `rawRows` land provider objects untouched in a native `JSON` column, so typed shapes are derived inside ClickHouse with ordinary views or materialized views instead of being mapped in pipeline code. + + `FetchContext` exposes source request and cancellation capabilities independently of checkpoint types; `ReadContext` extends it. + + Source operations run through `context.attempt`, which owns fetch permits, p-retry-shaped retry policy, `Retry-After`, cancellation, and failure classification (`HttpError.fromResponse` is the canonical boundary for fetch-based readers). Pipelines carry separate `maxStreams`, `maxFetches`, and `maxLoads` ceilings, executions have a duration budget, and the executor emits OpenTelemetry spans. + + `@chkit/core` gains a singular `entry` config field, mutually exclusive with `schema` globs: the module is imported once, its exported schema definitions are collected, and exported plugin-domain definitions are collected by their plugins. `@chkit/clickhouse` `insert()` accepts per-insert `settings`. + + Successful syncs rotate batch identity using the existing journal, while failed runs retain their replay identity. Execution cancellation also bounds journal I/O, stalled writes cannot report success, and loader construction failures release their permits. Ingestion requires a direct ClickHouse connection; incompatible host executors fail before any writes. Project `entry` and `schema` settings replace the inherited source mode when layering configuration. + +- Updated dependencies [65c90d6] +- Updated dependencies [75d15e9] +- Updated dependencies [3f1db03] +- Updated dependencies [f8238db] +- Updated dependencies [fedbf56] +- Updated dependencies [5a8d805] +- Updated dependencies [8296b8a] +- Updated dependencies [b501f5d] +- Updated dependencies [256ec62] + - @chkit/core@0.2.0-beta.8 + - @chkit/clickhouse@0.2.0-beta.8 diff --git a/packages/plugin-ingest/package.json b/packages/plugin-ingest/package.json index 33793881..da3dcc53 100644 --- a/packages/plugin-ingest/package.json +++ b/packages/plugin-ingest/package.json @@ -1,6 +1,6 @@ { "name": "@chkit/plugin-ingest", - "version": "0.1.2-beta.7", + "version": "0.2.0-beta.8", "description": "Scheduled pull ingestion into ClickHouse with journaled checkpoints for chkit", "license": "MIT", "author": "ObsessionDB", diff --git a/packages/plugin-obsessiondb/CHANGELOG.md b/packages/plugin-obsessiondb/CHANGELOG.md index 37e5374d..07955125 100644 --- a/packages/plugin-obsessiondb/CHANGELOG.md +++ b/packages/plugin-obsessiondb/CHANGELOG.md @@ -1,5 +1,31 @@ # @chkit/plugin-obsessiondb +## 0.2.0-beta.8 + +### Patch Changes + +- 3f9a246: Fix `backfill` mv_replay so it rebuilds **every** materialized view feeding the target table, not just the first. ClickHouse allows several MVs to share one destination table; previously only the first-declared MV was replayed and the rest were silently dropped, leaving the backfill incomplete. Each chunk now runs one `INSERT INTO target … SELECT … UNION ALL SELECT …` covering all matching MVs, so a single query id and idempotency token still cover the chunk. Single-MV plans are unchanged. +- 4ded781: Print the "Next steps" block once and with the correct runner for the selected package manager. `create-chkit` previously printed it twice — once package-manager-aware and once from onboarding with a hardcoded `bunx` — so `--package-manager npm` users were told to run `bunx chkit …`. Onboarding now derives the runner (`npx` / `pnpm dlx` / `yarn dlx` / `bunx`) from the package manager, and `create-chkit` only prints its own next-steps when onboarding is skipped, removing the duplicate. +- 75d15e9: Stop `chkit drift` from reporting `index_mismatch` for skip indexes it just created. Introspection read `system.data_skipping_indices.type`, which holds only the index name (`ngrambf_v1`), so every argument parsed as 0; it now reads `type_full` (`ngrambf_v1(3, 4096, 2, 0)`). chkit renders `INDEX name (expr)` and ClickHouse keeps those parentheses in `expr`, so the comparison now drops one pair when it encloses the whole expression. chkit-py introspection reads `type_full` as well. +- 4ded781: Make `--json` always emit a JSON object, never a bare JSON-encoded string. `printOutput` now wraps any plain string printed under `--json` in `{ schemaVersion, message }`, closing the whole class of bug at the serializer so no command can leak a bare string. `chkit obsessiondb whoami` gains a structured envelope (`status: logged_in | not_logged_in | session_expired`), and `chkit obsessiondb service list` emits a single object with a `services[]` array instead of one JSON line per service (which was not valid single-JSON). Previously these commands `JSON.stringify`'d a prose string (e.g. `"Not logged in…"`), breaking any pipe to `jq`. Text-mode output is unchanged. Note: this changes the `--json` output shape of `whoami` and `service list` from a string to an object. +- 4ded781: `chkit obsessiondb logout` now reports "No active session." when there are no stored credentials, instead of always printing "Logged out." (which implied it had ended a session that never existed). Logout stays idempotent and exits 0 either way; only the message changes. +- 3cc768d: Pin `@orpc/client` and `@orpc/contract` to 1.15.4 to fix prototype pollution (CVE-2026-28794) and include the subsequent deserializer validation fix (GHSA-4p2c-m292-ghmh). Keep exact versions and align the oRPC dependency family. +- Updated dependencies [f85f568] +- Updated dependencies [3f9a246] +- Updated dependencies [9ad23f9] +- Updated dependencies [65c90d6] +- Updated dependencies [75d15e9] +- Updated dependencies [3f1db03] +- Updated dependencies [f8238db] +- Updated dependencies [fedbf56] +- Updated dependencies [5a8d805] +- Updated dependencies [8296b8a] +- Updated dependencies [b501f5d] +- Updated dependencies [256ec62] + - @chkit/plugin-backfill@0.2.0-beta.8 + - @chkit/core@0.2.0-beta.8 + - @chkit/clickhouse@0.2.0-beta.8 + ## 0.1.2-beta.7 ### Patch Changes diff --git a/packages/plugin-obsessiondb/package.json b/packages/plugin-obsessiondb/package.json index 2f2d96b9..43685ef3 100644 --- a/packages/plugin-obsessiondb/package.json +++ b/packages/plugin-obsessiondb/package.json @@ -1,6 +1,6 @@ { "name": "@chkit/plugin-obsessiondb", - "version": "0.1.2-beta.7", + "version": "0.2.0-beta.8", "description": "ObsessionDB managed ClickHouse integration plugin for chkit", "license": "MIT", "author": "ObsessionDB", diff --git a/packages/plugin-pull/CHANGELOG.md b/packages/plugin-pull/CHANGELOG.md index f1924fee..d74e0c50 100644 --- a/packages/plugin-pull/CHANGELOG.md +++ b/packages/plugin-pull/CHANGELOG.md @@ -1,5 +1,60 @@ # @chkit/plugin-pull +## 0.2.0-beta.8 + +### Minor Changes + +- fedbf56: Support Kafka engine tables without MergeTree key clauses, with escaped literal + settings, pull round trips, and normalized drift/check comparisons. Preserve the + existing setting-string contract for other engines. Reject unsupported Kafka + changes before writing migration artifacts and document an explicit, destructive- + gated replacement workflow. Handle quoted delimiters and escaped trailing + backslashes in introspection and migration statement splitting. +- 256ec62: Support ClickHouse `text` indexes in schemas, migrations, pull, and drift. Text + indexes require a tokenizer and support preprocessing, postprocessing, phrase + search, and dictionary/posting-list options when supported by the server. + Granularity is automatic. Preserve whitespace and escapes inside SQL literals, + compare parameter order and SQL formatting consistently, and reject unsupported + or malformed metadata instead of silently losing settings. Includes Python parity + and live adversarial round-trip tests on ClickHouse 26.3 and 26.8. + +### Patch Changes + +- 65c90d6: Add `dictionary()` as a first-class ClickHouse schema primitive, mirroring `materializedView()` across the full lifecycle: DSL authoring, validation, canonicalization, SQL rendering, migration planning/diff, drift, `check`, destructive-op safety, `pull` introspection, and `codegen` typed interfaces. + + - `dictionary({ database, name, attributes, primaryKey, source, layout, lifetime, range?, settings?, comment? })` — attributes support `default`/`expression` (mutually exclusive), `hierarchical`, `bidirectional` (requires `hierarchical`), `injective`, and `isObjectId`. `range: { min, max }` renders `RANGE(MIN ... MAX ...)` for `RANGE_HASHED`/`COMPLEX_KEY_RANGE_HASHED` layouts, and `settings` renders `SETTINGS(...)`. + - ClickHouse has no `ALTER DICTIONARY`, so any structural change plans a single atomic `CREATE OR REPLACE DICTIONARY`. Dropping a dictionary is treated as destructive and blocked without `--allow-destructive`. + - Set `renamedFrom` on a dictionary (or pass `--rename-dictionary old_db.old=new_db.new` to `chkit generate`) to rename a dictionary via `RENAME DICTIONARY IF EXISTS ... TO ...` instead of a destructive drop + create. + - `chkit pull` introspects live dictionaries (including `RANGE`/`SETTINGS` and all attribute modifiers) into typed schema files, preserving ClickHouse's `[HIDDEN]` password redaction on `SOURCE(...)` credentials. A `SOURCE(...)` password change diffs and migrates like any other field change; `chkit generate` warns when a literal password is about to be written into migration SQL as plain text. `chkit pull` warns in two cases: when an introspected password comes back as `[HIDDEN]` (chkit can't recover the real value, so that dictionary's `source` is excluded from future diffs until it's replaced), and when ClickHouse is configured to reveal real passwords on introspection (`display_secrets_in_show_and_select` + `displaySecretsInShowAndSelect`), since that writes a plain-text credential into the generated schema file with no other indication. All warnings print to the console and are included as a `warnings` array in `--json` output. + - `codegen` generates a typed interface (and optional Zod schema) for each dictionary from its `attributes`, always included regardless of `includeViews`. + - `ON CLUSTER` mode stamps `ON CLUSTER ` onto every dictionary DDL statement, including `CREATE OR REPLACE DICTIONARY` and `RENAME DICTIONARY`. + +- 3f1db03: Support index-only projections (`PROJECTION p INDEX (a, b) TYPE basic`) end to end. `ProjectionDefinition` is now a union of the existing `{ name, query }` SELECT form and a new `{ name, index, type }` index-only form, which renders without the wrapping parens that made the SELECT form invalid for it. `chkit pull` previously parsed only the SELECT form and dropped index-only projections on the floor, so a pulled schema silently recreated the table without them; they now round-trip through pull, generate, migrate, and drift. + + Index expressions are normalized to the exact form ClickHouse stores — a single expression bare (`INDEX a`), several as a tuple (`INDEX (a, b)`), redundant parens peeled at every level, and a space after each argument separator — so `'(a)'` and `'a'`, or `'concat(x,y)'` and `'concat(x, y)'`, describe the same table and no longer read as drift. + + Two new validation errors guard the new form: `projection_ambiguous_kind` when an entry sets both `query` and `index` (which would otherwise silently discard the SELECT body), and `projection_empty_index` when the index expression is empty (which would otherwise emit invalid DDL). + +- 8296b8a: Fix two related pull/drift bugs around tables whose `ORDER BY` is declared alongside a projection or a derived primary key. + + `chkit pull` parsed table-level clauses (`ENGINE`, `ORDER BY`, `PRIMARY KEY`, `PARTITION BY`, `TTL`, `SETTINGS`) by matching the first keyword anywhere in `SHOW CREATE TABLE`. A projection whose `SELECT` body contains `ORDER BY` — or a column-level `TTL` — sits in the column list before those clauses, so the parser matched the inner keyword and swallowed the engine into `orderBy`/`primaryKey`, producing an invalid pulled schema (#190). Table-level clauses are now parsed only from the portion after the column list. + + `chkit drift` always reported `primary_key_mismatch` for any table whose `PRIMARY KEY` is derived from `ORDER BY`. ClickHouse omits the derived key from `SHOW CREATE TABLE`, but the schema carries it, so the two never matched. Drift now applies the same derivation to the live side, so a derived primary key reads clean while a genuine primary-key difference is still reported (#194). + +- 4ded781: Route `chkit pull` introspection through the host-provided executor instead of always opening its own ClickHouse connection. When an ObsessionDB service is selected, pull now introspects through the ObsessionDB API (the same executor `generate`/`migrate`/`status` use) rather than silently falling back to `http://localhost:8123` and failing with "connection refused" while printing `using service `. A direct ClickHouse target is unchanged, custom introspectors still open their own raw connection, and a run with no reachable target now errors with an actionable message instead of a misleading localhost fallback. +- b501f5d: Extract shared plugin command scaffolding into `@chkit/core`: new `createPluginRunner` (binds a plugin's config-error class once and wraps command `run` handlers in the shared error-to-exit-code envelope) and `withFactoryDefaults` (layers plugin-factory options under parsed data). The backfill, codegen, and pull plugins now use these helpers instead of private copies — no behavior change, but the plugins require the matching `@chkit/core` version. +- Updated dependencies [65c90d6] +- Updated dependencies [75d15e9] +- Updated dependencies [3f1db03] +- Updated dependencies [f8238db] +- Updated dependencies [fedbf56] +- Updated dependencies [5a8d805] +- Updated dependencies [8296b8a] +- Updated dependencies [b501f5d] +- Updated dependencies [256ec62] + - @chkit/core@0.2.0-beta.8 + - @chkit/clickhouse@0.2.0-beta.8 + ## 0.1.2-beta.7 ### Patch Changes diff --git a/packages/plugin-pull/package.json b/packages/plugin-pull/package.json index e21b1d59..fe5023e3 100644 --- a/packages/plugin-pull/package.json +++ b/packages/plugin-pull/package.json @@ -1,6 +1,6 @@ { "name": "@chkit/plugin-pull", - "version": "0.1.2-beta.7", + "version": "0.2.0-beta.8", "description": "Introspect live ClickHouse databases into chkit schema files", "license": "MIT", "author": "ObsessionDB",