From a39c7c7d5fcb35d9a4e9bd2f544b552227ad92e8 Mon Sep 17 00:00:00 2001 From: Steven Hildreth Date: Fri, 29 May 2026 07:12:36 -0500 Subject: [PATCH 1/6] docs: add 2026-05-28 release plans --- design/2026-05-28.RELEASE_PLANS.md | 1280 ++++++++++++++++++++++++++++ 1 file changed, 1280 insertions(+) create mode 100644 design/2026-05-28.RELEASE_PLANS.md diff --git a/design/2026-05-28.RELEASE_PLANS.md b/design/2026-05-28.RELEASE_PLANS.md new file mode 100644 index 0000000..b47dcaf --- /dev/null +++ b/design/2026-05-28.RELEASE_PLANS.md @@ -0,0 +1,1280 @@ +# Release Process Reset Plan + +**Date:** 2026-05-28 +**Status:** Proposed +**Scope:** GitHub Actions, release candidates, native artifacts, NuGet packages, +mobile artifacts, benchmark assets, docs, and release operator workflow. + +## 1. Problem Statement + +Creating a DecentDB release currently finds too many failures after a public +release tag exists. That creates the worst possible feedback loop: + +- A tag-triggered workflow starts from a commit that cannot be changed. +- The workflow spends a long time in validation before reaching artifact lanes. +- Failures in late platform lanes require either rerunning the same broken tag + contents or creating a new commit and deciding whether to move a public tag. +- NuGet packages are immutable once published, so a partially successful + release leaves permanent public state. +- GitHub Actions logs are remote, slow, and often only fully available after a + job or run completes. +- Release, NuGet, mobile, and benchmark workflows are separate enough to fail + independently, but similar enough that fixes and validation are duplicated. +- Some failures are not product failures at all. For example, the + `Web WASM Nightly` run on 2026-05-29 failed before any DecentDB build, + browser smoke, or benchmark step ran because `cargo install + wasm-bindgen-cli --version 0.2.114 --locked` hit a crates.io/curl EOF while + downloading `clap_derive`. The release process must separate this kind of + tool bootstrap or network failure from engine, binding, packaging, and + benchmark regressions. +- Nightly memory-safety failures can mix both categories in one workflow. The + `Memory Safety Nightly` run on 2026-05-29 had one job fail before memory + validation because crates.io returned HTTP 500 while downloading `lua-src`, + and another job fail after product validation because Valgrind reported + `148 bytes possibly lost` in the Python/C ABI path through + `ProcessCoordinator::begin_reader`. The release process must preserve that + distinction: dependency download failures are bootstrap/infra; Valgrind, + sanitizer, Miri, and leak findings after tests run are product signals until + reproduced, fixed, suppressed with justification, or explicitly waived. + +The release process should be redesigned so release-blocking GitHub Actions +failures are found before the final tag exists and before anything is published +to an external registry. + +The operational goal is simple: + +1. Build and validate a release candidate from a branch or exact SHA. +2. Fix any workflow, packaging, benchmark, or platform failures while the + version is still unpublished. +3. Produce a candidate artifact bundle with checksums and provenance. +4. Tag only a known-good commit. +5. Publish only artifacts that already passed candidate validation. + +## 2. Non-Goals + +- This plan does not weaken correctness validation. It moves validation earlier. +- This plan does not make tag history mutable. The desired state is fewer tag + rewrites, not more. +- This plan does not make benchmark regressions invisible. It separates + benchmark review from release publication. +- This plan does not require all expensive nightly checks to block every PR. +- This plan does not require self-hosted runners. They may help later, but the + first fix is workflow topology. + +## 3. Release Invariants + +These should become hard rules in automation and release documentation. + +1. A public `vX.Y.Z` tag is final. +2. A workflow rerun never picks up code or workflow changes made after the + tagged commit. +3. `Re-run failed jobs` is only useful for transient runner/network failures. + It is not a fix mechanism for workflow logic, linker errors, package layout + problems, missing artifacts, or script bugs. +4. External publication is a separate phase from candidate validation. +5. NuGet publication is manual/explicit until a candidate bundle has passed. +6. The release publish workflow should not compile native code. It should + consume a candidate bundle that already contains the exact release artifacts. +7. Every release artifact must include a manifest entry with: + - release version + - source commit SHA + - source ref + - producing workflow run ID + - target platform/RID + - artifact path + - SHA-256 checksum + - size + - toolchain versions where practical +8. The tag SHA, release candidate SHA, and artifact manifest SHA must match. +9. The root `CHANGELOG.md` placeholder must not be edited. Release notes live in + `docs/about/changelog.md`. +10. Release-critical workflows must not discover first-time tool bootstrap + failures from a public tag. External tools used by release candidates must + be pinned, version-checked, and either cached, restored from a trusted + artifact, or installed by an explicit bootstrap step with retries and clear + failure classification. +11. Scheduled and nightly workflow failures must be classified before they are + used as release evidence: + - `product`: DecentDB build, test, package, browser smoke, or benchmark + behavior failed. + - `tool_bootstrap`: a required compiler, CLI, browser, package manager, or + dependency download failed before product validation began. + - `runner_infra`: hosted runner, network, cache service, or GitHub platform + behavior failed outside the repository's control. + A release may require recent successful product-oriented nightly evidence, + but an infra-only nightly failure should not be treated as proof of a + DecentDB regression. +12. Dependency materialization must be visibly separate from product + validation. Cargo, npm, Python, Go, Maven/Gradle, Dart, and system package + downloads should happen in explicit bootstrap/fetch/install steps with + retry policy and logs. If a download fails before tests or analyzers run, + the result is `tool_bootstrap` or `runner_infra`; if the analyzer runs and + reports a memory, correctness, packaging, or benchmark failure, the result + is `product`. + +## 4. Current Workflow Inventory + +The repository currently has ten workflows: + +| Workflow | Current Role | Release Pain | Recommendation | +|---|---|---|---| +| `.github/workflows/ci.yml` | PR/main Rust lint, tests, web smoke | Useful, but not a complete release preflight | Keep as required PR gate; add workflow lint and targeted release metadata checks | +| `.github/workflows/docs.yml` | Deploy docs to Pages on docs changes | Separate from release docs validation | Keep deploy-only; release candidate should build docs but not deploy | +| `.github/workflows/release.yml` | Tag-triggered full validation, native artifacts, JDBC/DBeaver assets, GitHub release | Finds platform packaging failures after tag exists; validation delays artifact failure by ~1 hour | Split into candidate build and publish-from-candidate | +| `.github/workflows/nuget.yml` | Tag/manual NuGet build and optional publish | Builds from tag; can publish before other release artifacts are green | Fold into candidate/publish model; publish only candidate `.nupkg` artifacts | +| `.github/workflows/nuget-manual.yml` | Older manual NuGet tag workflow | Duplicates `nuget.yml` | Delete after replacement is in place | +| `.github/workflows/mobile-native-artifacts.yml` | Tag/manual Flutter/mobile native artifacts | Separate tag-triggered release surface with its own failure modes | Move artifact build into release candidate; keep optional manual diagnostic workflow or remove | +| `.github/workflows/benchmark-assets.yml` | Main/manual benchmark asset refresh and bot commit | Can fail main after release; benchmark narrative can block unrelated publish flow | Keep as benchmark review/update workflow; release candidate should run benchmark narrative validation but not auto-commit assets | +| `.github/workflows/python-benchmarks.yml` | Nightly/manual Python benchmark suite | Overlaps with benchmark-assets and pre-commit benchmark checks | Consolidate into benchmark-assets or rename as nightly benchmark research; do not make release-critical | +| `.github/workflows/memory-safety-nightly.yml` | Nightly sanitizers, Miri, leaks, Valgrind | Valuable, too expensive/flaky for every release tag; can mix dependency bootstrap failures with real memory findings | Keep nightly; candidate can require latest successful run within a window; classify bootstrap failures separately from sanitizer/Miri/Valgrind findings | +| `.github/workflows/web-wasm-nightly.yml` | Nightly browser OPFS smoke and benchmark | Duplicates `ci.yml` web smoke and release candidate web needs; can also fail before product validation because it installs `wasm-bindgen-cli` from crates.io in the job hot path | Keep nightly for extended browser benchmark; candidate should run a deterministic browser smoke slice; pin/cache/bootstrap browser and WASM tools separately | + +## 5. Target Workflow Set + +The target should be a smaller, clearer workflow set: + +1. `ci.yml` + - PR and main gate. + - Fast enough to run on normal development. + - Required before merge. + +2. `release-candidate.yml` + - Manual only. + - Runs against a branch, tag, or SHA before the final release tag exists. + - Builds every release artifact or a targeted subset. + - Runs dry-run publication checks. + - Uploads a candidate bundle and manifest. + - Does not publish to NuGet. + - Does not create or update a GitHub release. + - Does not push benchmark assets to main. + +3. `release-publish.yml` + - Manual only, or tag-triggered only after candidate manifest verification is + implemented. + - Takes `release_tag` and `candidate_run_id`. + - Verifies the tag SHA equals the candidate manifest SHA. + - Downloads artifacts from the candidate run. + - Verifies checksums. + - Creates/updates the GitHub release. + - Publishes NuGet packages only when an explicit boolean input is true. + - Does not compile code. + +4. `benchmark-assets.yml` + - Main/manual benchmark asset refresh. + - Produces review artifacts. + - Commits benchmark assets only when the benchmark narrative guard passes or + when a maintainer explicitly accepts the regression. + - Not responsible for release publication. + +5. `docs.yml` + - GitHub Pages deploy only. + - No release packaging behavior. + +6. `nightly-safety.yml` + - Replacement or rename target for memory safety checks. + - Can keep existing implementation. + +7. `nightly-browser.yml` + - Replacement or keep target for web WASM/browser benchmark. + - Can keep existing implementation. + +The target deletion/consolidation list: + +- Delete `nuget-manual.yml` after `release-candidate.yml` and + `release-publish.yml` cover dry-run and publish. +- Remove tag triggers from `nuget.yml`, or delete `nuget.yml` once NuGet is + fully handled by candidate/publish workflows. +- Remove tag triggers from `mobile-native-artifacts.yml`, or delete the workflow + after mobile artifacts are part of the candidate bundle. +- Rename or merge `python-benchmarks.yml` into the benchmark workflow family. + +## 6. Release Candidate Workflow Requirements + +`release-candidate.yml` should be the main release engineering workflow. + +### 6.1 Inputs + +Required inputs: + +- `release_version` + - Example: `2.8.1` + - Used for package versions and artifact names. + +- `release_ref` + - Default: current selected ref. + - Can be `main`, a release branch, a tag, or a full SHA. + +Optional inputs: + +- `platform` + - `all` + - `linux-x64` + - `linux-arm64` + - `macos-arm64` + - `windows-x64` + - `nuget` + - `mobile-android` + - `mobile-ios` + - `benchmarks` + - `docs` + - `web` + +- `validation_level` + - `fast` + - `full` + - `paranoid` + +- `publishable` + - Default false. + - When false, artifact names should include `candidate-`. + - When true, artifact names should use final release naming, but still not + publish. + +- `accept_benchmark_regression` + - Default false. + - When true, benchmark artifacts may be marked publishable despite narrative + guard failures, but the manifest must record that acceptance. + +### 6.2 Jobs + +The candidate workflow should contain these jobs. + +#### plan + +Responsibilities: + +- Normalize `release_version`. +- Validate version syntax. +- Resolve `release_ref` to an exact commit SHA. +- Generate the platform matrix. +- Emit a `candidate_id`, for example: + `v2.8.1-rcbuild--`. + +Outputs: + +- `version` +- `tag_name` +- `source_sha` +- `source_ref` +- `candidate_id` +- `matrix` + +#### workflow-lint + +Responsibilities: + +- Run `actionlint` over all workflows. +- Validate no release-critical workflow uses a tag trigger to publish directly. +- Validate `nuget-manual.yml` is absent once the new flow is adopted. +- Validate workflow action major versions are current enough for the runner + deprecation window. +- Validate release-critical workflows do not contain naked dynamic tool installs + such as `cargo install`, `go install`, `npm install -g`, or curl-piped + installer scripts unless they are wrapped by an approved repository script + that pins the version, verifies the installed binary, uses retry policy, and + emits clear bootstrap logs. +- Validate scheduled workflows that are used as release evidence classify + failures as product, tool bootstrap, or runner infrastructure. + +Implementation notes: + +- Install `actionlint` via a pinned known-good version. Avoid `@latest` in + release-critical workflow paths because it creates the same moving-target + bootstrap problem this plan is trying to remove. +- Add the same check to `scripts/do-pre-commit-checks.py --mode paranoid`. + +#### validate + +Responsibilities: + +- Run the repo's release validation at the requested level. +- For `full` or `paranoid`, this should be equivalent to: + `python scripts/do-pre-commit-checks.py --mode paranoid`. +- Upload logs even on failure. + +Rules: + +- This job can be skipped for targeted diagnostic runs. +- It must not be skipped for a publishable all-platform release candidate. + +#### native-artifacts + +Responsibilities: + +- Build native artifacts for: + - Linux x64 + - Linux arm64 + - macOS arm64 + - Windows x64 +- Build: + - native shared library + - CLI + - migrate tool + - JNI bridge + - JDBC jar + - DBeaver plugin zip + - Dart native asset package +- Package artifacts using final release names when `publishable=true`. +- Upload artifacts with a manifest fragment. + +Critical requirements: + +- The Windows JNI bridge must link against the DecentDB cdylib/import library, + not the Rust static library, unless the compiler/linker toolchain is MSVC + end-to-end. +- The job must support a single-platform matrix run. Windows packaging bugs + should be debuggable without Linux/macOS builds or the full validation suite. +- Every package step must avoid relative `..` paths in upload-artifact inputs. + +#### nuget-pack + +Responsibilities: + +- Build native RID artifacts or consume native artifacts from + `native-artifacts`. +- Pack every DecentDB NuGet package. +- Run package validation. +- Upload `.nupkg` and `.snupkg` files as candidate artifacts. + +Rules: + +- No push to NuGet in this job. +- Package versions must exactly match `release_version`. +- If the package version already exists on NuGet.org, fail before packing when + `publishable=true`. + +#### mobile-artifacts + +Responsibilities: + +- Build Android Flutter/native artifacts. +- Build iOS Flutter/native artifacts. +- Run mobile artifact smoke/guardrail checks. +- Upload mobile packages and manifest fragments. + +Rules: + +- This replaces the release-critical tag behavior in + `mobile-native-artifacts.yml`. +- Android and iOS must support targeted reruns. + +#### benchmark-review + +Responsibilities: + +- Run release benchmark narrative validation. +- Run raw `rust-baseline` cross-check. +- Generate benchmark review artifacts. +- Upload generated chart assets and narrative report. + +Rules: + +- A benchmark narrative failure should mark the candidate as not publishable by + default. +- It should not auto-commit benchmark assets. +- If `accept_benchmark_regression=true`, the manifest must include: + - the failed metrics + - the acceptance input + - the accepting run ID + +#### docs-build + +Responsibilities: + +- Build Rust docs. +- Build MkDocs site. +- Validate release-facing docs links if possible. + +Rules: + +- Do not deploy pages from candidate. +- The Pages deploy workflow remains separate. + +#### web-browser-smoke + +Responsibilities: + +- Build WASM package. +- Run deterministic browser OPFS smoke. +- Optionally run a short browser transport benchmark. +- Verify the exact `wasm-bindgen` CLI version before invoking it. + +Rules: + +- Extended browser benchmark remains nightly. +- Candidate smoke should be deterministic and bounded. +- Candidate and nightly browser jobs should use the same repository-owned + `wasm-bindgen` install/check script. +- Do not install `wasm-bindgen-cli` with a bare `cargo install` in the same + step that is treated as product validation. Restore a cached binary or + trusted tool artifact first; on cache miss, run an explicit bootstrap step + with retry policy and version verification. +- A failure before the DecentDB WASM crate builds is `tool_bootstrap` or + `runner_infra`, not a browser OPFS or engine regression. +- Upload bootstrap logs, `rustc --version --verbose`, `cargo --version`, + `node --version`, `npm --version`, and `wasm-bindgen --version` even when the + browser smoke itself is skipped. +- Keep the OPFS smoke and browser transport benchmark as separate steps so a + benchmark failure does not hide whether basic browser persistence still + works. + +#### manifest + +Responsibilities: + +- Download all manifest fragments. +- Verify expected artifact coverage. +- Verify checksums. +- Verify version strings. +- Verify package names. +- Verify source SHA. +- Emit one `release-candidate-manifest.json`. + +The manifest should have this rough shape: + +```json +{ + "schema": 1, + "project": "decentdb", + "version": "2.8.1", + "tag": "v2.8.1", + "source_sha": "", + "source_ref": "main", + "candidate_run_id": 123, + "created_at": "2026-05-28T00:00:00Z", + "publishable": false, + "benchmark_regression_accepted": false, + "artifacts": [ + { + "kind": "github-release-native", + "platform": "windows-x64", + "path": "decentdb-v2.8.1-Windows-x64.zip", + "sha256": "", + "bytes": 123 + } + ] +} +``` + +## 7. Release Publish Workflow Requirements + +`release-publish.yml` should publish from an existing candidate. It should not +build. + +### 7.1 Inputs + +Required: + +- `release_tag` + - Example: `v2.8.1` + +- `candidate_run_id` + - The successful `release-candidate.yml` run ID. + +Optional: + +- `publish_github_release` + - Default true. + +- `publish_nuget` + - Default false until maintainers are confident. + +- `publish_mobile` + - Default true if mobile artifacts are part of the release. + +- `allow_existing_github_release_update` + - Default false. + - Required for re-uploading GitHub release assets. + +### 7.2 Verification Jobs + +The workflow must: + +1. Resolve `release_tag` to `tag_sha`. +2. Download the candidate manifest from `candidate_run_id`. +3. Verify `manifest.source_sha == tag_sha`. +4. Verify `manifest.version` matches `release_tag`. +5. Download all candidate artifacts. +6. Verify every checksum in the manifest. +7. Verify no NuGet package version already exists before publishing. +8. Verify GitHub release asset names are complete and non-duplicated. + +### 7.3 Publish Jobs + +GitHub release publication: + +- Create a release if it does not exist. +- Upload artifacts from the candidate bundle. +- Do not rebuild artifacts. +- Attach the candidate manifest and checksum file. + +NuGet publication: + +- Push `.nupkg` files from the candidate bundle. +- Never rebuild packages in the publish job. +- Require `publish_nuget=true`. +- Fail if any package version already exists unless the package is already + listed and the workflow is only verifying state. + +## 8. Proposed Workflow Disposition + +### 8.1 `.github/workflows/release.yml` + +Current problem: + +- Does validation and artifact creation on tag push. +- A platform artifact failure arrives after the validation suite. +- The tagged commit cannot pick up fixes without moving the tag or making a new + version. + +Plan: + +- Convert it into `release-candidate.yml` or split the current file. +- Keep temporary diagnostic inputs while migrating: + - `build_platform` + - `run_validation` + - `create_release` +- Long term, remove GitHub release publication from this workflow and move it to + `release-publish.yml`. + +### 8.2 `.github/workflows/nuget.yml` + +Current problem: + +- Has both tag and manual behavior. +- Can publish NuGet packages before GitHub release artifacts are known-good. + +Plan: + +- Remove tag-triggered publishing. +- Move pack validation into `release-candidate.yml`. +- Move publish into `release-publish.yml`. +- Keep a dry-run mode until the new flow proves stable. + +### 8.3 `.github/workflows/nuget-manual.yml` + +Current problem: + +- Duplicates `nuget.yml`. + +Plan: + +- Delete after the new candidate/publish workflows are merged. +- Any unique logic should be ported into scripts, not copied into another + workflow. + +### 8.4 `.github/workflows/mobile-native-artifacts.yml` + +Current problem: + +- Separate tag trigger. +- Separate failure surface. +- Release artifacts can fail after other release components have passed. + +Plan: + +- Move Android/iOS artifact build into `release-candidate.yml`. +- Keep a manual-only mobile diagnostic workflow only if it remains useful. +- Remove the tag trigger. + +### 8.5 `.github/workflows/benchmark-assets.yml` + +Current problem: + +- Runs after main pushes and can fail while a release is in progress. +- It can auto-commit assets to main, which is useful but should not be confused + with release publication. + +Plan: + +- Keep as benchmark asset maintenance. +- Candidate workflow should run benchmark validation and upload review + artifacts. +- Publishing benchmark assets to main should remain separate from release + tagging. + +### 8.6 `.github/workflows/python-benchmarks.yml` + +Current problem: + +- Partial benchmark suite overlaps with benchmark-assets and pre-commit checks. + +Plan: + +- Decide whether it is still useful as a nightly research workflow. +- If yes, rename to `benchmark-nightly.yml` and make scope explicit. +- If no, delete and port any still-useful tests into benchmark-assets or + pre-commit paranoid mode. + +### 8.7 `.github/workflows/ci.yml` + +Current problem: + +- Useful PR gate but not release-complete. + +Plan: + +- Keep. +- Add workflow lint or make workflow lint a separate required job. +- Consider adding `scripts/validate_release_metadata.py` for version/changelog + changes when release files are touched. + +### 8.8 `.github/workflows/docs.yml` + +Current problem: + +- No major problem. It deploys Pages. + +Plan: + +- Keep deploy-only. +- Candidate workflow builds docs but does not deploy. + +### 8.9 Nightly Workflows + +Current workflows: + +- `memory-safety-nightly.yml` +- `web-wasm-nightly.yml` + +Plan: + +- Keep nightly. +- Do not require every release candidate to rerun full nightly suites. +- Candidate manifest should record whether the latest successful nightly on + `main` is recent enough, for example within seven days. +- Candidate manifest should also record the latest failed nightly classification + when the latest run failed. A `tool_bootstrap` or `runner_infra` failure is + operational debt, but it is different from a product regression. + +Specific rule for `memory-safety-nightly.yml`: + +- The memory-safety nightly must split dependency/bootstrap, build, and analyzer + execution in the logs. +- Installing Valgrind, heaptrack, Rust toolchains, Python packages, system + packages, and Cargo crate sources is bootstrap. +- Building DecentDB after dependencies are present is build validation. +- Running Valgrind, Miri, LeakSanitizer, AddressSanitizer, heap profiling + analysis, binding lifecycle leak tests, and long-running stress tests is + product validation. +- A dependency download failure before the analyzer runs is `tool_bootstrap` or + `runner_infra`, not a memory-safety finding. +- A Valgrind, sanitizer, Miri, leak threshold, heap analysis, or stress-test + failure after the analyzer starts is `product` until it is reproduced and + fixed, added to a narrowly justified suppression file, or explicitly waived + in the release manifest. +- Every memory-safety job should upload logs even when the build step fails. If + the analyzer was never invoked, the uploaded artifact should make that clear + so release review does not mistake an empty log set for a clean run. +- The candidate manifest should record the latest successful memory-safety + nightly run ID and, when the latest run failed, the failed job names and + classifications. + +The 2026-05-29 `Memory Safety Nightly` failure is the concrete example this +policy must handle: + +- `C ABI Valgrind` failed during `cargo build -p decentdb --quiet` because + `static.crates.io` returned HTTP 500 while downloading `lua-src`; classify as + `tool_bootstrap` or `runner_infra`. +- `Python Valgrind` ran the probe successfully, then Valgrind exited nonzero + because it reported `148 bytes possibly lost` in a path through + `decentdb::wal::coordination::ProcessCoordinator::begin_reader`; classify as + `product` until investigated. + +Specific rule for `web-wasm-nightly.yml`: + +- The browser nightly must split tool bootstrap from product validation. +- Installing or restoring `wasm-bindgen-cli`, Playwright/browser dependencies, + Node, and Rust targets is bootstrap. +- Building `bindings/web`, producing the DecentDB WASM artifact, running OPFS + smoke, and running browser transport benchmarks is product validation. +- If bootstrap fails, the run should fail loudly and upload logs, but release + notes and release readiness should not claim that DecentDB browser behavior + regressed. +- If product validation fails, the failure should block release readiness until + fixed or explicitly waived with a documented decision. + +The 2026-05-29 `Web WASM Nightly` failure is the concrete example this policy +must handle: `cargo install wasm-bindgen-cli --version 0.2.114 --locked` +downloaded the package, then failed fetching the transitive `clap_derive` +dependency because curl received an unexpected EOF from the peer. That failure +is a network/tool bootstrap failure, not evidence that OPFS, WASM, or DecentDB +engine behavior failed. + +## 9. Scripts To Add Or Refactor + +The goal is to move release behavior out of YAML wherever possible. + +### 9.1 `scripts/release/plan.py` + +Purpose: + +- Parse candidate inputs. +- Resolve platform matrix. +- Emit JSON for GitHub matrix and local use. + +Benefits: + +- The same matrix can be tested locally. +- Fewer giant YAML blocks. +- Less risk of invalid GitHub expression context usage. + +### 9.2 `scripts/release/build_native_artifact.sh` + +Purpose: + +- Build one native target package. +- Used by GitHub Actions and local diagnostics. + +Arguments: + +- `--platform linux-x64|linux-arm64|macos-arm64|windows-x64` +- `--version 2.8.1` +- `--out-dir .tmp/release-candidate/native` +- `--package-tag v2.8.1` or `candidate-` + +### 9.3 `scripts/release/build_jni_bridge.sh` + +Purpose: + +- Encapsulate JNI bridge compiler/linker selection. +- On Windows, choose a consistent toolchain: + - MinGW plus cdylib/import library, or + - MSVC `cl` plus MSVC import library. + +The script should print: + +- compiler path +- linker path +- selected DecentDB link artifact +- selected Java include directory + +This would have made the `decentdb.lib` vs `decentdb.dll.lib` problem obvious. + +### 9.4 `scripts/release/build_nuget_packages.sh` + +Purpose: + +- Build all NuGet packages from staged native artifacts. +- Validate package contents. +- Never publish. + +Arguments: + +- `--version` +- `--native-root` +- `--out-dir` + +### 9.5 `scripts/release/check_nuget_version_available.py` + +Purpose: + +- Query NuGet.org before packaging/publishing. +- Fail if the exact version already exists. + +Rules: + +- This check is mandatory before `publish_nuget=true`. +- For RC versions, it still checks exact version existence. + +### 9.6 `scripts/release/build_mobile_artifacts.sh` + +Purpose: + +- Wrapper around Android and iOS build scripts. +- Normalizes artifact naming. +- Writes manifest fragments. + +### 9.7 `scripts/release/write_manifest.py` + +Purpose: + +- Produce manifest fragments for each job. +- Compute SHA-256 and sizes. + +### 9.8 `scripts/release/validate_manifest.py` + +Purpose: + +- Verify candidate completeness. +- Verify required artifacts by platform. +- Verify checksums. +- Verify version strings. +- Verify release tag compatibility. + +### 9.9 `scripts/validate_workflows.py` + +Purpose: + +- Run `actionlint`. +- Enforce workflow policy: + - no direct package publication on tag push, except approved publish workflow + - no duplicate NuGet workflows + - no release workflows without manual targeted inputs + - no JavaScript action versions that rely on a deprecated Node runtime after + GitHub's announced runner cutoff + - no upload-artifact paths with `../` + - no release-critical or release-evidence workflow with an unwrapped dynamic + tool install + +This should run in: + +- `ci.yml` +- `scripts/do-pre-commit-checks.py --mode paranoid` +- release candidate `workflow-lint` + +### 9.10 `scripts/ci/ensure_wasm_bindgen.sh` + +Purpose: + +- Provide one repository-owned way to install or verify `wasm-bindgen-cli` for + web, WASM, browser smoke, and browser benchmark workflows. +- Avoid repeating fragile `cargo install wasm-bindgen-cli ...` snippets in + YAML. + +Rules: + +- The expected version must be pinned in one place, either in the script or in a + small checked-in tool version file. +- The script first checks whether `wasm-bindgen` exists and matches the expected + version. +- If the correct binary is present, the script exits without network access. +- If the binary is missing, the script installs the pinned version with + `--locked`, Cargo network retry settings, and a small explicit retry loop for + transient registry failures. +- The script prints `rustc --version --verbose`, `cargo --version`, the expected + `wasm-bindgen` version, and the final `wasm-bindgen --version`. +- On failure, the script exits with a message that labels the failure as + `tool_bootstrap`. +- Workflows should cache the resulting binary with a key that includes the + operating system, architecture, Rust host triple, Rust version, and + `wasm-bindgen` version. + +### 9.11 `scripts/ci/cargo_fetch_locked.sh` + +Purpose: + +- Materialize Cargo dependencies before release-evidence jobs enter product + validation. +- Make crates.io/network failures easy to classify as bootstrap or runner + infrastructure instead of hiding them inside Valgrind, sanitizer, benchmark, + or artifact build steps. + +Rules: + +- Run `cargo fetch --locked` with Cargo network retry settings before analyzer + or benchmark jobs. +- Print `rustc --version --verbose`, `cargo --version`, and the Cargo registry + protocol configuration. +- Use a small explicit retry loop for transient registry HTTP 5xx, EOF, and + timeout failures. +- On failure, exit with a message that labels the failure as `tool_bootstrap` or + `runner_infra`. +- Workflows should cache Cargo registry and git sources with keys that include + the operating system, architecture, Rust version, and `Cargo.lock` hash. +- Product validation steps should run only after this step succeeds, so a + dependency download outage cannot masquerade as a DecentDB regression. + +## 10. Local/Remote Debugging Strategy + +GitHub Actions cannot be perfectly reproduced locally, especially macOS, +Windows, and hosted runner image behavior. The next best thing is to make every +remote step small and targeted. + +### 10.1 Mandatory Targeted Dispatch + +Every release-critical workflow should support: + +- one platform +- no publish +- optional validation skip +- branch/SHA ref + +Example: + +```bash +gh workflow run release-candidate.yml \ + --ref main \ + -f release_version=2.8.1 \ + -f platform=windows-x64 \ + -f validation_level=fast \ + -f publishable=false +``` + +### 10.2 Command To Watch A Diagnostic Run + +```bash +sleep 10 +RUN_ID=$(gh run list \ + --workflow release-candidate.yml \ + --event workflow_dispatch \ + --branch main \ + --limit 1 \ + --json databaseId \ + --jq '.[0].databaseId') + +gh run watch "$RUN_ID" --exit-status +``` + +### 10.3 Pull Logs Immediately + +```bash +gh run view "$RUN_ID" --json jobs +gh run view "$RUN_ID" --job "" --log +``` + +For completed individual jobs when `gh run view --log` refuses because the run +is still active: + +```bash +gh api /repos/sphildreth/decentdb/actions/jobs//logs > .tmp/job.log +``` + +### 10.4 Rerun Policy + +Use `gh run rerun --failed` only for likely transient failures: + +- network download timed out +- GitHub artifact upload returned a transient service error +- package registry had a temporary 5xx + +Do not rerun as the primary fix for: + +- compile errors +- linker errors +- missing files +- invalid artifact paths +- failed tests +- invalid workflow expressions +- package version conflicts + +## 11. Release Operator Runbook + +This is the intended post-redesign release flow. + +### 11.1 Prepare Release Branch + +```bash +git switch main +git pull --ff-only origin main +git switch -c release/v2.8.1 + +./scripts/bump_version.sh 2.8.1 +python scripts/validate_release_metadata.py +python scripts/do-pre-commit-checks.py --mode fast +``` + +Open a PR: + +```bash +git push -u origin release/v2.8.1 +gh pr create \ + --base main \ + --head release/v2.8.1 \ + --title "Release v2.8.1" \ + --body "Prepare DecentDB v2.8.1." +``` + +### 11.2 Merge Release Branch + +After PR checks pass, merge normally through GitHub. + +Then update local main: + +```bash +git switch main +git pull --ff-only origin main +``` + +### 11.3 Run Full Release Candidate Before Tagging + +```bash +gh workflow run release-candidate.yml \ + --ref main \ + -f release_version=2.8.1 \ + -f platform=all \ + -f validation_level=full \ + -f publishable=true \ + -f accept_benchmark_regression=false +``` + +Watch: + +```bash +sleep 10 +RUN_ID=$(gh run list \ + --workflow release-candidate.yml \ + --event workflow_dispatch \ + --branch main \ + --limit 1 \ + --json databaseId \ + --jq '.[0].databaseId') + +gh run watch "$RUN_ID" --exit-status +``` + +### 11.4 If A Candidate Lane Fails + +Run only that lane: + +```bash +gh workflow run release-candidate.yml \ + --ref main \ + -f release_version=2.8.1 \ + -f platform=windows-x64 \ + -f validation_level=fast \ + -f publishable=false +``` + +Fix the branch through a PR, merge, and rerun the full candidate. Do not create +the tag yet. + +### 11.5 Tag Only After Candidate Passes + +Resolve the exact SHA from the candidate run: + +```bash +CANDIDATE_SHA=$(gh run view "$RUN_ID" \ + --json headSha \ + --jq '.headSha') + +git fetch origin main +git tag -a "v2.8.1" "$CANDIDATE_SHA" -m "DecentDB v2.8.1" +git push origin "refs/tags/v2.8.1" +``` + +### 11.6 Publish From Candidate + +```bash +gh workflow run release-publish.yml \ + --ref main \ + -f release_tag=v2.8.1 \ + -f candidate_run_id="$RUN_ID" \ + -f publish_github_release=true \ + -f publish_nuget=true \ + -f publish_mobile=true +``` + +Watch: + +```bash +sleep 10 +PUBLISH_RUN_ID=$(gh run list \ + --workflow release-publish.yml \ + --event workflow_dispatch \ + --branch main \ + --limit 1 \ + --json databaseId \ + --jq '.[0].databaseId') + +gh run watch "$PUBLISH_RUN_ID" --exit-status +``` + +## 12. Emergency Policy For A Partially Published Release + +If NuGet packages are already published but GitHub release artifacts failed: + +1. Do not immediately delete or move the release tag. +2. Determine whether the failed artifacts were built from the same commit as + the published packages. +3. If the fix requires source or workflow changes after package publication, + prefer a patch release, for example `v2.8.1`. +4. Only move a public tag if: + - no external package was published, or + - maintainers explicitly accept the provenance mismatch, and + - the release notes record what happened. + +Recommended default: + +- Use a patch release. +- Keep the failed release documented. +- Do not rewrite public release history after external registry publication. + +## 13. Branch Protection And Required Checks + +The redesigned process should work with branch protection: + +- PRs to `main` still run required `ci.yml`. +- Release candidate workflows run manually after merge to `main`. +- Tags are created only after candidate success. +- Publish workflow verifies tag SHA against the candidate manifest. + +Required PR checks should stay reasonably fast. Do not make all release +candidate jobs required on every PR. Instead: + +- Require CI. +- Require workflow lint. +- Require release metadata validation when release/version files change. +- Require candidate success only as an operator runbook step before tagging. + +## 14. Artifact Naming + +Final GitHub release assets should remain: + +- `decentdb-vX.Y.Z-Linux-x64.tar.gz` +- `decentdb-vX.Y.Z-Linux-arm64.tar.gz` +- `decentdb-vX.Y.Z-macOS-arm64.tar.gz` +- `decentdb-vX.Y.Z-Windows-x64.zip` +- `decentdb-dart-native-vX.Y.Z-Linux-x64.tar.gz` +- `decentdb-dart-native-vX.Y.Z-Linux-arm64.tar.gz` +- `decentdb-dart-native-vX.Y.Z-macOS-arm64.tar.gz` +- `decentdb-dart-native-vX.Y.Z-Windows-x64.zip` +- `decentdb-mobile-android-vX.Y.Z.zip` +- `decentdb-mobile-ios-vX.Y.Z.zip` +- `decentdb-jdbc-vX.Y.Z-Linux.jar` +- `decentdb-jdbc-vX.Y.Z-Linux-arm64.jar` +- `decentdb-jdbc-vX.Y.Z-macOS.jar` +- `decentdb-jdbc-vX.Y.Z-Windows.jar` +- `decentdb-dbeaver-vX.Y.Z-Linux.zip` +- `decentdb-dbeaver-vX.Y.Z-Linux-arm64.zip` +- `decentdb-dbeaver-vX.Y.Z-macOS.zip` +- `decentdb-dbeaver-vX.Y.Z-Windows.zip` +- `release-candidate-manifest-vX.Y.Z.json` +- `checksums-vX.Y.Z.sha256` + +Candidate-only artifacts can include: + +- `decentdb-candidate---Windows-x64.zip` + +But publishable candidate artifacts should use final names so the publish job +does not rename or rebuild anything. + +## 15. Acceptance Criteria + +The release process reset is complete when: + +1. A maintainer can run a Windows-only release artifact diagnostic without the + full validation suite. +2. A maintainer can run Android-only or iOS-only mobile artifact diagnostics. +3. A maintainer can dry-run NuGet packaging without publishing. +4. A full release candidate run creates every release artifact and a manifest + before any final tag is created. +5. A publish workflow can create the GitHub release from candidate artifacts + without compiling code. +6. NuGet publication uses candidate `.nupkg` artifacts and does not rebuild. +7. `nuget-manual.yml` is removed. +8. Tag-triggered workflows no longer perform first-time discovery of release + blockers. +9. `actionlint` runs in CI or paranoid pre-commit. +10. Release docs explain the candidate-then-tag-then-publish flow. +11. Release-evidence workflows, including browser/WASM nightly, classify + failures as product, tool bootstrap, or runner infrastructure. +12. The browser/WASM workflows no longer depend on an unwrapped one-shot + `cargo install wasm-bindgen-cli` in the product validation path. +13. Memory-safety workflows clearly distinguish dependency download/build + bootstrap failures from analyzer findings such as Valgrind leaks, + sanitizer failures, Miri failures, and stress-test failures. + +## 16. Implementation Phases + +### Phase 0: Stop The Immediate Bleeding + +Tasks: + +- Add targeted diagnostic inputs to the current release workflow. +- Fix Windows JNI linking so MinGW does not choose the MSVC Rust staticlib. +- Make benchmark narrative failures block asset publication without failing + unrelated release work. +- Add `actionlint` validation locally and in CI. +- Wrap `wasm-bindgen-cli` installation in a pinned, version-checking, + retry-capable script and update `web-wasm-nightly.yml` to use it. +- Add a Cargo dependency fetch/bootstrap script with retry/classification and + use it before memory-safety analyzers and other release-evidence jobs. +- Update browser/WASM workflow action versions for the current GitHub runner + deprecation window. + +### Phase 1: Extract Reusable Release Scripts + +Tasks: + +- Add `scripts/release/plan.py`. +- Add native artifact wrapper script. +- Add JNI bridge wrapper script. +- Add NuGet pack wrapper script. +- Add manifest writer/validator. +- Add workflow policy validation. +- Add nightly failure classification helpers for memory-safety and browser + workflows. + +### Phase 2: Add `release-candidate.yml` + +Tasks: + +- Implement manual candidate workflow. +- Support all-platform and single-platform modes. +- Support validation levels. +- Upload complete artifacts and manifest. +- Confirm candidate can reproduce current GitHub release assets. + +### Phase 3: Add `release-publish.yml` + +Tasks: + +- Implement candidate artifact download by run ID. +- Verify manifest and tag SHA. +- Publish GitHub release assets. +- Publish NuGet packages from existing candidate artifacts. +- Attach manifest and checksums. + +### Phase 4: Remove Duplicate And Tag-Triggered Publish Surfaces + +Tasks: + +- Delete `nuget-manual.yml`. +- Remove tag-triggered NuGet publishing from `nuget.yml` or delete `nuget.yml`. +- Remove tag-triggered mobile artifact publishing or fold it into candidate. +- Reduce `release.yml` to candidate/publish split or replace it entirely. + +### Phase 5: Document And Enforce + +Tasks: + +- Update development release docs. +- Update `README.md` release artifact notes if names change. +- Update `scripts/do-pre-commit-checks.py --mode paranoid` with workflow lint. +- Add a policy check that rejects direct tag-triggered package publication. +- Add a policy check that rejects unwrapped dynamic tool installs in + release-critical and release-evidence workflows. + +## 17. Open Decisions + +1. Should `release-publish.yml` be manual-only, or should a tag push trigger it + after verifying a matching successful candidate? + + Recommendation: manual-only first. Add tag automation only after the manifest + verification has proven stable. + +2. Should candidate artifacts be retained for 30 days or 90 days? + + Recommendation: 90 days for publishable candidates, 14 or 30 days for + diagnostic candidates. + +3. Should benchmark assets be part of the GitHub release artifact set? + + Recommendation: include benchmark narrative and generated charts in the + candidate artifact bundle, but keep checked-in README asset updates as a + separate PR/main workflow. + +4. Should mobile artifacts be attached to every GitHub release? + + Recommendation: yes if they are supported as release deliverables; otherwise + mark them experimental and keep them out of publish-blocking release assets. + +5. Should nightly memory-safety success be required for release publication? + + Recommendation: require a recent successful nightly by policy, not by + rerunning every sanitizer/Miri job in the release candidate. + +## 18. Summary + +The release process should stop treating the final tag as the first real +integration point. The new flow is: + +1. Merge release-prep changes. +2. Run a full release candidate from `main` before tagging. +3. Debug any failing lane with targeted candidate runs. +4. Tag the exact candidate SHA only after candidate success. +5. Publish GitHub and NuGet artifacts from the candidate bundle. + +This turns release debugging from an hour-long tag loop into a set of targeted +pre-tag checks with explicit provenance. From 6d6b215d113a86e00a13e527df26fa854113579f Mon Sep 17 00:00:00 2001 From: Steven Hildreth Date: Fri, 29 May 2026 07:23:20 -0500 Subject: [PATCH 2/6] feat(release): define automation rules for tool pinning and artifact retention - Require explicit Rust toolchain pinning to prevent silent cache invalidation. - Mandate centralized tool versioning in `tools/versions.toml`. - Establish retention policies (90/30/14 days) for candidate artifacts. - Require manual confirmation for NuGet publication to prevent accidental releases. - Specify concurrency group settings to ensure release workflows queue rather than cancel. --- design/2026-05-28.RELEASE_PLANS.md | 271 ++++++++++++++++++++++++++--- 1 file changed, 249 insertions(+), 22 deletions(-) diff --git a/design/2026-05-28.RELEASE_PLANS.md b/design/2026-05-28.RELEASE_PLANS.md index b47dcaf..e32f440 100644 --- a/design/2026-05-28.RELEASE_PLANS.md +++ b/design/2026-05-28.RELEASE_PLANS.md @@ -111,6 +111,33 @@ These should become hard rules in automation and release documentation. the result is `tool_bootstrap` or `runner_infra`; if the analyzer runs and reports a memory, correctness, packaging, or benchmark failure, the result is `product`. +13. Rust must be pinned with `rust-toolchain.toml`. Release-critical and + release-evidence workflows must not float on `stable` without an explicit + repository pin because a stable Rust release can silently invalidate caches, + change compiler behavior, and move binary compatibility under a release. +14. Non-Rust tool versions must be centralized in `tools/versions.toml`. + Workflows and helper scripts should read from that file or be validated + against it. At minimum it should cover: + - `wasm-bindgen-cli` + - `actionlint` + - Node.js + - Python + - .NET SDK + - Go + - Java + - Dart +15. Candidate artifact retention is a release invariant, not an open decision: + publishable candidate artifacts are retained for 90 days; normal diagnostic + candidates are retained for 30 days; narrow one-platform scratch + diagnostics may use 14 days when storage pressure matters. The manifest + should record the requested retention class. +16. NuGet publication is a point of no return. NuGet.org packages cannot be + deleted after publication, only unlisted. `publish_nuget=true` therefore + requires a separate manual confirmation or protected environment approval + after the candidate bundle and GitHub draft release are verified. +17. Release candidate and publish workflows must use per-version concurrency + groups with `cancel-in-progress: false`. A second release run for the same + version should queue rather than cancel or race with an in-progress run. ## 4. Current Workflow Inventory @@ -121,7 +148,7 @@ The repository currently has ten workflows: | `.github/workflows/ci.yml` | PR/main Rust lint, tests, web smoke | Useful, but not a complete release preflight | Keep as required PR gate; add workflow lint and targeted release metadata checks | | `.github/workflows/docs.yml` | Deploy docs to Pages on docs changes | Separate from release docs validation | Keep deploy-only; release candidate should build docs but not deploy | | `.github/workflows/release.yml` | Tag-triggered full validation, native artifacts, JDBC/DBeaver assets, GitHub release | Finds platform packaging failures after tag exists; validation delays artifact failure by ~1 hour | Split into candidate build and publish-from-candidate | -| `.github/workflows/nuget.yml` | Tag/manual NuGet build and optional publish | Builds from tag; can publish before other release artifacts are green | Fold into candidate/publish model; publish only candidate `.nupkg` artifacts | +| `.github/workflows/nuget.yml` | Tag push plus `workflow_dispatch` with `release_tag` and `publish_to_nuget` inputs for dry-run or publish | Builds from tag; can publish before other release artifacts are green | Fold into candidate/publish model; publish only candidate `.nupkg` artifacts | | `.github/workflows/nuget-manual.yml` | Older manual NuGet tag workflow | Duplicates `nuget.yml` | Delete after replacement is in place | | `.github/workflows/mobile-native-artifacts.yml` | Tag/manual Flutter/mobile native artifacts | Separate tag-triggered release surface with its own failure modes | Move artifact build into release candidate; keep optional manual diagnostic workflow or remove | | `.github/workflows/benchmark-assets.yml` | Main/manual benchmark asset refresh and bot commit | Can fail main after release; benchmark narrative can block unrelated publish flow | Keep as benchmark review/update workflow; release candidate should run benchmark narrative validation but not auto-commit assets | @@ -147,17 +174,24 @@ The target should be a smaller, clearer workflow set: - Does not publish to NuGet. - Does not create or update a GitHub release. - Does not push benchmark assets to main. + - Uses a per-version concurrency group. Publishable all-platform candidates + should serialize on `release-candidate-vX.Y.Z`; targeted diagnostics may + include the platform in the group only when they cannot mutate or overwrite + publishable candidate state. 3. `release-publish.yml` - Manual only, or tag-triggered only after candidate manifest verification is implemented. - Takes `release_tag` and `candidate_run_id`. - - Verifies the tag SHA equals the candidate manifest SHA. + - Dereferences annotated tags to the commit SHA and verifies that commit SHA + equals the candidate manifest SHA. - Downloads artifacts from the candidate run. - Verifies checksums. - - Creates/updates the GitHub release. + - Creates or updates a draft GitHub release first. - Publishes NuGet packages only when an explicit boolean input is true. - Does not compile code. + - Uses a per-version publish concurrency group such as + `release-publish-vX.Y.Z` with `cancel-in-progress: false`. 4. `benchmark-assets.yml` - Main/manual benchmark asset refresh. @@ -235,6 +269,17 @@ Optional inputs: - When true, benchmark artifacts may be marked publishable despite narrative guard failures, but the manifest must record that acceptance. +Concurrency: + +- Publishable candidates use `concurrency.group: + release-candidate-${{ inputs.release_version }}` and + `cancel-in-progress: false`. +- Targeted diagnostic candidates may use a narrower group such as + `release-candidate-${{ inputs.release_version }}-${{ inputs.platform }}` if + they only upload run-scoped diagnostic artifacts. +- No candidate workflow should overwrite a shared "latest candidate" artifact + outside the GitHub run's own artifact namespace. + ### 6.2 Jobs The candidate workflow should contain these jobs. @@ -249,6 +294,9 @@ Responsibilities: - Generate the platform matrix. - Emit a `candidate_id`, for example: `v2.8.1-rcbuild--`. + - The `` portion comes from `github.run_id`; this means + `candidate_id` is computed inside the workflow and cannot be derived from + dispatch inputs alone. Outputs: @@ -275,6 +323,9 @@ Responsibilities: emits clear bootstrap logs. - Validate scheduled workflows that are used as release evidence classify failures as product, tool bootstrap, or runner infrastructure. +- Validate Rust workflows use `rust-toolchain.toml` rather than a floating + `stable` pin unless a specific job is intentionally testing Rust beta/nightly. +- Validate workflow tool versions match `tools/versions.toml`. Implementation notes: @@ -296,6 +347,9 @@ Rules: - This job can be skipped for targeted diagnostic runs. - It must not be skipped for a publishable all-platform release candidate. +- When `publishable=true`, `validate` is mandatory and must succeed. The + workflow should fail during planning if a caller attempts to request + `publishable=true` with validation disabled or omitted. #### native-artifacts @@ -322,6 +376,16 @@ Critical requirements: - The Windows JNI bridge must link against the DecentDB cdylib/import library, not the Rust static library, unless the compiler/linker toolchain is MSVC end-to-end. +- On Windows, the JNI bridge script should detect and validate the link + artifact in this order: + - prefer `target/release/decentdb.dll.lib` or the import library generated + alongside `decentdb.dll` + - reject the Rust static library when using MinGW + - for MSVC, allow the MSVC import library only after verifying it corresponds + to `decentdb.dll` + - verify exports with `dumpbin /exports` on MSVC or `nm`/`objdump` on + GNU-compatible toolchains, and fail if the expected `ddb_*` symbols are not + present - The job must support a single-platform matrix run. Windows packaging bugs should be debuggable without Linux/macOS builds or the full validation suite. - Every package step must avoid relative `..` paths in upload-artifact inputs. @@ -342,6 +406,9 @@ Rules: - Package versions must exactly match `release_version`. - If the package version already exists on NuGet.org, fail before packing when `publishable=true`. +- The same version availability check must run again in `release-publish.yml` + immediately before `dotnet nuget push`; the pack-time check catches mistakes + early, and the publish-time check is the final race-condition guard. #### mobile-artifacts @@ -405,6 +472,10 @@ Rules: - Candidate smoke should be deterministic and bounded. - Candidate and nightly browser jobs should use the same repository-owned `wasm-bindgen` install/check script. +- `scripts/do-pre-commit-checks.py` must use the same helper for the + `web-wasm-browser-smoke` check. The current pre-commit checker duplicates the + `cargo build --target wasm32-unknown-unknown` plus `wasm-bindgen` flow, so it + should be the first local consumer of the shared script. - Do not install `wasm-bindgen-cli` with a bare `cargo install` in the same step that is treated as product validation. Restore a cached binary or trusted tool artifact first; on cache miss, run an explicit bootstrap step @@ -441,8 +512,21 @@ The manifest should have this rough shape: "source_sha": "", "source_ref": "main", "candidate_run_id": 123, + "candidate_id": "v2.8.1-rcbuild-123-abcdef0", "created_at": "2026-05-28T00:00:00Z", "publishable": false, + "retention_days": 30, + "tool_versions": { + "rustc": "", + "cargo": "", + "node": "22.x", + "python": "3.12.x", + "dotnet": "10.0.x", + "go": "1.25", + "java": "21", + "dart": "", + "wasm_bindgen": "0.2.114" + }, "benchmark_regression_accepted": false, "artifacts": [ { @@ -450,12 +534,25 @@ The manifest should have this rough shape: "platform": "windows-x64", "path": "decentdb-v2.8.1-Windows-x64.zip", "sha256": "", - "bytes": 123 + "bytes": 123, + "build_toolchain": { + "rustc": "", + "target": "x86_64-pc-windows-msvc" + } } ] } ``` +Rules: + +- `tool_versions` records the workflow-level tools used by the candidate. +- Each artifact may include `build_toolchain` when its compiler, target, RID, + linker, SDK, or packaging tool differs from the top-level tools. +- Optional new tool/version keys are backwards-compatible additions, but + removing or changing the meaning of existing keys requires a manifest schema + bump. + ## 7. Release Publish Workflow Requirements `release-publish.yml` should publish from an existing candidate. It should not @@ -475,10 +572,16 @@ Optional: - `publish_github_release` - Default true. + - Creates or updates a draft release unless `finalize_github_release=true`. - `publish_nuget` - Default false until maintainers are confident. +- `nuget_publish_confirmation` + - Required when `publish_nuget=true`. + - Must equal the normalized release tag, for example `v2.8.1`, or the + workflow must fail before any package push. + - `publish_mobile` - Default true if mobile artifacts are part of the release. @@ -486,13 +589,29 @@ Optional: - Default false. - Required for re-uploading GitHub release assets. +- `finalize_github_release` + - Default false. + - When false, assets are uploaded to a draft release for manual review. + - When true, the workflow may publish the release after all checks and any + required environment approvals pass. + +Concurrency: + +- Use `concurrency.group: release-publish-${{ inputs.release_tag }}` with + `cancel-in-progress: false`. +- The publish workflow should queue a second publish attempt for the same + version rather than race release asset creation or NuGet pushes. + ### 7.2 Verification Jobs The workflow must: -1. Resolve `release_tag` to `tag_sha`. +1. Resolve `release_tag` to the commit SHA, not the tag object SHA. + - For annotated tags, use `git rev-parse "$release_tag^{commit}"`. + - Do not compare the manifest against `git rev-parse "$release_tag^{tag}"`; + that is the annotated tag object, not the release commit. 2. Download the candidate manifest from `candidate_run_id`. -3. Verify `manifest.source_sha == tag_sha`. +3. Verify `manifest.source_sha == tag_commit_sha`. 4. Verify `manifest.version` matches `release_tag`. 5. Download all candidate artifacts. 6. Verify every checksum in the manifest. @@ -503,16 +622,21 @@ The workflow must: GitHub release publication: -- Create a release if it does not exist. +- Create a draft release if it does not exist. +- Update an existing draft only when `allow_existing_github_release_update=true`. - Upload artifacts from the candidate bundle. - Do not rebuild artifacts. - Attach the candidate manifest and checksum file. +- Keep the release in draft state by default so a maintainer can review notes + and assets before public publication. NuGet publication: - Push `.nupkg` files from the candidate bundle. - Never rebuild packages in the publish job. - Require `publish_nuget=true`. +- Require `nuget_publish_confirmation` to match the release tag and, preferably, + a protected environment approval for NuGet publication. - Fail if any package version already exists unless the package is already listed and the workflow is only verifying state. @@ -705,6 +829,50 @@ engine behavior failed. The goal is to move release behavior out of YAML wherever possible. +Structured planning, manifest, validation, and metadata checks should be Python +by default because they need good error messages, JSON handling, and +cross-platform behavior. Thin build wrappers may remain shell scripts when they +mainly orchestrate native tools, but they should emit machine-readable manifest +fragments through the Python manifest writer rather than hand-building JSON in +YAML or shell. + +### 9.0 Tool Version Files + +Required files: + +- `rust-toolchain.toml` +- `tools/versions.toml` + +`rust-toolchain.toml` should pin the default Rust channel used by CI, release, +nightly, local scripts, and cache keys. Jobs that intentionally use nightly or +beta, such as Miri or sanitizer jobs, must declare that exception explicitly. + +`tools/versions.toml` should centralize non-Rust tool versions. A first version +should include: + +```toml +[tools] +actionlint = "" +wasm_bindgen_cli = "0.2.114" +node = "22" +python = "3.12" +dotnet = "10.0.x" +go = "1.25" +java = "21" +dart = "" +``` + +Rules: + +- Workflows should read these values directly where practical. +- When GitHub Actions syntax makes direct reads awkward, `workflow-lint` should + verify that hardcoded workflow versions match `tools/versions.toml`. +- `scripts/do-pre-commit-checks.py --mode paranoid` should include the same + version consistency check. +- Cache keys for release-critical jobs should include the pinned Rust version + and relevant tool version from this file rather than a floating `stable` + label. + ### 9.1 `scripts/release/plan.py` Purpose: @@ -748,8 +916,12 @@ The script should print: - linker path - selected DecentDB link artifact - selected Java include directory +- detected toolchain family: `mingw` or `msvc` +- export verification command and result This would have made the `decentdb.lib` vs `decentdb.dll.lib` problem obvious. +It should fail with a clear diagnostic if both a Rust static library and a DLL +import library exist and the selected compiler family would pick the wrong one. ### 9.4 `scripts/release/build_nuget_packages.sh` @@ -774,7 +946,10 @@ Purpose: Rules: -- This check is mandatory before `publish_nuget=true`. +- This check is mandatory in `release-candidate.yml` before packing when + `publishable=true`. +- This check is mandatory again in `release-publish.yml` immediately before + package push when `publish_nuget=true`. - For RC versions, it still checks exact version existence. ### 9.6 `scripts/release/build_mobile_artifacts.sh` @@ -816,6 +991,9 @@ Purpose: - no upload-artifact paths with `../` - no release-critical or release-evidence workflow with an unwrapped dynamic tool install + - no floating Rust `stable` in release-critical workflows unless the job is + explicitly marked as a floating-toolchain compatibility check + - no hardcoded tool versions that drift from `tools/versions.toml` This should run in: @@ -834,8 +1012,7 @@ Purpose: Rules: -- The expected version must be pinned in one place, either in the script or in a - small checked-in tool version file. +- The expected version must come from `tools/versions.toml`. - The script first checks whether `wasm-bindgen` exists and matches the expected version. - If the correct binary is present, the script exits without network access. @@ -849,6 +1026,9 @@ Rules: - Workflows should cache the resulting binary with a key that includes the operating system, architecture, Rust host triple, Rust version, and `wasm-bindgen` version. +- `scripts/do-pre-commit-checks.py` should call this script before its + `web-wasm-browser-smoke` command, and that smoke check should stop requiring a + preinstalled `wasm-bindgen` binary outside the repository-managed helper. ### 9.11 `scripts/ci/cargo_fetch_locked.sh` @@ -871,7 +1051,8 @@ Rules: - On failure, exit with a message that labels the failure as `tool_bootstrap` or `runner_infra`. - Workflows should cache Cargo registry and git sources with keys that include - the operating system, architecture, Rust version, and `Cargo.lock` hash. + the operating system, architecture, pinned Rust version from + `rust-toolchain.toml`, and `Cargo.lock` hash. - Product validation steps should run only after this step succeeds, so a dependency download outage cannot masquerade as a DecentDB regression. @@ -1043,6 +1224,11 @@ git tag -a "v2.8.1" "$CANDIDATE_SHA" -m "DecentDB v2.8.1" git push origin "refs/tags/v2.8.1" ``` +The tag should be annotated for release provenance. The publish workflow must +dereference it with `v2.8.1^{commit}` when comparing against +`manifest.source_sha`; comparing against the annotated tag object SHA is a +release verification bug. + ### 11.6 Publish From Candidate ```bash @@ -1052,6 +1238,7 @@ gh workflow run release-publish.yml \ -f candidate_run_id="$RUN_ID" \ -f publish_github_release=true \ -f publish_nuget=true \ + -f nuget_publish_confirmation=v2.8.1 \ -f publish_mobile=true ``` @@ -1079,7 +1266,10 @@ If NuGet packages are already published but GitHub release artifacts failed: the published packages. 3. If the fix requires source or workflow changes after package publication, prefer a patch release, for example `v2.8.1`. -4. Only move a public tag if: +4. Do not assume NuGet can be rolled back. NuGet.org supports unlisting, not + true deletion. Unlisting may reduce discovery, but consumers with the exact + version can still restore it. +5. Only move a public tag if: - no external package was published, or - maintainers explicitly accept the provenance mismatch, and - the release notes record what happened. @@ -1089,6 +1279,8 @@ Recommended default: - Use a patch release. - Keep the failed release documented. - Do not rewrite public release history after external registry publication. +- If a NuGet package is invalid, unlist only after documenting the reason and + publishing a corrected patch version. ## 13. Branch Protection And Required Checks @@ -1097,7 +1289,8 @@ The redesigned process should work with branch protection: - PRs to `main` still run required `ci.yml`. - Release candidate workflows run manually after merge to `main`. - Tags are created only after candidate success. -- Publish workflow verifies tag SHA against the candidate manifest. +- Publish workflow dereferences the release tag to a commit SHA and verifies + that commit against the candidate manifest. Required PR checks should stay reasonably fast. Do not make all release candidate jobs required on every PR. Instead: @@ -1139,6 +1332,15 @@ Candidate-only artifacts can include: But publishable candidate artifacts should use final names so the publish job does not rename or rebuild anything. +Artifact upload retention: + +- Publishable candidate bundles: `retention-days: 90` +- Normal diagnostic candidate artifacts: `retention-days: 30` +- Narrow scratch diagnostics: `retention-days: 14` + +The manifest should include the retention class so a maintainer can understand +whether an artifact is expected to remain available long enough for publication. + ## 15. Acceptance Criteria The release process reset is complete when: @@ -1164,6 +1366,14 @@ The release process reset is complete when: 13. Memory-safety workflows clearly distinguish dependency download/build bootstrap failures from analyzer findings such as Valgrind leaks, sanitizer failures, Miri failures, and stress-test failures. +14. `rust-toolchain.toml` and `tools/versions.toml` exist and are used by + workflows, release scripts, and paranoid pre-commit checks. +15. The release candidate manifest records top-level `tool_versions`, optional + per-artifact `build_toolchain` data, and the artifact retention class. +16. `release-publish.yml` creates a draft GitHub release by default and + requires explicit confirmation or approval before NuGet publication. +17. Candidate and publish workflows serialize per version with + `cancel-in-progress: false`. ## 16. Implementation Phases @@ -1171,6 +1381,10 @@ The release process reset is complete when: Tasks: +- Add `rust-toolchain.toml` and make CI/release workflows use it instead of + floating `stable`. +- Add `tools/versions.toml` with pinned or explicitly ranged tool versions for + Rust-adjacent and binding toolchains. - Add targeted diagnostic inputs to the current release workflow. - Fix Windows JNI linking so MinGW does not choose the MSVC Rust staticlib. - Make benchmark narrative failures block asset publication without failing @@ -1178,6 +1392,8 @@ Tasks: - Add `actionlint` validation locally and in CI. - Wrap `wasm-bindgen-cli` installation in a pinned, version-checking, retry-capable script and update `web-wasm-nightly.yml` to use it. +- Refactor `scripts/do-pre-commit-checks.py` so `web-wasm-browser-smoke` uses + the same `wasm-bindgen` helper as CI/nightly. - Add a Cargo dependency fetch/bootstrap script with retry/classification and use it before memory-safety analyzers and other release-evidence jobs. - Update browser/WASM workflow action versions for the current GitHub runner @@ -1203,7 +1419,10 @@ Tasks: - Implement manual candidate workflow. - Support all-platform and single-platform modes. - Support validation levels. +- Add per-version concurrency with `cancel-in-progress: false`. - Upload complete artifacts and manifest. +- Include `tool_versions`, optional `build_toolchain`, retention class, and + nightly evidence classification in the manifest. - Confirm candidate can reproduce current GitHub release assets. ### Phase 3: Add `release-publish.yml` @@ -1211,9 +1430,12 @@ Tasks: Tasks: - Implement candidate artifact download by run ID. -- Verify manifest and tag SHA. +- Dereference annotated release tags to commit SHA and verify against manifest. +- Use per-version publish concurrency with `cancel-in-progress: false`. +- Create or update a draft GitHub release by default. - Publish GitHub release assets. - Publish NuGet packages from existing candidate artifacts. +- Require explicit NuGet confirmation/protected approval before pushing. - Attach manifest and checksums. ### Phase 4: Remove Duplicate And Tag-Triggered Publish Surfaces @@ -1235,32 +1457,37 @@ Tasks: - Add a policy check that rejects direct tag-triggered package publication. - Add a policy check that rejects unwrapped dynamic tool installs in release-critical and release-evidence workflows. +- Add a policy check that rejects workflow tool-version drift from + `tools/versions.toml`. ## 17. Open Decisions +Resolved by this plan: + +- Candidate artifact retention is 90 days for publishable candidates, 30 days + for normal diagnostics, and 14 days for narrow scratch diagnostics. +- GitHub releases are draft-first by default. +- NuGet publication requires explicit confirmation or protected approval because + package publication cannot be undone. + 1. Should `release-publish.yml` be manual-only, or should a tag push trigger it after verifying a matching successful candidate? Recommendation: manual-only first. Add tag automation only after the manifest verification has proven stable. -2. Should candidate artifacts be retained for 30 days or 90 days? - - Recommendation: 90 days for publishable candidates, 14 or 30 days for - diagnostic candidates. - -3. Should benchmark assets be part of the GitHub release artifact set? +2. Should benchmark assets be part of the GitHub release artifact set? Recommendation: include benchmark narrative and generated charts in the candidate artifact bundle, but keep checked-in README asset updates as a separate PR/main workflow. -4. Should mobile artifacts be attached to every GitHub release? +3. Should mobile artifacts be attached to every GitHub release? Recommendation: yes if they are supported as release deliverables; otherwise mark them experimental and keep them out of publish-blocking release assets. -5. Should nightly memory-safety success be required for release publication? +4. Should nightly memory-safety success be required for release publication? Recommendation: require a recent successful nightly by policy, not by rerunning every sanitizer/Miri job in the release candidate. From b6ae31a72e0d483bee320fe5631a0d20b5d285b2 Mon Sep 17 00:00:00 2001 From: Steven Hildreth Date: Fri, 29 May 2026 07:32:31 -0500 Subject: [PATCH 3/6] docs(release): expand release plan requirements and migration strategy Update the release plans documentation to include new automation rules regarding dependency locking, secret scoping, artifact categorization, and job timeouts. Additionally, add a migration action summary table to track the transition from legacy workflows to the new candidate/publish model. --- design/2026-05-28.RELEASE_PLANS.md | 298 ++++++++++++++++++++++++++--- 1 file changed, 271 insertions(+), 27 deletions(-) diff --git a/design/2026-05-28.RELEASE_PLANS.md b/design/2026-05-28.RELEASE_PLANS.md index e32f440..f762222 100644 --- a/design/2026-05-28.RELEASE_PLANS.md +++ b/design/2026-05-28.RELEASE_PLANS.md @@ -138,6 +138,28 @@ These should become hard rules in automation and release documentation. 17. Release candidate and publish workflows must use per-version concurrency groups with `cancel-in-progress: false`. A second release run for the same version should queue rather than cancel or race with an in-progress run. +18. Release builds must use the checked-in `Cargo.lock` with `--locked`. + Candidate manifests must record the `Cargo.lock` SHA-256 so the publish + step can prove it is publishing artifacts built from the reviewed dependency + graph. +19. Package publication secrets must only be available to publish jobs. Candidate, + validation, nightly, benchmark, and diagnostic jobs must not receive + `NUGET_API_KEY` or equivalent external registry credentials. Publication + secrets should be scoped through protected GitHub environments where + possible. +20. Candidate artifacts must be split by purpose: + - publish-critical artifacts: native packages, NuGet packages, mobile + release artifacts, manifests, and checksums + - review artifacts: benchmark reports, generated charts, docs previews, and + diagnostic logs + - scratch artifacts: targeted one-off diagnostic outputs + The manifest job must report per-artifact and per-group sizes. Review and + scratch artifacts should not be bundled with publish-critical artifacts just + to simplify downloads. +21. Release workflows must set explicit `timeout-minutes` on every job. Targeted + diagnostic jobs should fail fast enough to support iteration; publishable + all-platform candidates may run longer, but no lane should hang without a + bounded timeout and uploaded partial logs. ## 4. Current Workflow Inventory @@ -221,6 +243,22 @@ The target deletion/consolidation list: - Remove tag triggers from `mobile-native-artifacts.yml`, or delete the workflow after mobile artifacts are part of the candidate bundle. - Rename or merge `python-benchmarks.yml` into the benchmark workflow family. +- Keep the existing `release.yml` available as a manual diagnostic fallback + until the new flow has completed at least one real production release, but + remove or hard-disable its tag-triggered GitHub release publication before the + first candidate-then-tag release. The fallback must not race the new + publish-from-candidate flow after a tag push. + +Migration action summary: + +| Current Workflow | Migration Action | Retirement Criteria | +|---|---|---| +| `release.yml` | Convert to manual fallback, then replace with candidate/publish split | Remove after one successful production release through `release-candidate.yml` and `release-publish.yml` | +| `nuget.yml` | Move pack to candidate and publish to publish workflow | Remove tag publish path before NuGet secrets are scoped to publish only | +| `nuget-manual.yml` | Delete after unique dry-run behavior is ported | Delete when new NuGet dry-run and publish flow is green | +| `mobile-native-artifacts.yml` | Move Android/iOS release artifacts into candidate | Remove tag trigger before mobile artifacts are publish-critical | +| `benchmark-assets.yml` | Keep as asset maintenance; candidate only uploads review reports | Keep, but not as release publication | +| `python-benchmarks.yml` | Rename/merge/delete as benchmark research | Decide after benchmark coverage is represented in candidate/nightly jobs | ## 6. Release Candidate Workflow Requirements @@ -307,15 +345,14 @@ Outputs: - `candidate_id` - `matrix` -#### workflow-lint +#### workflow-policy Responsibilities: -- Run `actionlint` over all workflows. - Validate no release-critical workflow uses a tag trigger to publish directly. +- Validate the legacy `release.yml` tag-triggered publication path is disabled + before the first candidate-then-tag release. - Validate `nuget-manual.yml` is absent once the new flow is adopted. -- Validate workflow action major versions are current enough for the runner - deprecation window. - Validate release-critical workflows do not contain naked dynamic tool installs such as `cargo install`, `go install`, `npm install -g`, or curl-piped installer scripts unless they are wrapped by an approved repository script @@ -325,7 +362,28 @@ Responsibilities: failures as product, tool bootstrap, or runner infrastructure. - Validate Rust workflows use `rust-toolchain.toml` rather than a floating `stable` pin unless a specific job is intentionally testing Rust beta/nightly. +- Validate publish jobs have per-version concurrency with + `cancel-in-progress: false`. +- Validate package publication secrets are only referenced from approved + publish jobs and protected environments. +- Validate release jobs define explicit `timeout-minutes`. + +Rules: + +- `workflow-policy` is a hard gate for publishable candidates. +- Targeted diagnostic candidates may run it in advisory mode only when they do + not publish and do not create shared candidate artifacts. + +#### workflow-hygiene + +Responsibilities: + +- Run `actionlint` over all workflows. +- Validate workflow action major versions are current enough for the runner + deprecation window. - Validate workflow tool versions match `tools/versions.toml`. +- Validate no hardcoded tool versions drift from `tools/versions.toml`. +- Validate no upload-artifact paths use `../`. Implementation notes: @@ -334,13 +392,25 @@ Implementation notes: bootstrap problem this plan is trying to remove. - Add the same check to `scripts/do-pre-commit-checks.py --mode paranoid`. +Rules: + +- `workflow-hygiene` should be required in normal PR/CI before merge. +- In `release-candidate.yml`, `workflow-hygiene` may be a soft gate for + targeted diagnostics. It should still run and upload results so maintainers do + not ignore deprecation and drift warnings. + #### validate Responsibilities: +- Run `python scripts/validate_release_metadata.py`. - Run the repo's release validation at the requested level. -- For `full` or `paranoid`, this should be equivalent to: - `python scripts/do-pre-commit-checks.py --mode paranoid`. +- For a publishable all-platform candidate, the validation set should be + equivalent in coverage to `python scripts/do-pre-commit-checks.py --mode + paranoid`, but it does not need to literally invoke that local orchestration + script as one monolithic CI job. +- For targeted diagnostics, run the release-relevant subset for the selected + platform and surface area. - Upload logs even on failure. Rules: @@ -350,6 +420,14 @@ Rules: - When `publishable=true`, `validate` is mandatory and must succeed. The workflow should fail during planning if a caller attempts to request `publishable=true` with validation disabled or omitted. +- `validation_level=fast` is the default for single-platform diagnostics. + `validation_level=full` and `validation_level=paranoid` are intended for + publishable `platform=all` candidates or explicit release manager requests. +- `tools/versions.toml` must coexist with the existing `VERSION` file: + `VERSION` remains the product version source of truth, while + `tools/versions.toml` controls build and workflow toolchain versions. Release + metadata validation should check both without letting toolchain pins rewrite + the product version. #### native-artifacts @@ -436,8 +514,12 @@ Responsibilities: Rules: -- A benchmark narrative failure should mark the candidate as not publishable by - default. +- A benchmark narrative failure should mark the candidate as `needs_review`, not + as an automatic hard failure. GitHub-hosted benchmark variance is too high for + a single noisy regression to force a code change by itself. +- A publishable candidate may proceed only after a release operator reviews the + benchmark report. If the operator accepts a material regression or noisy + result, the manifest must record that decision. - It should not auto-commit benchmark assets. - If `accept_benchmark_regression=true`, the manifest must include: - the failed metrics @@ -499,6 +581,11 @@ Responsibilities: - Verify version strings. - Verify package names. - Verify source SHA. +- Verify `Cargo.lock` hash. +- Compute per-artifact, per-group, and total artifact sizes. +- Validate full completeness for publishable `platform=all` candidates. +- Emit partial manifests for targeted diagnostics. +- Record nightly evidence metadata gathered by `check_nightly_evidence.py`. - Emit one `release-candidate-manifest.json`. The manifest should have this rough shape: @@ -511,11 +598,19 @@ The manifest should have this rough shape: "tag": "v2.8.1", "source_sha": "", "source_ref": "main", + "cargo_lock_sha256": "", "candidate_run_id": 123, "candidate_id": "v2.8.1-rcbuild-123-abcdef0", "created_at": "2026-05-28T00:00:00Z", "publishable": false, + "partial": false, "retention_days": 30, + "artifact_size_bytes": { + "publish_critical": 123, + "review": 456, + "scratch": 0, + "total": 579 + }, "tool_versions": { "rustc": "", "cargo": "", @@ -528,9 +623,22 @@ The manifest should have this rough shape: "wasm_bindgen": "0.2.114" }, "benchmark_regression_accepted": false, + "nightly_evidence": { + "memory_safety": { + "latest_successful_run_id": 123456, + "latest_failed_run_id": null, + "latest_failed_classification": null + }, + "browser": { + "latest_successful_run_id": 123457, + "latest_failed_run_id": null, + "latest_failed_classification": null + } + }, "artifacts": [ { "kind": "github-release-native", + "group": "publish-critical", "platform": "windows-x64", "path": "decentdb-v2.8.1-Windows-x64.zip", "sha256": "", @@ -547,11 +655,29 @@ The manifest should have this rough shape: Rules: - `tool_versions` records the workflow-level tools used by the candidate. +- `cargo_lock_sha256` is required for publishable candidates. - Each artifact may include `build_toolchain` when its compiler, target, RID, linker, SDK, or packaging tool differs from the top-level tools. +- Artifact `group` must be one of `publish-critical`, `review`, or `scratch`. + Publish jobs consume only `publish-critical` artifacts plus manifests and + checksums. +- When `platform=all` and `publishable=true`, `partial` must be false and the + manifest validator must require every expected release artifact. +- For single-platform diagnostics, `partial` must be true and the manifest + validator should only require artifacts for the selected platform/surface. +- The manifest should include a configured size budget. Initial guidance: + publish-critical artifacts should be split by platform and package type rather + than uploaded as one large monolithic bundle; review artifacts should be + separate so benchmark/docs size growth cannot block native package download. - Optional new tool/version keys are backwards-compatible additions, but removing or changing the meaning of existing keys requires a manifest schema bump. +- `release-publish.yml` must reject a manifest with `schema` greater than the + publish workflow's supported schema version. +- Schema 1 consumers must ignore unknown keys, but must not ignore missing or + semantically invalid required keys. +- The supported manifest schema should be declared in the publish workflow or + manifest validation script so schema support can be reviewed as code. ## 7. Release Publish Workflow Requirements @@ -613,10 +739,16 @@ The workflow must: 2. Download the candidate manifest from `candidate_run_id`. 3. Verify `manifest.source_sha == tag_commit_sha`. 4. Verify `manifest.version` matches `release_tag`. -5. Download all candidate artifacts. -6. Verify every checksum in the manifest. -7. Verify no NuGet package version already exists before publishing. -8. Verify GitHub release asset names are complete and non-duplicated. +5. Reject manifests with `schema` greater than the publish workflow's supported + manifest schema. +6. Reject `partial=true` manifests. +7. Verify `cargo_lock_sha256` is present and matches the checked-in + `Cargo.lock` at `tag_commit_sha`. +8. Download all publish-critical candidate artifacts. +9. Verify every checksum in the manifest. +10. Verify no NuGet package version already exists before publishing. +11. Verify GitHub release asset names are complete and non-duplicated. +12. Verify nightly evidence fields are present or explicitly waived. ### 7.3 Publish Jobs @@ -640,6 +772,18 @@ NuGet publication: - Fail if any package version already exists unless the package is already listed and the workflow is only verifying state. +Secrets and environments: + +- `NUGET_API_KEY` should be available only to the NuGet publish job and only + through a protected environment. +- Candidate, validation, benchmark, nightly, docs, and artifact build jobs + should not have access to package publication secrets. +- During migration, remove NuGet publication secret access from legacy + `release.yml`, `nuget.yml`, and `nuget-manual.yml` before relying on the new + publish workflow. +- Secret rotation should be part of the migration checklist if a key was exposed + to broader workflows during the old release process. + ## 8. Proposed Workflow Disposition ### 8.1 `.github/workflows/release.yml` @@ -654,10 +798,16 @@ Current problem: Plan: - Convert it into `release-candidate.yml` or split the current file. +- Before the first candidate-then-tag release, remove the tag trigger or add a + hard guard so pushing `vX.Y.Z` does not create a GitHub release from the old + workflow. Otherwise the old tag-triggered `release.yml` can race the new + `release-publish.yml`. - Keep temporary diagnostic inputs while migrating: - `build_platform` - `run_validation` - `create_release` +- Keep it operational as a manual fallback until the new flow has successfully + shipped one production release. - Long term, remove GitHub release publication from this workflow and move it to `release-publish.yml`. @@ -770,6 +920,11 @@ Plan: - Candidate manifest should also record the latest failed nightly classification when the latest run failed. A `tool_bootstrap` or `runner_infra` failure is operational debt, but it is different from a product regression. +- `release-candidate.yml` should gather this data through a repository script, + not ad hoc YAML. The script should query GitHub Actions for the relevant + nightly workflows on `main`, return run IDs/conclusions/timestamps, and + classify the latest failure when possible. +- The candidate manifest records nightly run metadata, not nightly artifacts. Specific rule for `memory-safety-nightly.yml`: @@ -865,13 +1020,19 @@ dart = "" Rules: - Workflows should read these values directly where practical. -- When GitHub Actions syntax makes direct reads awkward, `workflow-lint` should - verify that hardcoded workflow versions match `tools/versions.toml`. +- When GitHub Actions syntax makes direct reads awkward, `workflow-hygiene` + should verify that hardcoded workflow versions match `tools/versions.toml`. - `scripts/do-pre-commit-checks.py --mode paranoid` should include the same version consistency check. - Cache keys for release-critical jobs should include the pinned Rust version and relevant tool version from this file rather than a floating `stable` label. +- `scripts/validate_release_metadata.py` remains the existing release metadata + guard for product version consistency. It should be extended, or accompanied + by a helper it calls, to validate that `tools/versions.toml` exists and does + not conflict with workflow/toolchain pins. The `VERSION` file remains the + product version source of truth; `tools/versions.toml` is only for build and + workflow tools. ### 9.1 `scripts/release/plan.py` @@ -976,30 +1137,39 @@ Purpose: - Verify checksums. - Verify version strings. - Verify release tag compatibility. +- Validate `partial` manifest rules: + - `publishable=true` requires `partial=false` and full artifact coverage + - targeted diagnostics may emit `partial=true` +- Reject manifests with unsupported schema versions when running in publish + verification mode. ### 9.9 `scripts/validate_workflows.py` Purpose: - Run `actionlint`. -- Enforce workflow policy: +- Enforce workflow policy checks: - no direct package publication on tag push, except approved publish workflow - no duplicate NuGet workflows - no release workflows without manual targeted inputs - - no JavaScript action versions that rely on a deprecated Node runtime after - GitHub's announced runner cutoff - - no upload-artifact paths with `../` - no release-critical or release-evidence workflow with an unwrapped dynamic tool install - no floating Rust `stable` in release-critical workflows unless the job is explicitly marked as a floating-toolchain compatibility check + - no package publication secrets outside approved publish jobs + - required release workflows have per-version concurrency +- Enforce workflow hygiene checks: + - no JavaScript action versions that rely on a deprecated Node runtime after + GitHub's announced runner cutoff + - no upload-artifact paths with `../` - no hardcoded tool versions that drift from `tools/versions.toml` This should run in: - `ci.yml` - `scripts/do-pre-commit-checks.py --mode paranoid` -- release candidate `workflow-lint` +- release candidate `workflow-policy` +- release candidate `workflow-hygiene` ### 9.10 `scripts/ci/ensure_wasm_bindgen.sh` @@ -1044,6 +1214,10 @@ Rules: - Run `cargo fetch --locked` with Cargo network retry settings before analyzer or benchmark jobs. +- Support `--dry-run`, which should run lockfile consistency checks such as + `cargo metadata --locked` without entering expensive product validation. This + gives workflow lint and metadata validation a quick way to detect + `Cargo.toml`/`Cargo.lock` drift. - Print `rustc --version --verbose`, `cargo --version`, and the Cargo registry protocol configuration. - Use a small explicit retry loop for transient registry HTTP 5xx, EOF, and @@ -1056,6 +1230,34 @@ Rules: - Product validation steps should run only after this step succeeds, so a dependency download outage cannot masquerade as a DecentDB regression. +### 9.12 `scripts/release/check_nightly_evidence.py` + +Purpose: + +- Query GitHub Actions for recent nightly workflow results on `main`. +- Return structured JSON for the candidate manifest. +- Classify latest failures as `product`, `tool_bootstrap`, `runner_infra`, or + `unknown` when logs are insufficient. + +Inputs: + +- `--repo sphildreth/decentdb` +- `--branch main` +- `--workflow memory-safety-nightly.yml` +- `--workflow web-wasm-nightly.yml` +- `--max-age-days 7` +- `--out .tmp/release-candidate/nightly-evidence.json` + +Rules: + +- The script records run IDs, conclusions, timestamps, head SHAs, failed job + names, and classification. +- It should not download or re-upload nightly artifacts. +- If GitHub API access fails, the candidate should record + `nightly_evidence.status=unavailable`; publish can require a manual waiver. +- A successful publishable candidate should either include recent successful + nightly evidence or a documented waiver in the manifest. + ## 10. Local/Remote Debugging Strategy GitHub Actions cannot be perfectly reproduced locally, especially macOS, @@ -1129,6 +1331,21 @@ Do not rerun as the primary fix for: - invalid workflow expressions - package version conflicts +### 10.5 Candidate Workflow Fallback + +The new candidate workflow itself can fail because of YAML syntax, expression +context mistakes, matrix generation bugs, or GitHub Actions feature drift. The +migration must keep a fallback while the new path proves itself: + +- Keep the current `release.yml` callable by `workflow_dispatch` as a manual + diagnostic fallback. +- Disable its tag-triggered publication path before using the new + candidate-then-tag process. +- Do not grant the fallback workflow NuGet publication secrets after secrets are + moved to the new publish workflow. +- Retire the fallback only after one successful production release has gone + through `release-candidate.yml` and `release-publish.yml`. + ## 11. Release Operator Runbook This is the intended post-redesign release flow. @@ -1218,7 +1435,22 @@ Resolve the exact SHA from the candidate run: CANDIDATE_SHA=$(gh run view "$RUN_ID" \ --json headSha \ --jq '.headSha') +``` + +Before pushing the tag, verify that legacy tag-triggered publication workflows +are disabled or guarded: + +```bash +python scripts/validate_workflows.py --release-policy +gh workflow view release.yml --yaml | rg 'refs/tags|tags:|softprops/action-gh-release' +``` + +The policy check should fail if the old `release.yml` can still publish from a +tag push. Do not push the tag until that is resolved. +Create the annotated tag only after the policy check passes: + +```bash git fetch origin main git tag -a "v2.8.1" "$CANDIDATE_SHA" -m "DecentDB v2.8.1" git push origin "refs/tags/v2.8.1" @@ -1325,6 +1557,12 @@ Final GitHub release assets should remain: - `release-candidate-manifest-vX.Y.Z.json` - `checksums-vX.Y.Z.sha256` +For platform-specific JDBC and DBeaver artifacts, the manifest `platform` field +should use the same platform/RID vocabulary as native artifacts where possible +(`linux-x64`, `linux-arm64`, `macos-arm64`, `windows-x64`) even if the filename +keeps the current human-facing suffix (`Linux`, `Linux-arm64`, `macOS`, +`Windows`). + Candidate-only artifacts can include: - `decentdb-candidate---Windows-x64.zip` @@ -1381,19 +1619,23 @@ The release process reset is complete when: Tasks: +- Fix Windows JNI linking so MinGW does not choose the MSVC Rust staticlib. +- Wrap `wasm-bindgen-cli` installation in a pinned, version-checking, + retry-capable script and update `web-wasm-nightly.yml` to use it. +- Refactor `scripts/do-pre-commit-checks.py` so `web-wasm-browser-smoke` uses + the same `wasm-bindgen` helper as CI/nightly. +- Remove or hard-disable tag-triggered GitHub release publication in the legacy + `release.yml` before the first candidate-then-tag release. - Add `rust-toolchain.toml` and make CI/release workflows use it instead of floating `stable`. - Add `tools/versions.toml` with pinned or explicitly ranged tool versions for Rust-adjacent and binding toolchains. +- Extend `scripts/validate_release_metadata.py` to validate the new tool version + files while preserving `VERSION` as the product version source of truth. - Add targeted diagnostic inputs to the current release workflow. -- Fix Windows JNI linking so MinGW does not choose the MSVC Rust staticlib. -- Make benchmark narrative failures block asset publication without failing - unrelated release work. +- Make benchmark narrative failures produce `needs_review` candidate metadata + without failing unrelated release work or auto-committing assets. - Add `actionlint` validation locally and in CI. -- Wrap `wasm-bindgen-cli` installation in a pinned, version-checking, - retry-capable script and update `web-wasm-nightly.yml` to use it. -- Refactor `scripts/do-pre-commit-checks.py` so `web-wasm-browser-smoke` uses - the same `wasm-bindgen` helper as CI/nightly. - Add a Cargo dependency fetch/bootstrap script with retry/classification and use it before memory-safety analyzers and other release-evidence jobs. - Update browser/WASM workflow action versions for the current GitHub runner @@ -1411,6 +1653,7 @@ Tasks: - Add workflow policy validation. - Add nightly failure classification helpers for memory-safety and browser workflows. +- Add `scripts/release/check_nightly_evidence.py`. ### Phase 2: Add `release-candidate.yml` @@ -1453,7 +1696,8 @@ Tasks: - Update development release docs. - Update `README.md` release artifact notes if names change. -- Update `scripts/do-pre-commit-checks.py --mode paranoid` with workflow lint. +- Update `scripts/do-pre-commit-checks.py --mode paranoid` with workflow policy + and workflow hygiene checks. - Add a policy check that rejects direct tag-triggered package publication. - Add a policy check that rejects unwrapped dynamic tool installs in release-critical and release-evidence workflows. From d78dfb0caa27a5443a273bac32a6f61b059b7439 Mon Sep 17 00:00:00 2001 From: Steven Hildreth Date: Fri, 29 May 2026 07:44:02 -0500 Subject: [PATCH 4/6] docs(release): simplify release process and operator workflow Refactor the release plan documentation to focus on the two-stage release model (candidate vs. publish). Streamline the description of infrastructure failures and update the workflow recommendation table to reflect improved hygiene and metadata checks. --- design/2026-05-28.RELEASE_PLANS.md | 113 +++++++++++++++++------------ 1 file changed, 67 insertions(+), 46 deletions(-) diff --git a/design/2026-05-28.RELEASE_PLANS.md b/design/2026-05-28.RELEASE_PLANS.md index f762222..a1493a5 100644 --- a/design/2026-05-28.RELEASE_PLANS.md +++ b/design/2026-05-28.RELEASE_PLANS.md @@ -20,22 +20,10 @@ release tag exists. That creates the worst possible feedback loop: job or run completes. - Release, NuGet, mobile, and benchmark workflows are separate enough to fail independently, but similar enough that fixes and validation are duplicated. -- Some failures are not product failures at all. For example, the - `Web WASM Nightly` run on 2026-05-29 failed before any DecentDB build, - browser smoke, or benchmark step ran because `cargo install - wasm-bindgen-cli --version 0.2.114 --locked` hit a crates.io/curl EOF while - downloading `clap_derive`. The release process must separate this kind of - tool bootstrap or network failure from engine, binding, packaging, and - benchmark regressions. -- Nightly memory-safety failures can mix both categories in one workflow. The - `Memory Safety Nightly` run on 2026-05-29 had one job fail before memory - validation because crates.io returned HTTP 500 while downloading `lua-src`, - and another job fail after product validation because Valgrind reported - `148 bytes possibly lost` in the Python/C ABI path through - `ProcessCoordinator::begin_reader`. The release process must preserve that - distinction: dependency download failures are bootstrap/infra; Valgrind, - sanitizer, Miri, and leak findings after tests run are product signals until - reproduced, fixed, suppressed with justification, or explicitly waived. +- Some failures are not product failures at all. Tool bootstrap, network, and + hosted runner failures must be separated from engine, binding, packaging, and + benchmark regressions. Concrete examples are recorded later under nightly + workflow policy. The release process should be redesigned so release-blocking GitHub Actions failures are found before the final tag exists and before anything is published @@ -50,6 +38,25 @@ The operational goal is simple: 4. Tag only a known-good commit. 5. Publish only artifacts that already passed candidate validation. +### Operator-Facing Outcome + +The simplified release process has only two release buttons: + +1. `release-candidate.yml` + - Builds and validates artifacts. + - Produces a manifest and checksums. + - Has no public side effects. + +2. `release-publish.yml` + - Verifies a tag against a successful candidate. + - Creates or updates a draft GitHub release. + - Optionally publishes NuGet after explicit confirmation. + - Does not compile or package code. + +Everything else in this plan exists to make those two buttons reliable and +boring. If a proposed task does not make candidate creation or candidate-based +publication simpler, it should be deferred. + ## 2. Non-Goals - This plan does not weaken correctness validation. It moves validation earlier. @@ -167,7 +174,7 @@ The repository currently has ten workflows: | Workflow | Current Role | Release Pain | Recommendation | |---|---|---|---| -| `.github/workflows/ci.yml` | PR/main Rust lint, tests, web smoke | Useful, but not a complete release preflight | Keep as required PR gate; add workflow lint and targeted release metadata checks | +| `.github/workflows/ci.yml` | PR/main Rust lint, tests, web smoke | Useful, but not a complete release preflight | Keep as required PR gate; add workflow hygiene and targeted release metadata checks | | `.github/workflows/docs.yml` | Deploy docs to Pages on docs changes | Separate from release docs validation | Keep deploy-only; release candidate should build docs but not deploy | | `.github/workflows/release.yml` | Tag-triggered full validation, native artifacts, JDBC/DBeaver assets, GitHub release | Finds platform packaging failures after tag exists; validation delays artifact failure by ~1 hour | Split into candidate build and publish-from-candidate | | `.github/workflows/nuget.yml` | Tag push plus `workflow_dispatch` with `release_tag` and `publish_to_nuget` inputs for dry-run or publish | Builds from tag; can publish before other release artifacts are green | Fold into candidate/publish model; publish only candidate `.nupkg` artifacts | @@ -202,8 +209,7 @@ The target should be a smaller, clearer workflow set: publishable candidate state. 3. `release-publish.yml` - - Manual only, or tag-triggered only after candidate manifest verification is - implemented. + - Manual only for the first implementation. - Takes `release_tag` and `candidate_run_id`. - Dereferences annotated tags to the commit SHA and verifies that commit SHA equals the candidate manifest SHA. @@ -260,6 +266,27 @@ Migration action summary: | `benchmark-assets.yml` | Keep as asset maintenance; candidate only uploads review reports | Keep, but not as release publication | | `python-benchmarks.yml` | Rename/merge/delete as benchmark research | Decide after benchmark coverage is represented in candidate/nightly jobs | +### 5.1 Minimum Viable Release Flow + +The first implementation should optimize for removing the painful tag loop, not +for building the perfect release platform. The minimum useful cut is: + +1. Disable legacy tag-triggered publication paths that can race the new flow. +2. Add a manual `release-candidate.yml` that can build all publish-critical + artifacts, upload checksums, and emit a manifest. +3. Add targeted candidate dispatch for one platform or one package family, so + Windows, mobile, NuGet, or browser failures can be debugged without rerunning + everything. +4. Add a manual `release-publish.yml` that verifies the candidate manifest, + verifies the tag commit, creates a draft GitHub release, uploads existing + artifacts, and optionally publishes NuGet after confirmation. +5. Keep the old `release.yml` as a manual diagnostic fallback only until one + production release succeeds through the new flow. + +Tool pinning, nightly evidence, workflow policy checks, artifact size reporting, +and rich manifest metadata are important hardening work. They should support the +minimum flow above rather than obscure it. + ## 6. Release Candidate Workflow Requirements `release-candidate.yml` should be the main release engineering workflow. @@ -889,7 +916,7 @@ Current problem: Plan: - Keep. -- Add workflow lint or make workflow lint a separate required job. +- Add workflow hygiene checks or make them a separate required job. - Consider adding `scripts/validate_release_metadata.py` for version/changelog changes when release files are touched. @@ -1216,7 +1243,7 @@ Rules: or benchmark jobs. - Support `--dry-run`, which should run lockfile consistency checks such as `cargo metadata --locked` without entering expensive product validation. This - gives workflow lint and metadata validation a quick way to detect + gives workflow hygiene and metadata validation a quick way to detect `Cargo.toml`/`Cargo.lock` drift. - Print `rustc --version --verbose`, `cargo --version`, and the Cargo registry protocol configuration. @@ -1528,7 +1555,8 @@ Required PR checks should stay reasonably fast. Do not make all release candidate jobs required on every PR. Instead: - Require CI. -- Require workflow lint. +- Require workflow hygiene checks such as `actionlint` in normal CI. +- Require workflow policy checks when release or workflow files change. - Require release metadata validation when release/version files change. - Require candidate success only as an operator runbook step before tagging. @@ -1704,37 +1732,30 @@ Tasks: - Add a policy check that rejects workflow tool-version drift from `tools/versions.toml`. -## 17. Open Decisions +## 17. Resolved Release Defaults -Resolved by this plan: +The release process uses these defaults: +- `release-publish.yml` is manual-only for the first implementation. Tag-trigger + automation can be reconsidered after the manifest verification path has + proven stable. - Candidate artifact retention is 90 days for publishable candidates, 30 days for normal diagnostics, and 14 days for narrow scratch diagnostics. - GitHub releases are draft-first by default. - NuGet publication requires explicit confirmation or protected approval because package publication cannot be undone. - -1. Should `release-publish.yml` be manual-only, or should a tag push trigger it - after verifying a matching successful candidate? - - Recommendation: manual-only first. Add tag automation only after the manifest - verification has proven stable. - -2. Should benchmark assets be part of the GitHub release artifact set? - - Recommendation: include benchmark narrative and generated charts in the - candidate artifact bundle, but keep checked-in README asset updates as a - separate PR/main workflow. - -3. Should mobile artifacts be attached to every GitHub release? - - Recommendation: yes if they are supported as release deliverables; otherwise - mark them experimental and keep them out of publish-blocking release assets. - -4. Should nightly memory-safety success be required for release publication? - - Recommendation: require a recent successful nightly by policy, not by - rerunning every sanitizer/Miri job in the release candidate. +- Benchmark narrative reports and generated charts are included in the + candidate artifact bundle as review artifacts. Checked-in README/site + benchmark asset updates remain a separate PR/main workflow concern and are + not part of release publication. +- Mobile artifacts are attached to every GitHub release when they are supported + release deliverables. If a future mobile platform is experimental, mark it as + experimental in the manifest and keep it out of publish-blocking artifacts + until it is promoted. +- Release publication requires recent successful nightly memory-safety evidence + by policy. The candidate workflow records nightly evidence metadata; it does + not rerun every sanitizer/Miri/Valgrind job. A missing or failing nightly + requires an explicit documented waiver before publication. ## 18. Summary From 7d86a15b7954ba5bf707c9800cce8f579bc962f4 Mon Sep 17 00:00:00 2001 From: Steven Hildreth Date: Tue, 9 Jun 2026 12:54:43 -0500 Subject: [PATCH 5/6] feat(dotnet): add maintenance helpers and query diagnostics - Add `DecentDBMaintenance` helpers for WAL status, checkpoint, compact, and vacuum operations. - Add `ExplainQuery` extension for `EXPLAIN` and `EXPLAIN ANALYZE` diagnostics. - Improve error messaging for unsupported database format versions. - Add EF Core regression tests for indexed string equality. - Bump version to 2.9.0 across all bindings and documentation. --- Cargo.lock | 10 +- Cargo.toml | 2 +- VERSION | 2 +- benchmarks/rust-baseline/Cargo.lock | 4 +- bindings/dart/dart/pubspec.yaml | 2 +- bindings/dart/examples/console/pubspec.lock | 2 +- .../examples/console_complex/pubspec.lock | 2 +- .../examples/flutter_desktop/pubspec.lock | 2 +- bindings/dart/flutter/android/build.gradle | 2 +- bindings/dart/flutter/example/pubspec.lock | 4 +- bindings/dart/flutter/example/pubspec.yaml | 2 +- .../dart/flutter/ios/decentdb_flutter.podspec | 2 +- bindings/dart/flutter/pubspec.lock | 2 +- bindings/dart/flutter/pubspec.yaml | 2 +- bindings/dotnet/README.md | 13 + .../ShowcaseScenarioImplementations.cs | 4 +- bindings/dotnet/examples/README.md | 2 +- .../src/DecentDB.AdoNet/DecentDBConnection.cs | 16 +- .../DecentDBConnectionExtensions.cs | 43 +++ .../DecentDB.AdoNet/DecentDBMaintenance.cs | 277 ++++++++++++++++++ .../DecentDBMaintenanceResults.cs | 127 ++++++++ .../src/DecentDB.AdoNet/DecentDBQueryPlan.cs | 32 ++ bindings/dotnet/src/DecentDB.AdoNet/README.md | 30 ++ .../IndexedStringEqualityRegressionTests.cs | 150 ++++++++++ .../DecentDB.Tests/ExplainAnalyzeTests.cs | 28 ++ .../tests/DecentDB.Tests/MaintenanceTests.cs | 149 ++++++++++ .../dbeaver-extension/META-INF/MANIFEST.MF | 4 +- bindings/java/dbeaver-extension/build.gradle | 2 +- bindings/java/driver/build.gradle | 2 +- .../com/decentdb/jdbc/DecentDBDriver.java | 2 +- bindings/node/decentdb/package-lock.json | 4 +- bindings/node/decentdb/package.json | 2 +- bindings/node/knex-decentdb/package-lock.json | 6 +- bindings/node/knex-decentdb/package.json | 2 +- bindings/python/pyproject.toml | 2 +- design/FUTURE_WINS.md | 4 +- docs/about/changelog.md | 20 ++ docs/user-guide/benchmarks.md | 2 +- tests/bindings/dart/pubspec.lock | 2 +- 39 files changed, 924 insertions(+), 41 deletions(-) create mode 100644 bindings/dotnet/src/DecentDB.AdoNet/DecentDBMaintenanceResults.cs create mode 100644 bindings/dotnet/src/DecentDB.AdoNet/DecentDBQueryPlan.cs create mode 100644 bindings/dotnet/tests/DecentDB.EntityFrameworkCore.Tests/IndexedStringEqualityRegressionTests.cs diff --git a/Cargo.lock b/Cargo.lock index b3438b7..137485a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -831,7 +831,7 @@ dependencies = [ [[package]] name = "decentdb" -version = "2.8.0" +version = "2.9.0" dependencies = [ "base64 0.22.1", "chacha20", @@ -867,7 +867,7 @@ dependencies = [ [[package]] name = "decentdb-benchmark" -version = "2.8.0" +version = "2.9.0" dependencies = [ "anyhow", "clap", @@ -881,7 +881,7 @@ dependencies = [ [[package]] name = "decentdb-cli" -version = "2.8.0" +version = "2.9.0" dependencies = [ "anyhow", "clap", @@ -895,7 +895,7 @@ dependencies = [ [[package]] name = "decentdb-migrate" -version = "2.8.0" +version = "2.9.0" dependencies = [ "anyhow", "clap", @@ -1859,7 +1859,7 @@ dependencies = [ [[package]] name = "libpg_query_sys" -version = "2.8.0" +version = "2.9.0" dependencies = [ "pg_query", ] diff --git a/Cargo.toml b/Cargo.toml index 9ba6941..8e4cd3b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,7 +12,7 @@ exclude = [ resolver = "2" [workspace.package] -version = "2.8.0" +version = "2.9.0" edition = "2021" authors = ["Steven Hildreth"] license = "Apache-2.0" diff --git a/VERSION b/VERSION index 834f262..c8e38b6 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.8.0 +2.9.0 diff --git a/benchmarks/rust-baseline/Cargo.lock b/benchmarks/rust-baseline/Cargo.lock index fee31fe..242f996 100644 --- a/benchmarks/rust-baseline/Cargo.lock +++ b/benchmarks/rust-baseline/Cargo.lock @@ -336,7 +336,7 @@ dependencies = [ [[package]] name = "decentdb" -version = "2.8.0" +version = "2.9.0" dependencies = [ "base64", "chacha20", @@ -683,7 +683,7 @@ dependencies = [ [[package]] name = "libpg_query_sys" -version = "2.8.0" +version = "2.9.0" dependencies = [ "pg_query", ] diff --git a/bindings/dart/dart/pubspec.yaml b/bindings/dart/dart/pubspec.yaml index 0180f41..8c71f29 100644 --- a/bindings/dart/dart/pubspec.yaml +++ b/bindings/dart/dart/pubspec.yaml @@ -1,6 +1,6 @@ name: decentdb description: Dart FFI bindings for the Rust DecentDB C ABI. -version: 2.8.0 +version: 2.9.0 repository: https://github.com/sphildreth/decentdb homepage: https://github.com/sphildreth/decentdb/tree/main/bindings/dart diff --git a/bindings/dart/examples/console/pubspec.lock b/bindings/dart/examples/console/pubspec.lock index eb36a2d..938b582 100644 --- a/bindings/dart/examples/console/pubspec.lock +++ b/bindings/dart/examples/console/pubspec.lock @@ -7,7 +7,7 @@ packages: path: "../../dart" relative: true source: path - version: "2.8.0" + version: "2.9.0" ffi: dependency: transitive description: diff --git a/bindings/dart/examples/console_complex/pubspec.lock b/bindings/dart/examples/console_complex/pubspec.lock index eb36a2d..938b582 100644 --- a/bindings/dart/examples/console_complex/pubspec.lock +++ b/bindings/dart/examples/console_complex/pubspec.lock @@ -7,7 +7,7 @@ packages: path: "../../dart" relative: true source: path - version: "2.8.0" + version: "2.9.0" ffi: dependency: transitive description: diff --git a/bindings/dart/examples/flutter_desktop/pubspec.lock b/bindings/dart/examples/flutter_desktop/pubspec.lock index eb36a2d..938b582 100644 --- a/bindings/dart/examples/flutter_desktop/pubspec.lock +++ b/bindings/dart/examples/flutter_desktop/pubspec.lock @@ -7,7 +7,7 @@ packages: path: "../../dart" relative: true source: path - version: "2.8.0" + version: "2.9.0" ffi: dependency: transitive description: diff --git a/bindings/dart/flutter/android/build.gradle b/bindings/dart/flutter/android/build.gradle index fd5b76f..154d859 100644 --- a/bindings/dart/flutter/android/build.gradle +++ b/bindings/dart/flutter/android/build.gradle @@ -3,7 +3,7 @@ plugins { } group = 'dev.decentdb.decentdb_flutter' -version = '2.8.0' +version = '2.9.0' android { namespace 'dev.decentdb.decentdb_flutter' diff --git a/bindings/dart/flutter/example/pubspec.lock b/bindings/dart/flutter/example/pubspec.lock index 200a884..59925bf 100644 --- a/bindings/dart/flutter/example/pubspec.lock +++ b/bindings/dart/flutter/example/pubspec.lock @@ -71,14 +71,14 @@ packages: path: "../../dart" relative: true source: path - version: "2.8.0" + version: "2.9.0" decentdb_flutter: dependency: "direct main" description: path: ".." relative: true source: path - version: "2.8.0" + version: "2.9.0" fake_async: dependency: transitive description: diff --git a/bindings/dart/flutter/example/pubspec.yaml b/bindings/dart/flutter/example/pubspec.yaml index e586005..8fc3aef 100644 --- a/bindings/dart/flutter/example/pubspec.yaml +++ b/bindings/dart/flutter/example/pubspec.yaml @@ -1,7 +1,7 @@ name: decentdb_flutter_example description: Reference Flutter mobile app for DecentDB. publish_to: none -version: 2.8.0 +version: 2.9.0 environment: sdk: ^3.0.0 diff --git a/bindings/dart/flutter/ios/decentdb_flutter.podspec b/bindings/dart/flutter/ios/decentdb_flutter.podspec index 8313105..b18939d 100644 --- a/bindings/dart/flutter/ios/decentdb_flutter.podspec +++ b/bindings/dart/flutter/ios/decentdb_flutter.podspec @@ -1,6 +1,6 @@ Pod::Spec.new do |s| s.name = 'decentdb_flutter' - s.version = '2.8.0' + s.version = '2.9.0' s.summary = 'Flutter mobile integration helpers for DecentDB.' s.description = 'Provides Flutter registration and native artifact wiring for the DecentDB Dart FFI package.' s.homepage = 'https://github.com/sphildreth/decentdb' diff --git a/bindings/dart/flutter/pubspec.lock b/bindings/dart/flutter/pubspec.lock index e0ed518..3bf8809 100644 --- a/bindings/dart/flutter/pubspec.lock +++ b/bindings/dart/flutter/pubspec.lock @@ -71,7 +71,7 @@ packages: path: "../dart" relative: true source: path - version: "2.8.0" + version: "2.9.0" fake_async: dependency: transitive description: diff --git a/bindings/dart/flutter/pubspec.yaml b/bindings/dart/flutter/pubspec.yaml index 28a7693..99a767e 100644 --- a/bindings/dart/flutter/pubspec.yaml +++ b/bindings/dart/flutter/pubspec.yaml @@ -1,6 +1,6 @@ name: decentdb_flutter description: Flutter mobile integration helpers for the DecentDB Dart FFI package. -version: 2.8.0 +version: 2.9.0 publish_to: none repository: https://github.com/sphildreth/decentdb homepage: https://github.com/sphildreth/decentdb/tree/main/bindings/dart/flutter diff --git a/bindings/dotnet/README.md b/bindings/dotnet/README.md index 18005ec..5181819 100644 --- a/bindings/dotnet/README.md +++ b/bindings/dotnet/README.md @@ -30,8 +30,10 @@ This directory contains the official .NET bindings for DecentDB: | Connection pooling | ❌ (MicroOrm interprets `Pooling`) | ✅ | ❌ | | View querying (keyless DTO) | ✅ | ✅ (`QueryRawAsync`) | ✅ (`DbQuery` / keyless entity) | | Diagnostic events (`SqlExecuting`/`SqlExecuted`) | ✅ | ❌ | ✅ (EF Core logging) | +| Query plan helper (`ExplainQuery`) | ✅ | ❌ | ✅ (`ToQueryString` + ADO.NET helper) | | `GetSchema("Indexes")` with `IS_PRIMARY_KEY` | ✅ | N/A | N/A | | `DeleteDatabaseFiles` helper | ✅ | N/A | N/A | +| File maintenance (`GetWalStatus`, `CheckpointAsync`, `CompactAsync`, `VacuumAsync`) | ✅ | N/A | N/A | | Model pre-building cache | N/A | N/A | ✅ (`DecentDBModelBuilder`) | | Compiled query support | N/A | N/A | ✅ (`EF.CompileQuery`) | | Correlated aggregate rewrite | N/A | N/A | ✅ (infrastructure; rewrite deferred) | @@ -69,6 +71,17 @@ Supported keys: The `DecentDBConnection.DeleteDatabaseFiles(path)` helper deletes the database file and all sidecar files (`.wal`, `-wal`, `-shm`, `.coord`) safely. +The `DecentDBMaintenance` helper exposes binding-native maintenance operations: +`GetWalStatus(path)`, `CheckpointAsync(path)`, `CompactAsync(source, target)`, +and `VacuumAsync(path, createBackup: true)`. These helpers use the .NET binding +directly. The older `VacuumAtomicAsync(path, cliPath, ...)` remains available +for legacy executable-backed vacuum workflows. + +For query-plan diagnostics, call `connection.ExplainQuery(sql)` or +`connection.ExplainQuery(sql, analyze: true)` on an open ADO.NET connection. +The returned plan includes the generated `EXPLAIN` SQL, plan lines, joined text, +and elapsed diagnostic duration. + `DecentDB.Native.DecentDB.ExecuteQueued(sql)` exposes the native queued path for self-contained SQL, and `WriteQueueMetrics()` returns queue counters. ADO.NET commands keep prepared-statement execution on the direct path until the C ABI diff --git a/bindings/dotnet/examples/DecentDb.ShowCase/ShowcaseScenarioImplementations.cs b/bindings/dotnet/examples/DecentDb.ShowCase/ShowcaseScenarioImplementations.cs index 3a13099..9ce4bd7 100644 --- a/bindings/dotnet/examples/DecentDb.ShowCase/ShowcaseScenarioImplementations.cs +++ b/bindings/dotnet/examples/DecentDb.ShowCase/ShowcaseScenarioImplementations.cs @@ -712,14 +712,14 @@ private static async Task DemonstrateOperationalBehaviors(ShowcaseScenarioContex inMemory.SaveAs(maintenanceDbPath); } - var vacuumed = await DecentDBMaintenance.VacuumAtomicAsync(maintenanceDbPath); + var vacuumResult = await DecentDBMaintenance.VacuumAsync(maintenanceDbPath); using var verify = scenario.CreateOpenConnection(maintenanceDbPath); using var verifyCommand = verify.CreateCommand(); verifyCommand.CommandText = "SELECT COUNT(*) FROM maintenance_demo"; var copiedRows = Convert.ToInt64(verifyCommand.ExecuteScalar()); scenario.WriteLine($" SaveAs copied rows: {copiedRows}"); - scenario.WriteLine($" VacuumAtomicAsync: {vacuumed}"); + scenario.WriteLine($" VacuumAsync replaced database: {vacuumResult.DatabaseExisted}"); scenario.WriteLine(); } private static async Task DemonstrateConcurrencyControl(ShowcaseScenarioContext scenario) diff --git a/bindings/dotnet/examples/README.md b/bindings/dotnet/examples/README.md index 6b6f84b..0fe6903 100644 --- a/bindings/dotnet/examples/README.md +++ b/bindings/dotnet/examples/README.md @@ -163,7 +163,7 @@ connection.GetSchema("Indexes") // Index information ### Database Maintenance - **Checkpoint** - Flush WAL to main database file - **SaveAs** - Export database to new file -- **VacuumAtomicAsync** - Compact a file-backed database with the maintenance helper +- **VacuumAsync** - Compact a file-backed database with the maintenance helper ### Provider Ergonomics - `DecentDBConnectionStringBuilder` can be passed directly to `UseDecentDB(...)` diff --git a/bindings/dotnet/src/DecentDB.AdoNet/DecentDBConnection.cs b/bindings/dotnet/src/DecentDB.AdoNet/DecentDBConnection.cs index 05ef223..0584dea 100644 --- a/bindings/dotnet/src/DecentDB.AdoNet/DecentDBConnection.cs +++ b/bindings/dotnet/src/DecentDB.AdoNet/DecentDBConnection.cs @@ -168,10 +168,24 @@ public override void Open() catch (Exception ex) { _state = ConnectionState.Closed; - throw new InvalidOperationException($"Failed to open database: {ex.Message}", ex); + throw new InvalidOperationException(CreateOpenFailureMessage(path, ex), ex); } } + private static string CreateOpenFailureMessage(string path, Exception exception) + { + var message = exception.Message; + if (message.Contains("unsupported database format version", StringComparison.OrdinalIgnoreCase)) + { + return "Failed to open database: unsupported DecentDB file format. " + + "Use a compatible DecentDB engine, run decentdb-migrate when a migration path is available, " + + "or rebuild/export the database with the current engine. " + + $"Path: {path}. Native error: {message}"; + } + + return $"Failed to open database: {message}"; + } + public override Task OpenAsync(CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); diff --git a/bindings/dotnet/src/DecentDB.AdoNet/DecentDBConnectionExtensions.cs b/bindings/dotnet/src/DecentDB.AdoNet/DecentDBConnectionExtensions.cs index 7051578..2479a15 100644 --- a/bindings/dotnet/src/DecentDB.AdoNet/DecentDBConnectionExtensions.cs +++ b/bindings/dotnet/src/DecentDB.AdoNet/DecentDBConnectionExtensions.cs @@ -1,4 +1,7 @@ using System; +using System.Collections.Generic; +using System.Data; +using System.Diagnostics; namespace DecentDB.AdoNet; @@ -9,4 +12,44 @@ public static void Checkpoint(this DecentDBConnection connection) if (connection == null) throw new ArgumentNullException(nameof(connection)); connection.Checkpoint(); } + + /// + /// Executes EXPLAIN or EXPLAIN ANALYZE for a SQL statement and returns the plan text. + /// + /// An open DecentDB connection. + /// The SQL statement to inspect. + /// When true, executes EXPLAIN ANALYZE and includes actual metrics. + public static DecentDBQueryPlan ExplainQuery( + this DecentDBConnection connection, + string sql, + bool analyze = false) + { + if (connection == null) throw new ArgumentNullException(nameof(connection)); + if (string.IsNullOrWhiteSpace(sql)) + throw new ArgumentException("SQL cannot be null or empty.", nameof(sql)); + if (connection.State != ConnectionState.Open) + throw new InvalidOperationException("Connection is not open."); + + var explainSql = analyze ? $"EXPLAIN ANALYZE {sql}" : $"EXPLAIN {sql}"; + var stopwatch = Stopwatch.StartNew(); + var lines = new List(); + + using (var command = connection.CreateCommand()) + { + command.CommandText = explainSql; + using var reader = command.ExecuteReader(); + while (reader.Read()) + { + lines.Add(Convert.ToString(reader.GetValue(0)) ?? string.Empty); + } + } + + stopwatch.Stop(); + return new DecentDBQueryPlan( + sql, + explainSql, + analyze, + lines, + stopwatch.Elapsed); + } } diff --git a/bindings/dotnet/src/DecentDB.AdoNet/DecentDBMaintenance.cs b/bindings/dotnet/src/DecentDB.AdoNet/DecentDBMaintenance.cs index 61b71ae..1a971cf 100644 --- a/bindings/dotnet/src/DecentDB.AdoNet/DecentDBMaintenance.cs +++ b/bindings/dotnet/src/DecentDB.AdoNet/DecentDBMaintenance.cs @@ -11,6 +11,206 @@ namespace DecentDB.AdoNet /// public static class DecentDBMaintenance { + /// + /// Returns file-size diagnostics for a DecentDB database and known WAL sidecars. + /// + /// The path to the DecentDB database file. + public static DecentDBWalStatus GetWalStatus(string databasePath) + { + var fullPath = NormalizeDatabasePath(databasePath); + return new DecentDBWalStatus( + fullPath, + FileLengthOrZero(fullPath), + fullPath + "-wal", + FileLengthOrZero(fullPath + "-wal"), + fullPath + ".wal", + FileLengthOrZero(fullPath + ".wal"), + fullPath + ".coord", + FileLengthOrZero(fullPath + ".coord")); + } + + /// + /// Opens the database and performs a DecentDB checkpoint without invoking the CLI. + /// + /// The path to the DecentDB database file. + /// A token to cancel before the operation starts. + public static Task CheckpointAsync( + string databasePath, + CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + + var fullPath = NormalizeDatabasePath(databasePath); + var before = GetWalStatus(fullPath); + if (!File.Exists(fullPath)) + { + return Task.FromResult(new DecentDBCheckpointResult( + fullPath, + databaseExisted: false, + before, + before, + TimeSpan.Zero)); + } + + var stopwatch = Stopwatch.StartNew(); + using (var connection = OpenConnection(fullPath)) + { + connection.Checkpoint(); + } + + stopwatch.Stop(); + var after = GetWalStatus(fullPath); + return Task.FromResult(new DecentDBCheckpointResult( + fullPath, + databaseExisted: true, + before, + after, + stopwatch.Elapsed)); + } + + /// + /// Saves a compact copy of a database to a new destination file without invoking the CLI. + /// + /// The source DecentDB database file. + /// The destination DecentDB database file. + /// Whether to delete an existing destination and its sidecars first. + /// A token to cancel before the operation starts. + public static Task CompactAsync( + string sourceDatabasePath, + string destinationDatabasePath, + bool overwrite = false, + CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + + var sourcePath = NormalizeDatabasePath(sourceDatabasePath); + var destinationPath = NormalizeDatabasePath(destinationDatabasePath); + if (PathsEqual(sourcePath, destinationPath)) + { + throw new ArgumentException("Destination database path must be different from the source path.", nameof(destinationDatabasePath)); + } + + var sourceBytes = FileLengthOrZero(sourcePath); + if (!File.Exists(sourcePath)) + { + return Task.FromResult(new DecentDBCompactResult( + sourcePath, + destinationPath, + sourceExisted: false, + sourceBytes, + destinationBytes: 0, + TimeSpan.Zero)); + } + + var destinationDirectory = Path.GetDirectoryName(destinationPath); + if (!string.IsNullOrEmpty(destinationDirectory) && !Directory.Exists(destinationDirectory)) + { + Directory.CreateDirectory(destinationDirectory); + } + + var destinationArtifactsExist = HasDatabaseArtifacts(destinationPath); + if (destinationArtifactsExist) + { + if (!overwrite) + { + throw new IOException($"Destination database or sidecar artifacts already exist: {destinationPath}"); + } + + DecentDBConnection.DeleteDatabaseFiles(destinationPath); + } + + var stopwatch = Stopwatch.StartNew(); + using (var connection = OpenConnection(sourcePath)) + { + connection.SaveAs(destinationPath); + } + + stopwatch.Stop(); + return Task.FromResult(new DecentDBCompactResult( + sourcePath, + destinationPath, + sourceExisted: true, + sourceBytes, + FileLengthOrZero(destinationPath), + stopwatch.Elapsed)); + } + + /// + /// Compacts a database to a temporary file and replaces the original without invoking the CLI. + /// Ensure no other connections are open to the database file before running. + /// + /// The path to the DecentDB database file. + /// If true, renames the original database file with a .bak extension instead of deleting it. + /// A token to cancel before each operation phase starts. + public static async Task VacuumAsync( + string databasePath, + bool createBackup = false, + CancellationToken cancellationToken = default) + { + cancellationToken.ThrowIfCancellationRequested(); + + var fullPath = NormalizeDatabasePath(databasePath); + var before = GetWalStatus(fullPath); + if (!File.Exists(fullPath)) + { + return new DecentDBVacuumResult( + fullPath, + databaseExisted: false, + backupCreated: false, + backupPath: null, + before, + before, + TimeSpan.Zero); + } + + var stopwatch = Stopwatch.StartNew(); + var tempPath = fullPath + ".vacuum_tmp"; + var backupPath = fullPath + ".bak"; + + try + { + await CheckpointAsync(fullPath, cancellationToken).ConfigureAwait(false); + cancellationToken.ThrowIfCancellationRequested(); + + DecentDBConnection.DeleteDatabaseFiles(tempPath); + await CompactAsync(fullPath, tempPath, overwrite: true, cancellationToken).ConfigureAwait(false); + cancellationToken.ThrowIfCancellationRequested(); + + if (createBackup) + { + if (File.Exists(backupPath)) + { + File.Delete(backupPath); + } + + File.Move(fullPath, backupPath); + } + else + { + File.Delete(fullPath); + } + + DeleteSidecars(fullPath); + File.Move(tempPath, fullPath); + DeleteSidecars(tempPath); + stopwatch.Stop(); + + return new DecentDBVacuumResult( + fullPath, + databaseExisted: true, + backupCreated: createBackup, + backupPath: createBackup ? backupPath : null, + before, + GetWalStatus(fullPath), + stopwatch.Elapsed); + } + catch + { + DecentDBConnection.DeleteDatabaseFiles(tempPath); + throw; + } + } + private static string ResolveCliExecutablePath(string cliExecutablePath) { if (Path.IsPathRooted(cliExecutablePath) && File.Exists(cliExecutablePath)) @@ -46,6 +246,83 @@ private static string ResolveCliExecutablePath(string cliExecutablePath) return cliExecutablePath; } + private static string NormalizeDatabasePath(string databasePath) + { + if (string.IsNullOrWhiteSpace(databasePath)) + throw new ArgumentException("Database path cannot be null or empty.", nameof(databasePath)); + + return Path.GetFullPath(databasePath); + } + + private static bool PathsEqual(string left, string right) + { + var comparison = OperatingSystem.IsWindows() + ? StringComparison.OrdinalIgnoreCase + : StringComparison.Ordinal; + return string.Equals(left, right, comparison); + } + + private static DecentDBConnection OpenConnection(string databasePath) + { + var builder = new DecentDBConnectionStringBuilder + { + DataSource = databasePath + }; + var connection = new DecentDBConnection(builder.ConnectionString); + connection.Open(); + return connection; + } + + private static long FileLengthOrZero(string path) + { + try + { + return File.Exists(path) ? new FileInfo(path).Length : 0; + } + catch (IOException) + { + return 0; + } + catch (UnauthorizedAccessException) + { + return 0; + } + } + + private static bool HasDatabaseArtifacts(string databasePath) + { + return File.Exists(databasePath) || + File.Exists(databasePath + "-wal") || + File.Exists(databasePath + ".wal") || + File.Exists(databasePath + "-shm") || + File.Exists(databasePath + ".coord"); + } + + private static void DeleteSidecars(string databasePath) + { + TryDelete(databasePath + "-wal"); + TryDelete(databasePath + ".wal"); + TryDelete(databasePath + "-shm"); + TryDelete(databasePath + ".coord"); + } + + private static void TryDelete(string path) + { + try + { + if (File.Exists(path)) + { + File.Delete(path); + } + } + catch (FileNotFoundException) + { + } + catch (DirectoryNotFoundException) + { + } + } + private static IEnumerable CandidateCliPaths(string root) { if (OperatingSystem.IsWindows()) diff --git a/bindings/dotnet/src/DecentDB.AdoNet/DecentDBMaintenanceResults.cs b/bindings/dotnet/src/DecentDB.AdoNet/DecentDBMaintenanceResults.cs new file mode 100644 index 0000000..42ccc86 --- /dev/null +++ b/bindings/dotnet/src/DecentDB.AdoNet/DecentDBMaintenanceResults.cs @@ -0,0 +1,127 @@ +using System; + +namespace DecentDB.AdoNet; + +/// +/// File-size snapshot for a DecentDB database and its WAL sidecars. +/// +public sealed class DecentDBWalStatus +{ + public DecentDBWalStatus( + string databasePath, + long databaseBytes, + string dashWalPath, + long dashWalBytes, + string dottedWalPath, + long dottedWalBytes, + string coordinationPath, + long coordinationBytes) + { + DatabasePath = databasePath; + DatabaseBytes = databaseBytes; + DashWalPath = dashWalPath; + DashWalBytes = dashWalBytes; + DottedWalPath = dottedWalPath; + DottedWalBytes = dottedWalBytes; + CoordinationPath = coordinationPath; + CoordinationBytes = coordinationBytes; + } + + public string DatabasePath { get; } + public long DatabaseBytes { get; } + public string DashWalPath { get; } + public long DashWalBytes { get; } + public string DottedWalPath { get; } + public long DottedWalBytes { get; } + public string CoordinationPath { get; } + public long CoordinationBytes { get; } + public long TotalWalBytes => checked(DashWalBytes + DottedWalBytes); + public bool HasWal => TotalWalBytes > 0; +} + +/// +/// Result from an in-process checkpoint against a DecentDB database file. +/// +public sealed class DecentDBCheckpointResult +{ + public DecentDBCheckpointResult( + string databasePath, + bool databaseExisted, + DecentDBWalStatus before, + DecentDBWalStatus after, + TimeSpan duration) + { + DatabasePath = databasePath; + DatabaseExisted = databaseExisted; + Before = before; + After = after; + Duration = duration; + } + + public string DatabasePath { get; } + public bool DatabaseExisted { get; } + public DecentDBWalStatus Before { get; } + public DecentDBWalStatus After { get; } + public TimeSpan Duration { get; } +} + +/// +/// Result from an in-process compact/save-as operation. +/// +public sealed class DecentDBCompactResult +{ + public DecentDBCompactResult( + string sourceDatabasePath, + string destinationDatabasePath, + bool sourceExisted, + long sourceBytes, + long destinationBytes, + TimeSpan duration) + { + SourceDatabasePath = sourceDatabasePath; + DestinationDatabasePath = destinationDatabasePath; + SourceExisted = sourceExisted; + SourceBytes = sourceBytes; + DestinationBytes = destinationBytes; + Duration = duration; + } + + public string SourceDatabasePath { get; } + public string DestinationDatabasePath { get; } + public bool SourceExisted { get; } + public long SourceBytes { get; } + public long DestinationBytes { get; } + public TimeSpan Duration { get; } +} + +/// +/// Result from an in-process vacuum/compact operation. +/// +public sealed class DecentDBVacuumResult +{ + public DecentDBVacuumResult( + string databasePath, + bool databaseExisted, + bool backupCreated, + string? backupPath, + DecentDBWalStatus before, + DecentDBWalStatus after, + TimeSpan duration) + { + DatabasePath = databasePath; + DatabaseExisted = databaseExisted; + BackupCreated = backupCreated; + BackupPath = backupPath; + Before = before; + After = after; + Duration = duration; + } + + public string DatabasePath { get; } + public bool DatabaseExisted { get; } + public bool BackupCreated { get; } + public string? BackupPath { get; } + public DecentDBWalStatus Before { get; } + public DecentDBWalStatus After { get; } + public TimeSpan Duration { get; } +} diff --git a/bindings/dotnet/src/DecentDB.AdoNet/DecentDBQueryPlan.cs b/bindings/dotnet/src/DecentDB.AdoNet/DecentDBQueryPlan.cs new file mode 100644 index 0000000..65c861a --- /dev/null +++ b/bindings/dotnet/src/DecentDB.AdoNet/DecentDBQueryPlan.cs @@ -0,0 +1,32 @@ +using System; +using System.Collections.Generic; + +namespace DecentDB.AdoNet; + +/// +/// Query-plan output captured from DecentDB's EXPLAIN or EXPLAIN ANALYZE statement. +/// +public sealed class DecentDBQueryPlan +{ + public DecentDBQueryPlan( + string sql, + string explainSql, + bool analyze, + IReadOnlyList lines, + TimeSpan duration) + { + Sql = sql; + ExplainSql = explainSql; + Analyze = analyze; + Lines = lines; + Duration = duration; + Text = string.Join(Environment.NewLine, lines); + } + + public string Sql { get; } + public string ExplainSql { get; } + public bool Analyze { get; } + public IReadOnlyList Lines { get; } + public string Text { get; } + public TimeSpan Duration { get; } +} diff --git a/bindings/dotnet/src/DecentDB.AdoNet/README.md b/bindings/dotnet/src/DecentDB.AdoNet/README.md index 238b5f1..27c9eef 100644 --- a/bindings/dotnet/src/DecentDB.AdoNet/README.md +++ b/bindings/dotnet/src/DecentDB.AdoNet/README.md @@ -6,6 +6,7 @@ This package provides: - `DecentDBConnection` / `DecentDBCommand` / `DecentDBDataReader` - `DecentDBConnectionStringBuilder` +- `DecentDBMaintenance` for checkpoint, WAL status, compact, and vacuum helpers ## Install @@ -34,6 +35,35 @@ Bare paths (e.g., `"/tmp/mydb.ddb"`) are also accepted by `DecentDBConnection`'s Use `DecentDBConnection.DeleteDatabaseFiles(path)` to safely delete the database file and all sidecar files (`.wal`, `-wal`, `-shm`, `.coord`) in the correct order. This prevents stale WAL or coordination artifacts when recreating databases. +## Maintenance helpers + +Use `DecentDBMaintenance` for file-path based maintenance through the .NET +binding: + +```csharp +var before = DecentDBMaintenance.GetWalStatus(path); +var checkpoint = await DecentDBMaintenance.CheckpointAsync(path); +var compact = await DecentDBMaintenance.CompactAsync(path, compactedPath); +var vacuum = await DecentDBMaintenance.VacuumAsync(path, createBackup: true); +``` + +`VacuumAsync(...)` uses the .NET binding directly by checkpointing, saving a +compact temporary copy, and replacing the original file. `VacuumAtomicAsync(...)` +remains available for legacy executable-backed offline vacuum flows. + +## Query diagnostics + +Use `ExplainQuery` on an open `DecentDBConnection` to capture `EXPLAIN` or +`EXPLAIN ANALYZE` output without writing command boilerplate: + +```csharp +var plan = connection.ExplainQuery( + "SELECT * FROM artists WHERE musicbrainz_id_raw = $1", + analyze: true); + +Console.WriteLine(plan.Text); +``` + ## Notes - The native engine library is shipped as a NuGet runtime native asset under `runtimes/{rid}/native/`. diff --git a/bindings/dotnet/tests/DecentDB.EntityFrameworkCore.Tests/IndexedStringEqualityRegressionTests.cs b/bindings/dotnet/tests/DecentDB.EntityFrameworkCore.Tests/IndexedStringEqualityRegressionTests.cs new file mode 100644 index 0000000..ec9b077 --- /dev/null +++ b/bindings/dotnet/tests/DecentDB.EntityFrameworkCore.Tests/IndexedStringEqualityRegressionTests.cs @@ -0,0 +1,150 @@ +using System.Globalization; +using DecentDB.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore; +using Xunit; + +namespace DecentDB.EntityFrameworkCore.Tests; + +public sealed class IndexedStringEqualityRegressionTests : IDisposable +{ + private const int RowCount = 6_000; + private const int TargetId = 4_321; + + private readonly string _dbPath = Path.Combine( + Path.GetTempPath(), + $"test_ef_indexed_string_equality_{Guid.NewGuid():N}.ddb"); + + public void Dispose() + { + TryDelete(_dbPath); + TryDelete(_dbPath + "-wal"); + } + + [Fact] + public void ExactEquality_OnLargeIndexedNormalizedNameAndRawId_ReturnsExpectedRow() + { + SeedData(); + + using var context = CreateContext(); + var targetNormalizedName = BuildNormalizedName(TargetId); + var targetRawId = BuildRawId(TargetId); + + var byNormalizedNameQuery = context.Artists + .AsNoTracking() + .Where(x => x.NameNormalized == targetNormalizedName); + var byRawIdQuery = context.Artists + .AsNoTracking() + .Where(x => x.RawId == targetRawId); + + AssertExactEqualityPredicate(byNormalizedNameQuery.ToQueryString(), "name_normalized", "targetNormalizedName"); + AssertExactEqualityPredicate(byRawIdQuery.ToQueryString(), "raw_id", "targetRawId"); + + var byNormalizedName = byNormalizedNameQuery.Single(); + var byRawId = byRawIdQuery.Single(); + + Assert.Equal(RowCount, context.Artists.Count()); + Assert.Equal(TargetId, byNormalizedName.Id); + Assert.Equal(TargetId, byRawId.Id); + Assert.Equal(targetRawId, byNormalizedName.RawId); + Assert.Equal(targetNormalizedName, byRawId.NameNormalized); + } + + private IndexedStringDbContext CreateContext() + { + var optionsBuilder = new DbContextOptionsBuilder(); + optionsBuilder.UseDecentDB($"Data Source={_dbPath}"); + return new IndexedStringDbContext(optionsBuilder.Options); + } + + private void SeedData() + { + using var context = CreateContext(); + Assert.True(context.Database.EnsureCreated()); + + var artists = Enumerable.Range(1, RowCount) + .Select(id => new IndexedArtist + { + Id = id, + Name = $"Artist {id.ToString("D5", CultureInfo.InvariantCulture)}", + NameNormalized = BuildNormalizedName(id), + RawId = BuildRawId(id) + }) + .ToArray(); + + context.Artists.AddRange(artists); + context.SaveChanges(); + } + + private static string BuildNormalizedName(int id) + { + var stableId = id.ToString("D5", CultureInfo.InvariantCulture); + var repeatedToken = new string((char)('a' + (id % 26)), 384); + return $"melodee normalized artist {stableId} {repeatedToken} canonical {stableId}"; + } + + private static string BuildRawId(int id) + { + var stableId = id.ToString("D12", CultureInfo.InvariantCulture); + return $"musicbrainz:artist:00000000-0000-0000-0000-{stableId}"; + } + + private static void AssertExactEqualityPredicate(string sql, string columnName, string parameterName) + { + Assert.Contains("WHERE", sql, StringComparison.OrdinalIgnoreCase); + Assert.Contains($"\"{columnName}\"", sql, StringComparison.Ordinal); + Assert.Contains(parameterName, sql, StringComparison.Ordinal); + Assert.Contains(" = ", sql, StringComparison.Ordinal); + Assert.DoesNotContain("LIKE", sql, StringComparison.OrdinalIgnoreCase); + } + + private static void TryDelete(string path) + { + if (File.Exists(path)) + { + File.Delete(path); + } + } + + private sealed class IndexedStringDbContext : DbContext + { + public IndexedStringDbContext(DbContextOptions options) + : base(options) + { + } + + public DbSet Artists => Set(); + + protected override void OnModelCreating(ModelBuilder modelBuilder) + { + modelBuilder.Entity(entity => + { + entity.ToTable("ef_indexed_string_artists"); + entity.HasKey(x => x.Id); + + entity.Property(x => x.Id) + .HasColumnName("id") + .ValueGeneratedNever(); + entity.Property(x => x.Name) + .HasColumnName("name"); + entity.Property(x => x.NameNormalized) + .HasColumnName("name_normalized"); + entity.Property(x => x.RawId) + .HasColumnName("raw_id"); + + entity.HasIndex(x => x.NameNormalized) + .HasDatabaseName("ix_ef_indexed_string_artists_name_normalized"); + entity.HasIndex(x => x.RawId) + .HasDatabaseName("ux_ef_indexed_string_artists_raw_id") + .IsUnique(); + }); + } + } + + private sealed class IndexedArtist + { + public int Id { get; set; } + public string Name { get; set; } = string.Empty; + public string NameNormalized { get; set; } = string.Empty; + public string RawId { get; set; } = string.Empty; + } +} diff --git a/bindings/dotnet/tests/DecentDB.Tests/ExplainAnalyzeTests.cs b/bindings/dotnet/tests/DecentDB.Tests/ExplainAnalyzeTests.cs index fa30376..00ea979 100644 --- a/bindings/dotnet/tests/DecentDB.Tests/ExplainAnalyzeTests.cs +++ b/bindings/dotnet/tests/DecentDB.Tests/ExplainAnalyzeTests.cs @@ -128,4 +128,32 @@ public void Explain_WithoutAnalyze_NoActualMetrics() Assert.DoesNotContain("Actual Rows:", planText); Assert.DoesNotContain("Actual Time:", planText); } + + [Fact] + public void ExplainQueryExtension_ReturnsPlanTextAndDuration() + { + using var conn = OpenConnection(); + using var cmd = conn.CreateCommand(); + cmd.CommandText = "CREATE TABLE t (id INT, name TEXT)"; + cmd.ExecuteNonQuery(); + cmd.CommandText = "INSERT INTO t VALUES (1, 'Alice'), (2, 'Bob')"; + cmd.ExecuteNonQuery(); + + var plan = conn.ExplainQuery("SELECT * FROM t WHERE id = 1", analyze: true); + + Assert.True(plan.Analyze); + Assert.Equal("SELECT * FROM t WHERE id = 1", plan.Sql); + Assert.Equal("EXPLAIN ANALYZE SELECT * FROM t WHERE id = 1", plan.ExplainSql); + Assert.NotEmpty(plan.Lines); + Assert.Contains("Actual Rows:", plan.Text); + Assert.True(plan.Duration >= TimeSpan.Zero); + } + + [Fact] + public void ExplainQueryExtension_ClosedConnection_Throws() + { + using var conn = new DecentDBConnection($"Data Source={_dbPath}"); + + Assert.Throws(() => conn.ExplainQuery("SELECT 1")); + } } diff --git a/bindings/dotnet/tests/DecentDB.Tests/MaintenanceTests.cs b/bindings/dotnet/tests/DecentDB.Tests/MaintenanceTests.cs index 8878b81..51f646a 100644 --- a/bindings/dotnet/tests/DecentDB.Tests/MaintenanceTests.cs +++ b/bindings/dotnet/tests/DecentDB.Tests/MaintenanceTests.cs @@ -42,6 +42,134 @@ public async Task VacuumAtomicAsync_NonExistentFile_ReturnsFalse() Assert.False(result); } + [Fact] + public void GetWalStatus_MissingDatabase_ReturnsZeroSizes() + { + var dbPath = Path.Combine(Path.GetTempPath(), $"test_wal_status_none_{Guid.NewGuid():N}.ddb"); + + var status = DecentDBMaintenance.GetWalStatus(dbPath); + + Assert.Equal(Path.GetFullPath(dbPath), status.DatabasePath); + Assert.Equal(0, status.DatabaseBytes); + Assert.Equal(0, status.TotalWalBytes); + Assert.False(status.HasWal); + } + + [Fact] + public async Task CheckpointAsync_ValidFile_CheckpointsWithoutCli() + { + var dbPath = Path.Combine(Path.GetTempPath(), $"test_checkpoint_binding_{Guid.NewGuid():N}.ddb"); + + try + { + SeedDatabase(dbPath); + + var result = await DecentDBMaintenance.CheckpointAsync(dbPath); + + Assert.True(result.DatabaseExisted); + Assert.Equal(Path.GetFullPath(dbPath), result.DatabasePath); + Assert.True(File.Exists(dbPath)); + Assert.True(result.After.TotalWalBytes <= result.Before.TotalWalBytes); + } + finally + { + DecentDBConnection.DeleteDatabaseFiles(dbPath); + } + } + + [Fact] + public async Task CompactAsync_ValidFile_CreatesReadableCopyWithoutCli() + { + var dbPath = Path.Combine(Path.GetTempPath(), $"test_compact_binding_{Guid.NewGuid():N}.ddb"); + var compactedPath = dbPath + ".compact"; + + try + { + SeedDatabase(dbPath); + + var result = await DecentDBMaintenance.CompactAsync(dbPath, compactedPath); + + Assert.True(result.SourceExisted); + Assert.True(result.SourceBytes > 0); + Assert.True(result.DestinationBytes > 0); + Assert.Equal(1L, CountRows(compactedPath)); + } + finally + { + DecentDBConnection.DeleteDatabaseFiles(dbPath); + DecentDBConnection.DeleteDatabaseFiles(compactedPath); + } + } + + [Fact] + public async Task CompactAsync_SameSourceAndDestination_Throws() + { + var dbPath = Path.Combine(Path.GetTempPath(), $"test_compact_same_{Guid.NewGuid():N}.ddb"); + + try + { + SeedDatabase(dbPath); + + await Assert.ThrowsAsync(() => + DecentDBMaintenance.CompactAsync(dbPath, dbPath, overwrite: true)); + } + finally + { + DecentDBConnection.DeleteDatabaseFiles(dbPath); + } + } + + [Fact] + public async Task CompactAsync_DestinationSidecarExists_RequiresOverwrite() + { + var dbPath = Path.Combine(Path.GetTempPath(), $"test_compact_sidecar_{Guid.NewGuid():N}.ddb"); + var compactedPath = dbPath + ".compact"; + + try + { + SeedDatabase(dbPath); + await File.WriteAllTextAsync(compactedPath + "-wal", "stale"); + + await Assert.ThrowsAsync(() => + DecentDBMaintenance.CompactAsync(dbPath, compactedPath)); + + var result = await DecentDBMaintenance.CompactAsync(dbPath, compactedPath, overwrite: true); + + Assert.True(result.SourceExisted); + Assert.Equal(1L, CountRows(compactedPath)); + } + finally + { + DecentDBConnection.DeleteDatabaseFiles(dbPath); + DecentDBConnection.DeleteDatabaseFiles(compactedPath); + } + } + + [Fact] + public async Task VacuumAsync_ValidFile_ReplacesDatabaseAndCanCreateBackupWithoutCli() + { + var dbPath = Path.Combine(Path.GetTempPath(), $"test_vacuum_binding_{Guid.NewGuid():N}.ddb"); + + try + { + SeedDatabase(dbPath); + + var result = await DecentDBMaintenance.VacuumAsync(dbPath, createBackup: true); + + Assert.True(result.DatabaseExisted); + Assert.True(result.BackupCreated); + Assert.True(File.Exists(dbPath)); + Assert.NotNull(result.BackupPath); + Assert.True(File.Exists(result.BackupPath)); + Assert.Equal(1L, CountRows(dbPath)); + } + finally + { + DecentDBConnection.DeleteDatabaseFiles(dbPath); + if (File.Exists(dbPath + ".bak")) File.Delete(dbPath + ".bak"); + } + } + [Fact] public async Task VacuumAtomicAsync_ValidFile_PerformsVacuum() { @@ -95,4 +223,25 @@ public async Task VacuumAtomicAsync_ValidFile_PerformsVacuum() if (File.Exists(dbPath + ".bak")) File.Delete(dbPath + ".bak"); } } + + private static void SeedDatabase(string dbPath) + { + using var conn = new DecentDBConnection($"Data Source={dbPath};WAL Auto Checkpoint=0"); + conn.Open(); + using var cmd = conn.CreateCommand(); + cmd.CommandText = "CREATE TABLE MaintenanceTest (Id INTEGER PRIMARY KEY, Val TEXT);"; + cmd.ExecuteNonQuery(); + + cmd.CommandText = "INSERT INTO MaintenanceTest (Id, Val) VALUES (1, 'Hello');"; + cmd.ExecuteNonQuery(); + } + + private static long CountRows(string dbPath) + { + using var conn = new DecentDBConnection($"Data Source={dbPath}"); + conn.Open(); + using var cmd = conn.CreateCommand(); + cmd.CommandText = "SELECT COUNT(*) FROM MaintenanceTest;"; + return Convert.ToInt64(cmd.ExecuteScalar()); + } } diff --git a/bindings/java/dbeaver-extension/META-INF/MANIFEST.MF b/bindings/java/dbeaver-extension/META-INF/MANIFEST.MF index dd8b215..f15257d 100644 --- a/bindings/java/dbeaver-extension/META-INF/MANIFEST.MF +++ b/bindings/java/dbeaver-extension/META-INF/MANIFEST.MF @@ -2,7 +2,7 @@ Manifest-Version: 1.0 Bundle-ManifestVersion: 2 Bundle-Name: DecentDB DBeaver Extension Bundle-SymbolicName: org.jkiss.dbeaver.ext.decentdb;singleton:=true -Bundle-Version: 2.8.0 +Bundle-Version: 2.9.0 Bundle-Activator: org.jkiss.dbeaver.ext.decentdb.DecentDBActivator Bundle-Vendor: DecentDB Contributors Require-Bundle: org.eclipse.core.runtime, @@ -11,5 +11,5 @@ Require-Bundle: org.eclipse.core.runtime, org.jkiss.dbeaver.ext.generic Bundle-RequiredExecutionEnvironment: JavaSE-17 Bundle-ClassPath: ., - lib/decentdb-jdbc-2.8.0.jar + lib/decentdb-jdbc-2.9.0.jar Export-Package: org.jkiss.dbeaver.ext.decentdb.model diff --git a/bindings/java/dbeaver-extension/build.gradle b/bindings/java/dbeaver-extension/build.gradle index edcfa78..739cd02 100644 --- a/bindings/java/dbeaver-extension/build.gradle +++ b/bindings/java/dbeaver-extension/build.gradle @@ -3,7 +3,7 @@ plugins { } group = 'org.jkiss.dbeaver.ext' -version = '2.8.0' +version = '2.9.0' java { sourceCompatibility = JavaVersion.VERSION_21 diff --git a/bindings/java/driver/build.gradle b/bindings/java/driver/build.gradle index 1d9ca1b..0a51344 100644 --- a/bindings/java/driver/build.gradle +++ b/bindings/java/driver/build.gradle @@ -3,7 +3,7 @@ plugins { } group = 'com.decentdb' -version = '2.8.0' +version = '2.9.0' def repoRoot = file("${rootProject.projectDir}/../..") def nativeLibDirPath = project.findProperty('nativeLibDir') ?: diff --git a/bindings/java/driver/src/main/java/com/decentdb/jdbc/DecentDBDriver.java b/bindings/java/driver/src/main/java/com/decentdb/jdbc/DecentDBDriver.java index 2499e04..6e48403 100644 --- a/bindings/java/driver/src/main/java/com/decentdb/jdbc/DecentDBDriver.java +++ b/bindings/java/driver/src/main/java/com/decentdb/jdbc/DecentDBDriver.java @@ -28,7 +28,7 @@ public final class DecentDBDriver implements Driver { public static final String URL_PREFIX = "jdbc:decentdb:"; - public static final String DRIVER_VERSION = "2.8.0"; + public static final String DRIVER_VERSION = "2.9.0"; public static final int DRIVER_MAJOR_VERSION = 1; public static final int DRIVER_MINOR_VERSION = 8; diff --git a/bindings/node/decentdb/package-lock.json b/bindings/node/decentdb/package-lock.json index f7eb682..298295c 100644 --- a/bindings/node/decentdb/package-lock.json +++ b/bindings/node/decentdb/package-lock.json @@ -1,12 +1,12 @@ { "name": "decentdb-native", - "version": "2.8.0", + "version": "2.9.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "decentdb-native", - "version": "2.8.0", + "version": "2.9.0", "devDependencies": { "node-gyp": "^12.2.0" } diff --git a/bindings/node/decentdb/package.json b/bindings/node/decentdb/package.json index e236814..588ad84 100644 --- a/bindings/node/decentdb/package.json +++ b/bindings/node/decentdb/package.json @@ -1,6 +1,6 @@ { "name": "decentdb-native", - "version": "2.8.0", + "version": "2.9.0", "private": true, "description": "DecentDB Node.js native addon (N-API) + thin JS wrapper", "main": "index.js", diff --git a/bindings/node/knex-decentdb/package-lock.json b/bindings/node/knex-decentdb/package-lock.json index d776e6c..1da88d5 100644 --- a/bindings/node/knex-decentdb/package-lock.json +++ b/bindings/node/knex-decentdb/package-lock.json @@ -1,12 +1,12 @@ { "name": "knex-decentdb", - "version": "2.8.0", + "version": "2.9.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "knex-decentdb", - "version": "2.8.0", + "version": "2.9.0", "dependencies": { "decentdb-native": "file:../decentdb" }, @@ -16,7 +16,7 @@ }, "../decentdb": { "name": "decentdb-native", - "version": "2.8.0", + "version": "2.9.0", "devDependencies": { "node-gyp": "^12.2.0" } diff --git a/bindings/node/knex-decentdb/package.json b/bindings/node/knex-decentdb/package.json index b0e222a..5013176 100644 --- a/bindings/node/knex-decentdb/package.json +++ b/bindings/node/knex-decentdb/package.json @@ -1,6 +1,6 @@ { "name": "knex-decentdb", - "version": "2.8.0", + "version": "2.9.0", "private": true, "description": "Knex client/dialect for DecentDB", "main": "index.js", diff --git a/bindings/python/pyproject.toml b/bindings/python/pyproject.toml index efaa46c..5093f82 100644 --- a/bindings/python/pyproject.toml +++ b/bindings/python/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "decentdb" -version = "2.8.0" +version = "2.9.0" description = "Python DB-API 2.0 driver and SQLAlchemy dialect for DecentDB" readme = "README.md" authors = [ diff --git a/design/FUTURE_WINS.md b/design/FUTURE_WINS.md index 6a10faf..95f22c3 100644 --- a/design/FUTURE_WINS.md +++ b/design/FUTURE_WINS.md @@ -111,8 +111,8 @@ Status values: - `BACKLOG`: valuable, but not part of the near-term implementation path. Future version values are planning buckets, not release commitments. The latest -public release in this repository is `2.8.0`. `vNext` means the first release -bucket after `2.8.0` only when scope is explicitly accepted. `vNext+1` and +public release in this repository is `2.9.0`. `vNext` means the first release +bucket after `2.9.0` only when scope is explicitly accepted. `vNext+1` and `vNext+2` are follow-on planning buckets, not exact semantic versions. Roadmap lifecycle: once a Future Win is 100% implemented, tested, and diff --git a/docs/about/changelog.md b/docs/about/changelog.md index 59efb50..cae9d14 100644 --- a/docs/about/changelog.md +++ b/docs/about/changelog.md @@ -5,6 +5,26 @@ All notable changes to DecentDB will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased] + +## [2.9.0] - [2026-06-09] + +### Added + +- Added .NET ADO.NET maintenance helpers for WAL status snapshots, + binding-native checkpoints, compact save-as copies, and in-process vacuum without + requiring an external executable. +- Added a .NET ADO.NET `ExplainQuery` helper for `EXPLAIN` and + `EXPLAIN ANALYZE` diagnostics. +- Added .NET EF Core regression coverage for exact equality on large indexed + string columns modeled after Melodee's MusicBrainz lookup workload. + +### Changed + +- Improved .NET ADO.NET open failures for unsupported database format versions + with guidance to use a compatible engine, run `decentdb-migrate` when + available, or rebuild/export with the current engine. + ## [2.8.0] - [2026-05-28] ### Added diff --git a/docs/user-guide/benchmarks.md b/docs/user-guide/benchmarks.md index 9cd62d8..18b6484 100644 --- a/docs/user-guide/benchmarks.md +++ b/docs/user-guide/benchmarks.md @@ -13,7 +13,7 @@ This page collects the current Python embedded comparison charts and a plain-lan | Engine | Version stamp | Source | | --- | --- | --- | -| DecentDB | 2.8.0 | Workspace package version | +| DecentDB | 2.9.0 | Workspace package version | | SQLite (`SQLite_wal_full`) | 3.52.0 | Benchmark-reported engine version | | DuckDB | 1.5.1 | Benchmark-reported engine version | | H2 (`JDBC`) | 2.2.224 | Benchmark-reported engine version | diff --git a/tests/bindings/dart/pubspec.lock b/tests/bindings/dart/pubspec.lock index 778194a..c33289b 100644 --- a/tests/bindings/dart/pubspec.lock +++ b/tests/bindings/dart/pubspec.lock @@ -7,7 +7,7 @@ packages: path: "../../../bindings/dart/dart" relative: true source: path - version: "2.8.0" + version: "2.9.0" ffi: dependency: "direct main" description: From 61ef83da319d57ae5b7e927cff926a4c00eccfca Mon Sep 17 00:00:00 2001 From: Steven Hildreth Date: Tue, 9 Jun 2026 13:14:38 -0500 Subject: [PATCH 6/6] perf(wal): shrink reader-slot registry capacity when empty Optimize memory usage in the WAL coordination layer by calling `shrink_to_fit` on the process-local reader-slot registry when all slots have been cleared. This addresses memory safety concerns identified by Valgrind during nightly runs by ensuring retained capacity is released. --- crates/decentdb/src/wal/coordination.rs | 3 +++ docs/about/changelog.md | 6 ++++++ 2 files changed, 9 insertions(+) diff --git a/crates/decentdb/src/wal/coordination.rs b/crates/decentdb/src/wal/coordination.rs index a898f64..fe619a7 100644 --- a/crates/decentdb/src/wal/coordination.rs +++ b/crates/decentdb/src/wal/coordination.rs @@ -876,6 +876,9 @@ impl Drop for ProcessReaderGuard { coord_path: self.coordinator.inner.coord_path.clone(), slot: self.slot, }); + if slots.is_empty() { + slots.shrink_to_fit(); + } } let _ = self.coordinator.clear_reader_slot(self.slot); } diff --git a/docs/about/changelog.md b/docs/about/changelog.md index cae9d14..dc9eeea 100644 --- a/docs/about/changelog.md +++ b/docs/about/changelog.md @@ -9,6 +9,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [2.9.0] - [2026-06-09] +### Fixed + +- Fixed Memory Safety Nightly Valgrind failures by releasing retained capacity + from the process-local WAL reader-slot registry after the final local reader + slot is removed. + ### Added - Added .NET ADO.NET maintenance helpers for WAL status snapshots,