diff --git a/.clang-tidy b/.clang-tidy index 45f3a6c7b2..4e2f5427f6 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -52,7 +52,7 @@ CheckOptions: readability-magic-numbers.IgnoredIntegerValues: "" readability-magic-numbers.IgnoredFloatingPointValues: "" bugprone-easily-swappable-parameters.MinimumLength: 3 - misc-include-cleaner.IgnoreHeaders: "sys/.*|mach/.*|dispatch/.*|_types/.*|Availability\\.h|secure/.*|_.*\\.h|zconf\\.h" + misc-include-cleaner.IgnoreHeaders: "sys/.*|asm-generic/.*|mach/.*|dispatch/.*|_types/.*|Availability\\.h|secure/.*|_.*\\.h|zconf\\.h" readability-function-cognitive-complexity.Threshold: 25 readability-function-size.StatementThreshold: 200 readability-function-size.LineThreshold: 400 diff --git a/.github/workflows/_security.yml b/.github/workflows/_security.yml index 638274f425..cd3f7589d4 100644 --- a/.github/workflows/_security.yml +++ b/.github/workflows/_security.yml @@ -43,6 +43,14 @@ jobs: codeql-gate: runs-on: ubuntu-latest timeout-minutes: 240 + # Without security-events:read this job inherits the workflow's + # contents:read, the code-scanning alert API answers 403, and the gate + # below reported "0 alerts" on every run it has ever made. actions:read is + # what the CodeQL-run poll above needs for the same reason. + permissions: + contents: read + security-events: read + actions: read steps: - name: Wait for CodeQL on current commit (max 150 min) env: @@ -78,17 +86,68 @@ jobs: - name: Check for open code scanning alerts env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # A PR is judged by its own CodeQL analysis (refs/pull/N/merge); + # without a ref the API answers for the default branch, so a fix in + # the PR could never clear the gate before it merged. + ALERT_REF: ${{ github.event_name == 'pull_request' && format('refs/pull/{0}/merge', github.event.pull_request.number) || '' }} run: | + # FAIL CLOSED. The previous form was `gh api ... 2>/dev/null || echo "0"`, + # which looks defensive and is the opposite. gh writes the API error + # BODY to stdout, so a 403 made ALERTS the string + # {"message":"Resource not accessible by integration",...,"status":"403"}0 + # every `[` comparison then failed as non-integer, the `if` took its + # false branch, and the step printed "CodeQL gate passed (0 alerts)" + # and exited 0. That is exactly how a security gate should NEVER fail: + # silently, in the direction of "everything is fine". An unreadable + # alert list is now an error, not a zero. + # NOTE: this runs in a command substitution, so it must `return`, not + # `exit` -- an `exit` here would only end the subshell and let the + # caller continue with an empty count, recreating the very bug this + # replaces. The callers below check the status explicitly. + # Scope: CodeQL alerts only, outside vendored third-party code. + # Scorecard findings are repository-hygiene signals reported by their + # own workflow, and alerts in vendored grammars/libraries are + # upstream code this repo does not patch, so neither blocks a merge. + # --paginate: the API returns 30 alerts per page by default, so an + # unpaginated count silently stopped at 30. + ALERTS_URL='repos/${{ github.repository }}/code-scanning/alerts?state=open&per_page=100' + [ -n "$ALERT_REF" ] && ALERTS_URL="$ALERTS_URL&ref=$ALERT_REF" + GATE_FILTER='.[] | select(.tool.name == "CodeQL") + | select((.most_recent_instance.location.path // "") + | test("^(internal/cbm/)?vendored/") | not)' + count_open_alerts() { + local body status + body=$(gh api --paginate \ + "$ALERTS_URL" \ + --jq "$GATE_FILTER | .number" 2>/dev/null) + status=$? + if [ $status -ne 0 ]; then + echo "BLOCKED: cannot read code scanning alerts (gh exit $status)." >&2 + echo " Refusing to treat an unreadable alert list as zero." >&2 + echo " Most likely cause: the job lacks security-events:read." >&2 + return 1 + fi + local count + count=$(printf '%s' "$body" | grep -c . || true) + case "$count" in + ''|*[!0-9]*) + echo "BLOCKED: alert count was not a number: ${count:-}" >&2 + return 1 + ;; + esac + printf '%s' "$count" + } echo "Waiting 60s for alert API to settle..." sleep 60 - ALERTS=$(gh api 'repos/${{ github.repository }}/code-scanning/alerts?state=open' --jq 'length' 2>/dev/null || echo "0") + ALERTS=$(count_open_alerts) || exit 1 sleep 15 - ALERTS2=$(gh api 'repos/${{ github.repository }}/code-scanning/alerts?state=open' --jq 'length' 2>/dev/null || echo "0") + ALERTS2=$(count_open_alerts) || exit 1 [ "$ALERTS" -lt "$ALERTS2" ] && ALERTS=$ALERTS2 if [ "$ALERTS" -gt 0 ]; then echo "BLOCKED: $ALERTS open alert(s)" - gh api 'repos/${{ github.repository }}/code-scanning/alerts?state=open' \ - --jq '.[] | " #\(.number) [\(.rule.security_severity_level // .rule.severity)] \(.rule.id) — \(.most_recent_instance.location.path):\(.most_recent_instance.location.start_line)"' 2>/dev/null || true + gh api --paginate \ + "$ALERTS_URL" \ + --jq "$GATE_FILTER"' | " #\(.number) [\(.rule.security_severity_level // .rule.severity)] \(.rule.id) — \(.most_recent_instance.location.path):\(.most_recent_instance.location.start_line)"' 2>/dev/null || true exit 1 fi - echo "=== CodeQL gate passed (0 alerts) ===" + echo "=== CodeQL gate passed (0 gating CodeQL alerts) ===" diff --git a/.github/workflows/_test.yml b/.github/workflows/_test.yml index f8f39a88e1..3539173ccf 100644 --- a/.github/workflows/_test.yml +++ b/.github/workflows/_test.yml @@ -484,6 +484,17 @@ jobs: env: CCACHE_DIR: ${{ github.workspace }}/.ccache + # The harness prints only each failing suite's last 15 lines, which cuts + # off sanitizer reports. Keep the full per-suite logs of a failed shard. + - name: Upload test logs (on failure) + if: failure() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: test-logs-${{ matrix.os }}-${{ matrix.msystem }}-${{ strategy.job-index }} + path: build/c/test-logs/*.log + retention-days: 7 + if-no-files-found: ignore + - name: Shard manifest (completeness proof) if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 diff --git a/.github/workflows/claude-code-review.yml b/.github/workflows/claude-code-review.yml index 37e66f3fdb..9a738e97a3 100644 --- a/.github/workflows/claude-code-review.yml +++ b/.github/workflows/claude-code-review.yml @@ -10,6 +10,9 @@ on: # - "src/**/*.js" # - "src/**/*.jsx" +# Deny by default; each job grants only what it needs. +permissions: {} + jobs: claude-review: # Optional: Filter by PR author @@ -27,13 +30,13 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 with: fetch-depth: 1 - name: Run Claude Code Review id: claude-review - uses: anthropics/claude-code-action@v1 + uses: anthropics/claude-code-action@fd1c128679612beff4ca259c78021c506e8aa7a7 # v1 with: claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} plugin_marketplaces: 'https://github.com/anthropics/claude-code.git' diff --git a/.github/workflows/claude.yml b/.github/workflows/claude.yml index 6b15fac7af..033187fff2 100644 --- a/.github/workflows/claude.yml +++ b/.github/workflows/claude.yml @@ -10,6 +10,9 @@ on: pull_request_review: types: [submitted] +# Deny by default; each job grants only what it needs. +permissions: {} + jobs: claude: if: | @@ -26,13 +29,13 @@ jobs: actions: read # Required for Claude to read CI results on PRs steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 with: fetch-depth: 1 - name: Run Claude Code id: claude - uses: anthropics/claude-code-action@v1 + uses: anthropics/claude-code-action@fd1c128679612beff4ca259c78021c506e8aa7a7 # v1 with: claude_code_oauth_token: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} diff --git a/.github/workflows/dry-run.yml b/.github/workflows/dry-run.yml index 43a6b6bd98..4cf2ec46c1 100644 --- a/.github/workflows/dry-run.yml +++ b/.github/workflows/dry-run.yml @@ -58,6 +58,13 @@ jobs: # ── Security (independent island — does not block main pipeline) ── security: uses: ./.github/workflows/_security.yml + # The called job requests security-events:read to count code-scanning + # alerts; a reusable workflow cannot request more than its caller grants, + # so withholding this here fails the whole run at startup. + permissions: + contents: read + security-events: read + actions: read secrets: inherit # ── Lint (cppcheck + clang-format) ──────────────────────────── diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 6694d2a053..e10e6a259a 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -22,6 +22,13 @@ concurrency: jobs: security: uses: ./.github/workflows/_security.yml + # The called job requests security-events:read to count code-scanning + # alerts; a reusable workflow cannot request more than its caller grants, + # so withholding this here fails the whole run at startup. + permissions: + contents: read + security-events: read + actions: read secrets: inherit lint: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 33bf3bcd1e..794f67581b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -44,6 +44,13 @@ jobs: # ── Security (starts immediately, blocks only verify) ─────────── security: uses: ./.github/workflows/_security.yml + # The called job requests security-events:read to count code-scanning + # alerts; a reusable workflow cannot request more than its caller grants, + # so withholding this here fails the whole run at startup. + permissions: + contents: read + security-events: read + actions: read secrets: inherit # ── 0. Preflight: refuse malformed dispatch inputs ────────────── diff --git a/BACKLOG.md b/BACKLOG.md index 27e3dcd05b..9615f4e889 100644 --- a/BACKLOG.md +++ b/BACKLOG.md @@ -10,6 +10,18 @@ ## Open +- [ ] Add opt-in, single-run index metrics for performance diagnosis. + - Extend `index` with a one-shot flag (for example `--metrics-out PATH`), + rather than enabling persistent daemon timing by default. + - Emit versioned JSON for that invocation only: source revision, index mode, + discovery/index/staging/publish wall times, queue and lock waits, worker + peak/aggregate memory, files/definitions/edges, DB size, and outcome. + - Keep ordinary stdout and daemon logs quiet; no source contents or secrets + may enter the result. Reuse this result schema from the planned + `benchmark` command rather than introducing a second format. + - Acceptance: a normal index produces no metrics artifact; an opted-in + index produces one complete artifact even for a failed or cancelled run. + - [ ] Selectively evaluate remaining upstream fixes from `../codebase-memory-mcp` before merging into the CLI fork. Do not cherry-pick MCP-only changes or overwrite CP89/release-tooling work; apply diff --git a/Jenkinsfile b/Jenkinsfile index a44666e820..881352c81c 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -4,8 +4,10 @@ pipeline { parameters { string(name: 'CBM_RELEASE_VERSION', defaultValue: '', description: 'Optional version passed to the release build, for example v0.11.0-rc.2') - string(name: 'CBM_TEST_SUITES', defaultValue: 'cli daemon daemon_ipc pipeline', - description: 'Focused incremental test suites; leave empty for the full gate') + string(name: 'CBM_TEST_SUITES', defaultValue: '', + description: '''Focused incremental test suites; leave blank for the full gate. +Reference list (space-separated): +arena hash_table dyn_array str_intern log str_util workspace platform diagnostics complexity subprocess private_file_lock lock_registry dump_verify ac extraction extraction_inheritance extraction_imports parse_coverage grammar_regression grammar_labels grammar_imports store_nodes store_edges store_search store_bulk store_pragmas store_checkpoint dump_verify_io cypher index_supervisor daemon project_lock version_cohort daemon_version daemon_runtime daemon_application daemon_bootstrap daemon_ipc language userconfig gitignore git_context discover graph_buffer registry pipeline importance index_format pipeline_semantic_manifest_repro call_reference_contract call_reference_language_complex_contract repro_call_scope_usages repro_call_argument_usages repro_reference_precision repro_call_argument_matrix_a repro_call_argument_matrix_b repro_call_node_behaviors repro_language_registry repro_call_node_manifest repro_lsp_ordered_signatures repro_lsp_ordered_local repro_ts_overload_return_chains repro_harness_cleanup repro_runner_filter cross_repo index_resilience fqn route_canon path_alias watcher lz4 zstd sqlite_writer artifact scope type_rep go_lsp c_lsp php_lsp cs_lsp cs_lsp_bench perl_lsp py_lsp kotlin_lsp rust_lsp py_lsp_bench py_lsp_stress py_lsp_scale ts_lsp java_lsp java_lsp_coverage store_arch traces configlink infrascan cli agent_clients legacy_agent_profiles config_json_like config_toml_edit config_yaml_edit config_text_edit activation_transaction system_info worker_pool parallel repro_lexical_binding_precision slab_alloc mem ui httpd security yaml semantic ast_profile simhash stack_overflow_a stack_overflow_b stack_overflow_c integration lang_contract edge_imports edge_structural lsp_resolution_probe node_creation_probe edge_types_probe convergence_probe matrix_known_classes matrix_new_constructs grammar_probe_a grammar_probe_b grammar_probe_c grammar_probe_d grammar_probe_e grammar_probe_f grammar_probe_g incremental''') } options { @@ -19,6 +21,7 @@ pipeline { steps { sh ''' set -eu + rm -f .jenkins-installed-cli scripts/lint.sh --ci CLANG_FORMAT=clang-format-20 ''' } @@ -60,6 +63,17 @@ pipeline { } } } + stage('Stop active CBM daemon') { + steps { + sh ''' + set -eu + installed_cli="$HOME/.local/bin/codebase-memory-cli" + if [ -x "$installed_cli" ]; then + "$installed_cli" daemon stop >/dev/null 2>&1 || true + fi + ''' + } + } stage('Build') { steps { sh ''' @@ -72,6 +86,17 @@ pipeline { ''' } } + stage('Install built CLI') { + steps { + sh ''' + set -eu + install_dir="$HOME/.local/bin" + mkdir -p "$install_dir" + install -m 0755 build/c/codebase-memory-cli "$install_dir/codebase-memory-cli" + printf '%s\n' "$install_dir/codebase-memory-cli" > .jenkins-installed-cli + ''' + } + } stage('Prepare test runner') { when { expression { !params.CBM_TEST_SUITES?.trim() } @@ -135,12 +160,47 @@ pipeline { sh 'CBM_TEST_PHASE=post CBM_TEST_SEAM_BINARY="$WORKSPACE/build/c-seams/codebase-memory-cli" scripts/test.sh' } } - stage('Focused tests') { + stage('Prepare focused test runner') { + when { + expression { params.CBM_TEST_SUITES?.trim() } + } + steps { + sh ''' + set -eu + make -j"$(nproc)" -f Makefile.cbm build/c/test-runner + ''' + } + } + stage('Focused test shards') { when { expression { params.CBM_TEST_SUITES?.trim() } } steps { - sh 'scripts/test.sh --suites "$CBM_TEST_SUITES"' + script { + def suites = params.CBM_TEST_SUITES.trim().split(/[\s,]+/).findAll { it } + if (!suites || !suites.every { it ==~ /[a-z0-9_]+/ }) { + error('CBM_TEST_SUITES must contain only suite names') + } + + // Daemon-family suites share runtime/endpoint assumptions; + // keep them in one quiet shard and distribute the rest. + def daemonNames = ['cli', 'daemon', 'daemon_ipc', 'daemon_runtime', + 'daemon_application', 'daemon_bootstrap', + 'index_supervisor', 'watcher'] + def daemon = suites.findAll { it in daemonNames } + def other = suites.findAll { !(it in daemonNames) } + def groups = [daemon, other].findAll { !it.isEmpty() } + def branches = [:] + groups.eachWithIndex { group, index -> + def shardSuites = group.join(' ') + branches["Focused shard ${index + 1}/${groups.size()}"] = { + withEnv(["CBM_FOCUSED_SUITE_ARGS=${shardSuites}"]) { + sh 'build/c/test-runner $CBM_FOCUSED_SUITE_ARGS' + } + } + } + parallel branches + } } } stage('Package wrappers') { @@ -199,4 +259,15 @@ pipeline { } } } + post { + always { + sh ''' + set -eu + if [ -s .jenkins-installed-cli ]; then + installed_cli=$(cat .jenkins-installed-cli) + "$installed_cli" daemon start >/dev/null 2>&1 || true + fi + ''' + } + } } diff --git a/docs/RELEASE_QUALIFICATION_PLAN.md b/docs/RELEASE_QUALIFICATION_PLAN.md index 78f51d75b6..999bdd9b81 100644 --- a/docs/RELEASE_QUALIFICATION_PLAN.md +++ b/docs/RELEASE_QUALIFICATION_PLAN.md @@ -2,6 +2,36 @@ Status: **ACTIVE — supersedes migration-era remaining-work sequencing** +## Readiness inventory — 2026-10-01 + +The assigned readiness worker confirmed through the public GitHub API that +`ngallodev/codebase-memory-cli` is public, `main` points to +`38153854c46f237b700d68b60a65c1a1078c78c5`, the releases collection is empty, +and open PR #12 (`release-tooling`, head +`fd9fbe9c46c214c41e2f176d0278702afa87e4b7`) overlaps release tooling. PR #12 is +an external change under review and must be inspected before any later +implementation overlaps it. The latest 100 public Actions runs contained no +workflow named `Release`; this does not establish private secret or runner +configuration. + +The source contract audit confirms that both installers resolve stable bytes +through `/releases/latest/download`, request `checksums.txt`, and map to the +workflow's eight canonical release target tuples. The release workflow requires +successful lint, build, smoke and soak before creating a draft; it holds drafts +for qualification by default. Promotion validates exact release bytes and +source identity, then publishes npm/PyPI before un-drafting GitHub. Therefore a +GitHub-only stable promotion would require a reviewed workflow change; absent +evidence that fork-owned npm/PyPI credentials and namespaces are intended, do +not publish there. + +This inventory used unauthenticated public API reads because the worker had no +GitHub CLI login or `GH_TOKEN`/`GITHUB_TOKEN`. Repository secret names, Actions +variables, trusted-publisher configuration, fork scanner access, and runner +availability remain unverified. The CLI graph could not be queried: its cache +resolver returned `secure CLI coordination could not be created (cache-resolve)` +and an isolated `CBM_CACHE_DIR` probe reported a version-cohort conflict. No +repository index or primary checkout was changed by those probes. + ## Purpose The CLI-first migration is architecture-complete enough that remaining work is release qualification, not further MCP migration. Structural changes after this point require evidence from build, reliability, performance, native-Windows, usability, or release validation. @@ -204,3 +234,11 @@ The dispatch record binds the clean source commit, explicit RC tag, release work ### Qualified source identity External qualification evidence must bind the exact release tag to its immutable source commit (`release.source_commit`) in addition to the Windows archive and executable SHA-256 values. The promotion workflow checks out the exact tag and rejects evidence whose source commit differs. + +## First stable release and installer readiness + +The current shell and PowerShell installers resolve `https://github.com/ngallodev/codebase-memory-cli/releases/latest/download`. A draft release, an RC, or a GitHub release marked prerelease does not make that endpoint serve a stable release. The first working public installer therefore requires a qualified, non-prerelease, non-draft `vX.Y.Z` GitHub release with the installer-expected archives and checksums attached. Select the version explicitly after checking both the release and tag namespaces are unused; `v0.11.0-rc.1` remains a recommendation for a held candidate, not a selected version. + +After promotion, verify from an unauthenticated client that `/releases/latest` resolves to that stable tag, every installer-requested asset is present, and each direct download returns the expected archive bytes. Run fresh installs using those public URLs in a real WSL2 Linux distro and on native Windows. For each supported architecture, record installer transcript, asset/hash, `--version`, and a small fixture index/status/search/snippet result. The Linux host build is not WSL evidence; Windows ARM64 routing must be exercised on a native ARM64 machine or explicitly remain unqualified. Existing artifact, checksum, archive-layout and installer safety checks stay required. + +The current promotion workflow always publishes npm and PyPI before un-drafting GitHub. If the fork does not own those package namespaces or cannot use their trusted publishers, treat that as a fork configuration/ownership blocker. Before any publication, review a minimal GitHub-only promotion option that keeps exact-candidate qualification verification and exposes an explicit choice to omit registry publication. Do not add that route as an unreviewed readiness change and do not silently skip failed registry publication. diff --git a/graph-ui/package-lock.json b/graph-ui/package-lock.json index 7768b039a4..15c477b4c4 100644 --- a/graph-ui/package-lock.json +++ b/graph-ui/package-lock.json @@ -33,7 +33,7 @@ "tailwindcss": "^4.1.0", "typescript": "^5.7.0", "vite": "^6.4.3", - "vitest": "^4.1.0" + "vitest": "^4.1.11" } }, "node_modules/@adobe/css-tools": { @@ -3603,16 +3603,16 @@ } }, "node_modules/@vitest/expect": { - "version": "4.1.8", - "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-4.1.8.tgz", - "integrity": "sha512-h3nDO677RDLEGlBxyQ5CW8RlMThSKSRLUePLOx09gNIWRL40edgA1GCZSZgf1W55MFAG6/Sw14KeaAnqv0NKdQ==", + "version": "4.1.11", + "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-4.1.11.tgz", + "integrity": "sha512-VX2x5vNJXET47KAFzwERI+KRMtTTCSWTfSMKsW7JsUsXV4psq++e3DvZpuTDOpHcxytiDs6p2nhVb2tVDiiUYw==", "dev": true, "license": "MIT", "dependencies": { "@standard-schema/spec": "^1.1.0", "@types/chai": "^5.2.2", - "@vitest/spy": "4.1.8", - "@vitest/utils": "4.1.8", + "@vitest/spy": "4.1.11", + "@vitest/utils": "4.1.11", "chai": "^6.2.2", "tinyrainbow": "^3.1.0" }, @@ -3621,13 +3621,13 @@ } }, "node_modules/@vitest/mocker": { - "version": "4.1.8", - "resolved": "https://registry.npmjs.org/@vitest/mocker/-/mocker-4.1.8.tgz", - "integrity": "sha512-LEiN/xe4OSIbKe9HQIp5OC24agGD9J5CnmMgsLohVVoOPWL9a2sBoR6VBx43jQZb7Kr1l4RCuyCJzcAa0+dojw==", + "version": "4.1.11", + "resolved": "https://registry.npmjs.org/@vitest/mocker/-/mocker-4.1.11.tgz", + "integrity": "sha512-2XJVD55d1o5AZous5CCGKS74g/riOj9odEt2bQpCVZeblHyHdnMeFl4jl0XjU21stf4mbjUkew2eXQZt65g5CQ==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/spy": "4.1.8", + "@vitest/spy": "4.1.11", "estree-walker": "^3.0.3", "magic-string": "^0.30.21" }, @@ -3648,9 +3648,9 @@ } }, "node_modules/@vitest/pretty-format": { - "version": "4.1.8", - "resolved": "https://registry.npmjs.org/@vitest/pretty-format/-/pretty-format-4.1.8.tgz", - "integrity": "sha512-9GasEBxpZ1VYIpqHf/0+YGg121uSNwCKOJqIrTwWP/TB7DmFCiaBpNl3aPZzoLWfWkuqhbH8vJIVobZkvdo2cA==", + "version": "4.1.11", + "resolved": "https://registry.npmjs.org/@vitest/pretty-format/-/pretty-format-4.1.11.tgz", + "integrity": "sha512-yiZzPbGTS9Sr/JpFl8zHrcIkAofNbFV6k21vIgQN/cY/oxZeXhJv5sc/MBJ5jFKWmWs+oJHw0UXLZjmf931+Vw==", "dev": true, "license": "MIT", "dependencies": { @@ -3661,13 +3661,13 @@ } }, "node_modules/@vitest/runner": { - "version": "4.1.8", - "resolved": "https://registry.npmjs.org/@vitest/runner/-/runner-4.1.8.tgz", - "integrity": "sha512-EmVxeBAfMJvycdjd6Hm+RbFBbA9fKvo0Kx37hNpBYoYeavH3RNsBXWDooR1mgD52dCrxIIuP7UotpfiwOikvcg==", + "version": "4.1.11", + "resolved": "https://registry.npmjs.org/@vitest/runner/-/runner-4.1.11.tgz", + "integrity": "sha512-LztvUgdwMNJMIkj3hQnnxiC2Xy1zNxq928W/xhjCLaNCzqTZOudjwbQf6v9IntZGPw132i2Lq2rgTRZHD3JHNw==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/utils": "4.1.8", + "@vitest/utils": "4.1.11", "pathe": "^2.0.3" }, "funding": { @@ -3675,14 +3675,14 @@ } }, "node_modules/@vitest/snapshot": { - "version": "4.1.8", - "resolved": "https://registry.npmjs.org/@vitest/snapshot/-/snapshot-4.1.8.tgz", - "integrity": "sha512-acfZboRmAIf05DEKcBQy33VXojFJjtUdLyo7oOmV9kebb2xdU01UknNiPuPZoJZQyO7DF0gZdTGTpeAzET9QPQ==", + "version": "4.1.11", + "resolved": "https://registry.npmjs.org/@vitest/snapshot/-/snapshot-4.1.11.tgz", + "integrity": "sha512-pN7ikn1ON7h8ee4gIAp4AzyK+zBtJPzVbqOgu5LCEh4VaJVbPQcgYQYJIMGQPXVeJJq1fnfazis7a5pFNPahog==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/pretty-format": "4.1.8", - "@vitest/utils": "4.1.8", + "@vitest/pretty-format": "4.1.11", + "@vitest/utils": "4.1.11", "magic-string": "^0.30.21", "pathe": "^2.0.3" }, @@ -3691,9 +3691,9 @@ } }, "node_modules/@vitest/spy": { - "version": "4.1.8", - "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-4.1.8.tgz", - "integrity": "sha512-6EevtBp6OZOPF7bmz36HrGMeP3txgVSrgebWxHOafDXGkhIzfXK14f8KF6MuFfgXXUeHxmpD3BQxkV00/3s5mA==", + "version": "4.1.11", + "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-4.1.11.tgz", + "integrity": "sha512-apNa/prQy2qCeywhnixOHPRCgGNhvg7T4Dapfl1GahLp/R+uhBm5cPyFoNVyqsNd2h1nJxL6BqqdIjiABL60YA==", "dev": true, "license": "MIT", "funding": { @@ -3701,13 +3701,13 @@ } }, "node_modules/@vitest/utils": { - "version": "4.1.8", - "resolved": "https://registry.npmjs.org/@vitest/utils/-/utils-4.1.8.tgz", - "integrity": "sha512-uOJamYALNhfJ6iolExyQM40yIQwDqYnkKtQ5VCiSe17E33H0aQ/u+1GlRuz4LZBk6Mm3sg90G9hEbmEt37C1Zg==", + "version": "4.1.11", + "resolved": "https://registry.npmjs.org/@vitest/utils/-/utils-4.1.11.tgz", + "integrity": "sha512-zTCVGpyFsGWBhllOyKlTw/vnr6D9qxsfSDyfbyZmTyjHw5N/VuvzHpHoQjm2ZJzn4RJgx5w4r7V0er69CmLgPQ==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/pretty-format": "4.1.8", + "@vitest/pretty-format": "4.1.11", "convert-source-map": "^2.0.0", "tinyrainbow": "^3.1.0" }, @@ -6141,19 +6141,19 @@ } }, "node_modules/vitest": { - "version": "4.1.8", - "resolved": "https://registry.npmjs.org/vitest/-/vitest-4.1.8.tgz", - "integrity": "sha512-flY6ScbCIt9HThs+C5HS7jvGOB560DJtk/Z15IQROTA6zEy49Nh8T/dofWTQL+n3vswqn87sbJNiuqw1SDp5Ig==", + "version": "4.1.11", + "resolved": "https://registry.npmjs.org/vitest/-/vitest-4.1.11.tgz", + "integrity": "sha512-fhACrNXUidIbGSBr5FlbuBkO7VWC1ZyLl0DO4CU2DrQoAPxX84Ysxs+HeGQpii5lZWV1Q4gBZTTu49mF+A6Edw==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/expect": "4.1.8", - "@vitest/mocker": "4.1.8", - "@vitest/pretty-format": "4.1.8", - "@vitest/runner": "4.1.8", - "@vitest/snapshot": "4.1.8", - "@vitest/spy": "4.1.8", - "@vitest/utils": "4.1.8", + "@vitest/expect": "4.1.11", + "@vitest/mocker": "4.1.11", + "@vitest/pretty-format": "4.1.11", + "@vitest/runner": "4.1.11", + "@vitest/snapshot": "4.1.11", + "@vitest/spy": "4.1.11", + "@vitest/utils": "4.1.11", "es-module-lexer": "^2.0.0", "expect-type": "^1.3.0", "magic-string": "^0.30.21", @@ -6181,12 +6181,12 @@ "@edge-runtime/vm": "*", "@opentelemetry/api": "^1.9.0", "@types/node": "^20.0.0 || ^22.0.0 || >=24.0.0", - "@vitest/browser-playwright": "4.1.8", - "@vitest/browser-preview": "4.1.8", - "@vitest/browser-webdriverio": "4.1.8", - "@vitest/coverage-istanbul": "4.1.8", - "@vitest/coverage-v8": "4.1.8", - "@vitest/ui": "4.1.8", + "@vitest/browser-playwright": "4.1.11", + "@vitest/browser-preview": "4.1.11", + "@vitest/browser-webdriverio": "4.1.11", + "@vitest/coverage-istanbul": "4.1.11", + "@vitest/coverage-v8": "4.1.11", + "@vitest/ui": "4.1.11", "happy-dom": "*", "jsdom": "*", "vite": "^6.0.0 || ^7.0.0 || ^8.0.0" diff --git a/graph-ui/package.json b/graph-ui/package.json index b1aa3998bc..b023560e8a 100644 --- a/graph-ui/package.json +++ b/graph-ui/package.json @@ -37,7 +37,7 @@ "tailwindcss": "^4.1.0", "typescript": "^5.7.0", "vite": "^6.4.3", - "vitest": "^4.1.0" + "vitest": "^4.1.11" }, "overrides": { "form-data": ">=4.0.6", diff --git a/scripts/hooks/pre-commit b/scripts/hooks/pre-commit index 94fe2f2e72..c7bd94d299 100755 --- a/scripts/hooks/pre-commit +++ b/scripts/hooks/pre-commit @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -euo pipefail -# Pre-commit hook: linters + security audit + build + tests. +# Pre-commit hook: format staged C-family files. # # Activated automatically via scripts/setup.sh or manually: # git config core.hooksPath scripts/hooks @@ -29,16 +29,3 @@ if [ "${#FORMAT_FILES[@]}" -gt 0 ]; then "$FORMATTER" -i "${FORMAT_FILES[@]}" git add -- "${FORMAT_FILES[@]}" fi - -echo "pre-commit: running all linters in parallel..." -# clang-tidy runs diff-scoped (lint-tidy-diff, see scripts/lint-tidy-diff.sh): -# the full lint-tidy sweep surfaces thousands of pre-existing findings that -# have nothing to do with this commit and would block every contributor who -# has clang-tidy on PATH. `make lint-tidy` remains available for a full audit. -make -j3 -f Makefile.cbm lint-ci lint-tidy-diff CLANG_FORMAT="$FORMATTER" - -echo "pre-commit: security audit (source-level)..." -scripts/security-audit.sh - -echo "pre-commit: building and running tests..." -make -j$(sysctl -n hw.ncpu 2>/dev/null || nproc) -f Makefile.cbm test diff --git a/scripts/jenkins-local-job.xml b/scripts/jenkins-local-job.xml index 6a0b41b42a..739b5814f3 100644 --- a/scripts/jenkins-local-job.xml +++ b/scripts/jenkins-local-job.xml @@ -1,6 +1,6 @@ - Fast incremental codebase-memory-cli release-tooling build and test. + Fast selective-shard codebase-memory-cli release-tooling build and test. false @@ -10,7 +10,7 @@ CBM_TEST_SUITES cli daemon daemon_ipc pipeline - Focused incremental test suites for release-tooling. Leave empty only to run the full gate manually. + Selected suites are built once and run in safe parallel shards. Leave empty only to run the full gate manually. true diff --git a/scripts/security-allowlist.txt b/scripts/security-allowlist.txt index f8bbb80864..5853d5d96e 100644 --- a/scripts/security-allowlist.txt +++ b/scripts/security-allowlist.txt @@ -29,6 +29,12 @@ NETWORK:src/main.c:connect:1:loopback-v4:main_daemon_ctl_socket_connected:single # Format: WRITE:file:expected-count:justification WRITE:src/main.c:3:supervised worker response path; cache-scoped hook notice marker; test-seam-only browser-open marker at the caller-supplied test path WRITE:src/daemon/application.c:1:recovery truncates only its private uniquely-created per-job marker file +WRITE:src/cli/config_json_like.c:1:Windows exclusive-create (wbx) sibling temp file for atomic agent config replace +WRITE:src/cli/config_text_edit.c:1:Windows exclusive-create (w+bx) sibling temp file for atomic agent config replace +WRITE:src/cli/config_yaml_edit.c:1:Windows exclusive-create (wbx) sibling temp file for atomic agent config replace +WRITE:src/foundation/compat_fs.c:1:stream_copy_file fallback of the copy helper; destination chosen by the caller +WRITE:src/foundation/mem.c:1:opt-in allocator stats dump to the user-set CBM_MEM_STATS_OUT path +WRITE:src/operations/index.c:1:index error log in the cache logs/ directory, or the user-set CBM_INDEX_LOG path # ── CLI: update command (user-initiated, interactive) ────────────────────── src/cli/cli.c:cbm_popen:sha256 checksum verification (update cmd) diff --git a/scripts/test.sh b/scripts/test.sh index eda03911f7..fa7e0ee51d 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -247,6 +247,9 @@ bash "$ROOT/tests/test_soak_daemon_recovery_contract.sh" echo "=== Step 0e: Windows launcher bundle contract ===" bash "$ROOT/tests/test_windows_bundle_contract.sh" +echo "=== Step 0e3: security gate fail-closed contract ===" +bash "$ROOT/tests/test_security_gate_fail_closed.sh" + echo "=== Step 0f: tree-sitter runtime Makefile dependencies ===" bash "$ROOT/tests/test_makefile_ts_runtime_dependencies.sh" diff --git a/src/cli/cli.c b/src/cli/cli.c index 5697fc0d50..32165324d6 100644 --- a/src/cli/cli.c +++ b/src/cli/cli.c @@ -1652,14 +1652,21 @@ static bool cbm_json_mcp_command_path_probe_safe(const char *command) { #endif #ifdef CBM_CLI_ENABLE_TEST_API +#ifdef _WIN32 +/* Read only by the Windows command-path probe. */ static CBM_TLS int *g_mcp_command_path_probe_counter = NULL; +#endif bool cbm_legacy_command_path_probe_safe_for_testing(const char *command, bool windows) { return cbm_json_mcp_command_path_probe_safe_for_platform(command, windows); } void cbm_set_mcp_command_path_probe_counter_for_testing(int *counter) { +#ifdef _WIN32 g_mcp_command_path_probe_counter = counter; +#else + (void)counter; +#endif } #endif @@ -3551,11 +3558,14 @@ static int cbm_remove_grok_mcp_owned(const char *binary_path, const char *config /* #929: extensionless bash shims under %USERPROFILE%\\.claude\\hooks trigger * the "How do you want to open this file?" dialog when editors (Cursor) scan * the hooks dir, and cannot execute without bash anyway. Windows installs - * .cmd scripts; the extensionless legacy files are removed on upgrade. */ + * .cmd scripts, and the extensionless twin is removed on upgrade/uninstall + * when its bytes match an owned script. */ #define CMM_HOOK_GATE_SCRIPT "codebase-memory-cli-discovery-gate.cmd" #else #define CMM_HOOK_GATE_SCRIPT "codebase-memory-cli-discovery-gate" #endif +/* Extensionless name: the Windows legacy twin, identical to the current name on POSIX. */ +#define CMM_HOOK_GATE_SCRIPT_LEGACY "codebase-memory-cli-discovery-gate" /* Hard backstop in settings.json; the binary also self-bounds with an * in-process deadline well under this. */ #define CMM_HOOK_TIMEOUT_SEC 5 @@ -4250,14 +4260,21 @@ int cbm_upsert_claude_hooks(const char *settings_path) { char command[CLI_BUF_8K]; char previous_command[CLI_BUF_8K]; char released_command[CLI_BUF_8K]; + char previous_legacy_command[CLI_BUF_8K]; + char released_legacy_command[CLI_BUF_8K]; if (cbm_resolve_hook_command(CMM_HOOK_GATE_SCRIPT, command, sizeof(command)) != CLI_OK || cbm_resolve_previous_hook_command(CMM_HOOK_GATE_SCRIPT, previous_command, sizeof(previous_command)) != CLI_OK || cbm_resolve_released_hook_command(CMM_HOOK_GATE_SCRIPT, released_command, - sizeof(released_command)) != CLI_OK) { + sizeof(released_command)) != CLI_OK || + cbm_resolve_previous_hook_command(CMM_HOOK_GATE_SCRIPT_LEGACY, previous_legacy_command, + sizeof(previous_legacy_command)) != CLI_OK || + cbm_resolve_released_hook_command(CMM_HOOK_GATE_SCRIPT_LEGACY, released_legacy_command, + sizeof(released_legacy_command)) != CLI_OK) { return CLI_ERR; } - const char *const old_commands[] = {released_command, previous_command, NULL}; + const char *const old_commands[] = {released_command, previous_command, released_legacy_command, + previous_legacy_command, NULL}; int search_result = upsert_hooks_json((hooks_upsert_args_t){ .settings_path = settings_path, .hook_event = "PreToolUse", @@ -4284,14 +4301,21 @@ int cbm_remove_claude_hooks(const char *settings_path) { char command[CLI_BUF_8K]; char previous_command[CLI_BUF_8K]; char released_command[CLI_BUF_8K]; + char previous_legacy_command[CLI_BUF_8K]; + char released_legacy_command[CLI_BUF_8K]; if (cbm_resolve_hook_command(CMM_HOOK_GATE_SCRIPT, command, sizeof(command)) != CLI_OK || cbm_resolve_previous_hook_command(CMM_HOOK_GATE_SCRIPT, previous_command, sizeof(previous_command)) != CLI_OK || cbm_resolve_released_hook_command(CMM_HOOK_GATE_SCRIPT, released_command, - sizeof(released_command)) != CLI_OK) { + sizeof(released_command)) != CLI_OK || + cbm_resolve_previous_hook_command(CMM_HOOK_GATE_SCRIPT_LEGACY, previous_legacy_command, + sizeof(previous_legacy_command)) != CLI_OK || + cbm_resolve_released_hook_command(CMM_HOOK_GATE_SCRIPT_LEGACY, released_legacy_command, + sizeof(released_legacy_command)) != CLI_OK) { return CLI_ERR; } - const char *const old_commands[] = {released_command, previous_command, NULL}; + const char *const old_commands[] = {released_command, previous_command, released_legacy_command, + previous_legacy_command, NULL}; int search_result = remove_hooks_json((hooks_remove_args_t){ .settings_path = settings_path, .hook_event = "PreToolUse", @@ -4740,12 +4764,6 @@ static int cbm_remove_owned_hook_script(const char *path, const char *expected_c released_script_count); } -/* Install the search-augmenter shim to ~/.claude/hooks/. - * The shim is a thin wrapper that delegates to ` hook-augment`, - * which adds graph context to Grep/Glob/Bash search calls. It NEVER blocks a tool call: - * a missing/old/hung binary results in a silent exit 0 (issue #362/#288). - * The CLI-owned filename is intentionally distinct from MCP-era hook files so - * side-by-side installs never adopt or rewrite the MCP hook namespace. */ /* #929 (Windows): remove the pre-.cmd extensionless twin only when its bytes * match a current or released installer-owned script. Modified/foreign files * at the reserved path are preserved. POSIX keeps the extensionless name, @@ -4769,6 +4787,12 @@ static int cbm_remove_owned_legacy_hook_script(const char *hooks_dir, const char } #endif +/* Install the search-augmenter shim to ~/.claude/hooks/. + * The shim is a thin wrapper that delegates to ` hook-augment`, + * which adds graph context to Grep/Glob/Bash search calls. It NEVER blocks a tool call: + * a missing/old/hung binary results in a silent exit 0 (issue #362/#288). + * The CLI-owned filename is intentionally distinct from MCP-era hook files so + * side-by-side installs never adopt or rewrite the MCP hook namespace. */ bool cbm_install_hook_gate_script(const char *home, const char *binary_path) { if (!home || !binary_path) { return false; @@ -4802,6 +4826,12 @@ bool cbm_install_hook_gate_script(const char *home, const char *binary_path) { char released_scripts[CMM_RELEASED_GATE_VARIANT_COUNT][CLI_BUF_8K]; const char *legacy[CMM_RELEASED_GATE_VARIANT_COUNT] = {0}; size_t legacy_count = cbm_build_released_gate_candidates(binary_path, released_scripts, legacy); +#ifdef _WIN32 + if (cbm_remove_owned_legacy_hook_script(hooks_dir, CMM_HOOK_GATE_SCRIPT_LEGACY, script, legacy, + legacy_count) != CLI_OK) { + return false; + } +#endif return cbm_write_owned_hook_script_with_legacy(script_path, script, legacy, legacy_count); } @@ -4811,6 +4841,7 @@ bool cbm_install_hook_gate_script(const char *home, const char *binary_path) { #else #define CMM_SESSION_REMINDER_SCRIPT "codebase-memory-cli-session-reminder" #endif +#define CMM_SESSION_REMINDER_SCRIPT_LEGACY "codebase-memory-cli-session-reminder" static bool cbm_install_session_reminder_script(const char *home, const char *binary_path) { if (!home || !binary_path) { @@ -4842,6 +4873,13 @@ static bool cbm_install_session_reminder_script(const char *home, const char *bi sizeof(script)) != CLI_OK) { return false; } +#ifdef _WIN32 + const char *const legacy[] = {cmm_legacy_mcp_session_script}; + if (cbm_remove_owned_legacy_hook_script(hooks_dir, CMM_SESSION_REMINDER_SCRIPT_LEGACY, script, + legacy, 1U) != CLI_OK) { + return false; + } +#endif return cbm_write_owned_hook_script(script_path, script); } @@ -4850,14 +4888,23 @@ static int cbm_upsert_session_hooks(const char *settings_path) { char command[CLI_BUF_8K]; char previous_command[CLI_BUF_8K]; char released_command[CLI_BUF_8K]; + char previous_legacy_command[CLI_BUF_8K]; + char released_legacy_command[CLI_BUF_8K]; if (cbm_resolve_hook_command(CMM_SESSION_REMINDER_SCRIPT, command, sizeof(command)) != CLI_OK || cbm_resolve_previous_hook_command(CMM_SESSION_REMINDER_SCRIPT, previous_command, sizeof(previous_command)) != CLI_OK || cbm_resolve_released_hook_command(CMM_SESSION_REMINDER_SCRIPT, released_command, - sizeof(released_command)) != CLI_OK) { + sizeof(released_command)) != CLI_OK || + cbm_resolve_previous_hook_command(CMM_SESSION_REMINDER_SCRIPT_LEGACY, + previous_legacy_command, + sizeof(previous_legacy_command)) != CLI_OK || + cbm_resolve_released_hook_command(CMM_SESSION_REMINDER_SCRIPT_LEGACY, + released_legacy_command, + sizeof(released_legacy_command)) != CLI_OK) { return CLI_ERR; } - const char *const old_commands[] = {released_command, previous_command, NULL}; + const char *const old_commands[] = {released_command, previous_command, released_legacy_command, + previous_legacy_command, NULL}; int rc = 0; for (int i = 0; i < NUM_DIRS; i++) { if (upsert_hooks_json((hooks_upsert_args_t){.settings_path = settings_path, @@ -4878,14 +4925,23 @@ static int cbm_remove_session_hooks(const char *settings_path) { char command[CLI_BUF_8K]; char previous_command[CLI_BUF_8K]; char released_command[CLI_BUF_8K]; + char previous_legacy_command[CLI_BUF_8K]; + char released_legacy_command[CLI_BUF_8K]; if (cbm_resolve_hook_command(CMM_SESSION_REMINDER_SCRIPT, command, sizeof(command)) != CLI_OK || cbm_resolve_previous_hook_command(CMM_SESSION_REMINDER_SCRIPT, previous_command, sizeof(previous_command)) != CLI_OK || cbm_resolve_released_hook_command(CMM_SESSION_REMINDER_SCRIPT, released_command, - sizeof(released_command)) != CLI_OK) { + sizeof(released_command)) != CLI_OK || + cbm_resolve_previous_hook_command(CMM_SESSION_REMINDER_SCRIPT_LEGACY, + previous_legacy_command, + sizeof(previous_legacy_command)) != CLI_OK || + cbm_resolve_released_hook_command(CMM_SESSION_REMINDER_SCRIPT_LEGACY, + released_legacy_command, + sizeof(released_legacy_command)) != CLI_OK) { return CLI_ERR; } - const char *const old_commands[] = {released_command, previous_command, NULL}; + const char *const old_commands[] = {released_command, previous_command, released_legacy_command, + previous_legacy_command, NULL}; int rc = 0; for (int i = 0; i < NUM_DIRS; i++) { if (remove_hooks_json((hooks_remove_args_t){.settings_path = settings_path, @@ -4979,6 +5035,7 @@ static bool cbm_has_complete_claude_session_hooks(const char *home) { #else #define CMM_SUBAGENT_REMINDER_SCRIPT "codebase-memory-cli-subagent-reminder" #endif +#define CMM_SUBAGENT_REMINDER_SCRIPT_LEGACY "codebase-memory-cli-subagent-reminder" static bool cbm_install_subagent_reminder_script(const char *home, const char *binary_path) { if (!home || !binary_path) { @@ -5010,6 +5067,13 @@ static bool cbm_install_subagent_reminder_script(const char *home, const char *b sizeof(script)) != CLI_OK) { return false; } +#ifdef _WIN32 + const char *const legacy[] = {cmm_legacy_mcp_subagent_script}; + if (cbm_remove_owned_legacy_hook_script(hooks_dir, CMM_SUBAGENT_REMINDER_SCRIPT_LEGACY, script, + legacy, 1U) != CLI_OK) { + return false; + } +#endif return cbm_write_owned_hook_script(script_path, script); } @@ -5064,15 +5128,24 @@ int cbm_upsert_claude_subagent_hooks(const char *settings_path) { char command[CLI_BUF_8K]; char previous_command[CLI_BUF_8K]; char released_command[CLI_BUF_8K]; + char previous_legacy_command[CLI_BUF_8K]; + char released_legacy_command[CLI_BUF_8K]; if (cbm_resolve_hook_command(CMM_SUBAGENT_REMINDER_SCRIPT, command, sizeof(command)) != CLI_OK || cbm_resolve_previous_hook_command(CMM_SUBAGENT_REMINDER_SCRIPT, previous_command, sizeof(previous_command)) != CLI_OK || cbm_resolve_released_hook_command(CMM_SUBAGENT_REMINDER_SCRIPT, released_command, - sizeof(released_command)) != CLI_OK) { + sizeof(released_command)) != CLI_OK || + cbm_resolve_previous_hook_command(CMM_SUBAGENT_REMINDER_SCRIPT_LEGACY, + previous_legacy_command, + sizeof(previous_legacy_command)) != CLI_OK || + cbm_resolve_released_hook_command(CMM_SUBAGENT_REMINDER_SCRIPT_LEGACY, + released_legacy_command, + sizeof(released_legacy_command)) != CLI_OK) { return CLI_ERR; } - const char *const old_commands[] = {released_command, previous_command, NULL}; + const char *const old_commands[] = {released_command, previous_command, released_legacy_command, + previous_legacy_command, NULL}; /* matcher "*" is the natural choice a user would also pick for their own * catch-all SubagentStart hook, so claim ownership by command too — never * clobber or remove a foreign "*" entry. */ @@ -5089,15 +5162,24 @@ int cbm_remove_claude_subagent_hooks(const char *settings_path) { char command[CLI_BUF_8K]; char previous_command[CLI_BUF_8K]; char released_command[CLI_BUF_8K]; + char previous_legacy_command[CLI_BUF_8K]; + char released_legacy_command[CLI_BUF_8K]; if (cbm_resolve_hook_command(CMM_SUBAGENT_REMINDER_SCRIPT, command, sizeof(command)) != CLI_OK || cbm_resolve_previous_hook_command(CMM_SUBAGENT_REMINDER_SCRIPT, previous_command, sizeof(previous_command)) != CLI_OK || cbm_resolve_released_hook_command(CMM_SUBAGENT_REMINDER_SCRIPT, released_command, - sizeof(released_command)) != CLI_OK) { + sizeof(released_command)) != CLI_OK || + cbm_resolve_previous_hook_command(CMM_SUBAGENT_REMINDER_SCRIPT_LEGACY, + previous_legacy_command, + sizeof(previous_legacy_command)) != CLI_OK || + cbm_resolve_released_hook_command(CMM_SUBAGENT_REMINDER_SCRIPT_LEGACY, + released_legacy_command, + sizeof(released_legacy_command)) != CLI_OK) { return CLI_ERR; } - const char *const old_commands[] = {released_command, previous_command, NULL}; + const char *const old_commands[] = {released_command, previous_command, released_legacy_command, + previous_legacy_command, NULL}; return remove_hooks_json((hooks_remove_args_t){.settings_path = settings_path, .hook_event = "SubagentStart", .matcher_str = "*", @@ -9685,12 +9767,24 @@ static void uninstall_claude_code(const char *home, bool dry_run) { static const struct { const char *name; + const char *legacy_name; const char *prefix; } hook_types[] = { - {CMM_HOOK_GATE_SCRIPT, cmm_gate_script_prefix}, - {CMM_SESSION_REMINDER_SCRIPT, cmm_session_script_prefix}, - {CMM_SUBAGENT_REMINDER_SCRIPT, cmm_subagent_script_prefix}, + {CMM_HOOK_GATE_SCRIPT, CMM_HOOK_GATE_SCRIPT_LEGACY, cmm_gate_script_prefix}, + {CMM_SESSION_REMINDER_SCRIPT, CMM_SESSION_REMINDER_SCRIPT_LEGACY, + cmm_session_script_prefix}, + {CMM_SUBAGENT_REMINDER_SCRIPT, CMM_SUBAGENT_REMINDER_SCRIPT_LEGACY, + cmm_subagent_script_prefix}, }; +#ifdef _WIN32 + /* Released shapes only identify the extensionless Windows twin. */ + char released_gate[CMM_RELEASED_GATE_VARIANT_COUNT][CLI_BUF_8K]; + const char *gate_legacy[CMM_RELEASED_GATE_VARIANT_COUNT] = {0}; + size_t gate_legacy_count = + cbm_build_released_gate_candidates(installed_binary, released_gate, gate_legacy); + const char *const session_legacy[] = {cmm_legacy_mcp_session_script}; + const char *const subagent_legacy[] = {cmm_legacy_mcp_subagent_script}; +#endif char hooks_dir[CLI_BUF_1K]; snprintf(hooks_dir, sizeof(hooks_dir), "%s/hooks", config_dir); for (size_t i = 0; i < sizeof(hook_types) / sizeof(hook_types[0]); i++) { @@ -9704,7 +9798,23 @@ static void uninstall_claude_code(const char *home, bool dry_run) { cbm_remove_owned_hook_script(script_path, expected, NULL, 0U) < CLI_OK) { record_agent_config_error(true, "Claude Code", "hook_script_uninstall", hook_types[i].name); + continue; + } +#ifdef _WIN32 + /* Keyed by name so reordering hook_types cannot mismatch lists. */ + bool is_gate = strcmp(hook_types[i].legacy_name, CMM_HOOK_GATE_SCRIPT_LEGACY) == 0; + bool is_session = + strcmp(hook_types[i].legacy_name, CMM_SESSION_REMINDER_SCRIPT_LEGACY) == 0; + const char *const *legacy = is_gate ? gate_legacy + : is_session ? session_legacy + : subagent_legacy; + size_t legacy_count = is_gate ? gate_legacy_count : 1U; + if (cbm_remove_owned_legacy_hook_script(hooks_dir, hook_types[i].legacy_name, expected, + legacy, legacy_count) != CLI_OK) { + record_agent_config_error(true, "Claude Code", "legacy_hook_script_uninstall", + hook_types[i].legacy_name); } +#endif } } printf(" removed CLI-owned PreToolUse + SessionStart + SubagentStart hooks\n"); diff --git a/src/cypher/cypher.c b/src/cypher/cypher.c index f77f93a956..a63a4ab691 100644 --- a/src/cypher/cypher.c +++ b/src/cypher/cypher.c @@ -1996,6 +1996,13 @@ static int parse_post_where(parser_t *p, cbm_query_t *q, // NOLINT(misc-no-recur q->union_next = sub.query; sub.query = NULL; cbm_parse_free(&sub); + /* The branch after UNION was parsed by a SEPARATE parser over a slice + * of these tokens, so this parser's cursor never moved past the UNION + * keyword. That sub-parse now refuses to succeed with anything left + * over, so everything from here to the end is accounted for. Move the + * cursor to the end to say so, or cbm_parse's end-of-input check reads + * a fully parsed UNION query as unfinished. */ + p->pos = p->count; } return 0; } @@ -2059,6 +2066,36 @@ int cbm_parse(const cbm_token_t *tokens, int token_count, // NOLINT(misc-no-recu return CBM_NOT_FOUND; } + /* Every token must be consumed. The grammar accepts at most one WITH and + * treats RETURN as optional, so a query with a second WITH stage — or any + * typo after RETURN — used to stop parsing there and succeed anyway. The + * dropped tail took the filter and the RETURN with it, and the engine + * answered from the fragment it had parsed, using its default projection. + * That reported success and returned wrong rows, which is worse than a + * refusal because nothing tells the caller to look. Refuse instead. */ + if (peek(&p)->type != TOK_EOF) { + /* Only mention the one-WITH limit when a standalone WITH really is + * sitting in the part we could not read. Saying it every time points + * a reader at WITH when the problem is a typo. A WITH straight after + * STARTS is the STARTS WITH operator rather than a clause, the same + * guard parse_post_where uses. */ + const char *hint = ""; + for (int i = p.pos; i < p.count; i++) { + if (p.tokens[i].type == TOK_WITH && + (i == 0 || p.tokens[i - SKIP_ONE].type != TOK_STARTS)) { + hint = " Note that only one WITH clause is supported."; + break; + } + } + snprintf(p.error, sizeof(p.error), + "unexpected input at pos %d ('%s') — the query was not fully " + "parsed.%s", + peek(&p)->pos, peek(&p)->text ? peek(&p)->text : "", hint); + out->error = heap_strdup(p.error); + cbm_query_free(q); + return CBM_NOT_FOUND; + } + out->query = q; return 0; } diff --git a/src/daemon/ipc.c b/src/daemon/ipc.c index 41a0225b9c..9204a1c5b6 100644 --- a/src/daemon/ipc.c +++ b/src/daemon/ipc.c @@ -4366,6 +4366,18 @@ static bool win_sid_trusted(win_security_t *security, PSID sid) { win_sid_is_trusted_installer((const uint8_t *)sid, (size_t)sid_length); } +#ifdef CBM_ENABLE_TEST_SEAMS +bool cbm_daemon_ipc_win_sid_trusted_for_testing(void *sid) { + win_security_t security; + if (!win_security_init(&security)) { + return false; + } + bool trusted = win_sid_trusted(&security, (PSID)sid); + win_security_destroy(&security); + return trusted; +} +#endif + /* AppContainer identities: package SIDs (S-1-15-2-*) and capability SIDs * (S-1-15-3-*), under the APP_PACKAGE identifier authority (15). * diff --git a/src/foundation/compat_fs.c b/src/foundation/compat_fs.c index 345efbd73f..e3bd12d947 100644 --- a/src/foundation/compat_fs.c +++ b/src/foundation/compat_fs.c @@ -23,11 +23,13 @@ #include #include -#include /* _wmkdir */ -#include /* errno for spawn-failure logging */ -#include /* _O_RDONLY */ -#include /* _wunlink, _open_osfhandle, _close */ -#include /* intptr_t */ +#include /* _wmkdir */ +#include /* errno for spawn-failure logging */ +#include /* _O_RDONLY */ +#include /* _wunlink, _open_osfhandle, _close */ +#include /* _SH_DENYRW */ +#include /* _S_IREAD */ +#include /* intptr_t */ #include "foundation/log.h" #include "foundation/win_utf8.h" @@ -75,6 +77,18 @@ cbm_dir_t *cbm_opendir(const char *path) { d->find_handle = FindFirstFileW(d->wide_pattern, &d->find_data); if (d->find_handle == INVALID_HANDLE_VALUE) { + /* Map the Win32 error so callers can tell a missing directory + * (ENOENT) from a real failure, as on POSIX. */ + DWORD werr = GetLastError(); + if (werr == ERROR_FILE_NOT_FOUND || werr == ERROR_PATH_NOT_FOUND) { + errno = ENOENT; + } else if (werr == ERROR_ACCESS_DENIED) { + errno = EACCES; + } else if (werr == ERROR_DIRECTORY) { + errno = ENOTDIR; + } else { + errno = EIO; + } free(d); return NULL; } @@ -583,6 +597,26 @@ int cbm_rmdir(const char *path) { return ret; } +int cbm_lockfile_open(const char *path, bool create) { + wchar_t *wpath = cbm_path_to_wide(path); + if (!wpath) { + errno = EINVAL; + return CBM_NOT_FOUND; + } + int flags = _O_RDWR | _O_BINARY | _O_NOINHERIT | (create ? _O_CREAT : 0); + /* _SH_DENYRW: every other open of this file, from any process, fails + * with EACCES until this descriptor closes -- including at death. */ + int fd = _wsopen(wpath, flags, _SH_DENYRW, _S_IREAD | _S_IWRITE); + free(wpath); + return fd; +} + +void cbm_lockfile_close(int fd) { + if (fd >= 0) { + (void)_close(fd); + } +} + /* Build a properly-quoted Windows command line from an argv array. * Returns a heap-allocated wide string, or NULL on allocation failure. * Quoting follows the MSVC CRT convention: arguments containing spaces, @@ -732,6 +766,7 @@ int cbm_exec_no_shell(const char *const *argv) { #include #include #include +#include #include #include #include @@ -985,6 +1020,28 @@ int cbm_rmdir(const char *path) { return rmdir(path); } +int cbm_lockfile_open(const char *path, bool create) { + int flags = O_RDWR | O_CLOEXEC | O_NOFOLLOW | (create ? O_CREAT : 0); + int fd = open(path, flags, S_IRUSR | S_IWUSR); + if (fd < 0) { + return CBM_NOT_FOUND; + } + int rc = flock(fd, LOCK_EX | LOCK_NB); + if (rc != 0) { + int saved = errno; + (void)close(fd); + errno = saved; + return CBM_NOT_FOUND; + } + return fd; +} + +void cbm_lockfile_close(int fd) { + if (fd >= 0) { + (void)close(fd); + } +} + int cbm_exec_no_shell(const char *const *argv) { if (!argv || !argv[0]) { return CBM_NOT_FOUND; diff --git a/src/foundation/compat_fs.h b/src/foundation/compat_fs.h index d8bd02912a..7acb9e3e9a 100644 --- a/src/foundation/compat_fs.h +++ b/src/foundation/compat_fs.h @@ -91,6 +91,18 @@ int cbm_canonical_path(const char *path, char *out, size_t out_sz); /* Delete an empty directory. Returns 0 on success. */ int cbm_rmdir(const char *path); +/* Exclusive lock file. Opens `path` and takes an exclusive lock the KERNEL + * releases on any process death -- POSIX open(O_CLOEXEC|O_NOFOLLOW) + + * flock(LOCK_EX|LOCK_NB); Windows _wsopen with _SH_DENYRW (deny every other + * open) and _O_NOINHERIT -- so ownership never outlives its holder and is + * never inherited by a spawned child. `create` false never creates the file. + * Returns the descriptor, or -1 with errno: EWOULDBLOCK/EAGAIN (POSIX) or + * EACCES (Windows sharing violation) when another holder is live, ENOENT + * when absent, otherwise the open error. Never use this on a SQLite file: + * on macOS an flock conflicts with SQLite's own fcntl byte locks. */ +int cbm_lockfile_open(const char *path, bool create); +void cbm_lockfile_close(int fd); + /* Open a file by UTF-8 path. * On Windows, converts to wide-char and calls _wfopen so paths with * non-ASCII characters (accents, CJK, etc.) are handled correctly. diff --git a/src/foundation/sha256.c b/src/foundation/sha256.c index 937e363e67..2991d507c0 100644 --- a/src/foundation/sha256.c +++ b/src/foundation/sha256.c @@ -25,7 +25,10 @@ static const uint32_t K[64] = { #define SIG1(x) (ROTR(x, 17) ^ ROTR(x, 19) ^ ((x) >> 10)) static void sha256_transform(cbm_sha256_ctx *c, const uint8_t *data) { - uint32_t m[64]; + /* Scratch comes from the context, not this frame: a 256-byte local here + * is fake-stacked by ASan's use-after-return mode on every 64-byte block. + * Keep one schedule per hash instead of allocating one per block. */ + uint32_t *m = c->sched; for (int i = 0, j = 0; i < 16; i++, j += 4) { m[i] = ((uint32_t)data[j] << 24) | ((uint32_t)data[j + 1] << 16) | ((uint32_t)data[j + 2] << 8) | (uint32_t)data[j + 3]; diff --git a/src/foundation/sha256.h b/src/foundation/sha256.h index a34a741d31..7951890333 100644 --- a/src/foundation/sha256.h +++ b/src/foundation/sha256.h @@ -17,6 +17,10 @@ typedef struct { uint64_t bitlen; uint8_t buf[64]; size_t buflen; + /* Message-schedule scratch for one compression round. Keeping it in the + * context avoids ASan's use-after-return fake-stack allocation for a + * 256-byte transform-frame local on every input block. */ + uint32_t sched[64]; } cbm_sha256_ctx; void cbm_sha256_init(cbm_sha256_ctx *c); diff --git a/src/foundation/workspace.c b/src/foundation/workspace.c index 8fd5c503dd..ec6f8c1d75 100644 --- a/src/foundation/workspace.c +++ b/src/foundation/workspace.c @@ -8,6 +8,9 @@ #include "foundation/compat_fs.h" #include "foundation/platform.h" #include "foundation/sha256.h" +#ifdef _WIN32 +#include "foundation/win_utf8.h" +#endif #include diff --git a/src/main.c b/src/main.c index e83aa60c53..918c0fb910 100644 --- a/src/main.c +++ b/src/main.c @@ -67,6 +67,7 @@ enum { #include "foundation/diagnostics.h" #include "foundation/platform.h" #include "foundation/workspace.h" +#include "foundation/str_util.h" #include "foundation/compat.h" #include "foundation/compat_fs.h" #include "foundation/compat_thread.h" @@ -712,6 +713,26 @@ static char *main_canonical_cli_args_with_project(const char *tool_name, const c return NULL; } } + const char *cache_dir = cbm_resolve_cache_dir(); + if (!cache_dir || !cbm_validate_project_name(default_project)) { + return NULL; + } + char db_path[MAIN_PATH_CAP]; + int db_len = snprintf(db_path, sizeof(db_path), "%s/%s.db", cache_dir, default_project); + if (db_len <= 0 || (size_t)db_len >= sizeof(db_path)) { + return NULL; + } + cbm_store_t *store = cbm_store_open_path_query(db_path); + if (!store) { + return NULL; + } + cbm_project_t indexed = {0}; + bool indexed_ok = cbm_store_get_project(store, default_project, &indexed) == CBM_STORE_OK; + cbm_project_free_fields(&indexed); + cbm_store_close(store); + if (!indexed_ok) { + return NULL; + } const char *project = default_project; yyjson_mut_doc *mut = yyjson_doc_mut_copy(doc, NULL); yyjson_doc_free(doc); @@ -2129,6 +2150,48 @@ static void main_daemon_ctl_print_clients(const uint32_t *pids, uint8_t count, u } } +static void main_daemon_ctl_print_status_json(bool active, + const cbm_daemon_runtime_status_t *status) { + if (!active) { + printf("{\"active\":false,\"status\":\"not_running\"}\n"); + return; + } + yyjson_mut_doc *doc = yyjson_mut_doc_new(NULL); + yyjson_mut_val *root = doc ? yyjson_mut_obj(doc) : NULL; + if (!root) { + if (doc) { + yyjson_mut_doc_free(doc); + } + (void)fprintf(stderr, "error: could not allocate status JSON\n"); + return; + } + yyjson_mut_doc_set_root(doc, root); + bool ok = true; + ok = ok && yyjson_mut_obj_add_bool(doc, root, "active", true); + ok = ok && yyjson_mut_obj_add_str(doc, root, "status", "active"); + ok = ok && yyjson_mut_obj_add_bool(doc, root, "permanent", status->permanent); + ok = ok && yyjson_mut_obj_add_uint(doc, root, "pid", (unsigned long)status->daemon_pid); + ok = ok && yyjson_mut_obj_add_uint(doc, root, "clients", (unsigned)status->client_count); + ok = ok && yyjson_mut_obj_add_uint(doc, root, "committed_clients", + (unsigned)status->committed_clients); + ok = ok && yyjson_mut_obj_add_bool(doc, root, "stopping", status->stopping); + ok = ok && yyjson_mut_obj_add_strcpy(doc, root, "version", status->semantic_version); + ok = ok && yyjson_mut_obj_add_strcpy(doc, root, "build", status->build_fingerprint); + if (!ok) { + yyjson_mut_doc_free(doc); + (void)fprintf(stderr, "error: could not build status JSON\n"); + return; + } + char *json = yyjson_mut_write(doc, 0, NULL); + yyjson_mut_doc_free(doc); + if (!json) { + (void)fprintf(stderr, "error: could not serialize status JSON\n"); + return; + } + printf("%s\n", json); + free(json); +} + #ifdef _WIN32 typedef SOCKET main_daemon_ctl_socket_t; #define MAIN_DAEMON_CTL_BAD_SOCKET INVALID_SOCKET @@ -2559,6 +2622,7 @@ static int main_run_daemon_ctl(int argc, char **argv, const cbm_daemon_ipc_endpo const char *executable_path) { const char *subcommand = NULL; bool open_browser = false; + bool json_output = false; int requested_port = 0; bool arguments_valid = true; for (int index = 1; index < argc; index++) { @@ -2570,6 +2634,8 @@ static int main_run_daemon_ctl(int argc, char **argv, const cbm_daemon_ipc_endpo subcommand = argv[index]; } else if (strcmp(argv[index], "--open") == 0) { open_browser = true; + } else if (strcmp(argv[index], "--json") == 0) { + json_output = true; } else if (strncmp(argv[index], "--port=", 7) == 0) { requested_port = atoi(argv[index] + 7); if (requested_port <= 0 || requested_port >= MAIN_MAX_PORT) { @@ -2584,7 +2650,7 @@ static int main_run_daemon_ctl(int argc, char **argv, const cbm_daemon_ipc_endpo } if (!arguments_valid || !subcommand) { (void)fprintf(stderr, "usage: codebase-memory-cli daemon " - "[--open] [--port=N]\n"); + "[--json] [--open] [--port=N]\n"); return EXIT_FAILURE; } @@ -2594,6 +2660,10 @@ static int main_run_daemon_ctl(int argc, char **argv, const cbm_daemon_ipc_endpo if (strcmp(subcommand, "status") == 0) { if (!active) { + if (json_output) { + main_daemon_ctl_print_status_json(false, &status); + return EXIT_FAILURE; + } if (status.muted_endpoint_holder_pid != 0) { /* Alive process, dead runtime (2026-08-29 zombie class). * "not running" here sent the operator hunting through @@ -2612,6 +2682,10 @@ static int main_run_daemon_ctl(int argc, char **argv, const cbm_daemon_ipc_endpo "commands and hooks skip the per-command startup cost.\n"); return EXIT_FAILURE; } + if (json_output) { + main_daemon_ctl_print_status_json(true, &status); + return EXIT_SUCCESS; + } printf("daemon: active (%s)\n", status.permanent ? "permanent" : "session-managed"); printf(" pid: %lu\n", (unsigned long)status.daemon_pid); printf(" build: %s (%.12s...)\n", status.semantic_version, status.build_fingerprint); diff --git a/src/operations/index.c b/src/operations/index.c index 6d58c5a70e..86e65c854e 100644 --- a/src/operations/index.c +++ b/src/operations/index.c @@ -4,6 +4,7 @@ #include "operations/project_arg.h" #include "foundation/compat_fs.h" +#include "foundation/compat.h" #include "foundation/constants.h" #include "foundation/dump_verify.h" #include "foundation/log.h" @@ -56,6 +57,85 @@ static bool index_bool_arg(const char *args_json, const char *key) { return out; } +static bool index_db_path(const char *project, char *out, size_t out_size); + +static const char *index_mode_name(const char *mode_text) { + if (mode_text && !strcmp(mode_text, "fast")) { + return "fast"; + } + if (mode_text && !strcmp(mode_text, "moderate")) { + return "moderate"; + } + return "full"; +} + +static bool index_write_metrics(const char *path, const char *mode, const char *project, + const cbm_pipeline_t *pipeline, int rc) { + if (!path || !path[0]) + return true; + cbm_pipeline_metrics_t timing = {0}; + cbm_pipeline_get_metrics(pipeline, &timing); + int nodes = 0, edges = 0; + cbm_pipeline_get_committed_counts(pipeline, &nodes, &edges); + char db[CBM_SZ_1K] = {0}; + struct stat st = {0}; + bool have_db = index_db_path(project, db, sizeof(db)) && stat(db, &st) == 0; + yyjson_mut_doc *doc = yyjson_mut_doc_new(NULL); + yyjson_mut_val *root = doc ? yyjson_mut_obj(doc) : NULL; + yyjson_mut_val *times = doc ? yyjson_mut_obj(doc) : NULL, + *waits = doc ? yyjson_mut_obj(doc) : NULL; + yyjson_mut_val *memory = doc ? yyjson_mut_obj(doc) : NULL, + *counts = doc ? yyjson_mut_obj(doc) : NULL; + if (!doc || !root || !times || !waits || !memory || !counts) { + if (doc) + yyjson_mut_doc_free(doc); + return false; + } + yyjson_mut_doc_set_root(doc, root); + yyjson_mut_obj_add_uint(doc, root, "schema_version", 1); +#ifdef CBM_VERSION + yyjson_mut_obj_add_str(doc, root, "source_revision", CBM_VERSION); +#else + yyjson_mut_obj_add_str(doc, root, "source_revision", "unknown"); +#endif + yyjson_mut_obj_add_str(doc, root, "index_mode", mode ? mode : "full"); + yyjson_mut_obj_add_str(doc, root, "outcome", + rc == 0 ? "success" + : rc == CBM_PIPELINE_ABORT_PRESERVE_DB ? "cancelled" + : "failed"); + yyjson_mut_obj_add_uint(doc, times, "discovery_ms", timing.discovery_ms); + yyjson_mut_obj_add_uint(doc, times, "index_ms", timing.index_ms); + yyjson_mut_obj_add_uint(doc, times, "staging_ms", timing.staging_ms); + yyjson_mut_obj_add_uint(doc, times, "publish_ms", timing.publish_ms); + yyjson_mut_obj_add_val(doc, root, "wall_times", times); + yyjson_mut_obj_add_uint(doc, waits, "queue_ms", 0); + yyjson_mut_obj_add_uint(doc, waits, "lock_ms", 0); + yyjson_mut_obj_add_val(doc, root, "waits", waits); + yyjson_mut_obj_add_uint(doc, memory, "worker_peak_rss_bytes", cbm_mem_peak_rss()); + yyjson_mut_obj_add_uint(doc, memory, "worker_aggregate_rss_bytes", cbm_mem_rss()); + yyjson_mut_obj_add_val(doc, root, "memory", memory); + yyjson_mut_obj_add_int(doc, counts, "files", timing.files); + yyjson_mut_obj_add_int(doc, counts, "definitions", nodes > 0 ? nodes : 0); + yyjson_mut_obj_add_int(doc, counts, "edges", edges > 0 ? edges : 0); + yyjson_mut_obj_add_uint(doc, counts, "database_bytes", have_db ? (uint64_t)st.st_size : 0); + yyjson_mut_obj_add_val(doc, root, "counts", counts); + char *json = yyjson_mut_write(doc, 0, NULL); + yyjson_mut_doc_free(doc); + if (!json) + return false; + char temporary[CBM_SZ_4K]; + int formatted = snprintf(temporary, sizeof(temporary), "%s.tmp.XXXXXX", path); + int fd = formatted > 0 && (size_t)formatted < sizeof(temporary) ? cbm_mkstemp(temporary) : -1; + FILE *file = fd >= 0 ? fdopen(fd, "wb") : NULL; + bool wrote = file && fputs(json, file) >= 0; + bool closed = file && fclose(file) == 0; + bool ok = wrote && closed && cbm_rename_replace(temporary, path) == 0; + if (!ok && fd >= 0) + (void)cbm_unlink(temporary); + free(json); + return ok; +} + static cbm_operation_result_t index_text_error(const char *message) { return cbm_operation_result_copy(message ? message : "index operation failed", true); } @@ -117,7 +197,7 @@ static bool index_resolve_session_path(const cbm_operation_runtime_t *runtime, c return true; } -static char *index_args_with_repo_path(const char *args_json, const char *repo_path) { +static char *index_args_with_string(const char *args_json, const char *key, const char *value) { const char *json = args_json ? args_json : "{}"; yyjson_doc *source = yyjson_read(json, strlen(json), 0); yyjson_val *source_root = source ? yyjson_doc_get_root(source) : NULL; @@ -134,8 +214,8 @@ static char *index_args_with_repo_path(const char *args_json, const char *repo_p yyjson_mut_doc_free(copy); return NULL; } - (void)yyjson_mut_obj_remove_key(root, "repo_path"); - if (!yyjson_mut_obj_add_strcpy(copy, root, "repo_path", repo_path)) { + (void)yyjson_mut_obj_remove_key(root, key); + if (!yyjson_mut_obj_add_strcpy(copy, root, key, value)) { yyjson_mut_doc_free(copy); return NULL; } @@ -144,6 +224,10 @@ static char *index_args_with_repo_path(const char *args_json, const char *repo_p return out; } +static char *index_args_with_repo_path(const char *args_json, const char *repo_path) { + return index_args_with_string(args_json, "repo_path", repo_path); +} + static bool index_db_path(const char *project, char *out, size_t out_size) { const char *cache = cbm_resolve_cache_dir(); return project && cbm_validate_project_name(project) && cache && cache[0] && @@ -456,27 +540,80 @@ static bool index_build_success(yyjson_mut_doc *doc, yyjson_mut_val *root, const return degraded; } +static void index_fill_run_response(yyjson_mut_doc *doc, yyjson_mut_val *root, const char *project, + const char *repo_path, bool persistence, + cbm_pipeline_t *pipeline, int rc, char **excluded, + int excluded_count, const cbm_file_error_t *errors, + int error_count, bool metrics_failed) { + yyjson_mut_obj_add_strcpy(doc, root, "project", project ? project : ""); + if (rc == 0) { + char logfile[CBM_SZ_1K] = {0}; + bool has_log = index_write_log(project, errors, error_count, logfile, sizeof(logfile)); + bool degraded = + index_build_success(doc, root, project, repo_path, persistence, pipeline, excluded, + excluded_count, errors, error_count, has_log ? logfile : NULL); + yyjson_mut_obj_add_str(doc, root, "status", degraded ? "degraded" : "indexed"); + if (metrics_failed) { + yyjson_mut_obj_add_str(doc, root, "metrics_error", "failed to write metrics_out"); + } + if (cbm_pipeline_had_format_migration(pipeline)) { + yyjson_mut_obj_add_bool(doc, root, "format_migration", true); + } + } else { + yyjson_mut_obj_add_str(doc, root, "status", "error"); + yyjson_mut_obj_add_str(doc, root, "hint", + "Pipeline failed. Check repo_path exists and contains source files. " + "Try mode='fast' for a quicker diagnostic run."); + } +} + +static char *index_encode_run_response(const char *project, const char *repo_path, bool persistence, + cbm_pipeline_t *pipeline, int rc, char **excluded, + int excluded_count, const cbm_file_error_t *errors, + int error_count, bool metrics_failed) { + yyjson_mut_doc *doc = yyjson_mut_doc_new(NULL); + yyjson_mut_val *root = doc ? yyjson_mut_obj(doc) : NULL; + if (!root) { + if (doc) { + yyjson_mut_doc_free(doc); + } + return NULL; + } + yyjson_mut_doc_set_root(doc, root); + index_fill_run_response(doc, root, project, repo_path, persistence, pipeline, rc, excluded, + excluded_count, errors, error_count, metrics_failed); + char *payload = yyjson_mut_write(doc, 0, NULL); + yyjson_mut_doc_free(doc); + return payload; +} + static cbm_operation_result_t index_run_physical(const char *repo_path, const char *args_json, const cbm_operation_runtime_t *runtime) { char *mode_text = index_string_arg(args_json, "mode"); char *name = index_string_arg(args_json, "name"); + char *metrics_out = index_string_arg(args_json, "metrics_out"); char *mutation_project = cbm_project_name_from_path(name && name[0] ? name : repo_path); if (!mutation_project) { free(mode_text); free(name); + free(metrics_out); return index_text_error("could not resolve index project name"); } if (!runtime || !runtime->mutation_begin || !runtime->mutation_end || !runtime->mutation_begin(runtime->mutation_context, mutation_project)) { free(mode_text); free(name); + free(metrics_out); free(mutation_project); return index_text_error("index operation blocked by another mutation for this project"); } if (runtime->cancelled && runtime->cancelled(runtime->cancelled_context)) { + (void)index_write_metrics(metrics_out, index_mode_name(mode_text), mutation_project, NULL, + CBM_PIPELINE_ABORT_PRESERVE_DB); runtime->mutation_end(runtime->mutation_context, mutation_project); free(mode_text); free(name); + free(metrics_out); free(mutation_project); return index_text_error("index operation cancelled for this request"); } @@ -485,12 +622,15 @@ static cbm_operation_result_t index_run_physical(const char *repo_path, const ch mode = CBM_MODE_FAST; else if (mode_text && !strcmp(mode_text, "moderate")) mode = CBM_MODE_MODERATE; + const char *mode_name = index_mode_name(mode_text); free(mode_text); bool persistence = index_bool_arg(args_json, "persistence"); cbm_pipeline_t *pipeline = cbm_pipeline_new(repo_path, NULL, mode); if (!pipeline) { + (void)index_write_metrics(metrics_out, mode_name, mutation_project, NULL, CBM_NOT_FOUND); runtime->mutation_end(runtime->mutation_context, mutation_project); free(name); + free(metrics_out); free(mutation_project); return index_text_error("failed to create pipeline"); } @@ -498,6 +638,7 @@ static cbm_operation_result_t index_run_physical(const char *repo_path, const ch cbm_pipeline_free(pipeline); runtime->mutation_end(runtime->mutation_context, mutation_project); free(name); + free(metrics_out); free(mutation_project); return index_text_error("invalid project name"); } @@ -519,39 +660,16 @@ static cbm_operation_result_t index_run_physical(const char *repo_path, const ch int error_count = 0; cbm_pipeline_get_file_errors(pipeline, &errors, &error_count); cbm_mem_collect(); + bool metrics_failed = !index_write_metrics(metrics_out, mode_name, project, pipeline, rc); + if (metrics_failed) { + cbm_log_warn("index.metrics_write_failed", "path", metrics_out); + } + free(metrics_out); if (runtime->project_invalidate) runtime->project_invalidate(runtime->project_invalidate_context, project); - yyjson_mut_doc *doc = yyjson_mut_doc_new(NULL); - yyjson_mut_val *root = doc ? yyjson_mut_obj(doc) : NULL; - if (!doc || !root) { - if (doc) - yyjson_mut_doc_free(doc); - if (!cbm_index_worker_active()) - cbm_pipeline_free(pipeline); - runtime->mutation_end(runtime->mutation_context, mutation_project); - free(project); - free(mutation_project); - return index_text_error("result allocation failed"); - } - yyjson_mut_doc_set_root(doc, root); - yyjson_mut_obj_add_strcpy(doc, root, "project", project ? project : ""); - if (rc == 0) { - char logfile[CBM_SZ_1K] = {0}; - bool has_log = index_write_log(project, errors, error_count, logfile, sizeof(logfile)); - bool degraded = - index_build_success(doc, root, project, repo_path, persistence, pipeline, excluded, - excluded_count, errors, error_count, has_log ? logfile : NULL); - yyjson_mut_obj_add_str(doc, root, "status", degraded ? "degraded" : "indexed"); - if (cbm_pipeline_had_format_migration(pipeline)) - yyjson_mut_obj_add_bool(doc, root, "format_migration", true); - } else { - yyjson_mut_obj_add_str(doc, root, "status", "error"); - yyjson_mut_obj_add_str(doc, root, "hint", - "Pipeline failed. Check repo_path exists and contains source files. " - "Try mode='fast' for a quicker diagnostic run."); - } - char *payload = yyjson_mut_write(doc, 0, NULL); - yyjson_mut_doc_free(doc); + char *payload = + index_encode_run_response(project, repo_path, persistence, pipeline, rc, excluded, + excluded_count, errors, error_count, metrics_failed); if (cbm_index_worker_active()) cbm_log_info("index.worker.fast_exit", "skip", "pipeline_free"); else @@ -563,6 +681,79 @@ static cbm_operation_result_t index_run_physical(const char *repo_path, const ch : index_text_error("result encoding failed"); } +/* Resolve metrics_out, require its parent directory to exist inside the allowed + * roots, and rewrite the worker argument to the canonical parent + basename so + * the worker writes exactly the validated path. */ +static bool index_validate_metrics_out(const char *json, const cbm_operation_runtime_t *runtime, + const char *allowed_root, char **worker_args, char *err, + size_t err_size) { + char *metrics = index_string_arg(json, "metrics_out"); + if (!metrics || !metrics[0]) { + free(metrics); + return true; + } + cbm_normalize_path_sep(metrics); + if (!index_resolve_session_path(runtime, &metrics)) { + free(metrics); + (void)snprintf(err, err_size, "failed to resolve metrics_out"); + return false; + } + char *slash = strrchr(metrics, '/'); + const char *base = metrics; + if (slash) { + base = slash; + base++; + } + if (!base[0] || !strcmp(base, ".") || !strcmp(base, "..")) { + free(metrics); + (void)snprintf(err, err_size, "metrics_out must name a file"); + return false; + } + char parent[CBM_SZ_4K]; + size_t parent_len = slash ? (size_t)(slash - metrics) : 0; + if (!slash) { + (void)snprintf(parent, sizeof(parent), "."); + } else if (parent_len == 0) { + (void)snprintf(parent, sizeof(parent), "/"); + } else if (parent_len >= sizeof(parent)) { + free(metrics); + (void)snprintf(err, err_size, "metrics_out path too long"); + return false; + } else { + memcpy(parent, metrics, parent_len); + parent[parent_len] = '\0'; + } + char real[CBM_SZ_4K]; + if (!cbm_canonical_path(parent, real, sizeof(real))) { + free(metrics); + (void)snprintf(err, err_size, "metrics_out parent directory does not exist"); + return false; + } + cbm_normalize_path_sep(real); + if (!cbm_workspace_root_allowed(real, cbm_workspace_home_dir(), cbm_workspace_cache_dir(), + allowed_root, err, err_size)) { + free(metrics); + return false; + } + char final_path[CBM_SZ_4K]; + const char *last_sep = strrchr(real, '/'); + int n = snprintf(final_path, sizeof(final_path), "%s%s%s", real, + last_sep && !strcmp(last_sep, "/") ? "" : "/", base); + free(metrics); + if (n <= 0 || (size_t)n >= sizeof(final_path)) { + (void)snprintf(err, err_size, "metrics_out path too long"); + return false; + } + char *rewritten = index_args_with_string(*worker_args, "metrics_out", final_path); + if (!rewritten) { + (void)snprintf(err, err_size, "failed to prepare index request"); + return false; + } + free(*worker_args); + *worker_args = rewritten; + return true; +} + cbm_operation_result_t cbm_index_operation_execute(const char *args_json, const cbm_operation_runtime_t *runtime) { const char *json = args_json ? args_json : "{}"; @@ -599,6 +790,12 @@ cbm_operation_result_t cbm_index_operation_execute(const char *args_json, free(repo_path); return index_text_error("failed to prepare index request"); } + if (!index_validate_metrics_out(json, runtime, allowed_root, &worker_args, boundary, + sizeof(boundary))) { + free(worker_args); + free(repo_path); + return index_text_error(boundary); + } if (runtime && runtime->index_execute) { cbm_operation_result_t out = runtime->index_execute(runtime->index_execute_context, repo_path, worker_args); diff --git a/src/operations/read.c b/src/operations/read.c index 92b9922cbe..bdb7c1b4e4 100644 --- a/src/operations/read.c +++ b/src/operations/read.c @@ -221,6 +221,21 @@ static cbm_operation_result_t execute_projects(const char *args_json) { if (!project_db_file(entry->name)) { continue; } + char db_path[4096]; + if (snprintf(db_path, sizeof(db_path), "%s/%s", cache_dir, entry->name) >= + (int)sizeof(db_path)) { + continue; + } + cbm_store_t *candidate = cbm_store_open_path_query(db_path); + char project_name[1024]; + bool valid = + candidate && primary_project_name(candidate, project_name, sizeof(project_name)); + if (candidate) { + cbm_store_close(candidate); + } + if (!valid) { + continue; + } if (count == capacity) { size_t next = capacity ? capacity * 2U : 32U; char **grown = realloc(names, next * sizeof(*names)); diff --git a/src/operations/tool_catalog.c b/src/operations/tool_catalog.c index 8eda320d1c..4fafc3c5f4 100644 --- a/src/operations/tool_catalog.c +++ b/src/operations/tool_catalog.c @@ -40,7 +40,9 @@ static const tool_def_t TOOLS[] = { "are normalized.\"}," "\"persistence\":{\"type\":\"boolean\",\"default\":false,\"description\":" "\"Write compressed artifact to .codebase-memory/graph.db.zst for team sharing. " - "Teammates can bootstrap from the artifact instead of full re-indexing.\"}" + "Teammates can bootstrap from the artifact instead of full re-indexing.\"}," + "\"metrics_out\":{\"type\":\"string\",\"description\":" + "\"Write one versioned, count-and-timing-only index metrics JSON artifact.\"}" "},\"required\":[\"repo_path\"]}"}, {"search_graph", "Search graph", diff --git a/src/pipeline/pipeline.c b/src/pipeline/pipeline.c index 6d3efdeeda..09627674e6 100644 --- a/src/pipeline/pipeline.c +++ b/src/pipeline/pipeline.c @@ -37,7 +37,9 @@ enum { CBM_DIR_PERMS = 0755, PL_RING = 4, PL_RING_MASK = 3, PL_SEQ_PASSES = 6 }; #include "foundation/compat_thread.h" #include "foundation/profile.h" #include "foundation/mem.h" +#include "foundation/secure_random.h" +#include #include #include #include @@ -50,6 +52,7 @@ enum { CBM_DIR_PERMS = 0755, PL_RING = 4, PL_RING_MASK = 3, PL_SEQ_PASSES = 6 }; #include #define cbm_pipeline_getpid _getpid #else +#include #include #define cbm_pipeline_getpid getpid #endif @@ -72,6 +75,8 @@ static atomic_bool g_persist_test_cancel_after_destination_prepare = false; static atomic_bool g_persist_test_fail_adr_capture = false; static cbm_pipeline_test_hook_fn g_persist_test_before_final_manifest = NULL; static void *g_persist_test_before_final_manifest_userdata = NULL; +static cbm_pipeline_test_hook_fn g_persist_test_after_stage_created = NULL; +static void *g_persist_test_after_stage_created_userdata = NULL; void cbm_pipeline_incremental_test_fail_after_stage_dump_once(void) { atomic_store(&g_persist_test_fail_after_stage_dump, true); @@ -105,6 +110,29 @@ void cbm_pipeline_persist_test_run_before_final_manifest(void) { } } +void cbm_pipeline_incremental_test_after_stage_created_once(cbm_pipeline_test_hook_fn hook, + void *userdata) { + g_persist_test_after_stage_created = hook; + g_persist_test_after_stage_created_userdata = userdata; +} + +/* Fired by create_staging_path() right after the stage's main file is created + * with O_EXCL -- and, in the current lock-before-visible ordering, after its + * sidecar lock is already held. A test hook installed here can run a + * concurrent sweep (another cbm_pipeline_run() against the same final_path) at + * this instant to prove the just-created stage survives it. Under the OLD + * create-then-lock ordering this was the unlocked TOCTOU window, so the same + * hook binds RED if that ordering ever regresses. */ +void cbm_pipeline_persist_test_run_after_stage_created(void) { + cbm_pipeline_test_hook_fn hook = g_persist_test_after_stage_created; + void *userdata = g_persist_test_after_stage_created_userdata; + g_persist_test_after_stage_created = NULL; + g_persist_test_after_stage_created_userdata = NULL; + if (hook) { + hook(userdata); + } +} + bool cbm_pipeline_persist_test_take_failure_after_stage_dump(void) { return atomic_exchange(&g_persist_test_fail_after_stage_dump, false); } @@ -124,6 +152,8 @@ void cbm_pipeline_persist_test_reset_faults(void) { atomic_store(&g_persist_test_fail_adr_capture, false); g_persist_test_before_final_manifest = NULL; g_persist_test_before_final_manifest_userdata = NULL; + g_persist_test_after_stage_created = NULL; + g_persist_test_after_stage_created_userdata = NULL; } #endif @@ -190,11 +220,21 @@ struct cbm_pipeline { /* Committed graph size at dump time (-1 = dump did not run). #334 gate axis. */ int committed_nodes; int committed_edges; + cbm_pipeline_metrics_t metrics; /* #769: set when a stale-format index was routed through the one-time * full rebuild, so the operation response can surface the migration. */ bool format_migration; + /* Recorded by cbm_pipeline_run for the staged run beneath it: whether + * the destination existed, and whether it was copied into the stage so + * that an incremental route has a real previous generation to work + * from. Without a copy the stage is the run's empty placeholder, and + * probing THAT for integrity is what reported every first index as + * "invalid_existing_db" (#1864). */ + bool final_existed; + bool existing_generation; + /* ADR (project_summaries) captured before a full-reindex DB delete, so it * can be restored after the rebuild. NULL when no ADR existed. Issue #516. */ char *saved_adr; @@ -506,6 +546,11 @@ void cbm_pipeline_get_committed_counts(const cbm_pipeline_t *p, int *nodes, int } } +void cbm_pipeline_get_metrics(const cbm_pipeline_t *p, cbm_pipeline_metrics_t *out) { + if (out) + *out = p ? p->metrics : (cbm_pipeline_metrics_t){0}; +} + void cbm_pipeline_set_committed_counts(cbm_pipeline_t *p, int nodes, int edges) { if (p) { p->committed_nodes = nodes; @@ -1402,6 +1447,16 @@ static int capture_existing_adr(cbm_pipeline_t *p, const char *db_path) { static int try_incremental_or_delete_db(cbm_pipeline_t *p, cbm_file_info_t *files, int file_count, const cbm_file_hash_t *baseline_manifest, int baseline_count, bool force_full_on_mismatch) { + if (!p->existing_generation) { + /* Nothing to be incremental against: a first index, or a + * destination that could not be copied (already reported as + * backup_failed_full_rebuild). The stage is an empty placeholder, + * not a database, so it is not probed -- "invalid_existing_db" + * stays reserved for a real copy that fails its integrity check. */ + cbm_log_info("pipeline.route", "path", "full", "reason", + p->final_existed ? "existing_db_backup_failed" : "no_existing_db"); + return CBM_PIPELINE_FORCE_FULL_REINDEX; + } char *db_path = resolve_db_path(p); if (!db_path) { return CBM_PIPELINE_FORCE_FULL_REINDEX; @@ -1527,15 +1582,177 @@ static bool promote_mode_to_existing_coverage(cbm_pipeline_t *p) { return promoted; } +/* A lock held by a live twin (or a sidecar denied for the same reason) is the + * only lock failure worth a fresh suffix. */ +static bool stage_lock_errno_retryable(int err) { + return err == EWOULDBLOCK || err == EAGAIN || err == EACCES; +} + +/* Exclusive-create the stage's main file. POSIX creates it 0600 regardless of + * umask: the stage holds the full index. Returns NULL with errno set. */ +static FILE *stage_main_file_create(const char *path) { +#ifdef _WIN32 + return cbm_fopen(path, "wbx"); +#else + int fd = open(path, O_WRONLY | O_CREAT | O_EXCL | O_CLOEXEC, S_IRUSR | S_IWUSR); + if (fd < 0) { + return NULL; + } + FILE *file = fdopen(fd, "wb"); + if (!file) { + int saved = errno; + (void)close(fd); + (void)unlink(path); + errno = saved; + } + return file; +#endif +} + /* Defined below, next to the other publication helpers. */ static char *create_staging_path(const char *final_path); +/* ── Stage ownership (#1839) ───────────────────────────────────── + * + * A stage used to be recognisable only by its name: the mkstemp descriptor + * was closed at once and nothing marked who was writing it. A worker killed + * mid-run (the daemon cancels with SIGTERM then SIGKILL after one second of + * grace, which a gigabyte backup or clone never finishes inside) left its + * full-size stage behind forever, and no later run could tell a dead stage + * from a live one -- so none tried. + * + * Ownership is now an exclusive kernel lock on the sidecar ".lock", + * held from minting until the stage is discarded or renamed into place. The + * kernel releases it on any death, so "can I take this lock?" is exactly + * "is this stage dead?" -- no pid, no mtime, no grace period. The lock lives + * on a sidecar rather than the stage itself because on macOS an flock on a + * file conflicts with SQLite's fcntl byte locks on that same file. + * + * The stage path is passed around as a plain string through publish and + * finalize, so the descriptor is kept in this per-process registry keyed by + * path, and released by the same helpers that remove the file. */ +typedef struct stage_owner { + char *stage_path; + int lock_fd; + struct stage_owner *next; +} stage_owner_t; + +static stage_owner_t *g_stage_owners = NULL; +static atomic_flag g_stage_owners_spin = ATOMIC_FLAG_INIT; + +static void stage_owners_lock(void) { + while (atomic_flag_test_and_set_explicit(&g_stage_owners_spin, memory_order_acquire)) {} +} + +static void stage_owners_unlock(void) { + atomic_flag_clear_explicit(&g_stage_owners_spin, memory_order_release); +} + +static char *stage_lock_sidecar_path(const char *stage_path) { + static const char suffix[] = ".lock"; + size_t len = strlen(stage_path); + if (len > SIZE_MAX - sizeof(suffix)) { + return NULL; + } + char *sidecar = (char *)malloc(len + sizeof(suffix)); + if (!sidecar) { + return NULL; + } + (void)snprintf(sidecar, len + sizeof(suffix), "%s%s", stage_path, suffix); + return sidecar; +} + +int cbm_pipeline_stage_lock_hold(const char *stage_path) { + if (!stage_path) { + return CBM_NOT_FOUND; + } + char *sidecar = stage_lock_sidecar_path(stage_path); + if (!sidecar) { + return CBM_NOT_FOUND; + } + int fd = cbm_lockfile_open(sidecar, true); + free(sidecar); + return fd; +} + +void cbm_pipeline_stage_lock_drop(const char *stage_path, int lock_fd) { + if (!stage_path || lock_fd < 0) { + return; + } + char *sidecar = stage_lock_sidecar_path(stage_path); + /* Close before unlinking: Windows refuses to delete an open file. The + * sidecar exists unlocked for that instant, but by every drop the stage + * itself is already gone (discarded or renamed), so there is nothing a + * sweeper could take from us. */ + cbm_lockfile_close(lock_fd); + if (sidecar) { + (void)cbm_unlink(sidecar); + free(sidecar); + } +} + +/* Record an already-held stage lock in the per-process owner table, keyed by + * path so publish/finalize/discard can release it later. On success the table + * owns lock_fd; on failure the caller still does and must drop it. + * + * The lock must ALREADY be held: create_staging_path() takes it before the + * stage's main file is created (so the file is never visible on disk without + * its lock), then hands the descriptor here. Re-taking the lock in this helper + * would self-conflict -- both flock() and Windows _SH_DENYRW deny a second + * acquire of the same sidecar even from this same process. */ +static bool stage_owner_adopt(const char *stage_path, int lock_fd) { + stage_owner_t *owner = (stage_owner_t *)malloc(sizeof(*owner)); + char *path_copy = strdup(stage_path); + if (!owner || !path_copy) { + free(owner); + free(path_copy); + return false; + } + owner->stage_path = path_copy; + owner->lock_fd = lock_fd; + stage_owners_lock(); + owner->next = g_stage_owners; + g_stage_owners = owner; + stage_owners_unlock(); + return true; +} + +/* Release ownership of a stage that no longer exists under this name. A path + * this process never registered is a no-op. */ +static void stage_owner_release(const char *stage_path) { + if (!stage_path) { + return; + } + stage_owner_t *found = NULL; + stage_owners_lock(); + for (stage_owner_t **link = &g_stage_owners; *link; link = &(*link)->next) { + if (strcmp((*link)->stage_path, stage_path) == 0) { + found = *link; + *link = found->next; + break; + } + } + stage_owners_unlock(); + if (!found) { + return; + } + cbm_pipeline_stage_lock_drop(found->stage_path, found->lock_fd); + free(found->stage_path); + free(found); +} + +/* Remove a stage's main file and SQLite sidecars, keeping ownership. */ +static void remove_stage_files(const char *stage_path) { + (void)cbm_unlink(stage_path); + (void)cbm_remove_db_sidecars(stage_path); +} + static void discard_generation_stage(const char *stage_path) { if (!stage_path) { return; } - cbm_unlink(stage_path); - cbm_remove_db_sidecars(stage_path); + remove_stage_files(stage_path); + stage_owner_release(stage_path); } typedef struct { @@ -1929,6 +2146,7 @@ int cbm_pipeline_finalize_staged_generation(char *stage_path, const char *final_ discard_generation_stage(stage_path); return CBM_PIPELINE_PERSIST_FAILED; } + stage_owner_release(stage_path); cbm_log_info("finalize.timing", "block", "rename", "elapsed_ms", itoa_buf((int)elapsed_ms(t_fin))); return 0; @@ -2247,6 +2465,8 @@ static int cbm_pipeline_run_staged(cbm_pipeline_t *p) { int rc = cbm_discover_ex2(p->repo_path, &opts, &files, &file_count, &p->excluded_dirs, &p->excluded_count, &p->ignored_files, &p->ignored_count, &p->ignored_total); + p->metrics.discovery_ms += elapsed_ms(t0); + p->metrics.files = file_count; if (rc != 0) { cbm_log_error("pipeline.err", "phase", "discover", "rc", itoa_buf(rc)); } @@ -2386,8 +2606,8 @@ static void cleanup_staging_db(const char *path) { if (!path) { return; } - (void)cbm_unlink(path); - (void)cbm_remove_db_sidecars(path); + remove_stage_files(path); + stage_owner_release(path); } static bool ensure_db_parent(const char *path) { @@ -2415,12 +2635,57 @@ static bool ensure_db_parent(const char *path) { return ok; } +/* A stage minted from another stage must remain a sibling of the live + * database, not become a nested .stage.A.stage.B artifact. */ +enum { + CBM_STAGE_SUFFIX_RANDOM_CHARS = 6, + CBM_STAGE_CREATE_ATTEMPTS = 128, + CBM_STAGE_OWNER_ERRNO_TEXT_SIZE = 16, + CBM_STAGE_SWEEP_BYTES_TEXT_SIZE = 32, + CBM_STAGE_NAME_LIST_INITIAL_CAPACITY = 8, + CBM_STAGE_NAME_LIST_GROWTH_FACTOR = 2, +}; +static const char cbm_stage_marker[] = ".stage."; + +static bool stage_suffix_at(const char *tail) { + size_t marker_len = strlen(cbm_stage_marker); + if (strncmp(tail, cbm_stage_marker, marker_len) != 0) { + return false; + } + const char *random = tail + marker_len; + for (int i = 0; i < CBM_STAGE_SUFFIX_RANDOM_CHARS; i++) { + if (!isalnum((unsigned char)random[i])) { + return false; + } + } + return random[CBM_STAGE_SUFFIX_RANDOM_CHARS] == '\0'; +} + +static size_t stage_root_length(const char *path) { + size_t len = strlen(path); + const size_t suffix_len = strlen(cbm_stage_marker) + CBM_STAGE_SUFFIX_RANDOM_CHARS; + if (len <= suffix_len) { + return len; + } + size_t root_len = len - suffix_len; + for (size_t i = root_len; i < len; i++) { + if (path[i] == '/' +#ifdef _WIN32 + || path[i] == '\\' +#endif + ) { + return len; + } + } + return stage_suffix_at(path + root_len) ? root_len : len; +} + static char *create_staging_path(const char *final_path) { if (!final_path) { return NULL; } static const char suffix[] = ".stage.XXXXXX"; - size_t final_len = strlen(final_path); + size_t final_len = stage_root_length(final_path); if (final_len > SIZE_MAX - sizeof(suffix)) { return NULL; } @@ -2440,17 +2705,88 @@ static char *create_staging_path(const char *final_path) { } memcpy(path, final_path, final_len); memcpy(path + final_len, suffix, sizeof(suffix)); - int fd = cbm_mkstemp(path); - if (fd < 0) { - free(path); - return NULL; - } -#ifdef _WIN32 - _close(fd); -#else - close(fd); + /* The six random chars sit directly after the ".stage." marker; each + * attempt overwrites the "XXXXXX" template in place. Alphanumerics only, + * matching stage_suffix_at()/stage_entry_stage_length() so the sweep + * recognises the minted name and its sidecars. */ + char *random_at = path + final_len + strlen(cbm_stage_marker); + static const char alphabet[] = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"; + + /* Lock BEFORE the stage becomes visible: take the sidecar lock first, then + * create the stage's main file with O_EXCL. The main file is therefore + * never present on disk without its lock already held, so a concurrent + * run's sweep_orphan_stages() against the same final_path can only ever + * find this stage lock-held -- it reaches the stage through the .lock + * sidecar too (stage_entry_stage_length() matches it), probes the lock, + * sees a live holder, and keeps it. That closes the old create->register + * window that let a racing sweep delete a live-but-unlocked stage (POSIX) + * or collide on the sidecar with EACCES (Windows) -- #2111's windows-guards + * red. A pre-lock-era orphan minted by an OLDER binary still carries no + * lock, so the sweep's ENOENT path still removes it (#1839 preserved). + * + * A minted suffix collides with an existing stage only about 1 in 62^6; + * retry a bounded number of times, the way mkstemp/mkdtemp do, then fail. */ + int last_errno = 0; + for (int attempt = 0; attempt < CBM_STAGE_CREATE_ATTEMPTS; attempt++) { + unsigned char rnd[CBM_STAGE_SUFFIX_RANDOM_CHARS]; + if (!cbm_secure_random(rnd, sizeof(rnd))) { + free(path); + return NULL; + } + for (size_t i = 0; i < sizeof(rnd); i++) { + random_at[i] = alphabet[rnd[i] % (sizeof(alphabet) - sizeof(alphabet[0]))]; + } + errno = 0; + int lock_fd = cbm_pipeline_stage_lock_hold(path); + if (lock_fd < 0) { + /* A live twin already owns this exact suffix's sidecar (EAGAIN / + * EACCES): mint a fresh suffix and try again rather than contend + * for this one. Any other errno is a real failure. */ + last_errno = errno; + if (stage_lock_errno_retryable(last_errno)) { + continue; + } + break; + } + FILE *main_file = stage_main_file_create(path); + if (!main_file) { + /* Save errno before the drop: close/unlink can clobber it. */ + last_errno = errno; + cbm_pipeline_stage_lock_drop(path, lock_fd); + if (last_errno == EEXIST) { + /* The suffix collided with a lock-less orphan's main file -- + * its sidecar was takeable, so it is not a live writer. Never + * inherit a stranger's bytes: the lock and sidecar are + * dropped, so mint a fresh suffix. The orphan's main file is + * left for a later sweep, which removes it as a pre-lock-era + * orphan. */ + continue; + } + break; + } + (void)fclose(main_file); +#if defined(CBM_INCREMENTAL_TEST_API) && CBM_INCREMENTAL_TEST_API + /* Main file now exists and its lock is already held. Under the OLD + * create-then-lock ordering this was the unlocked window; the + * concurrent-sweep test fires here to prove the stage now survives a + * racing sweep, and to bind RED if that ordering ever regresses. */ + cbm_pipeline_persist_test_run_after_stage_created(); #endif - return path; + if (!stage_owner_adopt(path, lock_fd)) { + cbm_pipeline_stage_lock_drop(path, lock_fd); + (void)cbm_unlink(path); + free(path); + return NULL; + } + return path; + } + /* Attempts exhausted or a non-retryable failure -- keep the observability + * the old stage_owner_register() emitted for a lock failure. */ + char errno_text[CBM_STAGE_OWNER_ERRNO_TEXT_SIZE]; + (void)snprintf(errno_text, sizeof(errno_text), "%d", last_errno); + cbm_log_warn("pipeline.stage", "action", "lock_failed", "errno", errno_text, "path", path); + free(path); + return NULL; } /* A backup-failed destination may still have the only recoverable WAL or @@ -2554,10 +2890,201 @@ static int export_after_publish(cbm_pipeline_t *p, const char *final_path) { return 0; } +/* ── Orphan sweep (#1839) ──────────────────────────────────────── + * + * Nothing on the worker-death path ever cleaned a stage up, so the sweep + * runs at the start of every run, before this run mints its own stage. It + * considers ONLY names of the exact minted shape for THIS database -- + * ".stage.<6 alphanumerics>" plus that stage's -wal/-shm/-journal + * and .lock sidecars -- never the live database, a quarantined .corrupt, or + * another project's files. A stage is removed when its ownership lock can be + * taken (its writer is dead, or the stage predates ownership) and kept when + * a live writer holds the lock. The pre-ownership case is the one honest + * gap: a stage an OLDER binary is still writing against this database has + * no lock and is swept; that writer's final rename then fails and it + * discards. The live database is never named here on either path. */ + +static const char *const cbm_stage_sidecar_tails[] = {"", "-wal", "-shm", "-journal", ".lock"}; + +/* If `name` is ".stage.<6 alphanumerics>", return the + * length of the stage name proper (without the tail); 0 otherwise. */ +static size_t stage_entry_stage_length(const char *name, const char *base, size_t base_len) { + if (strncmp(name, base, base_len) != 0) { + return 0; + } + const char *at = name + base_len; + size_t marker_len = strlen(cbm_stage_marker); + if (strncmp(at, cbm_stage_marker, marker_len) != 0) { + return 0; + } + at += marker_len; + for (int i = 0; i < CBM_STAGE_SUFFIX_RANDOM_CHARS; i++) { + /* NUL is not alphanumeric, so a short name fails here too. */ + if (!isalnum((unsigned char)at[i])) { + return 0; + } + } + at += CBM_STAGE_SUFFIX_RANDOM_CHARS; + for (size_t i = 0; i < sizeof(cbm_stage_sidecar_tails) / sizeof(cbm_stage_sidecar_tails[0]); + i++) { + if (strcmp(at, cbm_stage_sidecar_tails[i]) == 0) { + return (size_t)(at - name); + } + } + return 0; +} + +typedef struct { + char **names; + int count; + int cap; +} stage_name_list_t; + +/* Add a stage name once, however many of its files were listed. */ +static void stage_name_list_add(stage_name_list_t *list, const char *name, size_t len) { + for (int i = 0; i < list->count; i++) { + if (strlen(list->names[i]) == len && strncmp(list->names[i], name, len) == 0) { + return; + } + } + if (list->count == list->cap) { + int cap = list->cap ? list->cap * CBM_STAGE_NAME_LIST_GROWTH_FACTOR + : CBM_STAGE_NAME_LIST_INITIAL_CAPACITY; + char **grown = (char **)realloc(list->names, (size_t)cap * sizeof(*grown)); + if (!grown) { + return; + } + list->names = grown; + list->cap = cap; + } + char *copy = (char *)malloc(len + sizeof(*copy)); + if (!copy) { + return; + } + memcpy(copy, name, len); + copy[len] = '\0'; + list->names[list->count++] = copy; +} + +static int64_t stage_bytes_on_disk(const char *stage_path) { + static const char *const files[] = {"", "-wal", "-shm", "-journal"}; + int64_t total = 0; + char side[CBM_SZ_4K]; + for (size_t i = 0; i < sizeof(files) / sizeof(files[0]); i++) { + int n = snprintf(side, sizeof(side), "%s%s", stage_path, files[i]); + if (n <= 0 || (size_t)n >= sizeof(side)) { + continue; + } + cbm_path_info_t info; + if (cbm_path_info_utf8(side, &info) == 0 && info.is_regular) { + total += info.size; + } + } + return total; +} + +static void sweep_one_stage(const char *stage_path) { + char *sidecar = stage_lock_sidecar_path(stage_path); + if (!sidecar) { + return; + } + errno = 0; + int lock_fd = cbm_lockfile_open(sidecar, false); + int probe_errno = errno; + free(sidecar); + if (lock_fd < 0 && probe_errno != ENOENT) { + /* A non-ENOENT probe may be a live lock holder or an access fault; + * both must preserve the stage rather than risking a live writer. */ + char errno_text[CBM_STAGE_OWNER_ERRNO_TEXT_SIZE]; + (void)snprintf(errno_text, sizeof(errno_text), "%d", probe_errno); + cbm_log_warn("pipeline.stage", "action", "orphan_kept", "reason", "lock_unavailable", + "errno", errno_text, "path", stage_path); + return; + } + /* No owner: absent sidecar (an ENOENT probe) or a lock the kernel released + * with its writer. Ours now, from the lock down. + * + * The absent-sidecar (ENOENT) case is exactly a pre-lock-era orphan: a + * stage an OLDER binary minted with no sidecar at all (#1839 pins that + * these ARE swept). It is NOT an in-flight stage of a current run: since + * create_staging_path() now takes the sidecar lock BEFORE the stage's main + * file becomes visible on disk, a live stage always has its sidecar, so a + * concurrent sweep landing here for one would instead find the lock held + * above and keep it. Removing on ENOENT therefore reclaims genuine orphans + * without ever deleting a live stage (the create->register race behind + * #2111's windows-guards red is closed at the source). */ + int64_t bytes = stage_bytes_on_disk(stage_path); + remove_stage_files(stage_path); + if (lock_fd >= 0) { + cbm_pipeline_stage_lock_drop(stage_path, lock_fd); + } + char bytes_text[CBM_STAGE_SWEEP_BYTES_TEXT_SIZE]; + (void)snprintf(bytes_text, sizeof(bytes_text), "%lld", (long long)bytes); + cbm_log_info("pipeline.stage", "action", "orphan_removed", "bytes", bytes_text, "path", + stage_path); +} + +static void sweep_orphan_stages(const char *final_path) { + /* Directory part INCLUDING its trailing separator, so the stage paths + * are joined exactly as the final path was spelled. */ + size_t prefix_len = 0; + for (const char *c = final_path; *c; c++) { + if (*c == '/' +#ifdef _WIN32 + || *c == '\\' +#endif + ) { + prefix_len = (size_t)(c - final_path) + sizeof(*c); + } + } + const char *base = final_path + prefix_len; + size_t base_len = strlen(base); + if (base_len == 0) { + return; + } + char *dir_path = prefix_len ? (char *)malloc(prefix_len + sizeof(*dir_path)) : strdup("."); + if (!dir_path) { + return; + } + if (prefix_len) { + memcpy(dir_path, final_path, prefix_len); + dir_path[prefix_len] = '\0'; + } + cbm_dir_t *dir = cbm_opendir(dir_path); + if (!dir) { + free(dir_path); + return; + } + stage_name_list_t list = {0}; + cbm_dirent_t *entry; + while ((entry = cbm_readdir(dir)) != NULL) { + size_t stage_len = stage_entry_stage_length(entry->name, base, base_len); + if (stage_len) { + stage_name_list_add(&list, entry->name, stage_len); + } + } + cbm_closedir(dir); + for (int i = 0; i < list.count; i++) { + size_t name_len = strlen(list.names[i]); + char *stage_path = (char *)malloc(prefix_len + name_len + sizeof(*stage_path)); + if (stage_path) { + memcpy(stage_path, final_path, prefix_len); + memcpy(stage_path + prefix_len, list.names[i], name_len + sizeof(*stage_path)); + sweep_one_stage(stage_path); + free(stage_path); + } + free(list.names[i]); + } + free(list.names); + free(dir_path); +} + int cbm_pipeline_run(cbm_pipeline_t *p) { if (!p) { return CBM_NOT_FOUND; } + p->metrics = (cbm_pipeline_metrics_t){0}; + uint64_t staged_at = cbm_now_ms(); char *final_path = resolve_db_path(p); if (!final_path || !ensure_db_parent(final_path)) { free(final_path); @@ -2565,6 +3092,7 @@ int cbm_pipeline_run(cbm_pipeline_t *p) { } struct stat final_st; bool final_existed = stat(final_path, &final_st) == 0; + sweep_orphan_stages(final_path); char *staging_path = create_staging_path(final_path); if (!staging_path) { free(final_path); @@ -2577,10 +3105,15 @@ int cbm_pipeline_run(cbm_pipeline_t *p) { if (!backup_succeeded) { cbm_log_warn("pipeline.stage", "action", "backup_failed_full_rebuild", "path", final_path); - cleanup_staging_db(staging_path); + /* The copy is gone but the NAME stays ours: the rebuilt + * generation is renamed over it by the inner finalize and then + * published from it below, so its lock is held to the end. */ + remove_stage_files(staging_path); } } + p->final_existed = final_existed; + p->existing_generation = final_existed && backup_succeeded; char *configured_db_path = p->db_path; p->db_path = strdup(staging_path); if (!p->db_path) { @@ -2590,7 +3123,10 @@ int cbm_pipeline_run(cbm_pipeline_t *p) { free(final_path); return CBM_NOT_FOUND; } + p->metrics.staging_ms = cbm_now_ms() - staged_at; + uint64_t indexed_at = cbm_now_ms(); int rc = cbm_pipeline_run_staged(p); + p->metrics.index_ms = cbm_now_ms() - indexed_at; free(p->db_path); p->db_path = configured_db_path; @@ -2611,6 +3147,7 @@ int cbm_pipeline_run(cbm_pipeline_t *p) { free(final_path); return CBM_PIPELINE_ABORT_PRESERVE_DB; } + uint64_t published_at = cbm_now_ms(); if (seal_staging_db(staging_path) != 0) { cleanup_staging_db(staging_path); free(staging_path); @@ -2660,7 +3197,9 @@ int cbm_pipeline_run(cbm_pipeline_t *p) { return CBM_PIPELINE_PERSIST_FAILED; } + stage_owner_release(staging_path); rc = export_after_publish(p, final_path); + p->metrics.publish_ms = cbm_now_ms() - published_at; free(staging_path); free(final_path); return rc; diff --git a/src/pipeline/pipeline.h b/src/pipeline/pipeline.h index 45684e839c..1ae93954ba 100644 --- a/src/pipeline/pipeline.h +++ b/src/pipeline/pipeline.h @@ -30,6 +30,14 @@ typedef struct cbm_gbuf cbm_gbuf_t; typedef struct cbm_pipeline cbm_pipeline_t; +typedef struct { + uint64_t discovery_ms; + uint64_t index_ms; + uint64_t staging_ms; + uint64_t publish_ms; + int files; +} cbm_pipeline_metrics_t; + /* ── Index mode ─────────────────────────────────────────────────── */ #ifndef CBM_INDEX_MODE_T_DEFINED @@ -104,6 +112,9 @@ bool cbm_pipeline_had_format_migration(const cbm_pipeline_t *p); * Nodes are the #334 plausibility-gate axis; edges are informational only. */ void cbm_pipeline_get_committed_counts(const cbm_pipeline_t *p, int *nodes, int *edges); +/* One-run timings/counts for an opted-in caller; all fields are zero before run. */ +void cbm_pipeline_get_metrics(const cbm_pipeline_t *p, cbm_pipeline_metrics_t *out); + /* ── Per-file indexing failures (Stage 2 / Track B) ─────────────── */ /* One source file that was skipped during indexing. All strings are owned by diff --git a/src/pipeline/pipeline_internal.h b/src/pipeline/pipeline_internal.h index ef63427bbe..a70d243bf4 100644 --- a/src/pipeline/pipeline_internal.h +++ b/src/pipeline/pipeline_internal.h @@ -792,6 +792,17 @@ int cbm_pipeline_publish_staged(char *stage_path, const cbm_pipeline_generation_ * executor; the dump path uses it internally). malloc'd, caller frees. */ char *cbm_pipeline_create_staging_path(const char *final_path); +/* Stage ownership (#1839). Every stage minted by cbm_pipeline_create_staging_path + * is owned through an exclusive kernel lock on the sidecar ".lock" for + * as long as the stage exists; the lock -- and the sidecar -- go away when the + * stage is discarded or renamed into place, and the kernel drops the lock + * when the writer dies. Hold/drop are the same primitive, exposed so a test + * can stand in for a live writer. hold returns a descriptor >= 0, or -1 when + * another holder is live or the sidecar cannot be created. drop releases the + * lock and unlinks the sidecar. */ +int cbm_pipeline_stage_lock_hold(const char *stage_path); +void cbm_pipeline_stage_lock_drop(const char *stage_path, int lock_fd); + /* ── Delta-repair staging primitives (pipeline_delta.c) ────────── * Closure-route-only subsystem: clone the live generation, patch exactly * the repaired node/edge set, publish through the shared finalize leg. */ @@ -891,6 +902,15 @@ void cbm_pipeline_incremental_test_fail_adr_capture_once(void); typedef void (*cbm_pipeline_test_hook_fn)(void *userdata); void cbm_pipeline_incremental_test_before_final_manifest_once(cbm_pipeline_test_hook_fn hook, void *userdata); +/* Fires from create_staging_path(), right after the stage's main file is + * created with O_EXCL. In the current lock-before-visible ordering its sidecar + * lock is already held at this point, so a test hook installed here can run a + * concurrent sweep (via another cbm_pipeline_run() against the same + * final_path) and confirm the just-created stage survives it. Under the OLD + * create-then-lock ordering this was the unlocked window, so the hook also + * binds RED if that ordering regresses. */ +void cbm_pipeline_incremental_test_after_stage_created_once(cbm_pipeline_test_hook_fn hook, + void *userdata); cbm_incremental_route_t cbm_pipeline_incremental_test_last_route(void); void cbm_pipeline_incremental_test_reset_faults(void); @@ -900,6 +920,7 @@ bool cbm_pipeline_persist_test_take_failure_after_stage_dump(void); bool cbm_pipeline_persist_test_take_cancel_after_predump(void); bool cbm_pipeline_persist_test_take_cancel_after_destination_prepare(void); void cbm_pipeline_persist_test_run_before_final_manifest(void); +void cbm_pipeline_persist_test_run_after_stage_created(void); void cbm_pipeline_persist_test_reset_faults(void); #endif diff --git a/src/simhash/minhash.c b/src/simhash/minhash.c index dc42ef4f54..c07c350128 100644 --- a/src/simhash/minhash.c +++ b/src/simhash/minhash.c @@ -290,7 +290,12 @@ void cbm_minhash_to_hex(const cbm_minhash_t *fp, char *buf, int bufsize) { } int pos = 0; for (int i = 0; i < CBM_MINHASH_K; i++) { - pos += snprintf(buf + pos, (size_t)(bufsize - pos), "%08x", fp->values[i]); + int written = snprintf(buf + pos, (size_t)(bufsize - pos), "%08x", fp->values[i]); + if (written < 0 || written >= bufsize - pos) { + buf[pos] = '\0'; + return; + } + pos += written; } } diff --git a/tests/test_cli.c b/tests/test_cli.c index d03aedbde3..fd0635fe99 100644 --- a/tests/test_cli.c +++ b/tests/test_cli.c @@ -8841,9 +8841,9 @@ TEST(cli_windows_claude_lifecycle_migrates_only_exact_owned_legacy_state) { test_mkdirp(hooks_dir); const char *const script_names[] = { - "cbm-code-discovery-gate", - "cbm-session-reminder", - "cbm-subagent-reminder", + "codebase-memory-cli-discovery-gate", + "codebase-memory-cli-session-reminder", + "codebase-memory-cli-subagent-reminder", }; const char *foreign_script = "@echo off\r\necho user-owned-hook\r\n"; for (size_t i = 0U; i < sizeof(script_names) / sizeof(script_names[0]); i++) { @@ -8870,15 +8870,15 @@ TEST(cli_windows_claude_lifecycle_migrates_only_exact_owned_legacy_state) { char subagent_released[1024] = {0}; bool commands_ready = cbm_resolve_claude_hook_command_for_testing( - "cbm-session-reminder.cmd", true, session_current, sizeof(session_current)) == 0 && - cbm_resolve_claude_hook_command_for_testing("cbm-session-reminder", false, session_previous, + "codebase-memory-cli-session-reminder.cmd", true, session_current, sizeof(session_current)) == 0 && + cbm_resolve_claude_hook_command_for_testing("codebase-memory-cli-session-reminder", false, session_previous, sizeof(session_previous)) == 0 && cbm_resolve_claude_hook_command_for_testing( - "cbm-subagent-reminder.cmd", true, subagent_current, sizeof(subagent_current)) == 0 && + "codebase-memory-cli-subagent-reminder.cmd", true, subagent_current, sizeof(subagent_current)) == 0 && cbm_resolve_claude_hook_command_for_testing( - "cbm-subagent-reminder", false, subagent_previous, sizeof(subagent_previous)) == 0; - snprintf(session_released, sizeof(session_released), "%s/cbm-session-reminder", hooks_dir); - snprintf(subagent_released, sizeof(subagent_released), "%s/cbm-subagent-reminder", hooks_dir); + "codebase-memory-cli-subagent-reminder", false, subagent_previous, sizeof(subagent_previous)) == 0; + snprintf(session_released, sizeof(session_released), "%s/codebase-memory-cli-session-reminder", hooks_dir); + snprintf(subagent_released, sizeof(subagent_released), "%s/codebase-memory-cli-subagent-reminder", hooks_dir); const char *foreign_command = "cmd.exe /d /s /c user-owned-hook.cmd"; yyjson_mut_doc *initial_doc = yyjson_mut_doc_new(NULL); @@ -8908,7 +8908,7 @@ TEST(cli_windows_claude_lifecycle_migrates_only_exact_owned_legacy_state) { free(initial_json); yyjson_mut_doc_free(initial_doc); - int install_rc = seeded ? cbm_install_agent_configs(tmpdir, binary_path, false, false) : -1; + int install_rc = seeded ? cbm_install_agent_hooks_for_testing(tmpdir, binary_path, false, false) : -1; char *installed_settings = read_test_file_alloc(settings_path); yyjson_doc *installed_doc = installed_settings ? yyjson_read(installed_settings, strlen(installed_settings), 0) : NULL; @@ -8998,18 +8998,18 @@ TEST(cli_windows_claude_hook_scripts_migrate_and_uninstall_all_owned_shapes) { cbm_unsetenv("COPILOT_HOME"); const char *const legacy_names[] = { - "cbm-code-discovery-gate", - "cbm-session-reminder", - "cbm-subagent-reminder", + "codebase-memory-cli-discovery-gate", + "codebase-memory-cli-session-reminder", + "codebase-memory-cli-subagent-reminder", }; const char *const current_names[] = { - "cbm-code-discovery-gate.cmd", - "cbm-session-reminder.cmd", - "cbm-subagent-reminder.cmd", + "codebase-memory-cli-discovery-gate.cmd", + "codebase-memory-cli-session-reminder.cmd", + "codebase-memory-cli-subagent-reminder.cmd", }; char *current_scripts[sizeof(current_names) / sizeof(current_names[0])] = {0}; - int initial_install_rc = cbm_install_agent_configs(tmpdir, binary_path, false, false); + int initial_install_rc = cbm_install_agent_hooks_for_testing(tmpdir, binary_path, false, false); bool current_scripts_ready = initial_install_rc == 0; for (size_t i = 0U; i < sizeof(current_names) / sizeof(current_names[0]); i++) { char current_path[768]; @@ -9022,7 +9022,7 @@ TEST(cli_windows_claude_hook_scripts_migrate_and_uninstall_all_owned_shapes) { } int current_upgrade_rc = - current_scripts_ready ? cbm_install_agent_configs(tmpdir, binary_path, false, false) : -1; + current_scripts_ready ? cbm_install_agent_hooks_for_testing(tmpdir, binary_path, false, false) : -1; bool current_legacy_removed = current_upgrade_rc == 0; for (size_t i = 0U; i < sizeof(legacy_names) / sizeof(legacy_names[0]); i++) { char current_path[768]; @@ -9049,7 +9049,7 @@ TEST(cli_windows_claude_hook_scripts_migrate_and_uninstall_all_owned_shapes) { } int released_upgrade_rc = - released_ready ? cbm_install_agent_configs(tmpdir, binary_path, false, false) : -1; + released_ready ? cbm_install_agent_hooks_for_testing(tmpdir, binary_path, false, false) : -1; bool released_legacy_removed = released_upgrade_rc == 0; for (size_t i = 0U; i < sizeof(legacy_names) / sizeof(legacy_names[0]); i++) { char legacy_path[768]; diff --git a/tests/test_cypher.c b/tests/test_cypher.c index 96225c1bc6..e90830018a 100644 --- a/tests/test_cypher.c +++ b/tests/test_cypher.c @@ -209,6 +209,71 @@ TEST(cypher_parse_simple_node) { PASS(); } +/* Trailing input must be an error, never a silent drop. The parser used to + * stop at the first thing it did not understand and report success, so the + * engine answered from the fragment it had parsed. */ +TEST(cypher_parse_rejects_trailing_tokens) { + cbm_query_t *q = NULL; + char *err = NULL; + int rc = cbm_cypher_parse("MATCH (f:Function) RETURN f.name AS n BANANA SPLIT 99", &q, &err); + ASSERT_NEQ(rc, 0); + ASSERT_NOT_NULL(err); + ASSERT_NULL(q); + + /* The message must name what actually stopped the parse, and must not + * mention WITH: this query has no WITH in it anywhere. */ + ASSERT(strstr(err, "BANANA") != NULL); + ASSERT(strstr(err, "WITH") == NULL); + + free(err); + PASS(); +} + +/* Only one WITH is supported. A second one used to take the rest of the + * query with it — the filter and the RETURN both vanished, and every row + * came back unfiltered under the default projection. */ +TEST(cypher_parse_rejects_second_with_clause) { + cbm_query_t *q = NULL; + char *err = NULL; + int rc = cbm_cypher_parse("MATCH (f:Function) " + "OPTIONAL MATCH (a)-[:CALLS]->(f) " + "WITH f, count(a) AS calls " + "OPTIONAL MATCH (b)-[:USAGE]->(f) " + "WITH f, calls, count(b) AS usages " + "WHERE calls = 0 AND usages = 0 " + "RETURN f.name AS n", + &q, &err); + ASSERT_NEQ(rc, 0); + ASSERT_NOT_NULL(err); + ASSERT_NULL(q); + + /* Here the note earns its place. The parse stops at OPTIONAL, and the + * reason is the second WITH further along, which the reader cannot see + * from the stopping point alone. */ + ASSERT(strstr(err, "only one WITH clause is supported") != NULL); + + free(err); + PASS(); +} + +/* The guard must not reject a query that is simply finished. One WITH, a + * WHERE after it and a RETURN is the shape the grammar does support. */ +TEST(cypher_parse_accepts_single_with_clause) { + cbm_query_t *q = NULL; + char *err = NULL; + int rc = cbm_cypher_parse("MATCH (f:Function) " + "WITH f, f.name AS n " + "WHERE n = 'buildTree' " + "RETURN n", + &q, &err); + ASSERT_EQ(rc, 0); + ASSERT_NULL(err); + ASSERT_NOT_NULL(q); + + cbm_query_free(q); + PASS(); +} + TEST(cypher_parse_relationship_outbound) { cbm_query_t *q = NULL; char *err = NULL; @@ -4442,6 +4507,9 @@ SUITE(cypher) { RUN_TEST(cypher_lex_full_query); /* Parser */ RUN_TEST(cypher_parse_simple_node); + RUN_TEST(cypher_parse_rejects_trailing_tokens); + RUN_TEST(cypher_parse_rejects_second_with_clause); + RUN_TEST(cypher_parse_accepts_single_with_clause); RUN_TEST(cypher_parse_relationship_outbound); RUN_TEST(cypher_parse_relationship_inbound); RUN_TEST(cypher_parse_relationship_any); diff --git a/tests/test_daemon_bootstrap.c b/tests/test_daemon_bootstrap.c index 7b39b2e441..c598fbd511 100644 --- a/tests/test_daemon_bootstrap.c +++ b/tests/test_daemon_bootstrap.c @@ -398,6 +398,7 @@ TEST(daemon_bootstrap_runtime_dir_env_relocates_rendezvous) { char override_parent[BOOTSTRAP_TEST_PATH_CAP] = {0}; char canonical_override[BOOTSTRAP_TEST_PATH_CAP] = {0}; char canonical_explicit[BOOTSTRAP_TEST_PATH_CAP] = {0}; + char canonical_missing[BOOTSTRAP_TEST_PATH_CAP] = {0}; char relocated_runtime[BOOTSTRAP_TEST_PATH_CAP] = {0}; char explicit_runtime[BOOTSTRAP_TEST_PATH_CAP] = {0}; char missing_parent[BOOTSTRAP_TEST_PATH_CAP] = {0}; @@ -439,6 +440,9 @@ TEST(daemon_bootstrap_runtime_dir_env_relocates_rendezvous) { if (created) { (void)snprintf(created_runtime, sizeof(created_runtime), "%s", cbm_daemon_ipc_endpoint_runtime_dir(created)); + /* The endpoint reports the canonical path; on macOS the temp root is + * under /var, which resolves to /private/var. */ + (void)cbm_canonical_path(missing_parent, canonical_missing, sizeof(canonical_missing)); } /* Restore before asserting: a failed assertion returns immediately, and a @@ -473,7 +477,7 @@ TEST(daemon_bootstrap_runtime_dir_env_relocates_rendezvous) { ASSERT_TRUE(bootstrap_path_has_parent(explicit_runtime, canonical_explicit)); ASSERT_FALSE(bootstrap_path_has_parent(explicit_runtime, canonical_override)); ASSERT_NOT_NULL(created); - ASSERT_TRUE(bootstrap_path_has_parent(created_runtime, missing_parent)); + ASSERT_TRUE(bootstrap_path_has_parent(created_runtime, canonical_missing)); PASS(); } @@ -961,6 +965,8 @@ typedef struct { char parent[BOOTSTRAP_TEST_PATH_CAP]; cbm_daemon_build_identity_t identity; pid_t children[BOOTSTRAP_ENOSPC_MAX_CHILDREN]; + int exit_status[BOOTSTRAP_ENOSPC_MAX_CHILDREN]; + bool reaped[BOOTSTRAP_ENOSPC_MAX_CHILDREN]; size_t child_count; size_t spawn_calls; } bootstrap_enospc_host_t; @@ -996,19 +1002,32 @@ static bool bootstrap_enospc_host_spawn(void *opaque, int run_result = endpoint ? cbm_daemon_host_run(&config) : 0; _exit(run_result == -1 ? 0 : 50); } - state->children[state->child_count++] = child; + /* SYNCHRONOUS on purpose (ported from upstream). The daemon host writes + * its start-failure record and only then releases its lifetime reservation + * and exits, so once it is reaped "record on disk, reservation released" is + * a stable state. Returning while the host was still running let the + * client fail fast on the record and the reaper SIGKILL a host that had not + * exited yet, counting it as a nonzero exit on slow runners. */ + size_t slot = state->child_count++; + state->children[slot] = child; + int status = 0; + pid_t waited; + do { + waited = waitpid(child, &status, 0); + } while (waited < 0 && errno == EINTR); + state->reaped[slot] = waited == child; + state->exit_status[slot] = status; return true; } static void bootstrap_enospc_reap(bootstrap_enospc_host_t *state, int *nonzero_exits) { *nonzero_exits = 0; for (size_t i = 0; i < state->child_count; i++) { - int status = 0; - pid_t waited; - do { - waited = waitpid(state->children[i], &status, WNOHANG); - } while (waited < 0 && errno == EINTR); - if (waited == 0) { + int status = state->exit_status[i]; + if (!state->reaped[i]) { + /* Only reachable when the synchronous wait in the spawn itself + * failed; never leave a host behind. */ + pid_t waited; (void)kill(state->children[i], SIGKILL); do { waited = waitpid(state->children[i], &status, 0); diff --git a/tests/test_daemon_runtime.c b/tests/test_daemon_runtime.c index e8e530f77e..3859a52c14 100644 --- a/tests/test_daemon_runtime.c +++ b/tests/test_daemon_runtime.c @@ -570,6 +570,7 @@ static bool runtime_test_link_shared_image(const char *destination) { #endif } +#ifdef __linux__ /* only the Linux changed-image case restores the image */ static bool runtime_test_restore_shared_image(void) { if (!runtime_shared_image[0]) { return false; @@ -592,6 +593,7 @@ static bool runtime_test_restore_shared_image(void) { return truncate(runtime_shared_image, (off_t)runtime_shared_image_size) == 0; #endif } +#endif static void runtime_test_shared_image_cleanup(void) { if (runtime_shared_image[0]) { diff --git a/tests/test_index_resilience.c b/tests/test_index_resilience.c index 7fcb152e2c..9ae2bd79ae 100644 --- a/tests/test_index_resilience.c +++ b/tests/test_index_resilience.c @@ -92,7 +92,7 @@ static char *ri_slurp(const char *path) { * flow, capturing the raw response. Returns the opened graph store (NULL on * failure). Mirrors repro_harness.h's rh_open_indexed but keeps the response so * we can assert on skipped_count / skipped[] / logfile. */ -static cbm_store_t *ri_index_capture(RProj *lp, char **out_resp) { +static cbm_store_t *ri_index_capture_with(RProj *lp, const char *extra_args, char **out_resp) { lp->project = cbm_project_name_from_path(lp->tmpdir); if (!lp->project) { return NULL; @@ -110,8 +110,9 @@ static cbm_store_t *ri_index_capture(RProj *lp, char **out_resp) { if (!lp->srv) { return NULL; } - char args[700]; - snprintf(args, sizeof(args), "{\"repo_path\":\"%s\"}", lp->tmpdir); + char args[1400]; + snprintf(args, sizeof(args), "{\"repo_path\":\"%s\"%s}", lp->tmpdir, + extra_args ? extra_args : ""); char *resp = cbm_test_operation_execute(lp->srv, "index_repository", args); if (out_resp) { *out_resp = resp; @@ -121,6 +122,10 @@ static cbm_store_t *ri_index_capture(RProj *lp, char **out_resp) { return cbm_store_open_path(lp->dbpath); } +static cbm_store_t *ri_index_capture(RProj *lp, char **out_resp) { + return ri_index_capture_with(lp, NULL, out_resp); +} + /* ── Tests ──────────────────────────────────────────────────────── */ /* INV(oversized-reported): with the size cap set LOW, indexing a repo that @@ -793,6 +798,77 @@ TEST(index_relative_repo_path_canonicalized) { #endif /* !_WIN32 */ } +/* INV(metrics-out-validated): metrics_out must resolve inside the allowed + * roots. A path in a directory outside CBM_ALLOWED_ROOT is refused and nothing + * is written there. */ +TEST(index_metrics_out_outside_root_refused) { + RProj lp; + memset(&lp, 0, sizeof(lp)); + snprintf(lp.tmpdir, sizeof(lp.tmpdir), "/tmp/cbm_resil_XXXXXX"); + if (!cbm_mkdtemp(lp.tmpdir)) { + FAIL("mkdtemp failed"); + } + rh_to_fwd_slashes(lp.tmpdir); + char outside[256]; + snprintf(outside, sizeof(outside), "/tmp/cbm_resil_out_XXXXXX"); + if (!cbm_mkdtemp(outside)) { + FAIL("mkdtemp failed"); + } + rh_to_fwd_slashes(outside); + ri_write_text(lp.tmpdir, "good.py", "def alpha():\n return 1\n"); + + cbm_setenv("CBM_ALLOWED_ROOT", lp.tmpdir, 1); + char extra[300], metrics[300]; + snprintf(metrics, sizeof(metrics), "%s/metrics.json", outside); + snprintf(extra, sizeof(extra), ",\"metrics_out\":\"%s\"", metrics); + char *resp = NULL; + cbm_store_t *store = ri_index_capture_with(&lp, extra, &resp); + cbm_unsetenv("CBM_ALLOWED_ROOT"); + + ASSERT_NOT_NULL(resp); + ASSERT_NOT_NULL(strstr(resp, "outside the allowed root")); + FILE *probe = fopen(metrics, "rb"); + ASSERT_NULL(probe); + free(resp); + rmdir(outside); + rh_cleanup(&lp, store); + PASS(); +} + +/* INV(metrics-out-written): a successful index with metrics_out inside the + * repo writes parseable metrics JSON with the success outcome. */ +TEST(index_metrics_out_written) { + RProj lp; + memset(&lp, 0, sizeof(lp)); + snprintf(lp.tmpdir, sizeof(lp.tmpdir), "/tmp/cbm_resil_XXXXXX"); + if (!cbm_mkdtemp(lp.tmpdir)) { + FAIL("mkdtemp failed"); + } + rh_to_fwd_slashes(lp.tmpdir); + ri_write_text(lp.tmpdir, "good.py", "def alpha():\n return 1\n"); + + char extra[300], metrics[300]; + snprintf(metrics, sizeof(metrics), "%s/metrics.json", lp.tmpdir); + snprintf(extra, sizeof(extra), ",\"metrics_out\":\"%s\"", metrics); + char *resp = NULL; + cbm_store_t *store = ri_index_capture_with(&lp, extra, &resp); + ASSERT_NOT_NULL(resp); + free(resp); + + char *json = ri_slurp(metrics); + ASSERT_NOT_NULL(json); + yyjson_doc *d = yyjson_read(json, strlen(json), 0); + ASSERT_NOT_NULL(d); + yyjson_val *root = yyjson_doc_get_root(d); + ASSERT_STR_EQ(yyjson_get_str(yyjson_obj_get(root, "outcome")), "success"); + ASSERT_NOT_NULL(yyjson_obj_get(root, "counts")); + yyjson_doc_free(d); + free(json); + unlink(metrics); + rh_cleanup(&lp, store); + PASS(); +} + SUITE(index_resilience) { RUN_TEST(index_oversized_file_reported); RUN_TEST(index_clean_run_no_logfile); @@ -800,4 +876,6 @@ SUITE(index_resilience) { RUN_TEST(index_parse_partial_clears_on_fix); RUN_TEST(index_not_indexed_by_design_reported); RUN_TEST(index_relative_repo_path_canonicalized); + RUN_TEST(index_metrics_out_outside_root_refused); + RUN_TEST(index_metrics_out_written); } diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c index 9c830d8f4e..c4248bc4ba 100644 --- a/tests/test_pipeline.c +++ b/tests/test_pipeline.c @@ -1059,14 +1059,13 @@ static NamedEdgePropertyObservation observe_named_edge_callee_property( } observation.database_opened = true; - static const char sql[] = - "SELECT e.properties, json_valid(e.properties), " - "CASE WHEN json_valid(e.properties) " - "THEN json_extract(e.properties, '$.callee') END " - "FROM edges e " - "JOIN nodes src ON src.id=e.source_id AND src.project=e.project " - "JOIN nodes tgt ON tgt.id=e.target_id AND tgt.project=e.project " - "WHERE e.project=?1 AND e.type=?2 AND src.name=?3 AND tgt.name=?4;"; + static const char sql[] = "SELECT e.properties, json_valid(e.properties), " + "CASE WHEN json_valid(e.properties) " + "THEN json_extract(e.properties, '$.callee') END " + "FROM edges e " + "JOIN nodes src ON src.id=e.source_id AND src.project=e.project " + "JOIN nodes tgt ON tgt.id=e.target_id AND tgt.project=e.project " + "WHERE e.project=?1 AND e.type=?2 AND src.name=?3 AND tgt.name=?4;"; sqlite3_stmt *stmt = NULL; if (sqlite3_prepare_v2(db, sql, -1, &stmt, NULL) != SQLITE_OK || sqlite3_bind_text(stmt, 1, project, -1, SQLITE_TRANSIENT) != SQLITE_OK || @@ -1754,13 +1753,13 @@ TEST(pipeline_call_reference_sequential_parallel_edge_set_parity) { long_reference_name[0] = 'l'; long_reference_name[LONG_REFERENCE_NAME_LEN] = '\0'; char long_reference_source[1024]; - int long_reference_source_len = snprintf( - long_reference_source, sizeof(long_reference_source), - "package parity\n" - "func %s() {}\n" - "func longPropertiesReferenceAccept(callback func()) {}\n" - "func longPropertiesReferenceSite() { longPropertiesReferenceAccept(%s) }\n", - long_reference_name, long_reference_name); + int long_reference_source_len = + snprintf(long_reference_source, sizeof(long_reference_source), + "package parity\n" + "func %s() {}\n" + "func longPropertiesReferenceAccept(callback func()) {}\n" + "func longPropertiesReferenceSite() { longPropertiesReferenceAccept(%s) }\n", + long_reference_name, long_reference_name); if (long_reference_source_len <= 0 || (size_t)long_reference_source_len >= sizeof(long_reference_source)) { th_rmtree(tmp); @@ -1948,18 +1947,18 @@ TEST(pipeline_call_reference_sequential_parallel_edge_set_parity) { named_edge_count(sequential_store, sequential_project, "CALLS", shadow_controls[i].source_name, shadow_controls[i].target_name); } - sequential_long_reference = named_edge_count( - sequential_store, sequential_project, "CALL_REFERENCE", "longPropertiesReferenceSite", - long_reference_name); - sequential_long_usage = named_edge_count(sequential_store, sequential_project, "USAGE", - "longPropertiesReferenceSite", - long_reference_name); - sequential_long_calls = named_edge_count(sequential_store, sequential_project, "CALLS", - "longPropertiesReferenceSite", - long_reference_name); + sequential_long_reference = + named_edge_count(sequential_store, sequential_project, "CALL_REFERENCE", + "longPropertiesReferenceSite", long_reference_name); + sequential_long_usage = + named_edge_count(sequential_store, sequential_project, "USAGE", + "longPropertiesReferenceSite", long_reference_name); + sequential_long_calls = + named_edge_count(sequential_store, sequential_project, "CALLS", + "longPropertiesReferenceSite", long_reference_name); sequential_long_property = observe_named_edge_callee_property( - sequential_db_path, sequential_project, "CALL_REFERENCE", - "longPropertiesReferenceSite", long_reference_name, long_reference_name); + sequential_db_path, sequential_project, "CALL_REFERENCE", "longPropertiesReferenceSite", + long_reference_name, long_reference_name); cbm_store_close(sequential_store); } cbm_pipeline_free(sequential); @@ -1997,9 +1996,9 @@ TEST(pipeline_call_reference_sequential_parallel_edge_set_parity) { named_edge_count(parallel_store, parallel_project, "CALLS", shadow_controls[i].source_name, shadow_controls[i].target_name); } - parallel_long_reference = named_edge_count( - parallel_store, parallel_project, "CALL_REFERENCE", "longPropertiesReferenceSite", - long_reference_name); + parallel_long_reference = + named_edge_count(parallel_store, parallel_project, "CALL_REFERENCE", + "longPropertiesReferenceSite", long_reference_name); parallel_long_usage = named_edge_count(parallel_store, parallel_project, "USAGE", "longPropertiesReferenceSite", long_reference_name); parallel_long_calls = named_edge_count(parallel_store, parallel_project, "CALLS", @@ -2176,14 +2175,13 @@ static char *cx_order_signature(const char *db_path, const char *project, int *f const char *props = sorted[i]->properties_json ? sorted[i]->properties_json : "{}"; const char *tld = strstr(props, "\"transitive_loop_depth\":"); const char *rec = strstr(props, "\"recursive\":"); - int w = snprintf(sig + used, CX_ORDER_LINE_MAX, "%s %s %d %d %s %s %.*s %.*s\n", - sorted[i]->qualified_name ? sorted[i]->qualified_name : "", - sorted[i]->file_path ? sorted[i]->file_path : "", - sorted[i]->start_line, sorted[i]->end_line, - sorted[i]->name ? sorted[i]->name : "", - sorted[i]->label ? sorted[i]->label : "", - tld ? (int)strcspn(tld, ",}") : 0, tld ? tld : "", - rec ? (int)strcspn(rec, ",}") : 0, rec ? rec : ""); + int w = snprintf( + sig + used, CX_ORDER_LINE_MAX, "%s %s %d %d %s %s %.*s %.*s\n", + sorted[i]->qualified_name ? sorted[i]->qualified_name : "", + sorted[i]->file_path ? sorted[i]->file_path : "", sorted[i]->start_line, + sorted[i]->end_line, sorted[i]->name ? sorted[i]->name : "", + sorted[i]->label ? sorted[i]->label : "", tld ? (int)strcspn(tld, ",}") : 0, + tld ? tld : "", rec ? (int)strcspn(rec, ",}") : 0, rec ? rec : ""); if (w < 0) { w = 0; } else if (w >= CX_ORDER_LINE_MAX) { @@ -2304,7 +2302,7 @@ TEST(pipeline_complexity_props_independent_of_worker_order) { ASSERT_EQ(sequential_rc, 0); ASSERT_NOT_NULL(sequential_sig); ASSERT_GTE(sequential_funcs, copied); /* at least the one function per fixture file */ - ASSERT_TRUE(cycles_detected); /* the cycles must reach the pass at all */ + ASSERT_TRUE(cycles_detected); /* the cycles must reach the pass at all */ if (mismatch_run >= 0) { printf("\n parallel run %d diverges from sequential: %s\n", mismatch_run, diff); FAIL("complexity props depend on worker id order"); @@ -2722,8 +2720,8 @@ static void closure_probe_repo(const char *tmp) { } /* Fresh full reference build of the same tree into its own DB. */ -static void closure_fresh_full(const char *tmp, const char *db_path, int *out_nodes, - int *out_edges, int *out_ref_edges, const char *project_hint) { +static void closure_fresh_full(const char *tmp, const char *db_path, int *out_nodes, int *out_edges, + int *out_ref_edges, const char *project_hint) { *out_nodes = -1; *out_edges = -2; *out_ref_edges = -3; @@ -2740,9 +2738,9 @@ static void closure_fresh_full(const char *tmp, const char *db_path, int *out_no if (store) { *out_nodes = cbm_store_count_nodes(store, project); *out_edges = cbm_store_count_edges(store, project); - *out_ref_edges = named_edge_to_file_count(store, project, "CALL_REFERENCE", - "closureProbeCaller", "closureProbeHelper", - "lib.ts"); + *out_ref_edges = + named_edge_to_file_count(store, project, "CALL_REFERENCE", "closureProbeCaller", + "closureProbeHelper", "lib.ts"); cbm_store_close(store); } } @@ -2827,8 +2825,8 @@ TEST(pipeline_closure_repair_body_edit_converges_with_fresh_full) { ASSERT_NOT_NULL(store); repaired_nodes = cbm_store_count_nodes(store, project); repaired_edges = cbm_store_count_edges(store, project); - repaired_refs = named_edge_to_file_count(store, project, "CALL_REFERENCE", - "closureProbeCaller", "closureProbeHelper", "lib.ts"); + repaired_refs = named_edge_to_file_count(store, project, "CALL_REFERENCE", "closureProbeCaller", + "closureProbeHelper", "lib.ts"); cbm_store_close(store); char full_db[512]; @@ -2884,8 +2882,8 @@ TEST(pipeline_closure_repair_removed_def_drops_dependent_edge) { int repaired_refs = -1; cbm_store_t *store = cbm_store_open_path(db); ASSERT_NOT_NULL(store); - repaired_refs = named_edge_to_file_count(store, project, "CALL_REFERENCE", - "closureProbeCaller", "closureProbeHelper", "lib.ts"); + repaired_refs = named_edge_to_file_count(store, project, "CALL_REFERENCE", "closureProbeCaller", + "closureProbeHelper", "lib.ts"); int repaired_nodes = cbm_store_count_nodes(store, project); int repaired_edges = cbm_store_count_edges(store, project); cbm_store_close(store); @@ -3128,8 +3126,7 @@ TEST(pipeline_incremental_tsconfig_alias_change_matches_fresh_full) { * target_a.ts to target_b.ts. Since alias-config governance landed this * runs as a closure repair, and the convergence assertions below now * prove that route rather than being satisfied by a full rebuild. */ - ASSERT_EQ(cbm_pipeline_incremental_test_last_route(), - CBM_INCREMENTAL_ROUTE_CLOSURE_REPAIR); + ASSERT_EQ(cbm_pipeline_incremental_test_last_route(), CBM_INCREMENTAL_ROUTE_CLOSURE_REPAIR); const char *incremental_project = cbm_pipeline_project_name(incremental); cbm_store_t *incremental_store = cbm_store_open_path(incremental_db); ASSERT_NOT_NULL(incremental_store); @@ -3317,8 +3314,8 @@ TEST(pipeline_publication_never_uses_a_predictable_staging_path) { static const char canary[] = "canary-must-survive\n"; char canary_path[PREDICTABLE_CANARIES][640]; for (int i = 0; i < PREDICTABLE_CANARIES; i++) { - snprintf(canary_path[i], sizeof(canary_path[i]), "%s.stage.%ld.%d", db_path, - (long)getpid(), i + 1); + snprintf(canary_path[i], sizeof(canary_path[i]), "%s.stage.%ld.%d", db_path, (long)getpid(), + i + 1); ASSERT_EQ(th_write_file(canary_path[i], canary), 0); } @@ -3362,6 +3359,481 @@ TEST(pipeline_publication_never_uses_a_predictable_staging_path) { PASS(); } +static int count_substring(const char *haystack, const char *needle) { + int count = 0; + size_t needle_len = strlen(needle); + for (const char *at = strstr(haystack, needle); at; at = strstr(at + needle_len, needle)) { + count++; + } + return count; +} + +static int count_nested_stage_entries(const char *dir_path, const char *db_basename) { + cbm_dir_t *dir = cbm_opendir(dir_path); + if (!dir) { + return -1; + } + size_t base_len = strlen(db_basename); + int count = 0; + cbm_dirent_t *entry; + while ((entry = cbm_readdir(dir)) != NULL) { + if (strncmp(entry->name, db_basename, base_len) == 0 && + count_substring(entry->name + base_len, ".stage.") >= 2) { + count++; + } + } + cbm_closedir(dir); + return count; +} + +/* #1839: the outer run rewrites the pipeline's db_path to its stage, so the + * inner publication (dump or delta clone) minted ITS stage from a stage: + * .stage.A.stage.B, plus -wal/-shm under WAL. A generation's stage is a + * sibling of the live database, whichever path it is minted from; a database + * whose own basename merely contains ".stage." is not a stage and keeps its + * full name. */ +TEST(pipeline_stage_names_never_nest) { + char tmp[256]; + snprintf(tmp, sizeof(tmp), "/tmp/cbm_stage_nesting_XXXXXX"); + ASSERT_NOT_NULL(cbm_mkdtemp(tmp)); + char db_path[512]; + snprintf(db_path, sizeof(db_path), "%s/generation.db", tmp); + char odd_db[512]; + snprintf(odd_db, sizeof(odd_db), "%s/x.stage.y.db", tmp); + + char *outer = cbm_pipeline_create_staging_path(db_path); + ASSERT_NOT_NULL(outer); + char *inner = cbm_pipeline_create_staging_path(outer); + ASSERT_NOT_NULL(inner); + char *odd_stage = cbm_pipeline_create_staging_path(odd_db); + ASSERT_NOT_NULL(odd_stage); + + size_t db_len = strlen(db_path); + size_t odd_len = strlen(odd_db); + int outer_tokens = count_substring(outer, ".stage."); + int inner_tokens = count_substring(inner, ".stage."); + bool inner_is_sibling = + strncmp(inner, db_path, db_len) == 0 && strncmp(inner + db_len, ".stage.", 7) == 0; + bool inner_distinct = strcmp(inner, outer) != 0; + bool odd_keeps_basename = + strncmp(odd_stage, odd_db, odd_len) == 0 && strncmp(odd_stage + odd_len, ".stage.", 7) == 0; + int nested_entries = count_nested_stage_entries(tmp, "generation.db"); + + cbm_pipeline_discard_stage(inner); + cbm_pipeline_discard_stage(outer); + cbm_pipeline_discard_stage(odd_stage); + int leftover_entries = count_generation_stage_artifacts(tmp, "generation.db") + + count_generation_stage_artifacts(tmp, "x.stage.y.db"); + free(inner); + free(outer); + free(odd_stage); + th_rmtree(tmp); + + ASSERT_EQ(outer_tokens, 1); + ASSERT_EQ(inner_tokens, 1); + ASSERT_TRUE(inner_is_sibling); + ASSERT_TRUE(inner_distinct); + ASSERT_TRUE(odd_keeps_basename); + ASSERT_EQ(nested_entries, 0); + ASSERT_EQ(leftover_entries, 0); + PASS(); +} + +static bool path_exists(const char *path) { + cbm_path_info_t info; + return cbm_path_info_utf8(path, &info) == 0; +} + +/* #1839: a minted stage is OWNED through an exclusive kernel lock on its + * ".lock" sidecar for exactly as long as the stage exists. A second + * holder cannot take it while the writer is live; discarding the stage frees + * the name and removes the sidecar, so nothing stays behind. */ +TEST(pipeline_minted_stage_is_owned_until_released) { + char tmp[256]; + snprintf(tmp, sizeof(tmp), "/tmp/cbm_stage_owner_XXXXXX"); + ASSERT_NOT_NULL(cbm_mkdtemp(tmp)); + char db_path[512]; + snprintf(db_path, sizeof(db_path), "%s/generation.db", tmp); + + char *stage = cbm_pipeline_create_staging_path(db_path); + ASSERT_NOT_NULL(stage); + char lock_path[600]; + snprintf(lock_path, sizeof(lock_path), "%s.lock", stage); + bool lock_present_while_live = path_exists(lock_path); + int hold_while_live = cbm_pipeline_stage_lock_hold(stage); + if (hold_while_live >= 0) { + cbm_pipeline_stage_lock_drop(stage, hold_while_live); + } + + cbm_pipeline_discard_stage(stage); + bool stage_gone = !path_exists(stage); + bool lock_gone = !path_exists(lock_path); + + int hold_after_discard = cbm_pipeline_stage_lock_hold(stage); + int second_holder = cbm_pipeline_stage_lock_hold(stage); + if (second_holder >= 0) { + cbm_pipeline_stage_lock_drop(stage, second_holder); + } + cbm_pipeline_stage_lock_drop(stage, hold_after_discard); + int hold_after_drop = cbm_pipeline_stage_lock_hold(stage); + cbm_pipeline_stage_lock_drop(stage, hold_after_drop); + int leftovers = count_generation_stage_artifacts(tmp, "generation.db"); + free(stage); + th_rmtree(tmp); + + ASSERT_TRUE(lock_present_while_live); + ASSERT_EQ(hold_while_live, -1); + ASSERT_TRUE(stage_gone); + ASSERT_TRUE(lock_gone); + ASSERT_TRUE(hold_after_discard >= 0); + ASSERT_EQ(second_holder, -1); + ASSERT_TRUE(hold_after_drop >= 0); + ASSERT_EQ(leftovers, 0); + PASS(); +} + +static bool file_has_content(const char *path, const char *expected) { + FILE *f = cbm_fopen(path, "rb"); + if (!f) { + return false; + } + char buf[256] = {0}; + size_t n = fread(buf, 1, sizeof(buf) - 1, f); + (void)fclose(f); + return n == strlen(expected) && memcmp(buf, expected, n) == 0; +} + +/* #1839 / #1864: a stage a dead writer left beside a VALID database is swept + * by the next run and never influences its route. The 0-byte shape is what a + * worker killed before its backup wrote a page leaves behind; a stale -shm + * beside it is swept with it. */ +TEST(pipeline_stale_zero_byte_stage_beside_valid_db_routes_incremental_and_is_swept) { + char tmp[256]; + snprintf(tmp, sizeof(tmp), "/tmp/cbm_stale_stage_XXXXXX"); + ASSERT_NOT_NULL(cbm_mkdtemp(tmp)); + write_temp_file(tmp, "generation.py", "def StableGeneration():\n return 1\n"); + char db_path[512]; + snprintf(db_path, sizeof(db_path), "%s/generation.db", tmp); + + cbm_pipeline_incremental_test_reset_faults(); + cbm_pipeline_t *baseline = cbm_pipeline_new(tmp, db_path, CBM_MODE_FULL); + ASSERT_NOT_NULL(baseline); + ASSERT_EQ(cbm_pipeline_run(baseline), 0); + char project[256]; + snprintf(project, sizeof(project), "%s", cbm_pipeline_project_name(baseline)); + cbm_pipeline_free(baseline); + + char stale_stage[600]; + char stale_shm[600]; + snprintf(stale_stage, sizeof(stale_stage), "%s.stage.deadbe", db_path); + snprintf(stale_shm, sizeof(stale_shm), "%s.stage.deadbe-shm", db_path); + ASSERT_EQ(th_write_file(stale_stage, ""), 0); + ASSERT_EQ(th_write_file(stale_shm, "stale-shm"), 0); + + /* A body-only change: no added names, so the planner may repair. */ + write_temp_file(tmp, "generation.py", "def StableGeneration():\n return 2\n"); + cbm_pipeline_incremental_test_reset_faults(); + cbm_pipeline_t *incr = cbm_pipeline_new(tmp, db_path, CBM_MODE_FULL); + ASSERT_NOT_NULL(incr); + int incr_rc = cbm_pipeline_run(incr); + cbm_incremental_route_t route = cbm_pipeline_incremental_test_last_route(); + cbm_pipeline_free(incr); + + bool stale_stage_gone = !path_exists(stale_stage); + bool stale_shm_gone = !path_exists(stale_shm); + int stage_count = count_generation_stage_artifacts(tmp, "generation.db"); + int stable_count = -1; + int absent_count = -1; + observe_named_generation(db_path, project, "StableGeneration", "NeverDefined", &stable_count, + &absent_count); + cbm_pipeline_incremental_test_reset_faults(); + th_rmtree(tmp); + + ASSERT_EQ(incr_rc, 0); + ASSERT_TRUE(route != CBM_INCREMENTAL_ROUTE_FORCED_FULL); + ASSERT_TRUE(route != CBM_INCREMENTAL_ROUTE_NONE); + ASSERT_TRUE(stale_stage_gone); + ASSERT_TRUE(stale_shm_gone); + ASSERT_EQ(stage_count, 0); + ASSERT_EQ(stable_count, 1); + ASSERT_EQ(absent_count, 0); + PASS(); +} + +/* #1839: the sweep removes exactly the stages nobody owns. A dead writer's + * stage (main, -wal, and the unlocked .lock sidecar its death left) goes; a + * stage whose writer is LIVE -- here the test, holding its lock -- is kept + * byte for byte, and goes only once that lock is dropped. No timing: liveness + * is the kernel lock, nothing else. */ +TEST(pipeline_sweep_removes_dead_writer_stage_keeps_locked_stage) { + char tmp[256]; + snprintf(tmp, sizeof(tmp), "/tmp/cbm_stage_sweep_XXXXXX"); + ASSERT_NOT_NULL(cbm_mkdtemp(tmp)); + write_temp_file(tmp, "generation.py", "def StableGeneration():\n return 1\n"); + char db_path[512]; + snprintf(db_path, sizeof(db_path), "%s/generation.db", tmp); + + cbm_pipeline_incremental_test_reset_faults(); + cbm_pipeline_t *baseline = cbm_pipeline_new(tmp, db_path, CBM_MODE_FULL); + ASSERT_NOT_NULL(baseline); + ASSERT_EQ(cbm_pipeline_run(baseline), 0); + char project[256]; + snprintf(project, sizeof(project), "%s", cbm_pipeline_project_name(baseline)); + cbm_pipeline_free(baseline); + + char dead_stage[600]; + char dead_wal[600]; + char dead_lock[600]; + char live_stage[600]; + char live_lock[600]; + snprintf(dead_stage, sizeof(dead_stage), "%s.stage.aaaaaa", db_path); + snprintf(dead_wal, sizeof(dead_wal), "%s.stage.aaaaaa-wal", db_path); + snprintf(dead_lock, sizeof(dead_lock), "%s.stage.aaaaaa.lock", db_path); + snprintf(live_stage, sizeof(live_stage), "%s.stage.bbbbbb", db_path); + snprintf(live_lock, sizeof(live_lock), "%s.stage.bbbbbb.lock", db_path); + static const char live_bytes[] = "live-stage-bytes"; + ASSERT_EQ(th_write_file(dead_stage, "dead-stage-bytes"), 0); + ASSERT_EQ(th_write_file(dead_wal, "dead-wal"), 0); + ASSERT_EQ(th_write_file(dead_lock, ""), 0); + ASSERT_EQ(th_write_file(live_stage, live_bytes), 0); + int live_fd = cbm_pipeline_stage_lock_hold(live_stage); + ASSERT_TRUE(live_fd >= 0); + + write_temp_file(tmp, "generation.py", "def StableGeneration():\n return 2\n"); + cbm_pipeline_incremental_test_reset_faults(); + cbm_pipeline_t *first = cbm_pipeline_new(tmp, db_path, CBM_MODE_FULL); + ASSERT_NOT_NULL(first); + int first_rc = cbm_pipeline_run(first); + cbm_pipeline_free(first); + + bool dead_stage_gone = !path_exists(dead_stage); + bool dead_wal_gone = !path_exists(dead_wal); + bool dead_lock_gone = !path_exists(dead_lock); + bool live_kept = file_has_content(live_stage, live_bytes); + bool live_lock_kept = path_exists(live_lock); + int stable_after_first = -1; + int absent_after_first = -1; + observe_named_generation(db_path, project, "StableGeneration", "NeverDefined", + &stable_after_first, &absent_after_first); + + cbm_pipeline_stage_lock_drop(live_stage, live_fd); + cbm_pipeline_incremental_test_reset_faults(); + cbm_pipeline_t *second = cbm_pipeline_new(tmp, db_path, CBM_MODE_FULL); + ASSERT_NOT_NULL(second); + int second_rc = cbm_pipeline_run(second); + cbm_pipeline_free(second); + + bool live_gone = !path_exists(live_stage); + bool live_lock_gone = !path_exists(live_lock); + int stage_count = count_generation_stage_artifacts(tmp, "generation.db"); + int stable_after_second = -1; + int absent_after_second = -1; + observe_named_generation(db_path, project, "StableGeneration", "NeverDefined", + &stable_after_second, &absent_after_second); + cbm_pipeline_incremental_test_reset_faults(); + th_rmtree(tmp); + + ASSERT_EQ(first_rc, 0); + ASSERT_TRUE(dead_stage_gone); + ASSERT_TRUE(dead_wal_gone); + ASSERT_TRUE(dead_lock_gone); + ASSERT_TRUE(live_kept); + ASSERT_TRUE(live_lock_kept); + ASSERT_EQ(stable_after_first, 1); + ASSERT_EQ(absent_after_first, 0); + ASSERT_EQ(second_rc, 0); + ASSERT_TRUE(live_gone); + ASSERT_TRUE(live_lock_gone); + ASSERT_EQ(stage_count, 0); + ASSERT_EQ(stable_after_second, 1); + ASSERT_EQ(absent_after_second, 0); + PASS(); +} + +/* #2111 create->register TOCTOU guard. The bug this pins: create_staging_path() + * used to make the stage's main file visible (via mkstemp) BEFORE taking its + * sidecar lock, and sweep_orphan_stages() treats an absent lock sidecar + * (ENOENT) as a confirmed-dead writer (kernel released the lock on death). So a + * second, concurrent cbm_pipeline_run() against the SAME final_path, landing + * its own sweep in that narrow unlocked window, removed the first run's + * in-flight stage out from under it: every extraction pass still completed + * (none touch the stage file on disk), but the publish that followed found its + * own stage gone. Two writers racing the same project is a real scenario this + * PR's own sweep exists to clean up after (auto_index; the recently-fixed + * stale-rendezvous-recovery retry path) -- not hypothetical, and exactly the + * shape of #2111's windows-guards red (every pass logged success, nothing was + * ever committed, "Pipeline failed" surfaced generic; on Windows the sidecar + * collision surfaced as EACCES/errno=13). + * + * The fix takes the sidecar lock BEFORE the main file becomes visible, so this + * hook -- fired the instant the main file exists -- finds the stage already + * lock-protected and the racing sweep keeps it. The hook fires at the same + * point under the old ordering, where the lock was NOT yet held, so this test + * goes RED if that ordering ever regresses. + * + * The racing run is cancelled immediately so it never reaches ITS OWN + * publish -- isolating the sweep's effect on the first run's stage from the + * separate question of two full runs both completing for the same project. */ +typedef struct { + const char *tmp_dir; + const char *db_path; + bool stage_survived; +} racing_sweep_arg_t; + +static bool find_sole_stage_path(const char *dir, const char *db_basename, char *out, + size_t out_sz) { + cbm_dir_t *d = cbm_opendir(dir); + if (!d) { + return false; + } + char prefix[256]; + snprintf(prefix, sizeof(prefix), "%s.stage.", db_basename); + size_t prefix_len = strlen(prefix); + bool found = false; + cbm_dirent_t *entry; + while ((entry = cbm_readdir(d)) != NULL) { + size_t name_len = strlen(entry->name); + enum { STAGE_SUFFIX_RANDOM_CHARS = 6 }; /* mirrors CBM_STAGE_SUFFIX_RANDOM_CHARS */ + if (strncmp(entry->name, prefix, prefix_len) == 0 && + name_len == prefix_len + STAGE_SUFFIX_RANDOM_CHARS) { + snprintf(out, out_sz, "%s/%s", dir, entry->name); + found = true; + break; + } + } + cbm_closedir(d); + return found; +} + +static void *racing_sweep_thread(void *arg) { + racing_sweep_arg_t *a = (racing_sweep_arg_t *)arg; + cbm_pipeline_t *p = cbm_pipeline_new(a->tmp_dir, a->db_path, CBM_MODE_FULL); + if (p) { + /* Its own sweep_orphan_stages() runs at the very start, before this + * run mints its own stage -- exactly like the first run's. Cancel + * immediately: this run must reach the sweep and nothing past it. */ + cbm_pipeline_cancel(p); + (void)cbm_pipeline_run(p); + cbm_pipeline_free(p); + } + return NULL; +} + +/* Fired from inside the FIRST run's create_staging_path(), the instant its + * stage main file exists (under the fix, with its sidecar lock already held; + * under the old create-then-lock ordering, before the lock was taken): run a + * second, cancelled cbm_pipeline_run() for the same project synchronously on + * another thread, so its sweep has every chance to reach the stage before + * control returns to the first run. */ +static void racing_sweep_hook(void *userdata) { + racing_sweep_arg_t *a = (racing_sweep_arg_t *)userdata; + char stage_path[600] = {0}; + bool had_stage = + find_sole_stage_path(a->tmp_dir, "generation.db", stage_path, sizeof(stage_path)); + cbm_thread_t tid; + if (cbm_thread_create(&tid, 0, racing_sweep_thread, a) == 0) { + cbm_thread_join(&tid); + } + a->stage_survived = had_stage && path_exists(stage_path); +} + +TEST(pipeline_concurrent_sweep_must_not_remove_inflight_stage) { + char tmp[256]; + snprintf(tmp, sizeof(tmp), "/tmp/cbm_stage_race_XXXXXX"); + ASSERT_NOT_NULL(cbm_mkdtemp(tmp)); + write_temp_file(tmp, "generation.py", "def StableGeneration():\n return 1\n"); + char db_path[512]; + snprintf(db_path, sizeof(db_path), "%s/generation.db", tmp); + + cbm_pipeline_incremental_test_reset_faults(); + racing_sweep_arg_t race_arg = {.tmp_dir = tmp, .db_path = db_path, .stage_survived = false}; + cbm_pipeline_incremental_test_after_stage_created_once(racing_sweep_hook, &race_arg); + + cbm_pipeline_t *p = cbm_pipeline_new(tmp, db_path, CBM_MODE_FULL); + ASSERT_NOT_NULL(p); + char project[256]; + snprintf(project, sizeof(project), "%s", cbm_pipeline_project_name(p)); + int rc = cbm_pipeline_run(p); + cbm_pipeline_free(p); + + bool db_exists = path_exists(db_path); + int defined_count = -1; + int absent_count = -1; + observe_named_generation(db_path, project, "StableGeneration", "NeverDefined", &defined_count, + &absent_count); + int stage_count = count_generation_stage_artifacts(tmp, "generation.db"); + cbm_pipeline_incremental_test_reset_faults(); + th_rmtree(tmp); + + ASSERT_TRUE(race_arg.stage_survived); + ASSERT_EQ(rc, 0); + ASSERT_TRUE(db_exists); + ASSERT_EQ(defined_count, 1); + ASSERT_EQ(absent_count, 0); + ASSERT_EQ(stage_count, 0); + PASS(); +} + +static char g_route_log_capture[8192]; +static atomic_flag g_route_log_spin = ATOMIC_FLAG_INIT; + +/* Keeps only the route decisions; worker threads log too, and only the + * appends need serialising. */ +static void capture_route_log_sink(const char *line) { + if (!line || !strstr(line, "pipeline.route")) { + return; + } + while (atomic_flag_test_and_set_explicit(&g_route_log_spin, memory_order_acquire)) {} + size_t used = strlen(g_route_log_capture); + size_t avail = sizeof(g_route_log_capture) - used; + if (avail > 1) { + int n = snprintf(g_route_log_capture + used, avail, "%s\n", line); + if (n < 0 || (size_t)n >= avail) { + g_route_log_capture[sizeof(g_route_log_capture) - 1] = '\0'; + } + } + atomic_flag_clear_explicit(&g_route_log_spin, memory_order_release); +} + +/* #1864: a first index has no previous generation. The route probe used to + * open the run's own EMPTY stage placeholder, fail its integrity check, and + * warn "reason=invalid_existing_db" on every first index of every project, + * which the report read as a corrupted database and a crash loop. A fresh + * index says what it is. */ +TEST(pipeline_fresh_index_never_reports_invalid_existing_db) { + char tmp[256]; + snprintf(tmp, sizeof(tmp), "/tmp/cbm_fresh_route_XXXXXX"); + ASSERT_NOT_NULL(cbm_mkdtemp(tmp)); + write_temp_file(tmp, "generation.py", "def StableGeneration():\n return 1\n"); + char db_path[512]; + snprintf(db_path, sizeof(db_path), "%s/generation.db", tmp); + + g_route_log_capture[0] = '\0'; + CBMLogLevel previous_level = cbm_log_get_level(); + cbm_log_set_level(CBM_LOG_DEBUG); + cbm_log_set_sink(capture_route_log_sink); + cbm_pipeline_incremental_test_reset_faults(); + cbm_pipeline_t *fresh = cbm_pipeline_new(tmp, db_path, CBM_MODE_FULL); + int fresh_rc = fresh ? cbm_pipeline_run(fresh) : -1; + cbm_pipeline_free(fresh); + cbm_log_set_sink(NULL); + cbm_log_set_level(previous_level); + + bool invalid_reported = strstr(g_route_log_capture, "invalid_existing_db") != NULL; + bool fresh_reported = strstr(g_route_log_capture, "reason=no_existing_db") != NULL; + bool db_present = path_exists(db_path); + int stage_count = count_generation_stage_artifacts(tmp, "generation.db"); + cbm_pipeline_incremental_test_reset_faults(); + th_rmtree(tmp); + + ASSERT_EQ(fresh_rc, 0); + ASSERT_TRUE(!invalid_reported); + ASSERT_TRUE(fresh_reported); + ASSERT_TRUE(db_present); + ASSERT_EQ(stage_count, 0); + PASS(); +} + /* Discovery and extraction must describe the same immutable generation. A * source file created after extraction is not present in the original file * list, so merely re-hashing that list cannot detect the race. Publication @@ -6824,7 +7296,7 @@ TEST(pipeline_swift_cross_package_import) { cbm_edge_t *edges = NULL; int ec = 0; ASSERT_EQ(cbm_store_find_edges_by_source_type(s, importer.id, "IMPORTS", &edges, &ec), - CBM_STORE_OK); + CBM_STORE_OK); bool found_exact_edge = false; for (int i = 0; i < ec; i++) { @@ -6902,19 +7374,18 @@ TEST(pipeline_python_cross_module_call) { * unique_name (candidates==1) is #1572 and is not this claim. */ TEST(pipeline_cross_language_same_name_does_not_share_calls_issue725) { const char *files[] = {"store.py", "app.py", "web/src/pages/Editor.js"}; - const char *contents[] = { - "class Store:\n" - " def commit(self):\n" - " return True\n", + const char *contents[] = {"class Store:\n" + " def commit(self):\n" + " return True\n", - "from store import Store\n" - "\n" - "def save():\n" - " return Store().commit()\n", + "from store import Store\n" + "\n" + "def save():\n" + " return Store().commit()\n", - "export function commit() {\n" - " return 1;\n" - "}\n"}; + "export function commit() {\n" + " return 1;\n" + "}\n"}; if (setup_lang_repo(files, contents, 3) != 0) FAIL("tmpdir"); @@ -9523,7 +9994,8 @@ TEST(registry_confidence_suffix_match) { TEST(registry_receiver_chain_refuses_library_unique_name_issue1893) { cbm_registry_t *reg = cbm_registry_new(); cbm_registry_add(reg, "data", "HomeboxUI.PickedFile.data", "Variable"); - cbm_resolution_t r = cbm_registry_resolve(reg, "URLSession.shared.data", "HomeboxUI.Net", NULL, NULL, 0); + cbm_resolution_t r = + cbm_registry_resolve(reg, "URLSession.shared.data", "HomeboxUI.Net", NULL, NULL, 0); ASSERT_NULL(r.qualified_name); cbm_registry_free(reg); PASS(); @@ -9533,7 +10005,8 @@ TEST(registry_receiver_chain_refuses_library_suffix_match_issue1893) { cbm_registry_t *reg = cbm_registry_new(); cbm_registry_add(reg, "data", "HomeboxUI.PickedFile.data", "Variable"); cbm_registry_add(reg, "data", "HomeboxUI.Payload.data", "Variable"); - cbm_resolution_t r = cbm_registry_resolve(reg, "URLSession.shared.data", "HomeboxUI.Net", NULL, NULL, 0); + cbm_resolution_t r = + cbm_registry_resolve(reg, "URLSession.shared.data", "HomeboxUI.Net", NULL, NULL, 0); ASSERT_NULL(r.qualified_name); cbm_registry_free(reg); PASS(); @@ -9542,7 +10015,8 @@ TEST(registry_receiver_chain_refuses_library_suffix_match_issue1893) { TEST(registry_receiver_chain_keeps_project_extension_issue1893) { cbm_registry_t *reg = cbm_registry_new(); cbm_registry_add(reg, "startOfDayUTC", "AuthDTOs.Calendar.startOfDayUTC", "Method"); - cbm_resolution_t r = cbm_registry_resolve(reg, "Calendar.utcGregorian.startOfDayUTC", "HomeboxUI.Stats", NULL, NULL, 0); + cbm_resolution_t r = cbm_registry_resolve(reg, "Calendar.utcGregorian.startOfDayUTC", + "HomeboxUI.Stats", NULL, NULL, 0); ASSERT_STR_EQ(r.qualified_name, "AuthDTOs.Calendar.startOfDayUTC"); ASSERT_STR_EQ(r.strategy, "unique_name"); cbm_registry_free(reg); @@ -10310,17 +10784,16 @@ static const char *pkg_entries_entry_for(const cbm_pkg_entries_t *e, const char * above for the full end-to-end proof. */ TEST(pkgmap_swift_targets_registers_module) { - static const char src[] = - "// swift-tools-version:5.9\n" - "import PackageDescription\n" - "let package = Package(\n" - " name: \"Core\",\n" - " targets: [.target(name: \"Core\", dependencies: [])]\n" - ")\n"; + static const char src[] = "// swift-tools-version:5.9\n" + "import PackageDescription\n" + "let package = Package(\n" + " name: \"Core\",\n" + " targets: [.target(name: \"Core\", dependencies: [])]\n" + ")\n"; cbm_pkg_entries_t entries; cbm_pkg_entries_init(&entries); - bool ok = cbm_pkgmap_try_parse("Package.swift", "Core/Package.swift", src, - (int)strlen(src), &entries); + bool ok = cbm_pkgmap_try_parse("Package.swift", "Core/Package.swift", src, (int)strlen(src), + &entries); ASSERT_TRUE(ok); ASSERT_TRUE(pkg_entries_has_name(&entries, "Core")); ASSERT_STR_EQ(pkg_entries_entry_for(&entries, "Core"), "Core/Sources/Core"); @@ -10341,8 +10814,8 @@ TEST(pkgmap_swift_products_do_not_register_alias) { ")\n"; cbm_pkg_entries_t entries; cbm_pkg_entries_init(&entries); - bool ok = cbm_pkgmap_try_parse("Package.swift", "Core/Package.swift", src, - (int)strlen(src), &entries); + bool ok = cbm_pkgmap_try_parse("Package.swift", "Core/Package.swift", src, (int)strlen(src), + &entries); ASSERT_TRUE(ok); ASSERT_FALSE(pkg_entries_has_name(&entries, "CoreKit")); ASSERT_TRUE(pkg_entries_has_name(&entries, "CoreImpl")); @@ -10361,15 +10834,14 @@ TEST(pkgmap_swift_products_do_not_register_alias) { * fixture in this file happens to follow `name:` with `dependencies:` or a * comma, so this specific shape was previously untested and unnoticed. */ TEST(pkgmap_swift_target_name_immediately_before_close_paren) { - static const char src[] = - "let package = Package(\n" - " name: \"Core\",\n" - " targets: [.target(name: \"Core\")]\n" - ")\n"; + static const char src[] = "let package = Package(\n" + " name: \"Core\",\n" + " targets: [.target(name: \"Core\")]\n" + ")\n"; cbm_pkg_entries_t entries; cbm_pkg_entries_init(&entries); - bool ok = cbm_pkgmap_try_parse("Package.swift", "Core/Package.swift", src, - (int)strlen(src), &entries); + bool ok = cbm_pkgmap_try_parse("Package.swift", "Core/Package.swift", src, (int)strlen(src), + &entries); ASSERT_TRUE(ok); ASSERT_TRUE(pkg_entries_has_name(&entries, "Core")); ASSERT_STR_EQ(pkg_entries_entry_for(&entries, "Core"), "Core/Sources/Core"); @@ -10387,8 +10859,8 @@ TEST(pkgmap_swift_target_honors_literal_path) { ")\n"; cbm_pkg_entries_t entries; cbm_pkg_entries_init(&entries); - bool ok = cbm_pkgmap_try_parse("Package.swift", "Core/Package.swift", src, - (int)strlen(src), &entries); + bool ok = cbm_pkgmap_try_parse("Package.swift", "Core/Package.swift", src, (int)strlen(src), + &entries); ASSERT_TRUE(ok); ASSERT_TRUE(pkg_entries_has_name(&entries, "Core")); ASSERT_STR_EQ(pkg_entries_entry_for(&entries, "Core"), "Core/Vendor/CoreLegacy"); @@ -10402,16 +10874,15 @@ TEST(pkgmap_swift_target_honors_literal_path) { * target entirely (fail closed), even though its `name:` is a valid * literal. */ TEST(pkgmap_swift_target_computed_path_fails_closed) { - static const char src[] = - "let customPath = computePath()\n" - "let package = Package(\n" - " name: \"Core\",\n" - " targets: [.target(name: \"Core\", path: customPath)]\n" - ")\n"; + static const char src[] = "let customPath = computePath()\n" + "let package = Package(\n" + " name: \"Core\",\n" + " targets: [.target(name: \"Core\", path: customPath)]\n" + ")\n"; cbm_pkg_entries_t entries; cbm_pkg_entries_init(&entries); - bool ok = cbm_pkgmap_try_parse("Package.swift", "Core/Package.swift", src, - (int)strlen(src), &entries); + bool ok = cbm_pkgmap_try_parse("Package.swift", "Core/Package.swift", src, (int)strlen(src), + &entries); ASSERT_TRUE(ok); ASSERT_EQ(entries.count, 0); cbm_pkg_entries_free(&entries); @@ -10433,8 +10904,8 @@ TEST(pkgmap_swift_target_in_comment_or_string_not_registered) { ")\n"; cbm_pkg_entries_t entries; cbm_pkg_entries_init(&entries); - bool ok = cbm_pkgmap_try_parse("Package.swift", "App/Package.swift", src, - (int)strlen(src), &entries); + bool ok = + cbm_pkgmap_try_parse("Package.swift", "App/Package.swift", src, (int)strlen(src), &entries); ASSERT_TRUE(ok); ASSERT_TRUE(pkg_entries_has_name(&entries, "App")); ASSERT_FALSE(pkg_entries_has_name(&entries, "Decoy")); @@ -10463,8 +10934,8 @@ TEST(pkgmap_swift_dependencies_do_not_leak_entries) { ")\n"; cbm_pkg_entries_t entries; cbm_pkg_entries_init(&entries); - bool ok = cbm_pkgmap_try_parse("Package.swift", "App/Package.swift", src, - (int)strlen(src), &entries); + bool ok = + cbm_pkgmap_try_parse("Package.swift", "App/Package.swift", src, (int)strlen(src), &entries); ASSERT_TRUE(ok); ASSERT_TRUE(pkg_entries_has_name(&entries, "App")); ASSERT_FALSE(pkg_entries_has_name(&entries, "Core")); @@ -10479,18 +10950,17 @@ TEST(pkgmap_swift_dependencies_do_not_leak_entries) { * (Utils/UtilsPkg) name OTHER modules, not this manifest's own * products/targets, so neither mints an entry. */ TEST(pkgmap_swift_target_name_dependency_does_not_leak_entry) { - static const char src[] = - "let package = Package(\n" - " name: \"App\",\n" - " targets: [.target(name: \"App\", dependencies: [\n" - " \"Core\",\n" - " .product(name: \"Utils\", package: \"UtilsPkg\")\n" - " ])]\n" - ")\n"; + static const char src[] = "let package = Package(\n" + " name: \"App\",\n" + " targets: [.target(name: \"App\", dependencies: [\n" + " \"Core\",\n" + " .product(name: \"Utils\", package: \"UtilsPkg\")\n" + " ])]\n" + ")\n"; cbm_pkg_entries_t entries; cbm_pkg_entries_init(&entries); - bool ok = cbm_pkgmap_try_parse("Package.swift", "App/Package.swift", src, - (int)strlen(src), &entries); + bool ok = + cbm_pkgmap_try_parse("Package.swift", "App/Package.swift", src, (int)strlen(src), &entries); ASSERT_TRUE(ok); ASSERT_TRUE(pkg_entries_has_name(&entries, "App")); ASSERT_FALSE(pkg_entries_has_name(&entries, "Core")); @@ -13494,7 +13964,6 @@ TEST(pipeline_delta_patch_indexes_docstring_into_fts_body) { PASS(); } - /* End-to-end for #518/#519: source → docstring → properties JSON → nodes_fts * `body` → findable. Each layer has its own test; this one proves they connect. * It is also the guard on the size budget: build_def_props drops an oversized @@ -14045,6 +14514,12 @@ SUITE(pipeline_semantic_manifest_repro) { RUN_TEST(pipeline_git_context_change_forces_full_and_refreshes_branch); RUN_TEST(pipeline_global_extension_config_change_forces_full); RUN_TEST(pipeline_publication_never_uses_a_predictable_staging_path); + RUN_TEST(pipeline_stage_names_never_nest); + RUN_TEST(pipeline_minted_stage_is_owned_until_released); + RUN_TEST(pipeline_stale_zero_byte_stage_beside_valid_db_routes_incremental_and_is_swept); + RUN_TEST(pipeline_sweep_removes_dead_writer_stage_keeps_locked_stage); + RUN_TEST(pipeline_concurrent_sweep_must_not_remove_inflight_stage); + RUN_TEST(pipeline_fresh_index_never_reports_invalid_existing_db); RUN_TEST(pipeline_source_mutation_before_publication_preserves_previous_generation); RUN_TEST(pipeline_source_addition_before_publication_preserves_previous_generation); RUN_TEST(pipeline_tsconfig_mutation_before_publication_preserves_previous_generation); diff --git a/tests/test_security_gate_fail_closed.sh b/tests/test_security_gate_fail_closed.sh new file mode 100644 index 0000000000..0e79f2356d --- /dev/null +++ b/tests/test_security_gate_fail_closed.sh @@ -0,0 +1,150 @@ +#!/usr/bin/env bash +# The code-scanning gate must FAIL CLOSED. +# +# History. `codeql-gate` in .github/workflows/_security.yml counted open alerts +# with: +# +# ALERTS=$(gh api '.../code-scanning/alerts?state=open' --jq 'length' 2>/dev/null || echo "0") +# +# That reads as defensive and is the exact opposite. The job declared no +# `permissions:` block, so it inherited the workflow's `contents: read` and the +# alert API answered 403. `gh` writes the API error BODY to stdout, so `ALERTS` +# became the string +# +# {"message":"Resource not accessible by integration",...,"status":"403"}0 +# +# every `[ ... -gt ... ]` then failed as a non-integer comparison, the `if` took +# its false branch, and the step printed "CodeQL gate passed (0 alerts)" and +# exited 0. Observed live in a GREEN run (34700514102 / job 103571322660), while +# the repository had an open high-severity alert. +# +# A security gate that cannot read its input must stop the build, not wave it +# through. This test pins both halves of that: the permission that makes the +# read possible, and the fail-closed handling for when it still is not. +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" + +python3 - "$ROOT" <<'PY' +from __future__ import annotations + +import pathlib +import re +import sys + +root = pathlib.Path(sys.argv[1]) +failures: list[str] = [] + + +def require(condition: bool, message: str) -> None: + if not condition: + failures.append(message) + + +workflow = (root / ".github" / "workflows" / "_security.yml").read_text(encoding="utf-8") + +# Isolate the codeql-gate job so a permission granted to some OTHER job cannot +# satisfy this test. +gate = re.search(r"\n codeql-gate:\n(?P(?: .*\n|\n)*)", workflow) +require(gate is not None, "_security.yml no longer defines a codeql-gate job") +body = gate.group("body") if gate else "" + +# 1. The read has to be possible at all. +require( + re.search(r"^ permissions:\s*$", body, re.M) is not None, + "codeql-gate must declare its own permissions block -- inheriting the " + "workflow default leaves it without security-events access and the alert " + "API answers 403", +) +require( + re.search(r"^ security-events:\s*read\s*$", body, re.M) is not None, + "codeql-gate must request security-events: read, or the code-scanning " + "alert query 403s and the gate cannot see any alert", +) + +# 2. The failure direction. `|| echo "0"` on the alert query is the specific +# construct that turned an API error into a clean pass. +require( + re.search(r"code-scanning/alerts[^\n]*\|\|\s*echo", body) is None, + "the alert count must not fall back to a literal on error -- an unreadable " + "alert list is not zero alerts", +) +require( + "BLOCKED: cannot read code scanning alerts" in body, + "codeql-gate must fail with a named error when the alert API cannot be read", +) +require( + re.search(r"alert count was not a number", body) is not None, + "codeql-gate must reject a non-numeric alert count instead of comparing it", +) + +# 3. The count is produced in a command substitution, so a bare `exit` inside +# the helper would end only the subshell and let the caller proceed with an +# empty value -- the same class of silent pass. The callers must check. +assignments = re.findall(r"^\s*ALERTS2?=\$\(count_open_alerts\)(.*)$", body, re.M) +require( + len(assignments) >= 2, + "expected both alert samples to go through the checked helper", +) +require( + all("|| exit 1" in tail for tail in assignments), + "every count_open_alerts call must be followed by || exit 1 -- a failure " + "inside a command substitution does not stop the caller on its own", +) + +# 4. Every caller must GRANT what the job requests. A reusable workflow cannot +# request more permission than its caller holds: asking for +# security-events:read from a caller that grants only contents:read does not +# downgrade, it fails the entire run with a startup_failure before a single +# job executes. Checking only the called workflow is how this test passed +# while the PR pipeline never started. +# The caller list is DERIVED, never hardcoded. An earlier revision listed +# ("pr.yml", "release.yml") by hand and stayed green while dry-run.yml -- a +# third caller nobody had listed -- failed every run at startup. A test that +# only checks the callers you remembered cannot catch the one you forgot. +workflow_dir = root / ".github" / "workflows" +callers = sorted( + path.name + for path in workflow_dir.glob("*.yml") + if "uses: ./.github/workflows/_security.yml" in path.read_text(encoding="utf-8") +) +require( + len(callers) >= 3, + "expected at least 3 callers of _security.yml, found: " + + (", ".join(callers) or "none") + + " -- if a caller was deliberately removed, lower this floor on purpose", +) + +for caller_name in callers: + caller = (workflow_dir / caller_name).read_text(encoding="utf-8") + # The next sibling may be a comment line, not a key, so the lookahead has to + # accept any 2-space-indented non-space -- matching only ` \w` silently + # failed to find release.yml's job at all. + call = re.search( + r"\n security:\n(?P(?: .*\n|\n)*?)(?=\n? \S|\Z)", caller + ) + require(call is not None, f"{caller_name} no longer has a security: job") + call_body = call.group("body") if call else "" + require( + "uses: ./.github/workflows/_security.yml" in call_body, + f"{caller_name}'s security job must call _security.yml", + ) + require( + re.search(r"^ security-events:\s*read\s*$", call_body, re.M) is not None, + f"{caller_name} must grant security-events: read to the security job -- " + "a called workflow cannot request more than its caller grants, and the " + "run fails at startup rather than degrading", + ) + require( + re.search(r"^ actions:\s*read\s*$", call_body, re.M) is not None, + f"{caller_name} must grant actions: read to the security job", + ) + +if failures: + print("Security gate fail-closed contract FAILED:") + for failure in failures: + print(f" - {failure}") + sys.exit(1) + +print("Security gate fail-closed contract passed") +PY diff --git a/tests/windows/test_daemon_lifecycle.py b/tests/windows/test_daemon_lifecycle.py index 5b074fc46e..d90dc342ab 100644 --- a/tests/windows/test_daemon_lifecycle.py +++ b/tests/windows/test_daemon_lifecycle.py @@ -36,6 +36,17 @@ def output_text(result): return ((result.stdout or b"") + (result.stderr or b"")).decode("utf-8", "replace") +def make_probe_repo(work): + """A one-file repo for daemon-backed one-shots. Read-only commands such as + `projects` run in-process in the CLI-first build and never spawn a daemon, + so the cold/warm hint checks drive a mutating command (`index`).""" + repo = os.path.join(work, "probe-repo") + os.makedirs(repo, exist_ok=True) + with open(os.path.join(repo, "probe.py"), "w", encoding="utf-8") as handle: + handle.write("def probe():\n return 1\n") + return repo + + def force_kill(pid): if not pid: return @@ -66,7 +77,8 @@ def main(): return 1 print("PASS: status reports not-running before any daemon exists") - cold = run_cli(binary, cache, ["projects", "--json"]) + probe_repo = make_probe_repo(work) + cold = run_cli(binary, cache, ["index", probe_repo]) cold_text = output_text(cold) if cold.returncode != 0 or "daemon start" not in cold_text: print("RED: a cold one-shot cli command should succeed and hint at " @@ -84,7 +96,7 @@ def main(): return 1 print("PASS: daemon start reported permanent pid %d" % daemon_pid) - warm = run_cli(binary, cache, ["projects", "--json"]) + warm = run_cli(binary, cache, ["index", probe_repo]) warm_text = output_text(warm) if warm.returncode != 0 or "daemon start" in warm_text: print("RED: a warm cli one-shot should recycle the daemon without the " diff --git a/tests/windows/test_daemon_stability.py b/tests/windows/test_daemon_stability.py index 6e1a920b95..116158c39e 100644 --- a/tests/windows/test_daemon_stability.py +++ b/tests/windows/test_daemon_stability.py @@ -177,6 +177,17 @@ def section_start_status_port(binary, work): kill_pid(daemon_pid) +def make_probe_repo(work): + """A one-file repo for daemon-backed one-shots. Read-only commands such as + `projects` run in-process in the CLI-first build and never spawn a daemon, + so the cold/warm hint checks drive a mutating command (`index`).""" + repo = os.path.join(work, "probe-repo") + os.makedirs(repo, exist_ok=True) + with open(os.path.join(repo, "probe.py"), "w", encoding="utf-8") as handle: + handle.write("def probe():\n return 1\n") + return repo + + def section_crash_recovery(binary, work): cache = os.path.join(work, "cache-crash") os.makedirs(cache, exist_ok=True) @@ -193,7 +204,7 @@ def section_crash_recovery(binary, work): print("RED: after kill -9 of pid %d the stale daemon state never cleared " "(`daemon status` kept reporting it)" % daemon_pid) return False - cold = run_cli(binary, cache, ["projects", "--json"], timeout=90) + cold = run_cli(binary, cache, ["index", make_probe_repo(work)], timeout=90) if cold.returncode != 0 or "daemon start" not in out_text(cold): print("RED: a cold one-shot after the daemon crash should succeed with the " "startup-tax hint:\n%s" % out_text(cold)[:400])