diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index e83a82d06f..2430c67127 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -29,7 +29,7 @@ jobs: run: sudo apt-get update && sudo apt-get install -y zlib1g-dev - name: Initialize CodeQL - uses: github/codeql-action/init@b96794f015dfd88f77b49b1c93e0fa7110f94c63 # v4.38.0 + uses: github/codeql-action/init@1c5b675653bb5c22dbe9b12b556ec555138e09fd # v4.38.1 with: languages: c-cpp build-mode: manual @@ -38,6 +38,6 @@ jobs: run: scripts/build.sh - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@b96794f015dfd88f77b49b1c93e0fa7110f94c63 # v4.38.0 + uses: github/codeql-action/analyze@1c5b675653bb5c22dbe9b12b556ec555138e09fd # v4.38.1 with: category: "/language:c-cpp" diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml index f2d8bcb927..c2acbd8ace 100644 --- a/.github/workflows/scorecard.yml +++ b/.github/workflows/scorecard.yml @@ -27,6 +27,6 @@ jobs: publish_results: true - name: Upload SARIF results - uses: github/codeql-action/upload-sarif@b96794f015dfd88f77b49b1c93e0fa7110f94c63 # v4 + uses: github/codeql-action/upload-sarif@1c5b675653bb5c22dbe9b12b556ec555138e09fd # v4 with: sarif_file: results.sarif diff --git a/Makefile.cbm b/Makefile.cbm index 8e309c9ebc..d6fc84b99c 100644 --- a/Makefile.cbm +++ b/Makefile.cbm @@ -341,6 +341,7 @@ FOUNDATION_SRCS = \ EXTRACTION_SRCS = \ $(CBM_DIR)/cbm.c \ $(CBM_DIR)/extract_defs.c \ + $(CBM_DIR)/callable_sig.c \ $(CBM_DIR)/extract_calls.c \ $(CBM_DIR)/extract_imports.c \ $(CBM_DIR)/extract_usages.c \ @@ -438,6 +439,7 @@ PIPELINE_SRCS = \ src/pipeline/lsp_surface.c \ src/pipeline/pipeline_delta.c \ src/pipeline/path_alias.c \ + src/pipeline/laravel_routing.c \ src/pipeline/registry.c \ src/pipeline/pipeline.c \ src/pipeline/pipeline_incremental.c \ @@ -637,6 +639,7 @@ TEST_FOUNDATION_OBJS = $(patsubst %.c,$(BUILD_DIR)/%.test.o,$(TEST_FOUNDATION_SR TEST_EXTRACTION_SRCS = \ tests/test_extraction.c \ + tests/test_callable_sig.c \ tests/test_extraction_inheritance.c \ tests/test_extraction_imports.c \ tests/test_parse_coverage.c \ diff --git a/README.md b/README.md index e903dcf599..cf0445c76d 100644 --- a/README.md +++ b/README.md @@ -337,3 +337,5 @@ See [`SECURITY.md`](SECURITY.md) and [`docs/SECURITY-DISCLOSURE.md`](docs/SECURI ## License and upstream attribution MIT. The root [`LICENSE`](LICENSE) preserves the upstream `Copyright (c) 2025 DeusData` notice. See [`UPSTREAM_ATTRIBUTION.md`](UPSTREAM_ATTRIBUTION.md) for the explicit distinction between inherited DeusData work and modifications made in this fork. + +Uninstall keeps project indexes by default, including with `--yes`, `--no`, or noninteractive input. Use `--delete-indexes` to remove them explicitly; `--dry-run` previews the changes. diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md index bbfe10b801..d6f4621940 100644 --- a/docs/CONFIGURATION.md +++ b/docs/CONFIGURATION.md @@ -88,6 +88,7 @@ Current keys: | `auto_index` | `false` | Permit the retained warm runtime to auto-index newly encountered projects. Canonical CLI usage can always index explicitly with `codebase-memory-cli index`. | | `auto_index_limit` | `50000` | Maximum file count allowed for automatic indexing of a new project. | | `auto_watch` | `true` | Register an active project with the retained background watcher when a warm runtime/hook integration supplies project context. The ordinary one-shot CLI does not require this. | +| `watch_non_git` | `false` | Poll non-Git roots with the indexer discovery rules and an adaptive file-tree scan. Skipped paths, including `.codebase-memory/`, do not trigger indexing. The first poll after daemon startup reindexes once; later changes trigger one reindex. This walks the whole tree each poll and is read when the daemon starts. | | `watcher_enabled` | `true` | Master switch for the retained background watcher subsystem. Set `false` to stop its poll thread/project registration. Reindex manually with `codebase-memory-cli index` when disabled. | | `index_max_files` | `off` | Optional maximum number of accepted source files in one discovery run. | | `index_max_source_mb` | `off` | Optional maximum accepted source size in MiB in one discovery run. | diff --git a/install.ps1 b/install.ps1 index 28d091e7eb..3ada7f90ea 100644 --- a/install.ps1 +++ b/install.ps1 @@ -320,81 +320,102 @@ try { $Dest = Join-Path $InstallDir $BinName -# Retire the running installation before replacing it. Windows keeps an image -# lock on a running .exe: the file cannot be overwritten, but it CAN be renamed -# out of the way, which is what makes an in-place update possible from here. -if (Test-Path -LiteralPath $Dest -PathType Leaf) { - try { & $Dest daemon stop 2>&1 | Out-Null } catch { } - $retired = "$Dest.retired-$(Get-Date -Format yyyyMMddHHmmss)" - $renamed = $false - foreach ($attempt in 1..10) { - try { Move-Item -LiteralPath $Dest -Destination $retired -Force -ErrorAction Stop; $renamed = $true; break } - catch { Start-Sleep -Milliseconds 500 } +$retired = $null +$ActivationSucceeded = $false +try { + # Retire the running installation before replacing it. Windows keeps an image + # lock on a running .exe: the file cannot be overwritten, but it CAN be renamed + # out of the way, which is what makes an in-place update possible from here. + if (Test-Path -LiteralPath $Dest -PathType Leaf) { + try { & $Dest daemon stop 2>&1 | Out-Null } catch { } + $retired = "$Dest.retired-$([Guid]::NewGuid().ToString('N'))" + $renamed = $false + foreach ($attempt in 1..10) { + try { Move-Item -LiteralPath $Dest -Destination $retired -ErrorAction Stop; $renamed = $true; break } + catch { Start-Sleep -Milliseconds 500 } + } + if (-not $renamed) { + Write-Host "error: could not retire the existing $BinName - close all running" -ForegroundColor Red + Write-Host " codebase-memory-cli sessions and coding agents, then re-run." -ForegroundColor Red + Remove-Item -Recurse -Force $TmpDir -ErrorAction SilentlyContinue + exit 1 + } + # Keep the old image until activation and the installed-binary probe succeed. } - if (-not $renamed) { - Write-Host "error: could not retire the existing $BinName - close all running" -ForegroundColor Red - Write-Host " codebase-memory-cli sessions and coding agents, then re-run." -ForegroundColor Red + + $InstallArgs = @("install", "-y", "--force", "--dir=$InstallDir") + if ($SkipConfig) { $InstallArgs += "--skip-config" } + & $DownloadedBinary @InstallArgs + if ($LASTEXITCODE -ne 0) { + Write-Host "error: installation failed (exit code $LASTEXITCODE)" -ForegroundColor Red Remove-Item -Recurse -Force $TmpDir -ErrorAction SilentlyContinue exit 1 } - # A retired image stays locked until its last process exits; delete it when - # we can, and leave it for the next run when we cannot. Never fail here. - Remove-Item -LiteralPath $retired -Force -ErrorAction SilentlyContinue -} -Get-ChildItem -LiteralPath $InstallDir -Filter "$BinName.retired-*" -ErrorAction SilentlyContinue | - ForEach-Object { Remove-Item -LiteralPath $_.FullName -Force -ErrorAction SilentlyContinue } -$InstallArgs = @("install", "-y", "--force", "--dir=$InstallDir") -if ($SkipConfig) { $InstallArgs += "--skip-config" } -& $DownloadedBinary @InstallArgs -if ($LASTEXITCODE -ne 0) { - Write-Host "error: installation failed (exit code $LASTEXITCODE)" -ForegroundColor Red - Remove-Item -Recurse -Force $TmpDir -ErrorAction SilentlyContinue - exit 1 -} + # Place the installer beside the binary so `update` points at a local file + # rather than a URL, and so the next update runs THIS release's installer. + # + # Sourced from the archive we just checksum-verified, and published by rename + # rather than written over the live path. PowerShell parses a script fully + # before executing it, so self-overwrite is less hazardous here than it is for + # bash -- but rename costs nothing and keeps both platforms on one rule. + # Best effort: a failure here still leaves a working install. + $DownloadedInstaller = Join-Path $TmpDir "install.ps1" + if (Test-Path -LiteralPath $DownloadedInstaller -PathType Leaf) { + $InstallerDest = Join-Path $InstallDir "install.ps1" + $InstallerTmp = "$InstallerDest.new" + try { + Copy-Item -LiteralPath $DownloadedInstaller -Destination $InstallerTmp -Force -ErrorAction Stop + Move-Item -LiteralPath $InstallerTmp -Destination $InstallerDest -Force -ErrorAction Stop + Write-Host "Installed updater -> $InstallerDest" + } catch { + Remove-Item -LiteralPath $InstallerTmp -Force -ErrorAction SilentlyContinue + Write-Host "note: could not place install.ps1 in $InstallDir (update will explain where to find it)" + } + } -# Place the installer beside the binary so `update` points at a local file -# rather than a URL, and so the next update runs THIS release's installer. -# -# Sourced from the archive we just checksum-verified, and published by rename -# rather than written over the live path. PowerShell parses a script fully -# before executing it, so self-overwrite is less hazardous here than it is for -# bash -- but rename costs nothing and keeps both platforms on one rule. -# Best effort: a failure here still leaves a working install. -$DownloadedInstaller = Join-Path $TmpDir "install.ps1" -if (Test-Path -LiteralPath $DownloadedInstaller -PathType Leaf) { - $InstallerDest = Join-Path $InstallDir "install.ps1" - $InstallerTmp = "$InstallerDest.new" + # Verify + # Same PS 5.1 stderr-wrapping guard as the candidate probe above; the pre-seed + # matters MOST here, because prior successful native calls leave a stale + # $LASTEXITCODE=0 that a start-failure would otherwise inherit. try { - Copy-Item -LiteralPath $DownloadedInstaller -Destination $InstallerTmp -Force -ErrorAction Stop - Move-Item -LiteralPath $InstallerTmp -Destination $InstallerDest -Force -ErrorAction Stop - Write-Host "Installed updater -> $InstallerDest" + $ProbeEap = $ErrorActionPreference + try { + $ErrorActionPreference = "Continue" + $global:LASTEXITCODE = 1 + $ver = & $Dest --version 2>&1 + } finally { + $ErrorActionPreference = $ProbeEap + } + if ($LASTEXITCODE -ne 0) { throw "installed binary exited with $LASTEXITCODE" } + Write-Host "Installed: $ver" } catch { - Remove-Item -LiteralPath $InstallerTmp -Force -ErrorAction SilentlyContinue - Write-Host "note: could not place install.ps1 in $InstallDir (update will explain where to find it)" + Write-Host "error: installed binary failed to run" -ForegroundColor Red + Remove-Item -Recurse -Force $TmpDir + exit 1 } -} -# Verify -# Same PS 5.1 stderr-wrapping guard as the candidate probe above; the pre-seed -# matters MOST here, because prior successful native calls leave a stale -# $LASTEXITCODE=0 that a start-failure would otherwise inherit. -try { - $ProbeEap = $ErrorActionPreference - try { - $ErrorActionPreference = "Continue" - $global:LASTEXITCODE = 1 - $ver = & $Dest --version 2>&1 - } finally { - $ErrorActionPreference = $ProbeEap + $ActivationSucceeded = $true +} finally { + if (-not $ActivationSucceeded -and $retired -and + (Test-Path -LiteralPath $retired -PathType Leaf)) { + if (-not (Test-Path -LiteralPath $Dest)) { + try { + Move-Item -LiteralPath $retired -Destination $Dest -ErrorAction Stop + } catch { + Write-Host "error: could not restore $Dest; previous binary retained at $retired : $_" -ForegroundColor Red + } + } else { + # The inner activation may deliberately retain a published candidate + # after partial config/PATH changes. Do not override that decision or + # overwrite an unexpected path entry outside its activation lease. + Write-Host "note: keeping the published executable; previous binary retained at $retired" + } } - if ($LASTEXITCODE -ne 0) { throw "installed binary exited with $LASTEXITCODE" } - Write-Host "Installed: $ver" -} catch { - Write-Host "error: installed binary failed to run" -ForegroundColor Red - Remove-Item -Recurse -Force $TmpDir - exit 1 } +# Retired images are disposable only after the replacement has been verified. +Get-ChildItem -LiteralPath $InstallDir -Filter "$BinName.retired-*" -ErrorAction SilentlyContinue | + ForEach-Object { Remove-Item -LiteralPath $_.FullName -Force -ErrorAction SilentlyContinue } # Agent configuration was included in the candidate-owned activation window. if ($SkipConfig) { diff --git a/internal/cbm/callable_sig.c b/internal/cbm/callable_sig.c new file mode 100644 index 0000000000..202a589c5f --- /dev/null +++ b/internal/cbm/callable_sig.c @@ -0,0 +1,1199 @@ +/* + * callable_sig.c — see callable_sig.h for the identity contract (#2061). + * + * The builder renders the suffix into a growable arena buffer, then applies + * the spelling rules in a single character pass, then the length cap. The result is re-checked + * against cbm_qn_callable_base_len so a malformed parse can never mint a suffix the leaf splitters + * would misread: such a suffix degrades to its hashed form. + */ +#include "callable_sig.h" +#include "helpers.h" +#include +#include +#include +#include +#include + +enum { + SIG_DEPTH_LIMIT = 64, /* type subtree recursion bound */ + SIG_DECL_DEPTH = 8, /* C-family declarator chain bound */ + SIG_MAX_ENTRIES = 256, /* entries tracked for the cap; more => capped */ + SIG_INITIAL_CAP = 256, /* first render buffer; grows by doubling */ + SIG_HASH_HEX = 16, /* FNV-1a 64 rendered as hex */ + SIG_OPERATOR_LEN = 8, /* strlen("operator") */ + SIG_CONST_LEN = 5, /* strlen("const") */ + SIG_VOLATILE_LEN = 8, /* strlen("volatile") */ + SIG_ARITY_DIGITS = 16, /* "(%d)" scratch */ + SIG_CAPPED_TAIL = 1 + SIG_HASH_HEX + 1, /* "#" hex ")" */ +}; + +static const uint64_t SIG_FNV_OFFSET = 0xcbf29ce484222325ULL; /* FNV-1a 64 basis */ +static const uint64_t SIG_FNV_PRIME = 0x100000001b3ULL; +static const char SIG_OPERATOR_CHARS[] = "+-*/%^&|~!=<>?[]"; + +CBMCallableIdentity cbm_callable_identity(CBMLanguage lang) { + /* Every language keeps its historical QN until its enable change lands + * (with the index-format bump that change carries). The planned modes: + * Java/Kotlin/C#/C++/CUDA/Scala TYPED, Swift LABELED_TYPED, ObjC LABELED, + * dynamic tier-2 languages ARITY. */ + (void)lang; + return CBM_CALLABLE_ID_NONE; +} + +/* ── Output buffer ─────────────────────────────────────────────── */ + +typedef struct { + CBMArena *arena; + const char *src; + CBMLanguage lang; + CBMCallableIdentity mode; + char *buf; + size_t len; + size_t cap; + bool overflow; + int entries; + bool too_many; + size_t entry_end[SIG_MAX_ENTRIES]; + size_t open_off; /* offset of the parameter list's "(" */ + size_t close_off; /* offset just past its ")" */ + bool skip_names; /* nested C parameter lists: leave out parameter names */ +} sig_ctx_t; + +static bool sig_ident_char(unsigned char ch) { + return isalnum(ch) || ch == '_' || ch == '$' || ch >= 0x80; +} + +/* Grow by doubling inside the arena; the abandoned copies are bounded by the + * final size (geometric), and the arena is the extraction's scratch anyway. */ +static void sig_raw(sig_ctx_t *c, const char *s, size_t n) { + if (c->overflow) { + return; + } + if (c->len + n >= c->cap) { + size_t cap = c->cap ? c->cap : SIG_INITIAL_CAP; + while (c->len + n >= cap) { + cap *= 2; + } + char *grown = cbm_arena_alloc(c->arena, cap); + if (!grown) { + c->overflow = true; + return; + } + if (c->len > 0) + memcpy(grown, c->buf, c->len); + c->buf = grown; + c->cap = cap; + } + memcpy(c->buf + c->len, s, n); + c->len += n; + c->buf[c->len] = '\0'; +} + +static void sig_raw_str(sig_ctx_t *c, const char *s) { + sig_raw(c, s, strlen(s)); +} + +/* Append one source token: internal whitespace is dropped except as a single + * space between two identifier characters, which is also the rule at the + * boundary with what is already in the buffer. */ +static void sig_token(sig_ctx_t *c, const char *s, size_t n) { + bool pending_space = false; + for (size_t i = 0; i < n; i++) { + unsigned char ch = (unsigned char)s[i]; + if (isspace(ch)) { + pending_space = true; + continue; + } + bool first = c->len == 0 || c->buf[c->len - 1] == ' '; + if (!first && sig_ident_char((unsigned char)c->buf[c->len - 1]) && sig_ident_char(ch) && + (pending_space || i == 0)) { + sig_raw(c, " ", 1); + } + pending_space = false; + sig_raw(c, (const char *)&s[i], 1); + } +} + +static const char *sig_variadic_marker(CBMLanguage lang) { + return lang == CBM_LANG_JAVA ? "[]" : "~"; +} + +/* ── Type spelling ─────────────────────────────────────────────── */ + +static bool sig_skipped_kind(const char *kind) { + return strstr(kind, "comment") != NULL || strcmp(kind, "annotation") == 0 || + strcmp(kind, "marker_annotation") == 0 || strcmp(kind, "attribute_list") == 0 || + strcmp(kind, "attribute") == 0 || strcmp(kind, "attribute_declaration") == 0 || + strcmp(kind, "attribute_specifier") == 0; +} + +/* Whether a leaf is emitted: C-family nested parameter lists spell their + * declarators without the parameter names. */ +static void sig_leaf(sig_ctx_t *c, TSNode n) { + const char *kind = ts_node_type(n); + if (c->skip_names && + (strcmp(kind, "identifier") == 0 || strcmp(kind, "field_identifier") == 0)) { + return; + } + uint32_t s = ts_node_start_byte(n); + uint32_t e = ts_node_end_byte(n); + if (e - s == 3 && memcmp(c->src + s, "...", 3) == 0) { + sig_raw_str(c, sig_variadic_marker(c->lang)); + return; + } + sig_token(c, c->src + s, e - s); +} + +/* Render every leaf token of a type subtree, in order, with a cursor walk + * bounded at SIG_DEPTH_LIMIT. C# tuple element names are not part of the + * tuple's type identity and are skipped; Swift tuple labels are, and stay. */ +static void sig_type_tokens(sig_ctx_t *c, TSNode root) { + if (ts_node_is_null(root)) { + return; + } + bool in_tuple_element[SIG_DEPTH_LIMIT + 1] = {false}; + TSTreeCursor cur = ts_tree_cursor_new(root); + int depth = 0; + for (;;) { + TSNode n = ts_tree_cursor_current_node(&cur); + const char *kind = ts_node_type(n); + const char *field = depth > 0 ? ts_tree_cursor_current_field_name(&cur) : NULL; + bool skip = sig_skipped_kind(kind) || + (in_tuple_element[depth] && field && strcmp(field, "name") == 0); + if (!skip && ts_node_child_count(n) == 0) { + sig_leaf(c, n); + } else if (!skip && depth < SIG_DEPTH_LIMIT && ts_tree_cursor_goto_first_child(&cur)) { + depth++; + in_tuple_element[depth] = strcmp(kind, "tuple_element") == 0; + continue; + } + while (depth > 0 && !ts_tree_cursor_goto_next_sibling(&cur)) { + ts_tree_cursor_goto_parent(&cur); + depth--; + } + if (depth == 0) { + break; + } + } + ts_tree_cursor_delete(&cur); +} + +/* ── Entries ───────────────────────────────────────────────────── */ + +static bool sig_writes_types(const sig_ctx_t *c) { + return c->mode == CBM_CALLABLE_ID_TYPED || c->mode == CBM_CALLABLE_ID_LABELED_TYPED; +} + +static bool sig_writes_labels(const sig_ctx_t *c) { + return c->mode == CBM_CALLABLE_ID_LABELED_TYPED || c->mode == CBM_CALLABLE_ID_LABELED; +} + +static void sig_entry_open(sig_ctx_t *c) { + if (c->entries > 0 && sig_writes_types(c)) { + sig_raw(c, ",", 1); + } +} + +static void sig_entry_close(sig_ctx_t *c) { + if (c->entries < SIG_MAX_ENTRIES) { + c->entry_end[c->entries] = c->len; + } else { + c->too_many = true; + } + c->entries++; +} + +/* label (NULL = none -> "_") followed by ':' in the labeled modes. */ +static void sig_label(sig_ctx_t *c, const char *label, size_t n) { + if (!sig_writes_labels(c)) { + return; + } + if (label && n > 0) { + sig_raw(c, label, n); + } else { + sig_raw(c, "_", 1); + } + sig_raw(c, ":", 1); +} + +static void sig_label_node(sig_ctx_t *c, TSNode n) { + if (ts_node_is_null(n)) { + sig_label(c, NULL, 0); + return; + } + uint32_t s = ts_node_start_byte(n); + sig_label(c, c->src + s, ts_node_end_byte(n) - s); +} + +/* An entry holding one plain type node (NULL node -> "?"). */ +static void sig_entry_type(sig_ctx_t *c, TSNode type, const char *suffix) { + sig_entry_open(c); + sig_label(c, NULL, 0); + if (sig_writes_types(c)) { + size_t before = c->len; + sig_type_tokens(c, type); + if (c->len == before) { + sig_raw(c, "?", 1); + } + if (suffix) { + sig_raw_str(c, suffix); + } + } + sig_entry_close(c); +} + +/* Field lookup that tolerates a null node: the vendored + * ts_node_child_by_field_name dereferences self.tree, so the all-zero node + * would crash instead of returning the null node. */ +static TSNode sig_field(TSNode n, const char *field) { + if (ts_node_is_null(n)) { + return n; + } + return ts_node_child_by_field_name(n, field, (uint32_t)strlen(field)); +} + +static bool sig_node_text_is(const sig_ctx_t *c, TSNode n, const char *text) { + uint32_t s = ts_node_start_byte(n); + uint32_t e = ts_node_end_byte(n); + size_t len = strlen(text); + return e - s == len && memcmp(c->src + s, text, len) == 0; +} + +/* ── Java ──────────────────────────────────────────────────────── */ + +static void sig_java_params(sig_ctx_t *c, TSNode node) { + TSNode params = sig_field(node, "parameters"); + uint32_t count = ts_node_is_null(params) ? 0 : ts_node_named_child_count(params); + for (uint32_t i = 0; i < count; i++) { + TSNode p = ts_node_named_child(params, i); + const char *kind = ts_node_type(p); + if (strcmp(kind, "formal_parameter") == 0) { + sig_entry_open(c); + sig_label(c, NULL, 0); + if (sig_writes_types(c)) { + size_t before = c->len; + sig_type_tokens(c, sig_field(p, "type")); + if (c->len == before) { + sig_raw(c, "?", 1); + } + sig_type_tokens(c, sig_field(p, "dimensions")); /* C-style int a[] */ + } + sig_entry_close(c); + } else if (strcmp(kind, "spread_parameter") == 0) { + TSNode type = {0}; + uint32_t pc = ts_node_named_child_count(p); + for (uint32_t j = 0; j < pc; j++) { + TSNode ch = ts_node_named_child(p, j); + const char *ck = ts_node_type(ch); + if (strcmp(ck, "modifiers") != 0 && strcmp(ck, "variable_declarator") != 0 && + !sig_skipped_kind(ck)) { + type = ch; + break; + } + } + sig_entry_type(c, type, sig_variadic_marker(c->lang)); + } + /* receiver_parameter (`Foo this`) is not part of the signature. */ + } +} + +/* ── C# ────────────────────────────────────────────────────────── */ + +static void sig_cs_param(sig_ctx_t *c, TSNode p) { + sig_entry_open(c); + sig_label(c, NULL, 0); + if (sig_writes_types(c)) { + uint32_t pc = ts_node_named_child_count(p); + for (uint32_t j = 0; j < pc; j++) { + TSNode m = ts_node_named_child(p, j); + if (strcmp(ts_node_type(m), "modifier") == 0 && + (sig_node_text_is(c, m, "ref") || sig_node_text_is(c, m, "out") || + sig_node_text_is(c, m, "in") || sig_node_text_is(c, m, "readonly"))) { + sig_type_tokens(c, m); + } + } + size_t before = c->len; + sig_type_tokens(c, sig_field(p, "type")); + if (c->len == before) { + sig_raw(c, "?", 1); + } + } + sig_entry_close(c); +} + +static void sig_cs_params(sig_ctx_t *c, TSNode node) { + TSNode params = sig_field(node, "parameters"); + uint32_t count = ts_node_is_null(params) ? 0 : ts_node_child_count(params); + for (uint32_t i = 0; i < count; i++) { + TSNode p = ts_node_child(params, i); + const char *field = ts_node_field_name_for_child(params, i); + if (strcmp(ts_node_type(p), "parameter") == 0) { + sig_cs_param(c, p); + } else if (field && strcmp(field, "type") == 0) { + /* `params T[] rest`: the grammar inlines the parameter's type and + * name into the list; `params` itself does not change identity. */ + sig_entry_type(c, p, NULL); + } + } +} + +static void sig_cs_tparams(sig_ctx_t *c, TSNode node) { + TSNode tp = sig_field(node, "type_parameters"); + uint32_t count = ts_node_is_null(tp) ? 0 : ts_node_named_child_count(tp); + int written = 0; + for (uint32_t i = 0; i < count; i++) { + TSNode p = ts_node_named_child(tp, i); + if (strcmp(ts_node_type(p), "type_parameter") != 0) { + continue; + } + sig_raw(c, written ? "," : "<", 1); + sig_type_tokens(c, sig_field(p, "name")); + written++; + } + if (written) { + sig_raw(c, ">", 1); + } +} + +/* ── C / C++ ───────────────────────────────────────────────────── */ + +static TSNode sig_c_inner_declarator(TSNode d) { + TSNode next = sig_field(d, "declarator"); + if (!ts_node_is_null(next)) { + return next; + } + uint32_t count = ts_node_named_child_count(d); + for (uint32_t i = 0; i < count; i++) { + TSNode ch = ts_node_named_child(d, i); + if (strstr(ts_node_type(ch), "declarator") != NULL) { + return ch; + } + } + return (TSNode){0}; +} + +static bool sig_is_c_decl_kind(const char *k) { + return strcmp(k, "function_definition") == 0 || strcmp(k, "declaration") == 0 || + strcmp(k, "field_declaration") == 0 || strcmp(k, "template_declaration") == 0; +} + +/* The function_declarator of a C-family definition/declaration, looking + * through (nested) template_declarations and pointer/reference return + * declarators. */ +static TSNode sig_c_function_declarator(TSNode node) { + TSNode cur = node; + for (int depth = 0; depth < SIG_DECL_DEPTH && !ts_node_is_null(cur) && + strcmp(ts_node_type(cur), "template_declaration") == 0; + depth++) { + TSNode inner = {0}; + uint32_t count = ts_node_named_child_count(cur); + for (uint32_t i = 0; i < count && ts_node_is_null(inner); i++) { + TSNode ch = ts_node_named_child(cur, i); + if (sig_is_c_decl_kind(ts_node_type(ch))) { + inner = ch; + } + } + cur = inner; + } + for (int depth = 0; depth < SIG_DECL_DEPTH && !ts_node_is_null(cur); depth++) { + if (strcmp(ts_node_type(cur), "function_declarator") == 0) { + return cur; + } + cur = sig_c_inner_declarator(cur); + } + return (TSNode){0}; +} + +static void sig_c_qualifiers(sig_ctx_t *c, TSNode n) { + uint32_t count = ts_node_named_child_count(n); + for (uint32_t i = 0; i < count; i++) { + TSNode q = ts_node_named_child(n, i); + if (strcmp(ts_node_type(q), "type_qualifier") == 0) { + sig_type_tokens(c, q); + } + } +} + +static bool sig_c_has_qualifier(TSNode n) { + uint32_t count = ts_node_named_child_count(n); + for (uint32_t i = 0; i < count; i++) { + if (strcmp(ts_node_type(ts_node_named_child(n, i)), "type_qualifier") == 0) { + return true; + } + } + return false; +} + +static bool sig_c_is_name(TSNode d) { + if (ts_node_is_null(d)) { + return true; + } + const char *k = ts_node_type(d); + return strstr(k, "declarator") == NULL; +} + +static bool sig_c_is_param_kind(const char *k) { + return strcmp(k, "parameter_declaration") == 0 || + strcmp(k, "optional_parameter_declaration") == 0 || + strcmp(k, "variadic_parameter_declaration") == 0; +} + +static bool sig_c_is_void_list(const sig_ctx_t *c, TSNode params) { + if (ts_node_named_child_count(params) != 1) { + return false; + } + TSNode p = ts_node_named_child(params, 0); + TSNode type = sig_field(p, "type"); + return strcmp(ts_node_type(p), "parameter_declaration") == 0 && !ts_node_is_null(type) && + sig_node_text_is(c, type, "void") && ts_node_is_null(sig_field(p, "declarator")) && + !sig_c_has_qualifier(p); +} + +/* The parameter list of a function-typed parameter (`int (*cb)(int x)`), + * spelled from its tokens without parameter names or default values. */ +static void sig_c_nested_params(sig_ctx_t *c, TSNode fdecl) { + TSNode params = sig_field(fdecl, "parameters"); + sig_raw(c, "(", 1); + uint32_t count = ts_node_is_null(params) ? 0 : ts_node_child_count(params); + bool void_list = !ts_node_is_null(params) && sig_c_is_void_list(c, params); + int written = 0; + bool saved = c->skip_names; + c->skip_names = true; + for (uint32_t i = 0; i < count && !void_list; i++) { + TSNode p = ts_node_child(params, i); + bool is_ellipsis = !ts_node_is_named(p) && sig_node_text_is(c, p, "..."); + if (!is_ellipsis && !sig_c_is_param_kind(ts_node_type(p))) { + continue; + } + if (written++) { + sig_raw(c, ",", 1); + } + if (is_ellipsis) { + sig_raw_str(c, sig_variadic_marker(c->lang)); + continue; + } + sig_c_qualifiers(c, p); + sig_type_tokens(c, sig_field(p, "type")); + sig_type_tokens(c, sig_field(p, "declarator")); + } + c->skip_names = saved; + sig_raw(c, ")", 1); +} + +/* Declarator markers, outermost first: `*` (with its qualifiers unless it is + * the top-level one, which the language drops), `&`/`&&`, an array that + * decays to `*` at the top level, a pack/variadic as the variadic marker. + * A function declarator opens "(" and is closed — with its parameter list — + * after everything inside it, innermost first; one with nothing inside is a + * function parameter decaying to a pointer. Returns whether any marker was + * written, so the caller knows whether base cv-qualifiers are top-level. */ +static bool sig_c_markers(sig_ctx_t *c, TSNode d) { + TSNode pending[SIG_DECL_DEPTH]; + size_t pending_at[SIG_DECL_DEPTH]; + int npending = 0; + bool wrote = false; + for (int depth = 0; !sig_c_is_name(d) && depth < SIG_DECL_DEPTH; depth++) { + const char *k = ts_node_type(d); + TSNode inner = sig_c_inner_declarator(d); + if (strstr(k, "pointer_declarator") != NULL) { + sig_raw(c, "*", 1); + if (!sig_c_is_name(inner)) { + sig_c_qualifiers(c, d); /* not top-level: qualifiers are identity */ + } + } else if (strstr(k, "reference_declarator") != NULL) { + TSNode op = ts_node_child(d, 0); + bool rvalue = + !ts_node_is_null(op) && !ts_node_is_named(op) && sig_node_text_is(c, op, "&&"); + sig_raw_str(c, rvalue ? "&&" : "&"); + } else if (strstr(k, "array_declarator") != NULL) { + sig_raw_str(c, sig_c_is_name(inner) ? "*" : "[]"); + } else if (strstr(k, "function_declarator") != NULL) { + sig_raw(c, "(", 1); + pending[npending] = d; + pending_at[npending] = c->len; + npending++; + } else if (strcmp(k, "variadic_declarator") == 0) { + sig_raw_str(c, sig_variadic_marker(c->lang)); + } else { + d = inner; /* parenthesized_declarator and friends add nothing */ + continue; + } + wrote = true; + d = inner; + } + while (npending > 0) { + npending--; + if (c->len == pending_at[npending]) { + sig_raw(c, "*", 1); /* a function parameter decays to a pointer */ + } + sig_raw(c, ")", 1); + sig_c_nested_params(c, pending[npending]); + } + return wrote; +} + +/* One C-family parameter's type: [non-top-level cv] base-type markers. */ +static void sig_c_param_type(sig_ctx_t *c, TSNode p) { + TSNode declarator = sig_field(p, "declarator"); + /* Probe whether the declarator adds markers, then render in order: + * top-level base cv (no markers) is dropped like the language does. */ + size_t mark = c->len; + bool has_markers = sig_c_markers(c, declarator); + c->len = mark; + c->buf[c->len] = '\0'; + if (has_markers) { + sig_c_qualifiers(c, p); + } + TSNode type = sig_field(p, "type"); + const char *tk = ts_node_is_null(type) ? "" : ts_node_type(type); + TSNode tag_name = sig_field(type, "name"); + bool elaborated = (strcmp(tk, "struct_specifier") == 0 || strcmp(tk, "class_specifier") == 0 || + strcmp(tk, "union_specifier") == 0 || strcmp(tk, "enum_specifier") == 0) && + !ts_node_is_null(tag_name) && ts_node_is_null(sig_field(type, "body")); + size_t before = c->len; + sig_type_tokens(c, elaborated ? tag_name : type); + if (c->len == before) { + sig_raw(c, "?", 1); + } + (void)sig_c_markers(c, declarator); +} + +/* "(" entries ")" of the callable's own function_declarator. */ +static void sig_c_param_list(sig_ctx_t *c, TSNode fdecl) { + TSNode params = sig_field(fdecl, "parameters"); + c->open_off = c->len; + sig_raw(c, "(", 1); + uint32_t count = ts_node_is_null(params) ? 0 : ts_node_child_count(params); + bool void_list = !ts_node_is_null(params) && sig_c_is_void_list(c, params); + for (uint32_t i = 0; i < count && !void_list; i++) { + TSNode p = ts_node_child(params, i); + bool is_ellipsis = !ts_node_is_named(p) && sig_node_text_is(c, p, "..."); + if (!is_ellipsis && !sig_c_is_param_kind(ts_node_type(p))) { + continue; + } + sig_entry_open(c); + sig_label(c, NULL, 0); + if (sig_writes_types(c)) { + if (is_ellipsis) { + sig_raw_str(c, sig_variadic_marker(c->lang)); + } else { + sig_c_param_type(c, p); + } + } + sig_entry_close(c); + } + sig_raw(c, ")", 1); + c->close_off = c->len; +} + +static void sig_cpp_tparams(sig_ctx_t *c, TSNode node) { + TSNode tmpl = node; + if (strcmp(ts_node_type(tmpl), "template_declaration") != 0) { + tmpl = ts_node_parent(node); + if (ts_node_is_null(tmpl) || strcmp(ts_node_type(tmpl), "template_declaration") != 0) { + return; + } + } + TSNode list = sig_field(tmpl, "parameters"); + uint32_t count = ts_node_is_null(list) ? 0 : ts_node_named_child_count(list); + sig_raw(c, "<", 1); + int written = 0; + for (uint32_t i = 0; i < count; i++) { + TSNode p = ts_node_named_child(list, i); + const char *k = ts_node_type(p); + if (sig_skipped_kind(k)) { + continue; + } + if (written++) { + sig_raw(c, ",", 1); + } + /* Template identity depends on parameter kinds, not their names. */ + if (strcmp(k, "type_parameter_declaration") == 0 || + strcmp(k, "optional_type_parameter_declaration") == 0) { + sig_raw_str(c, "typename"); + } else if (strcmp(k, "variadic_type_parameter_declaration") == 0) { + sig_raw_str(c, "typename"); + sig_raw_str(c, sig_variadic_marker(c->lang)); + } else if (strcmp(k, "template_template_parameter_declaration") == 0) { + sig_raw_str(c, "template"); + } else { + sig_c_param_type(c, p); + } + } + sig_raw(c, ">", 1); +} + +/* `template void S::f()`: the template parameters belong to the + * class, and the in-class declaration `void f();` carries none — both must + * spell the same identity. */ +static bool sig_cpp_member_of_class_template(TSNode fdecl) { + TSNode name = ts_node_is_null(fdecl) ? fdecl : sig_field(fdecl, "declarator"); + for (int depth = 0; depth < SIG_DECL_DEPTH && !ts_node_is_null(name) && + strcmp(ts_node_type(name), "qualified_identifier") == 0; + depth++) { + TSNode scope = sig_field(name, "scope"); + if (!ts_node_is_null(scope) && strcmp(ts_node_type(scope), "template_type") == 0) { + return true; + } + name = sig_field(name, "name"); + } + return false; +} + +static void sig_cpp_cvref(sig_ctx_t *c, TSNode fdecl) { + uint32_t count = ts_node_named_child_count(fdecl); + for (uint32_t i = 0; i < count; i++) { + TSNode ch = ts_node_named_child(fdecl, i); + const char *k = ts_node_type(ch); + if (strcmp(k, "type_qualifier") == 0 || strcmp(k, "ref_qualifier") == 0) { + sig_type_tokens(c, ch); + } + } +} + +/* ── Kotlin ────────────────────────────────────────────────────── */ + +static bool sig_kotlin_is_vararg(const sig_ctx_t *c, TSNode mods) { + uint32_t count = ts_node_named_child_count(mods); + for (uint32_t i = 0; i < count; i++) { + TSNode m = ts_node_named_child(mods, i); + if (strcmp(ts_node_type(m), "parameter_modifier") == 0 && + sig_node_text_is(c, m, "vararg")) { + return true; + } + } + return false; +} + +static void sig_kotlin_param(sig_ctx_t *c, TSNode p, bool vararg) { + sig_entry_open(c); + sig_label(c, NULL, 0); + if (sig_writes_types(c)) { + size_t before = c->len; + uint32_t count = ts_node_named_child_count(p); + for (uint32_t i = 0; i < count; i++) { + TSNode ch = ts_node_named_child(p, i); + if (strcmp(ts_node_type(ch), "simple_identifier") != 0) { + sig_type_tokens(c, ch); /* type_modifiers + the type */ + } + } + if (c->len == before) { + sig_raw(c, "?", 1); + } + if (vararg) { + sig_raw_str(c, sig_variadic_marker(c->lang)); + } + } + sig_entry_close(c); +} + +static void sig_kotlin_params(sig_ctx_t *c, TSNode node) { + TSNode receiver = sig_field(node, "receiver"); + if (!ts_node_is_null(receiver)) { + /* An extension's receiver is part of its identity: String.f(Int) and + * Int.f(Int) share a base QN. */ + sig_entry_open(c); + sig_label(c, NULL, 0); + if (sig_writes_types(c)) { + sig_raw_str(c, "this:"); + sig_type_tokens(c, receiver); + } + sig_entry_close(c); + } + TSNode params = cbm_find_child_by_kind(node, "function_value_parameters"); + uint32_t count = ts_node_is_null(params) ? 0 : ts_node_named_child_count(params); + bool vararg = false; + for (uint32_t i = 0; i < count; i++) { + TSNode ch = ts_node_named_child(params, i); + const char *k = ts_node_type(ch); + if (strcmp(k, "parameter_modifiers") == 0) { + vararg = vararg || sig_kotlin_is_vararg(c, ch); + } else if (strcmp(k, "parameter") == 0) { + sig_kotlin_param(c, ch, vararg); + vararg = false; + } + } +} + +/* ── Swift ─────────────────────────────────────────────────────── */ + +static void sig_swift_param(sig_ctx_t *c, TSNode p) { + TSNode external = sig_field(p, "external_name"); + TSNode internal = {0}; + uint32_t count = ts_node_child_count(p); + for (uint32_t i = 0; i < count; i++) { + TSNode ch = ts_node_child(p, i); + const char *field = ts_node_field_name_for_child(p, i); + if (field && strcmp(field, "name") == 0 && + strcmp(ts_node_type(ch), "simple_identifier") == 0) { + internal = ch; + break; + } + } + sig_entry_open(c); + /* One name is both label and internal name; `_` means unlabeled. */ + sig_label_node(c, ts_node_is_null(external) ? internal : external); + if (sig_writes_types(c)) { + size_t before = c->len; + for (uint32_t i = 0; i < count; i++) { + TSNode ch = ts_node_child(p, i); + const char *field = ts_node_field_name_for_child(p, i); + if (field && strcmp(field, "external_name") == 0) { + continue; + } + if (ts_node_eq(ch, internal)) { + continue; + } + const char *k = ts_node_type(ch); + if (strcmp(k, "parameter_modifiers") == 0) { + uint32_t mc = ts_node_named_child_count(ch); + for (uint32_t j = 0; j < mc; j++) { + TSNode m = ts_node_named_child(ch, j); + uint32_t ms = ts_node_start_byte(m); + if (ts_node_end_byte(m) > ms && c->src[ms] != '@') { + sig_type_tokens(c, m); /* inout; @escaping etc. dropped */ + } + } + } else if (ts_node_is_named(ch)) { + sig_type_tokens(c, ch); + } else if (sig_node_text_is(c, ch, "...")) { + sig_raw_str(c, sig_variadic_marker(c->lang)); + } + } + if (c->len == before) { + sig_raw(c, "?", 1); + } + } + sig_entry_close(c); +} + +static void sig_swift_params(sig_ctx_t *c, TSNode node) { + uint32_t count = ts_node_named_child_count(node); + for (uint32_t i = 0; i < count; i++) { + TSNode ch = ts_node_named_child(node, i); + if (strcmp(ts_node_type(ch), "parameter") == 0) { + sig_swift_param(c, ch); + } + } +} + +/* ── Scala ─────────────────────────────────────────────────────── */ + +static void sig_scala_params(sig_ctx_t *c, TSNode node) { + /* Every parameter clause, flattened: all of them are erased into the + * method's JVM signature. */ + uint32_t count = ts_node_named_child_count(node); + for (uint32_t i = 0; i < count; i++) { + TSNode clause = ts_node_named_child(node, i); + if (strcmp(ts_node_type(clause), "parameters") != 0) { + continue; + } + uint32_t pc = ts_node_named_child_count(clause); + for (uint32_t j = 0; j < pc; j++) { + TSNode p = ts_node_named_child(clause, j); + if (strcmp(ts_node_type(p), "parameter") == 0) { + sig_entry_type(c, sig_field(p, "type"), NULL); + } + } + } +} + +/* ── Objective-C ───────────────────────────────────────────────── */ + +static void sig_objc_params(sig_ctx_t *c, TSNode node) { + /* Selector keywords are the method's direct `identifier` children: the + * first is the name, each later one labels the parameter after it. */ + uint32_t count = ts_node_child_count(node); + bool saw_name = false; + TSNode pending = {0}; + bool have_pending = false; + for (uint32_t i = 0; i < count; i++) { + TSNode ch = ts_node_child(node, i); + const char *k = ts_node_type(ch); + if (strcmp(k, "identifier") == 0) { + if (saw_name) { + pending = ch; + have_pending = true; + } + saw_name = true; + } else if (strcmp(k, "method_parameter") == 0) { + sig_entry_open(c); + if (have_pending) { + sig_label_node(c, pending); + } else { + sig_label(c, NULL, 0); + } + have_pending = false; + if (sig_writes_types(c)) { + TSNode mt = cbm_find_child_by_kind(ch, "method_type"); + TSNode tn = ts_node_is_null(mt) ? mt : cbm_find_child_by_kind(mt, "type_name"); + size_t before = c->len; + sig_type_tokens(c, ts_node_is_null(tn) ? mt : tn); + if (c->len == before) { + sig_raw(c, "?", 1); + } + } + sig_entry_close(c); + } else if (!ts_node_is_named(ch) && sig_node_text_is(c, ch, "...")) { + sig_entry_open(c); + sig_raw_str(c, sig_variadic_marker(c->lang)); + sig_entry_close(c); + } + } +} + +/* ── Generic ───────────────────────────────────────────────────── */ + +static void sig_generic_params(sig_ctx_t *c, TSNode node) { + TSNode params = sig_field(node, "parameters"); + uint32_t count = ts_node_is_null(params) ? 0 : ts_node_named_child_count(params); + for (uint32_t i = 0; i < count; i++) { + TSNode p = ts_node_named_child(params, i); + if (sig_skipped_kind(ts_node_type(p))) { + continue; + } + TSNode type = sig_field(p, "type"); + sig_entry_type(c, type, NULL); + } +} + +/* ── Spelling pass ─────────────────────────────────────────────── */ + +/* Qualified type paths keep their last segment; "->" becomes "=>"; any other + * '.' or "::" (a receiver function type, a leading global scope) is dropped, + * so the suffix contract holds whatever the grammar produced. */ +static size_t sig_spell(char *s, size_t n) { + size_t out = 0; + size_t i = 0; + while (i < n) { + size_t sep = 0; + if (s[i] == '.') { + sep = 1; + } else if (s[i] == ':' && i + 1 < n && s[i + 1] == ':') { + sep = 2; + } + if (sep) { + bool ident_before = out > 0 && sig_ident_char((unsigned char)s[out - 1]); + bool ident_after = i + sep < n && sig_ident_char((unsigned char)s[i + sep]); + if (ident_before && ident_after) { + while (out > 0 && sig_ident_char((unsigned char)s[out - 1])) { + out--; + } + } + i += sep; + continue; + } + if (s[i] == '-' && i + 1 < n && s[i + 1] == '>') { + s[out++] = '='; + s[out++] = '>'; + i += 2; + continue; + } + s[out++] = s[i++]; + } + s[out] = '\0'; + return out; +} + +static uint64_t sig_fnv64(const char *s, size_t n) { + uint64_t h = SIG_FNV_OFFSET; + for (size_t i = 0; i < n; i++) { + h ^= (unsigned char)s[i]; + h *= SIG_FNV_PRIME; + } + return h; +} + +/* "(" leading whole entries ")" ... "#)" within CBM_CALLABLE_SIG_MAX. */ +static const char *sig_capped(CBMArena *a, const sig_ctx_t *c, const char *full, size_t n) { + char tail[SIG_CAPPED_TAIL + 1]; + snprintf(tail, sizeof(tail), "#%016llx)", (unsigned long long)sig_fnv64(full, n)); + size_t keep = 0; /* bytes of entry text after the "(" */ + bool comma = sig_writes_types(c); + int limit = c->too_many ? SIG_MAX_ENTRIES : c->entries; + for (int i = 0; i < limit; i++) { + size_t end = c->entry_end[i]; + if (end < c->open_off + 1 || end > c->close_off) { + break; + } + size_t candidate = end - (c->open_off + 1); + if (1 + candidate + (comma ? 1 : 0) + SIG_CAPPED_TAIL > CBM_CALLABLE_SIG_MAX) { + break; + } + keep = candidate; + } + char *out = cbm_arena_alloc(a, CBM_CALLABLE_SIG_MAX + 1); + if (!out) { + return NULL; + } + size_t len = 0; + out[len++] = '('; + memcpy(out + len, c->buf + c->open_off + 1, keep); + len += keep; + if (keep && comma) { + out[len++] = ','; + } + memcpy(out + len, tail, SIG_CAPPED_TAIL); + len += SIG_CAPPED_TAIL; + out[len] = '\0'; + return out; +} + +/* ── Builder ───────────────────────────────────────────────────── */ + +static void sig_render(sig_ctx_t *c, TSNode node) { + switch (c->lang) { + case CBM_LANG_CPP: + case CBM_LANG_CUDA: + case CBM_LANG_C: { + TSNode fdecl = sig_c_function_declarator(node); + if (c->lang != CBM_LANG_C && !sig_cpp_member_of_class_template(fdecl)) { + sig_cpp_tparams(c, node); + } + if (ts_node_is_null(fdecl)) { + c->open_off = c->len; + sig_raw(c, "()", 2); + c->close_off = c->len; + return; + } + sig_c_param_list(c, fdecl); + sig_cpp_cvref(c, fdecl); + return; + } + default: + break; + } + if (c->lang == CBM_LANG_CSHARP) { + sig_cs_tparams(c, node); + } + c->open_off = c->len; + sig_raw(c, "(", 1); + switch (c->lang) { + case CBM_LANG_JAVA: + sig_java_params(c, node); + break; + case CBM_LANG_CSHARP: + sig_cs_params(c, node); + break; + case CBM_LANG_KOTLIN: + sig_kotlin_params(c, node); + break; + case CBM_LANG_SWIFT: + sig_swift_params(c, node); + break; + case CBM_LANG_SCALA: + sig_scala_params(c, node); + break; + case CBM_LANG_OBJC: + sig_objc_params(c, node); + break; + default: + sig_generic_params(c, node); + break; + } + sig_raw(c, ")", 1); + c->close_off = c->len; +} + +static bool sig_roundtrips(const char *suffix, size_t n) { + /* "f" + suffix must invert to the base "f". */ + char probe[CBM_CALLABLE_SIG_MAX + 2]; + if (n > CBM_CALLABLE_SIG_MAX) { + return false; + } + probe[0] = 'f'; + memcpy(probe + 1, suffix, n); + probe[n + 1] = '\0'; + return cbm_qn_callable_base_len(probe) == 1; +} + +/* "(#)": the identity of a signature that cannot be spelled safely. */ +static const char *sig_hashed(CBMArena *a, const char *full, size_t n) { + return cbm_arena_sprintf(a, "(#%016llx)", (unsigned long long)sig_fnv64(full, n)); +} + +const char *cbm_callable_sig_mode(CBMArena *a, TSNode func_node, const char *source, + CBMLanguage lang, CBMCallableIdentity mode) { + if (!a || !source || ts_node_is_null(func_node) || mode == CBM_CALLABLE_ID_NONE) { + return NULL; + } + sig_ctx_t c; + memset(&c, 0, sizeof(c)); + c.arena = a; + c.src = source; + c.lang = lang; + c.mode = mode; + sig_raw(&c, "", 0); + if (c.overflow || !c.buf) { + return NULL; + } + c.buf[0] = '\0'; + sig_render(&c, func_node); + if (c.overflow) { + return NULL; + } + if (mode == CBM_CALLABLE_ID_ARITY) { + char digits[SIG_ARITY_DIGITS]; + snprintf(digits, sizeof(digits), "(%d)", c.entries); + return cbm_arena_strdup(a, digits); + } + size_t full = c.len; + char *spelled = cbm_arena_strndup(a, c.buf, full); + if (!spelled) { + return NULL; + } + size_t n = sig_spell(spelled, full); + if (n <= CBM_CALLABLE_SIG_MAX) { + return sig_roundtrips(spelled, n) ? spelled : sig_hashed(a, spelled, n); + } + /* Over the cap: re-derive the entry offsets in spelled coordinates. The + * spelling pass only shortens and only looks one byte ahead, and every + * recorded offset sits before a ',' or ')', so spelling a prefix gives + * exactly that prefix's spelled length. */ + int limit = c.too_many ? SIG_MAX_ENTRIES : c.entries; + for (int i = 0; i < limit; i++) { + char *prefix = cbm_arena_strndup(a, c.buf, c.entry_end[i]); + c.entry_end[i] = prefix ? sig_spell(prefix, c.entry_end[i]) : 0; + } + char *open_prefix = cbm_arena_strndup(a, c.buf, c.open_off); + char *close_prefix = cbm_arena_strndup(a, c.buf, c.close_off); + c.open_off = open_prefix ? sig_spell(open_prefix, c.open_off) : 0; + c.close_off = close_prefix ? sig_spell(close_prefix, c.close_off) : 0; + c.buf = spelled; + c.len = n; + const char *capped = sig_capped(a, &c, spelled, n); + if (capped && sig_roundtrips(capped, strlen(capped))) { + return capped; + } + return sig_hashed(a, spelled, n); +} + +const char *cbm_callable_sig(CBMArena *a, TSNode func_node, const char *source, CBMLanguage lang) { + return cbm_callable_sig_mode(a, func_node, source, lang, cbm_callable_identity(lang)); +} + +/* ── Inverse ───────────────────────────────────────────────────── */ + +static bool sig_ends_with_operator(const char *qn, size_t end) { + if (end < SIG_OPERATOR_LEN || + memcmp(qn + end - SIG_OPERATOR_LEN, "operator", SIG_OPERATOR_LEN) != 0) { + return false; + } + size_t at = end - SIG_OPERATOR_LEN; + return at == 0 || !sig_ident_char((unsigned char)qn[at - 1]); +} + +/* Whether qn[0..end) ends in a leaf a suffix may follow: an identifier (but + * not a make `$(VAR)` reference), the '>' closing template parameters, a C++ + * `operator...` leaf, or operator symbols — alone (Swift/Scala `+`, `==`) or + * after a Scala `_` (`unary_!`). A path such as `app/(auth)`, an anonymous + * `(anon)` leaf or an empty leaf is never a callable followed by a suffix. */ +static bool sig_callable_leaf_before(const char *qn, size_t end) { + size_t start = end; + while (start > 0 && qn[start - 1] != '.') { + start--; + } + if (start == end) { + return false; + } + unsigned char last = (unsigned char)qn[end - 1]; + if (last == '>' || (sig_ident_char(last) && last != '$')) { + return true; + } + if (end - start >= SIG_OPERATOR_LEN && memcmp(qn + start, "operator", SIG_OPERATOR_LEN) == 0) { + return true; + } + size_t op = end; + while (op > start && strchr(SIG_OPERATOR_CHARS, qn[op - 1]) != NULL) { + op--; + } + return op < end && (op == start || qn[op - 1] == '_'); +} + +size_t cbm_qn_callable_base_len_named(const char *qn, const char *name) { + size_t base = cbm_qn_callable_base_len(qn); + if (!qn || !name) { + return base; + } + size_t len = strlen(qn); + size_t n = strlen(name); + if (base == len || n == 0 || base < n || memcmp(qn + base - n, name, n) != 0) { + return len; + } + size_t at = base - n; + bool anchored = + at == 0 || qn[at - 1] == '.' || (at >= 2 && qn[at - 1] == ':' && qn[at - 2] == ':'); + return anchored ? base : len; +} + +size_t cbm_qn_callable_base_len(const char *qn) { + if (!qn) { + return 0; + } + size_t len = strlen(qn); + /* Trailing cvref (C++): const / volatile / & / && after the ')'. */ + size_t t = len; + for (;;) { + if (t > 0 && qn[t - 1] == '&') { + t--; + } else if (t >= SIG_CONST_LEN && + memcmp(qn + t - SIG_CONST_LEN, "const", SIG_CONST_LEN) == 0) { + t -= SIG_CONST_LEN; + } else if (t >= SIG_VOLATILE_LEN && + memcmp(qn + t - SIG_VOLATILE_LEN, "volatile", SIG_VOLATILE_LEN) == 0) { + t -= SIG_VOLATILE_LEN; + } else { + break; + } + } + if (t == 0 || qn[t - 1] != ')') { + return len; + } + /* The balanced "(...)" that ends at t. */ + int depth = 0; + size_t p = t; + bool found = false; + while (p > 0) { + p--; + if (qn[p] == ')') { + depth++; + } else if (qn[p] == '(') { + depth--; + if (depth == 0) { + found = true; + break; + } + } + } + if (!found || !sig_callable_leaf_before(qn, p)) { + return len; + } + /* A bare C++ call-operator leaf `...operator()` is a name, not a suffix. */ + if (t - p == 2 && sig_ends_with_operator(qn, p)) { + return len; + } + /* Optional "" right before the "(". */ + if (qn[p - 1] == '>') { + int adepth = 0; + size_t q = p; + while (q > 0) { + q--; + if (qn[q] == '>') { + adepth++; + } else if (qn[q] == '<') { + adepth--; + if (adepth == 0) { + break; + } + } else if (qn[q] == '.') { + q = 0; + adepth = 1; + break; + } + } + if (adepth == 0 && q > 0 && qn[q - 1] != '.' && !sig_ends_with_operator(qn, q)) { + return q; + } + } + return p; +} diff --git a/internal/cbm/callable_sig.h b/internal/cbm/callable_sig.h new file mode 100644 index 0000000000..c7b4d64fc9 --- /dev/null +++ b/internal/cbm/callable_sig.h @@ -0,0 +1,83 @@ +/* + * callable_sig.h — signature-qualified callable identity (#2061). + * + * A callable's graph identity is + * + * callable_qn := base_qn [] (params) [cvref] + * + * so overloads that share a base QN (Java/C#/C++/Kotlin/Swift/Scala/ObjC) + * become distinct nodes instead of merging into one node that collects every + * overload's CALLS edges. The `name` column stays the bare name. + * + * Every language selects a mode (cbm_callable_identity). A language whose + * mode is CBM_CALLABLE_ID_NONE keeps its historical QN byte for byte; the + * suffix is applied only where a mode is enabled, and the SAME builder + * (cbm_callable_sig) serves the definition QN and the call-scope QN so the two + * can never disagree. + * + * Suffix contract (what cbm_qn_callable_base_len inverts): + * - it never contains '.', "::" or "->" — the separators every QN leaf + * splitter uses — so a splitter that first strips the suffix finds the + * same leaf as before; + * - it is whitespace-normalized: one space only between two identifier + * characters ("unsigned long", "const T&"), none elsewhere; + * - type spelling: comments, annotations and attributes are dropped; + * qualified type paths keep their last segment (java.util.List -> List, + * std::vector -> vector); "->" in function types is spelled "=>"; a + * variadic/rest marker is "[]" for Java (the JVM array it is) and "~" + * everywhere else; C++ top-level cv-qualifiers are dropped and array + * parameters decay to pointers, as the language itself does; + * - it is at most CBM_CALLABLE_SIG_MAX bytes. A longer signature keeps the + * leading whole parameter entries that fit and ends in "#<16 hex>)", the + * FNV-1a 64 of the full uncapped suffix, so identity is preserved. + */ +#ifndef CBM_CALLABLE_SIG_H +#define CBM_CALLABLE_SIG_H + +#include "cbm.h" +#include "arena.h" +#include "tree_sitter/api.h" +#include + +typedef enum { + CBM_CALLABLE_ID_NONE = 0, /* historical base QN, no suffix */ + CBM_CALLABLE_ID_TYPED, /* (int,String) */ + CBM_CALLABLE_ID_LABELED_TYPED, /* (_:Data,to:URL) Swift */ + CBM_CALLABLE_ID_LABELED, /* (_:style:) ObjC selector */ + CBM_CALLABLE_ID_ARITY, /* (2) dynamic langs */ +} CBMCallableIdentity; + +enum { CBM_CALLABLE_SIG_MAX = 200 }; + +/* The identity mode a language uses for callable QNs. Every language is + * CBM_CALLABLE_ID_NONE until its enable change lands (with its index-format + * bump). Kept as a side table, like cbm_string_dispatch_suffixes, rather than + * a CBMLangSpec field: the ~160 positional lang_specs rows would all have to + * spell the new member under -Wmissing-field-initializers. */ +CBMCallableIdentity cbm_callable_identity(CBMLanguage lang); + +/* The identity suffix for the callable at `func_node` in the language's own + * mode, or NULL when that mode is NONE (or the node carries no parameter + * information the mode can use). Arena-owned. */ +const char *cbm_callable_sig(CBMArena *a, TSNode func_node, const char *source, CBMLanguage lang); + +/* Same, with an explicit mode — the builder the enable changes and the golden + * tables exercise before a language is switched on. */ +const char *cbm_callable_sig_mode(CBMArena *a, TSNode func_node, const char *source, + CBMLanguage lang, CBMCallableIdentity mode); + +/* Length of the base QN, i.e. `qn` without its callable identity suffix; equal + * to strlen(qn) when there is none. Purely syntactic (the inverse of the + * builder's contract above); a bare C++ `operator()` leaf is recognised as a + * name, not a suffix. NULL -> 0. */ +size_t cbm_qn_callable_base_len(const char *qn); + +/* The inverse anchored to the callable's bare `name` (the node's name + * column): the suffix is stripped only when the base QN is `name` or ends in + * "." / "::" + `name`. Other QN text can end in ')' as well — a Java field + * whose name is its declarator `X = f()`, a make `$(VAR)` target — and only + * the anchor makes the inverse exact on it. Use this form wherever the name + * is known; NULL name -> the unanchored form. */ +size_t cbm_qn_callable_base_len_named(const char *qn, const char *name); + +#endif /* CBM_CALLABLE_SIG_H */ diff --git a/internal/cbm/cbm.c b/internal/cbm/cbm.c index c5f93c72a1..de6bbcfdb9 100644 --- a/internal/cbm/cbm.c +++ b/internal/cbm/cbm.c @@ -244,6 +244,11 @@ void cbm_typeassign_push(CBMTypeAssignArray *arr, CBMArena *a, CBMTypeAssign ta) arr->items[arr->count++] = ta; } +void cbm_fieldtype_push(CBMFieldTypeArray *arr, CBMArena *a, CBMFieldType ft) { + GROW_ARRAY(arr, a); + arr->items[arr->count++] = ft; +} + void cbm_stringref_push(CBMStringRefArray *arr, CBMArena *a, CBMStringRef sr) { GROW_ARRAY(arr, a); arr->items[arr->count++] = sr; diff --git a/internal/cbm/cbm.h b/internal/cbm/cbm.h index fb5e446a5d..dc89a6b2d6 100644 --- a/internal/cbm/cbm.h +++ b/internal/cbm/cbm.h @@ -246,6 +246,10 @@ typedef struct { * HTTP_CALLS edge to base + path. Tail fields: zero-init stays valid. */ const char *http_client; const char *http_base_url; + /* Callable identity (#2061): offset of the signature suffix inside + * qualified_name (base QN = the first qn_sig_off bytes); 0 = no suffix. + * Always 0 until a language enables its callable_identity mode. */ + uint32_t qn_sig_off; } CBMDefinition; /* Argument captured from a call expression */ @@ -304,10 +308,20 @@ typedef struct { // guard's unique-name exemption. Default false. } CBMCall; +// What an import statement names, when the syntax says so. Only PHP's +// `use function` / `use const` set a non-default kind today: they name a +// namespace MEMBER, so they must never be mapped to a class file (#1186). +typedef enum { + CBM_IMPORT_KIND_DEFAULT = 0, // module / class / namespace (language default) + CBM_IMPORT_KIND_FUNCTION, // PHP `use function A\b` + CBM_IMPORT_KIND_CONST, // PHP `use const A\B` +} CBMImportKind; + typedef struct { const char *local_name; // local alias or name const char *module_path; // resolved module path / QN bool is_default; // ES default import (`import X from "Y"`), JS/TS only (#1916) + CBMImportKind kind; // CBM_IMPORT_KIND_DEFAULT unless the syntax names a member kind } CBMImport; typedef enum { @@ -405,6 +419,17 @@ typedef struct { CBMChannelDirection direction; } CBMChannel; +/* Python: one annotated instance field of a class -- `x: T` or `x: T = v` in + * the class body, `self.x: T = v` in __init__, or `self.x = p` where `p` is an + * annotated __init__ parameter. Not a graph node: it only carries the field's + * declared type to the cross-file LSP, so `obj.x.m()` on a class imported + * from another file can be typed (#1277). */ +typedef struct { + const char *class_qn; // QN of the owning class + const char *field_name; // attribute name + const char *type_text; // raw annotation text, resolved later per file +} CBMFieldType; + // Rust: impl Trait for Struct typedef struct { const char *trait_name; // trait name (raw text) @@ -431,6 +456,10 @@ typedef struct { uint32_t site_start_byte; // exact source occurrence; end > start when present uint32_t site_end_byte; // exclusive byte offset in the source file CBMSourceOrigin source_origin; // raw source or C-family preprocessed buffer + /* Callable identity (#2061): the resolved overload's signature suffix, + * appended to callee_qn when the edge is written; NULL = none (always, + * until a language enables its callable_identity mode). */ + const char *callee_sig; } CBMResolvedCall; typedef struct { @@ -518,6 +547,12 @@ typedef struct { int cap; } CBMChannelArray; +typedef struct { + CBMFieldType *items; + int count; + int cap; +} CBMFieldTypeArray; + // Full extraction result for one file. typedef struct CBMFileResult { CBMArena arena; // owns local memory; composites may also retain child arenas below @@ -536,6 +571,7 @@ typedef struct CBMFileResult { CBMStringRefArray string_refs; // URL/config string literals from AST CBMInfraBindingArray infra_bindings; // topic→URL pairs from IaC configs CBMChannelArray channels; // Socket.IO / EventEmitter pub/sub participation + CBMFieldTypeArray field_types; // Python: annotated instance fields (#1277) const char *module_qn; // module qualified name const char *namespace_name; // declared namespace/package (Java/Kotlin/C#/PHP), NULL if none @@ -606,6 +642,11 @@ typedef struct CBMFileResult { // by cbm_free_result(); ordinary single-file results leave these zeroed. struct CBMFileResult **owned_results; int owned_result_count; + + /* The file's own doc, set on its File node: the Go package comment or + * the Rust inner docs (//!). NULL for other languages and undocumented + * files. */ + const char *module_doc; } CBMFileResult; // --- Enclosing function cache --- @@ -692,6 +733,11 @@ typedef struct { /* How many nodes the unified walk actually visited (whether or not it ran * out of budget) — the measurement the budget has to be expressed in. */ uint32_t walk_nodes_visited; + /* Doc-comment lookup state (extract_defs.c), NULL until first used and + * allocated in `scratch`: the memo of parents' child arrays and the Perl + * POD section index. */ + void *doc_memo; + void *doc_pod_index; } CBMExtractCtx; // --- Public API --- @@ -892,6 +938,7 @@ void cbm_rw_push(CBMRWArray *arr, CBMArena *a, CBMReadWrite rw); void cbm_typerefs_push(CBMTypeRefArray *arr, CBMArena *a, CBMTypeRef tr); void cbm_envaccess_push(CBMEnvAccessArray *arr, CBMArena *a, CBMEnvAccess ea); void cbm_typeassign_push(CBMTypeAssignArray *arr, CBMArena *a, CBMTypeAssign ta); +void cbm_fieldtype_push(CBMFieldTypeArray *arr, CBMArena *a, CBMFieldType ft); void cbm_stringref_push(CBMStringRefArray *arr, CBMArena *a, CBMStringRef sr); void cbm_infrabinding_push(CBMInfraBindingArray *arr, CBMArena *a, CBMInfraBinding ib); void cbm_impltrait_push(CBMImplTraitArray *arr, CBMArena *a, CBMImplTrait it); diff --git a/internal/cbm/extract_defs.c b/internal/cbm/extract_defs.c index bb56e2870f..ac7e092431 100644 --- a/internal/cbm/extract_defs.c +++ b/internal/cbm/extract_defs.c @@ -1265,79 +1265,1447 @@ static bool is_comment_node(const char *kind) { strcmp(kind, "line_comment") == 0 || strcmp(kind, "multiline_comment") == 0); } -// Extract comment text, truncating to MAX_COMMENT_LEN. -// #1017: snap the cut point back to a complete UTF-8 codepoint boundary. -static char *extract_comment_text(CBMArena *a, TSNode node, const char *source) { - char *text = cbm_node_text(a, node, source); - if (text && strlen(text) > MAX_COMMENT_LEN) { - size_t cut = MAX_COMMENT_LEN; - while (cut > 0 && ((unsigned char)text[cut] & 0xC0) == 0x80) - cut--; - text[cut] = '\0'; +// Python-specific: docstring as first expression_statement -> string in function body. +static const char *extract_python_docstring(CBMArena *a, TSNode node, const char *source) { + TSNode body = ts_node_child_by_field_name(node, TS_FIELD("body")); + if (ts_node_is_null(body) || ts_node_named_child_count(body) == 0) { + return NULL; + } + TSNode first = ts_node_named_child(body, 0); + if (ts_node_is_null(first) || strcmp(ts_node_type(first), "expression_statement") != 0) { + return NULL; + } + if (ts_node_named_child_count(first) == 0) { + return NULL; + } + TSNode str = ts_node_named_child(first, 0); + if (ts_node_is_null(str)) { + return NULL; + } + const char *sk = ts_node_type(str); + if (strcmp(sk, "string") == 0 || strcmp(sk, "concatenated_string") == 0) { + return cbm_node_text(a, str, source); + } + return NULL; +} + +/* ── Doc comments ───────────────────────────────────────────────────────── + * A definition's doc is the comment RUN directly above it, not only the one + * comment node before it: + * - a run is contiguous (no blank line) and of one marker style; a Go comment + * group may mix styles and drops go/ast directives (//go:generate, //line); + * - it sits above the item's attributes (Rust), its declaration or export + * wrapper (JS/TS, C#, Go var/const) or its typedef (C); + * - a comment on the previous code's last line is that code's trailing + * comment, and a Rust inner doc documents the module, never the item; + * - a nearer plain comment does not hide a doc-style comment directly above; + * - a run a blank line away is detached (Go: always; elsewhere: unless it is + * doc-style), and a run with no letter or digit (a banner) is no doc; + * - the whole run is kept: the node's properties buffer grows with the doc. + * Fields, variables, enum members and macros of code languages get the same + * doc, a Perl sub the POD section that names it, and a Go or Rust file its + * package comment or inner docs (on the File node). + * + * Leading trivia is read from the parent's child array, kept in a small + * per-file memo: tree-sitter has no parent pointers, so ts_node_parent and + * ts_node_prev_sibling are O(position), and one lookup per member of an + * N-member class body cost O(N^2) (65,536 fields in one dotnet test class). */ + +enum { + DOC_SPAN_COMMENT = 1, + DOC_SPAN_ATTR = 2, + DOC_SPAN_INIT_CAP = 8, + DOC_SPAN_GROW = 2, + DOC_MEMO_SLOTS = 8, /* parents whose child arrays stay cached */ + DOC_MEMO_WIDE = 32, /* a level wider than this is cached while descending */ + DOC_BISECT = 2, +}; + +/* Comment marker families (the slash forms are spelled out in words so this + * comment does not nest). */ +typedef enum { + DOC_CS_OTHER = 0, /* another family (--, ;, %) */ + DOC_CS_LINE, /* slash-slash, or four and more slashes */ + DOC_CS_LINE_DOC, /* slash-slash-slash */ + DOC_CS_LINE_BANG, /* slash-slash-bang */ + DOC_CS_BLOCK, /* plain block, empty block, star-banner block */ + DOC_CS_BLOCK_DOC, /* slash-star-star */ + DOC_CS_BLOCK_BANG, /* slash-star-bang */ + DOC_CS_HASH, /* hash */ +} doc_style_t; + +/* One trivia element as a byte span. erow is the EFFECTIVE end row: a node + * that ends at column 0 (a Rust line_comment includes its newline) ends on the + * row before. */ +typedef struct { + uint32_t sb; + uint32_t eb; + uint32_t srow; + uint32_t erow; + uint8_t kind; + uint8_t style; +} doc_span_t; + +typedef struct { + doc_span_t *items; /* trivia directly before the anchor, in source order */ + int count; + int cap; + bool code_before; /* a non-trivia sibling precedes items[0] */ + uint32_t code_erow; /* ... its effective end row */ + uint32_t code_eb; /* ... its end byte (Kotlin gap scan) */ + uint32_t code_end_row; /* ... its raw end row (Kotlin gap scan) */ +} doc_trivia_t; + +static CBMArena *doc_scratch(CBMExtractCtx *ctx) { + return ctx->scratch ? ctx->scratch : ctx->arena; +} + +static uint32_t doc_erow(TSNode n) { + TSPoint s = ts_node_start_point(n); + TSPoint e = ts_node_end_point(n); + return (e.column == 0 && e.row > s.row) ? e.row - SKIP_ONE : e.row; +} + +static bool doc_has_prefix(const char *p, uint32_t n, const char *lit) { + size_t len = strlen(lit); + return n >= len && memcmp(p, lit, len) == 0; +} + +static doc_style_t doc_style_of(const char *src, uint32_t sb, uint32_t eb) { + const char *p = src + sb; + uint32_t n = eb > sb ? eb - sb : 0; + if (doc_has_prefix(p, n, "////")) { + return DOC_CS_LINE; + } + if (doc_has_prefix(p, n, "///")) { + return DOC_CS_LINE_DOC; + } + if (doc_has_prefix(p, n, "//!")) { + return DOC_CS_LINE_BANG; + } + if (doc_has_prefix(p, n, "//")) { + return DOC_CS_LINE; + } + if (doc_has_prefix(p, n, "/**/") || doc_has_prefix(p, n, "/***")) { + return DOC_CS_BLOCK; + } + if (doc_has_prefix(p, n, "/**")) { + return DOC_CS_BLOCK_DOC; + } + if (doc_has_prefix(p, n, "/*!")) { + return DOC_CS_BLOCK_BANG; + } + if (doc_has_prefix(p, n, "/*")) { + return DOC_CS_BLOCK; + } + return doc_has_prefix(p, n, "#") ? DOC_CS_HASH : DOC_CS_OTHER; +} + +/* Languages whose toolchain has a doc-comment syntax distinct from a plain + * comment (javadoc / KDoc / JSDoc / PHPDoc, rustdoc, C# XML docs, Doxygen). */ +static bool doc_lang_has_doc_syntax(CBMLanguage lang) { + switch (lang) { + case CBM_LANG_RUST: + case CBM_LANG_C: + case CBM_LANG_CPP: + case CBM_LANG_CUDA: + case CBM_LANG_OBJC: + case CBM_LANG_CSHARP: + case CBM_LANG_SWIFT: + case CBM_LANG_DART: + case CBM_LANG_JAVA: + case CBM_LANG_KOTLIN: + case CBM_LANG_SCALA: + case CBM_LANG_GROOVY: + case CBM_LANG_JAVASCRIPT: + case CBM_LANG_TYPESCRIPT: + case CBM_LANG_TSX: + case CBM_LANG_ARKTS: + case CBM_LANG_PHP: + return true; + default: + return false; + } +} + +static bool doc_style_is_doc(CBMLanguage lang, int style) { + switch (lang) { + case CBM_LANG_C: + case CBM_LANG_CPP: + case CBM_LANG_CUDA: + case CBM_LANG_OBJC: + return style == DOC_CS_LINE_DOC || style == DOC_CS_LINE_BANG || style == DOC_CS_BLOCK_DOC || + style == DOC_CS_BLOCK_BANG; + case CBM_LANG_RUST: + case CBM_LANG_CSHARP: + case CBM_LANG_SWIFT: + case CBM_LANG_DART: + return style == DOC_CS_LINE_DOC || style == DOC_CS_BLOCK_DOC; + default: + return doc_lang_has_doc_syntax(lang) && style == DOC_CS_BLOCK_DOC; + } +} + +/* Field/Variable docs are for code languages: config languages (YAML, TOML, + * ...) mint a Variable per key, and a comment there is no API doc. */ +static bool doc_lang_member_docs(CBMLanguage lang) { + return doc_lang_has_doc_syntax(lang) || lang == CBM_LANG_GO || lang == CBM_LANG_PERL; +} + +static uint8_t doc_trivia_kind(CBMLanguage lang, const char *kind) { + if (is_comment_node(kind)) { + return DOC_SPAN_COMMENT; + } + /* A Rust #[attr] is a sibling between the doc and the item. */ + if (lang == CBM_LANG_RUST && strcmp(kind, "attribute_item") == 0) { + return DOC_SPAN_ATTR; + } + return 0; +} + +static doc_span_t doc_span_of(TSNode n, const char *src, uint8_t kind) { + doc_span_t sp; + sp.sb = ts_node_start_byte(n); + sp.eb = ts_node_end_byte(n); + sp.srow = ts_node_start_point(n).row; + sp.erow = doc_erow(n); + sp.kind = kind; + sp.style = (uint8_t)(kind == DOC_SPAN_COMMENT ? doc_style_of(src, sp.sb, sp.eb) : DOC_CS_OTHER); + return sp; +} + +static void doc_push_span(CBMArena *a, doc_trivia_t *t, const doc_span_t *sp) { + if (t->count == t->cap) { + int ncap = t->cap ? t->cap * DOC_SPAN_GROW : DOC_SPAN_INIT_CAP; + doc_span_t *grown = (doc_span_t *)cbm_arena_alloc(a, (size_t)ncap * sizeof(doc_span_t)); + if (!grown) { + return; + } + if (t->count > 0) { + memcpy(grown, t->items, (size_t)t->count * sizeof(doc_span_t)); + } + t->items = grown; + t->cap = ncap; + } + t->items[t->count++] = *sp; +} + +/* A non-trivia sibling before the anchor: trivia seen so far are not its. */ +static void doc_note_code(doc_trivia_t *t, TSNode code) { + t->count = 0; + t->code_before = true; + t->code_erow = doc_erow(code); + t->code_eb = ts_node_end_byte(code); + t->code_end_row = ts_node_end_point(code).row; +} + +/* End of the Kotlin block comment opening at src[i] (block comments nest). */ +static uint32_t doc_kotlin_block_end(const char *src, uint32_t i, uint32_t to, uint32_t *row) { + int depth = 0; + while (i < to) { + bool pair = i + SKIP_ONE < to; + if (pair && src[i] == '/' && src[i + SKIP_ONE] == '*') { + depth++; + i += PAIR_LEN; + continue; + } + if (pair && src[i] == '*' && src[i + SKIP_ONE] == '/') { + depth--; + i += PAIR_LEN; + if (depth == 0) { + return i; + } + continue; + } + if (src[i] == '\n') { + (*row)++; + } + i++; + } + return i; +} + +/* tree-sitter-kotlin's automatic-semicolon scanner can swallow a comment that + * sits between two declarations (seen before `enum class` and `abstract + * class`): it has no node and survives only as bytes in the gap between the + * siblings. Recover the comments from that gap. Any other byte is a token the + * tree does not show, and the run restarts after it. */ +static void doc_scan_kotlin_gap(CBMArena *a, const char *src, uint32_t from, uint32_t to, + uint32_t row, doc_trivia_t *t) { + uint32_t i = from; + while (i < to) { + char c = src[i]; + if (c == '\n') { + row++; + i++; + continue; + } + if (c == ' ' || c == '\t' || c == '\r' || c == '\f') { + i++; + continue; + } + if (c == '/' && i + SKIP_ONE < to && + (src[i + SKIP_ONE] == '/' || src[i + SKIP_ONE] == '*')) { + doc_span_t sp = {i, i, row, row, DOC_SPAN_COMMENT, DOC_CS_OTHER}; + if (src[i + SKIP_ONE] == '/') { + while (i < to && src[i] != '\n') { + i++; + } + } else { + i = doc_kotlin_block_end(src, i, to, &row); + } + sp.eb = i; + sp.erow = row; + sp.style = (uint8_t)doc_style_of(src, sp.sb, sp.eb); + doc_push_span(a, t, &sp); + continue; + } + t->count = 0; + t->code_before = true; + t->code_erow = row; + i++; + } +} + +static bool doc_space_byte(char c) { + return c == ' ' || c == '\t' || c == '\r' || c == '\n' || c == '\f'; +} + +/* Start of the (nesting) block comment whose close ends at src[p), or + * UINT32_MAX; *row moves to its start row. */ +static uint32_t doc_back_block_start(const char *src, uint32_t p, uint32_t *row) { + uint32_t q = p; + int depth = 0; + while (q >= PAIR_LEN) { + if (src[q - PAIR_LEN] == '*' && src[q - SKIP_ONE] == '/') { + depth++; + q -= PAIR_LEN; + continue; + } + if (src[q - PAIR_LEN] == '/' && src[q - SKIP_ONE] == '*') { + depth--; + q -= PAIR_LEN; + if (depth == 0) { + return q; + } + continue; + } + if (src[q - SKIP_ONE] == '\n' && *row > 0) { + (*row)--; + } + q--; + } + return UINT32_MAX; +} + +/* Start of a line comment that fills its line up to src[p), or UINT32_MAX. */ +static uint32_t doc_back_line_start(const char *src, uint32_t p) { + uint32_t ls = p; + while (ls > 0 && src[ls - SKIP_ONE] != '\n') { + ls--; + } + uint32_t k = ls; + while (k < p && (src[k] == ' ' || src[k] == '\t')) { + k++; + } + return (k + SKIP_ONE < p && src[k] == '/' && src[k + SKIP_ONE] == '/') ? k : UINT32_MAX; +} + +/* tree-sitter-kotlin can also fold such a comment into the PREVIOUS node's + * byte range (package_header ends after the comment that follows it), so + * neither a sibling nor the gap holds it. Scan backwards from the anchor over + * whitespace, block comments and whole-line line comments; stop at anything + * else. The spans come out nearest-first and are appended in source order. */ +static void doc_scan_kotlin_back(CBMArena *a, const char *src, uint32_t anchor_sb, + uint32_t anchor_row, doc_trivia_t *t) { + doc_trivia_t rev; + memset(&rev, 0, sizeof(rev)); + uint32_t p = anchor_sb; + uint32_t row = anchor_row; + for (;;) { + while (p > 0 && doc_space_byte(src[p - SKIP_ONE])) { + if (src[p - SKIP_ONE] == '\n' && row > 0) { + row--; + } + p--; + } + uint32_t srow = row; + uint32_t start = (p >= PAIR_LEN && src[p - PAIR_LEN] == '*' && src[p - SKIP_ONE] == '/') + ? doc_back_block_start(src, p, &srow) + : doc_back_line_start(src, p); + if (start == UINT32_MAX) { + break; + } + doc_span_t sp = { + start, p, srow, row, DOC_SPAN_COMMENT, (uint8_t)doc_style_of(src, start, p)}; + doc_push_span(a, &rev, &sp); + p = start; + row = srow; + } + for (int i = rev.count - SKIP_ONE; i >= 0; i--) { + doc_push_span(a, t, &rev.items[i]); + } + t->code_before = p > 0; + t->code_erow = row; +} + +/* The memo: the child arrays of the last few parents looked up. A node's index + * is a binary search on start bytes, and its parent is found by descending + * from the deepest cached ancestor (the root is always reachable): the same + * answers as the tree-sitter calls, O(log N) per lookup on wide levels. */ +typedef struct { + TSNode node; + TSNode *kids; + uint32_t *kid_sb; + uint32_t n; + uint32_t cap; + uint32_t stamp; + bool used; +} doc_memo_ent_t; + +typedef struct { + doc_memo_ent_t ent[DOC_MEMO_SLOTS]; + uint32_t clock; +} doc_memo_t; + +/* The cached entry of `parent`, or the slot to reuse for it (unused first, + * else least recently used). */ +static doc_memo_ent_t *doc_memo_slot(doc_memo_t *m, TSNode parent, bool *hit) { + doc_memo_ent_t *victim = &m->ent[0]; + for (int i = 0; i < DOC_MEMO_SLOTS; i++) { + doc_memo_ent_t *e = &m->ent[i]; + if (e->used && ts_node_eq(e->node, parent)) { + *hit = true; + return e; + } + if (victim->used && (!e->used || e->stamp < victim->stamp)) { + victim = e; + } + } + *hit = false; + return victim; +} + +/* Fill `e` with the children of `parent`; false when the arrays cannot grow. */ +static bool doc_memo_fill(CBMArena *a, doc_memo_ent_t *e, TSNode parent) { + uint32_t n = ts_node_child_count(parent); + if (n > e->cap) { + TSNode *kids = (TSNode *)cbm_arena_alloc(a, (size_t)n * sizeof(TSNode)); + uint32_t *sb = (uint32_t *)cbm_arena_alloc(a, (size_t)n * sizeof(uint32_t)); + if (!kids || !sb) { + e->used = false; + return false; + } + e->kids = kids; + e->kid_sb = sb; + e->cap = n; + } + uint32_t k = 0; + TSTreeCursor cur = ts_tree_cursor_new(parent); + if (ts_tree_cursor_goto_first_child(&cur)) { + do { + e->kids[k] = ts_tree_cursor_current_node(&cur); + e->kid_sb[k] = ts_node_start_byte(e->kids[k]); + k++; + } while (k < n && ts_tree_cursor_goto_next_sibling(&cur)); + } + ts_tree_cursor_delete(&cur); + e->node = parent; + e->n = k; + e->used = true; + return true; +} + +static const doc_memo_ent_t *doc_memo_get(CBMExtractCtx *ctx, TSNode parent) { + CBMArena *a = doc_scratch(ctx); + if (!ctx->doc_memo) { + doc_memo_t *fresh = (doc_memo_t *)cbm_arena_alloc(a, sizeof(doc_memo_t)); + if (!fresh) { + return NULL; + } + memset(fresh, 0, sizeof(*fresh)); + ctx->doc_memo = fresh; + } + doc_memo_t *m = (doc_memo_t *)ctx->doc_memo; + m->clock++; + bool hit = false; + doc_memo_ent_t *e = doc_memo_slot(m, parent, &hit); + if (!hit && !doc_memo_fill(a, e, parent)) { + return NULL; + } + e->stamp = m->clock; + return e; +} + +/* Index of `node` among the cached children, or -1. */ +static int doc_memo_index(const doc_memo_ent_t *e, TSNode node) { + uint32_t s = ts_node_start_byte(node); + uint32_t lo = 0; + uint32_t hi = e->n; + while (lo < hi) { + uint32_t mid = lo + ((hi - lo) / DOC_BISECT); + if (e->kid_sb[mid] < s) { + lo = mid + SKIP_ONE; + } else { + hi = mid; + } + } + for (uint32_t i = lo; i < e->n && e->kid_sb[i] == s; i++) { + if (ts_node_eq(e->kids[i], node)) { + return (int)i; + } + } + return CBM_NOT_FOUND; +} + +/* Index of the cached child whose byte range contains `node`, or -1. */ +static int doc_memo_containing(const doc_memo_ent_t *e, TSNode node) { + uint32_t s = ts_node_start_byte(node); + uint32_t en = ts_node_end_byte(node); + uint32_t lo = 0; + uint32_t hi = e->n; + while (lo < hi) { + uint32_t mid = lo + ((hi - lo) / DOC_BISECT); + if (e->kid_sb[mid] <= s) { + lo = mid + SKIP_ONE; + } else { + hi = mid; + } + } + for (int i = (int)lo - SKIP_ONE; i >= 0; i--) { + TSNode kid = e->kids[i]; + uint32_t ke = ts_node_end_byte(kid); + if (ke >= en && (ke > e->kid_sb[i] || ts_node_eq(kid, node))) { + return i; + } + if (ke <= s && e->kid_sb[i] < s) { + break; + } + } + return CBM_NOT_FOUND; +} + +/* The deepest cached node that contains `node` (the node itself excluded). */ +static const doc_memo_ent_t *doc_memo_ancestor(const doc_memo_t *m, TSNode node) { + const doc_memo_ent_t *best = NULL; + uint32_t ns = ts_node_start_byte(node); + uint32_t ne = ts_node_end_byte(node); + for (int i = 0; m && i < DOC_MEMO_SLOTS; i++) { + const doc_memo_ent_t *e = &m->ent[i]; + if (!e->used || ts_node_eq(e->node, node)) { + continue; + } + uint32_t es = ts_node_start_byte(e->node); + uint32_t ee = ts_node_end_byte(e->node); + if (es <= ns && ne <= ee && + (!best || ee - es < ts_node_end_byte(best->node) - ts_node_start_byte(best->node))) { + best = e; + } + } + return best; +} + +/* ts_node_parent() through the memo: descend from the deepest cached ancestor, + * binary search on wide levels (cached on the way), linear on narrow ones. */ +static TSNode doc_parent(CBMExtractCtx *ctx, TSNode node) { + if (ts_node_eq(node, ctx->root)) { + TSNode none = {0}; + return none; + } + const doc_memo_ent_t *e = doc_memo_ancestor((const doc_memo_t *)ctx->doc_memo, node); + if (!e) { + e = doc_memo_get(ctx, ctx->root); + } + while (e) { + int k = doc_memo_containing(e, node); + if (k < 0) { + break; + } + TSNode cur = e->kids[k]; + if (ts_node_eq(cur, node)) { + return e->node; + } + e = NULL; + while (!ts_node_is_null(cur)) { + if (ts_node_child_count(cur) > DOC_MEMO_WIDE) { + e = doc_memo_get(ctx, cur); + break; + } + TSNode next = ts_node_child_with_descendant(cur, node); + if (ts_node_is_null(next)) { + break; + } + if (ts_node_eq(next, node)) { + return cur; + } + cur = next; + } + } + return ts_node_parent(node); /* not reachable through the memo */ +} + +/* Trivia before the child at index mk of a cached child array. */ +static void doc_collect_cached(CBMExtractCtx *ctx, const doc_memo_ent_t *me, int mk, + doc_trivia_t *t) { + int j = mk - SKIP_ONE; + while (j >= 0 && doc_trivia_kind(ctx->language, ts_node_type(me->kids[j]))) { + j--; + } + if (j >= 0) { + doc_note_code(t, me->kids[j]); + } + for (int i = j + SKIP_ONE; i < mk; i++) { + uint8_t kind = doc_trivia_kind(ctx->language, ts_node_type(me->kids[i])); + doc_span_t sp = doc_span_of(me->kids[i], ctx->source, kind); + doc_push_span(doc_scratch(ctx), t, &sp); + } +} + +/* The same without the memo (its arrays could not be allocated): one cursor + * pass over the parent. False when `anchor` is not among its children. */ +static bool doc_collect_cursor(CBMExtractCtx *ctx, TSNode parent, TSNode anchor, doc_trivia_t *t) { + bool found = false; + TSTreeCursor cur = ts_tree_cursor_new(parent); + if (ts_tree_cursor_goto_first_child(&cur)) { + do { + TSNode ch = ts_tree_cursor_current_node(&cur); + if (ts_node_eq(ch, anchor)) { + found = true; + break; + } + uint8_t kind = doc_trivia_kind(ctx->language, ts_node_type(ch)); + if (kind) { + doc_span_t sp = doc_span_of(ch, ctx->source, kind); + doc_push_span(doc_scratch(ctx), t, &sp); + } else { + doc_note_code(t, ch); + } + } while (ts_tree_cursor_goto_next_sibling(&cur)); + } + ts_tree_cursor_delete(&cur); + return found; +} + +/* Kotlin comments the grammar hid: in the gap after the previous sibling, else + * folded into the previous node (see the two scanners above). */ +static void doc_collect_kotlin(CBMExtractCtx *ctx, TSNode parent, TSNode anchor, doc_trivia_t *t) { + CBMArena *a = doc_scratch(ctx); + uint32_t from = t->code_before ? t->code_eb : ts_node_start_byte(parent); + uint32_t row = t->code_before ? t->code_end_row : ts_node_start_point(parent).row; + uint32_t to = ts_node_start_byte(anchor); + if (from < to) { + doc_scan_kotlin_gap(a, ctx->source, from, to, row, t); + } + if (t->count == 0) { + doc_scan_kotlin_back(a, ctx->source, to, ts_node_start_point(anchor).row, t); + } +} + +/* Leading trivia of `anchor`, in source order. */ +static void doc_collect_trivia(CBMExtractCtx *ctx, TSNode anchor, doc_trivia_t *t) { + memset(t, 0, sizeof(*t)); + TSNode parent = doc_parent(ctx, anchor); + if (ts_node_is_null(parent)) { + return; + } + const doc_memo_ent_t *me = doc_memo_get(ctx, parent); + int mk = me ? doc_memo_index(me, anchor) : CBM_NOT_FOUND; + bool found = mk >= 0; + if (found) { + doc_collect_cached(ctx, me, mk, t); + } else { + found = doc_collect_cursor(ctx, parent, anchor, t); + } + if (!found) { + memset(t, 0, sizeof(*t)); + return; + } + if (t->count == 0 && ctx->language == CBM_LANG_KOTLIN) { + doc_collect_kotlin(ctx, parent, anchor, t); + } +} + +/* go/ast CommentGroup.Text: a line comment with no space after the slashes is + * a directive, not documentation: //line, //extern, //export and the + * //[a-z0-9]+:[a-z0-9] form (//go:generate, //nolint:errcheck, ...). */ +static bool doc_go_is_directive(const char *src, const doc_span_t *sp) { + const char *p = src + sp->sb; + uint32_t n = sp->eb > sp->sb ? sp->eb - sp->sb : 0; + if (n <= PAIR_LEN || p[0] != '/' || p[SKIP_ONE] != '/' || p[PAIR_LEN] == ' ') { + return false; + } + p += PAIR_LEN; + n -= PAIR_LEN; + if (doc_has_prefix(p, n, "line ") || doc_has_prefix(p, n, "extern ") || + doc_has_prefix(p, n, "export ")) { + return true; + } + uint32_t colon = 0; + while (colon < n && p[colon] != ':' && p[colon] != '\n') { + colon++; + } + if (colon == 0 || colon >= n || p[colon] != ':' || colon + SKIP_ONE >= n) { + return false; + } + for (uint32_t i = 0; i <= colon + SKIP_ONE; i++) { + char b = p[i]; + if (i != colon && !((b >= 'a' && b <= 'z') || (b >= '0' && b <= '9'))) { + return false; + } + } + return true; +} + +/* UTF-8 bytes of the banner characters (see doc_has_words). */ +enum { + DOC_U8_ASCII_END = 0x80, + DOC_U8_CONT_MASK = 0xC0, /* 10xxxxxx: continuation byte */ + DOC_U8_CONT = 0x80, + DOC_U8_LATIN1 = 0xC2, /* lead byte of U+0080..U+00BF */ + DOC_U8_MIDDLE_DOT = 0xB7, /* U+00B7 */ + DOC_U8_U2000 = 0xE2, /* lead byte of U+2000..U+2FFF */ + DOC_U8_PUNCT = 0x80, /* U+2000..U+203F */ + DOC_U8_DASH_FIRST = 0x90, /* U+2010 hyphen */ + DOC_U8_BULLET_LAST = 0xA7, /* U+2027 hyphenation point */ + DOC_U8_BOX_FIRST = 0x94, /* U+2500 box drawing */ + DOC_U8_SHAPES_LAST = 0x97, /* U+25FF, end of the geometric shapes */ + DOC_U8_LATIN1_LEN = 2, + DOC_U8_U2000_LEN = 3, +}; + +static bool doc_banner_char(const unsigned char *p, size_t left) { + if (left >= DOC_U8_LATIN1_LEN && p[0] == DOC_U8_LATIN1 && p[SKIP_ONE] == DOC_U8_MIDDLE_DOT) { + return true; + } + if (left < DOC_U8_U2000_LEN || p[0] != DOC_U8_U2000) { + return false; + } + if (p[SKIP_ONE] >= DOC_U8_BOX_FIRST && p[SKIP_ONE] <= DOC_U8_SHAPES_LAST) { + return true; + } + return p[SKIP_ONE] == DOC_U8_PUNCT && p[PAIR_LEN] >= DOC_U8_DASH_FIRST && + p[PAIR_LEN] <= DOC_U8_BULLET_LAST; +} + +/* A comment with no letter or digit is a divider or banner, not documentation. + * Any non-ASCII character counts as a letter except the ones banners are drawn + * with: U+00B7, the dashes and bullets U+2010..U+2027, and the box-drawing, + * block and geometric-shape characters U+2500..U+25FF. */ +static bool doc_has_words(const char *s, size_t n) { + const unsigned char *u = (const unsigned char *)s; + for (size_t i = 0; i < n; i++) { + if (u[i] < DOC_U8_ASCII_END) { + if (isalnum(u[i])) { + return true; + } + } else if ((u[i] & DOC_U8_CONT_MASK) != DOC_U8_CONT && !doc_banner_char(u + i, n - i)) { + return true; + } + } + return false; +} + +static bool doc_span_kept(const char *src, const doc_span_t *sp, bool go_directives) { + return sp->kind == DOC_SPAN_COMMENT && !(go_directives && doc_go_is_directive(src, sp)); +} + +/* Text of items[first..last]: one comment keeps its exact bytes; several are + * joined by newlines, each without its own trailing newline. NULL when nothing + * is left (Go directives only) or when no comment has a letter or digit. */ +static const char *doc_run_text(CBMExtractCtx *ctx, const doc_trivia_t *t, int first, int last, + bool go_directives) { + const char *src = ctx->source; + int kept = 0; + bool words = false; + size_t total = 0; + for (int k = first; k <= last; k++) { + const doc_span_t *sp = &t->items[k]; + if (!doc_span_kept(src, sp, go_directives)) { + continue; + } + total += (size_t)(sp->eb - sp->sb) + SKIP_ONE; + words = words || doc_has_words(src + sp->sb, sp->eb - sp->sb); + kept++; + } + if (kept == 0 || !words) { + return NULL; + } + char *buf = (char *)cbm_arena_alloc(ctx->arena, total + SKIP_ONE); + if (!buf) { + return NULL; + } + size_t w = 0; + for (int k = first; k <= last; k++) { + const doc_span_t *sp = &t->items[k]; + if (!doc_span_kept(src, sp, go_directives)) { + continue; + } + uint32_t eb = sp->eb; + while (kept > SKIP_ONE && eb > sp->sb && + (src[eb - SKIP_ONE] == '\n' || src[eb - SKIP_ONE] == '\r')) { + eb--; + } + if (w > 0) { + buf[w++] = '\n'; + } + memcpy(buf + w, src + sp->sb, eb - sp->sb); + w += eb - sp->sb; + } + buf[w] = '\0'; + return buf; +} + +/* Index of the comment nearest the item, above its Rust attributes, or -1. */ +static int doc_nearest_comment(const doc_trivia_t *t) { + int k = t->count - SKIP_ONE; + while (k >= 0 && t->items[k].kind == DOC_SPAN_ATTR) { + k--; + } + return (k >= 0 && t->items[k].kind == DOC_SPAN_COMMENT) ? k : CBM_NOT_FOUND; +} + +/* A nearer plain comment does not hide a doc-style comment directly above it + * (a javadoc, then `// NOSONAR`): the index of that doc comment, else `near`. */ +static int doc_prefer_doc_style(CBMLanguage lang, const doc_trivia_t *t, int near) { + if (!doc_lang_has_doc_syntax(lang) || doc_style_is_doc(lang, t->items[near].style)) { + return near; + } + for (int j = near - SKIP_ONE; j >= 0; j--) { + const doc_span_t *sp = &t->items[j]; + if (t->items[j + SKIP_ONE].srow > sp->erow + SKIP_ONE) { + break; /* a blank line */ + } + if (sp->kind == DOC_SPAN_ATTR) { + continue; + } + if (sp->kind != DOC_SPAN_COMMENT) { + break; + } + if (doc_style_is_doc(lang, sp->style)) { + return j; + } + } + return near; +} + +/* First index of the run that ends at `near`: contiguous, one style (a Go + * comment group mixes styles), not the previous code's trailing comment. */ +static int doc_run_first(CBMLanguage lang, const doc_trivia_t *t, int near) { + int style = t->items[near].style; + int first = near; + for (int j = near - SKIP_ONE; j >= 0; j--) { + const doc_span_t *sp = &t->items[j]; + if (sp->kind != DOC_SPAN_COMMENT || t->items[j + SKIP_ONE].srow > sp->erow + SKIP_ONE) { + break; /* an attribute or a blank line ends the run */ + } + if (lang != CBM_LANG_GO && sp->style != style) { + break; + } + if (j == 0 && t->code_before && sp->srow == t->code_erow) { + break; + } + first = j; + } + return first; +} + +/* The doc of an anchor from its leading trivia. */ +static const char *doc_from_trivia(CBMExtractCtx *ctx, const doc_trivia_t *t, + uint32_t anchor_srow) { + CBMLanguage lang = ctx->language; + int near = t->items ? doc_nearest_comment(t) : CBM_NOT_FOUND; + if (near < 0) { + return NULL; + } + near = doc_prefer_doc_style(lang, t, near); + int style = t->items[near].style; + if (lang == CBM_LANG_RUST && (style == DOC_CS_LINE_BANG || style == DOC_CS_BLOCK_BANG)) { + return NULL; /* an inner doc: it documents the enclosing module */ + } + if (near == 0 && t->code_before && t->items[0].srow == t->code_erow) { + return NULL; /* the previous code's trailing comment */ + } + uint32_t next_row = near + SKIP_ONE < t->count ? t->items[near + SKIP_ONE].srow : anchor_srow; + bool detached = next_row > t->items[near].erow + SKIP_ONE; + if (detached && (lang == CBM_LANG_GO || !doc_style_is_doc(lang, style))) { + return NULL; + } + return doc_run_text(ctx, t, doc_run_first(lang, t, near), near, lang == CBM_LANG_GO); +} + +static const char *doc_for_anchor(CBMExtractCtx *ctx, TSNode anchor) { + doc_trivia_t t; + doc_collect_trivia(ctx, anchor, &t); + return doc_from_trivia(ctx, &t, ts_node_start_point(anchor).row); +} + +static bool doc_kind_is(TSNode n, const char *kind) { + return !ts_node_is_null(n) && strcmp(ts_node_type(n), kind) == 0; +} + +/* JS/TS: a function or class expression is documented above its declarator, + * field, pair or assignment statement, a declarator above its declaration, + * and any of them above `declare` and `export`. */ +static TSNode doc_anchor_js(CBMExtractCtx *ctx, TSNode cur) { + const char *k = ts_node_type(cur); + if (strcmp(k, "arrow_function") == 0 || strcmp(k, "function_expression") == 0 || + strcmp(k, "generator_function") == 0 || strcmp(k, "class") == 0) { + TSNode p = doc_parent(ctx, cur); + if (doc_kind_is(p, "variable_declarator") || doc_kind_is(p, "public_field_definition") || + doc_kind_is(p, "field_definition") || doc_kind_is(p, "pair")) { + cur = p; + } else if (doc_kind_is(p, "assignment_expression")) { + TSNode pp = doc_parent(ctx, p); + if (doc_kind_is(pp, "expression_statement")) { + cur = pp; + } + } + } + if (doc_kind_is(cur, "variable_declarator")) { + TSNode p = doc_parent(ctx, cur); + if (doc_kind_is(p, "lexical_declaration") || doc_kind_is(p, "variable_declaration")) { + cur = p; + } + } + TSNode p = doc_parent(ctx, cur); + if (doc_kind_is(p, "ambient_declaration")) { + cur = p; + p = doc_parent(ctx, cur); + } + return doc_kind_is(p, "export_statement") ? p : cur; +} + +/* C#: a field is documented above its field declaration, a top-level + * function above its global statement. */ +static TSNode doc_anchor_csharp(CBMExtractCtx *ctx, TSNode cur) { + if (doc_kind_is(cur, "variable_declarator")) { + TSNode p = doc_parent(ctx, cur); + if (doc_kind_is(p, "variable_declaration")) { + cur = p; + } + } + if (doc_kind_is(cur, "variable_declaration")) { + TSNode p = doc_parent(ctx, cur); + if (doc_kind_is(p, "field_declaration") || doc_kind_is(p, "event_field_declaration")) { + cur = p; + } } - return text; + TSNode p = doc_parent(ctx, cur); + return doc_kind_is(p, "global_statement") ? p : cur; } -// Go-specific: type_spec/type_alias comment is before the parent type_declaration. -static const char *extract_go_type_docstring(CBMArena *a, TSNode node, const char *source) { - const char *kind = ts_node_type(node); - if (strcmp(kind, "type_spec") != 0 && strcmp(kind, "type_alias") != 0) { - return NULL; +/* C family: `typedef struct X {..} Y;` and `struct X {..} v;` are documented + * above the declaration. Only a specifier WITH a body is a definition; a + * bodiless one is a usage. */ +static TSNode doc_anchor_c(CBMExtractCtx *ctx, TSNode cur) { + if ((doc_kind_is(cur, "struct_specifier") || doc_kind_is(cur, "enum_specifier") || + doc_kind_is(cur, "union_specifier") || doc_kind_is(cur, "class_specifier")) && + !ts_node_is_null(ts_node_child_by_field_name(cur, TS_FIELD("body")))) { + TSNode p = doc_parent(ctx, cur); + if (doc_kind_is(p, "type_definition") || doc_kind_is(p, "declaration")) { + return p; + } } - TSNode parent = ts_node_parent(node); - if (ts_node_is_null(parent) || strcmp(ts_node_type(parent), "type_declaration") != 0) { + return cur; +} + +/* Go: `var x = 1` (no parentheses) is documented above the declaration. */ +static TSNode doc_anchor_go(CBMExtractCtx *ctx, TSNode cur) { + if (doc_kind_is(cur, "var_spec") || doc_kind_is(cur, "const_spec")) { + TSNode p = doc_parent(ctx, cur); + if ((doc_kind_is(p, "var_declaration") || doc_kind_is(p, "const_declaration")) && + ts_node_eq(ts_node_child(p, SECOND_CHILD_IDX), cur)) { + return p; + } + } + return cur; +} + +/* The node whose leading comments document `node`. */ +static TSNode doc_anchor(CBMExtractCtx *ctx, TSNode node) { + switch (ctx->language) { + case CBM_LANG_JAVASCRIPT: + case CBM_LANG_TYPESCRIPT: + case CBM_LANG_TSX: + case CBM_LANG_ARKTS: + return doc_anchor_js(ctx, node); + case CBM_LANG_CSHARP: + return doc_anchor_csharp(ctx, node); + case CBM_LANG_C: + case CBM_LANG_CPP: + case CBM_LANG_CUDA: + case CBM_LANG_OBJC: + return doc_anchor_c(ctx, node); + case CBM_LANG_GO: + return doc_anchor_go(ctx, node); + default: + return node; + } +} + +/* ── Perl POD: the =head / =item section that names the sub ── */ + +enum { POD_HEAD = 1, POD_ITEM, POD_OVER, POD_BACK, POD_CUT, POD_OTHER }; + +typedef struct { + uint32_t sb; /* command paragraph */ + uint32_t eb; + uint32_t wb; /* end of the command word: its argument starts here */ + uint8_t type; + uint8_t level; /* =headN: N; =item and =back: list depth */ +} pod_cmd_t; + +typedef struct { + const char *name; + size_t name_len; + uint32_t sb; + uint32_t eb; +} pod_sec_t; + +typedef struct { + pod_sec_t *items; + int count; + int cap; +} pod_index_t; + +static bool pod_line_is_blank(const char *src, uint32_t b, uint32_t e) { + for (uint32_t k = b; k < e; k++) { + if (src[k] != ' ' && src[k] != '\t' && src[k] != '\r') { + return false; + } + } + return true; +} + +/* Next POD paragraph (a maximal run of non-blank lines) in src[*pos, pe). */ +static bool pod_next_para(const char *src, uint32_t *pos, uint32_t pe, uint32_t *psb, + uint32_t *peb) { + uint32_t i = *pos; + bool in = false; + uint32_t s = 0; + while (i < pe) { + uint32_t e = i; + while (e < pe && src[e] != '\n') { + e++; + } + bool blank = pod_line_is_blank(src, i, e); + if (!blank && !in) { + in = true; + s = i; + } + if (blank && in) { + *psb = s; + *peb = i; + *pos = i; + return true; + } + i = e < pe ? e + SKIP_ONE : pe; + } + *pos = pe; + if (in) { + *psb = s; + *peb = pe; + } + return in; +} + +static bool pod_word_is(const char *w, size_t wl, const char *lit) { + return wl == strlen(lit) && strncmp(w, lit, wl) == 0; +} + +/* Classify one command paragraph; `depth` is the =over nesting. */ +static pod_cmd_t pod_classify(const char *src, uint32_t sb, uint32_t eb, int *depth) { + uint32_t wb = sb + SKIP_ONE; + while (wb < eb && (isalnum((unsigned char)src[wb]) || src[wb] == '_')) { + wb++; + } + const char *w = src + sb + SKIP_ONE; + size_t wl = wb - sb - SKIP_ONE; + pod_cmd_t c = {sb, eb, wb, POD_OTHER, 0}; + size_t head_len = strlen("head"); + if (wl == head_len + SKIP_ONE && strncmp(w, "head", head_len) == 0 && + isdigit((unsigned char)w[head_len])) { + c.type = POD_HEAD; + c.level = (uint8_t)(w[head_len] - '0'); + } else if (pod_word_is(w, wl, "item")) { + c.type = POD_ITEM; + c.level = (uint8_t)*depth; + } else if (pod_word_is(w, wl, "over")) { + c.type = POD_OVER; + (*depth)++; + } else if (pod_word_is(w, wl, "back")) { + c.type = POD_BACK; + c.level = (uint8_t)*depth; + if (*depth > 0) { + (*depth)--; + } + } else if (pod_word_is(w, wl, "cut")) { + c.type = POD_CUT; + } + return c; +} + +/* Command paragraphs of one POD block, in order. */ +static int pod_commands(CBMArena *a, const char *src, uint32_t pb, uint32_t pe, pod_cmd_t **out) { + int cap = DOC_SPAN_INIT_CAP; + int n = 0; + pod_cmd_t *cmds = (pod_cmd_t *)cbm_arena_alloc(a, (size_t)cap * sizeof(pod_cmd_t)); + *out = cmds; + if (!cmds) { + return 0; + } + int depth = 0; + uint32_t pos = pb; + uint32_t sb = 0; + uint32_t eb = 0; + while (pod_next_para(src, &pos, pe, &sb, &eb)) { + if (src[sb] != '=' || sb + SKIP_ONE >= eb || !isalpha((unsigned char)src[sb + SKIP_ONE])) { + continue; + } + if (n == cap) { + int ncap = cap * DOC_SPAN_GROW; + pod_cmd_t *grown = (pod_cmd_t *)cbm_arena_alloc(a, (size_t)ncap * sizeof(pod_cmd_t)); + if (!grown) { + break; + } + memcpy(grown, cmds, (size_t)n * sizeof(pod_cmd_t)); + cmds = grown; + cap = ncap; + *out = cmds; + } + cmds[n++] = pod_classify(src, sb, eb, &depth); + } + return n; +} + +/* End byte of the section that cmds[ci] (a =head or =item) opens. */ +static uint32_t pod_section_end(const pod_cmd_t *cmds, int n, int ci, uint32_t pe) { + const pod_cmd_t *c = &cmds[ci]; + for (int j = ci + SKIP_ONE; j < n; j++) { + const pod_cmd_t *d = &cmds[j]; + if (d->type == POD_CUT) { + return d->sb; + } + if (d->type == POD_HEAD && (c->type == POD_ITEM || d->level <= c->level)) { + return d->sb; + } + if (c->type == POD_ITEM && (d->type == POD_ITEM || d->type == POD_BACK) && + d->level == c->level) { + return d->sb; + } + } + return pe; +} + +/* A heading's text with POD formatting codes (X<..>, X<< .. >>) removed and + * newlines and tabs as spaces. */ +static char *pod_plain_heading(CBMArena *a, const char *src, uint32_t b, uint32_t e) { + char *buf = (char *)cbm_arena_alloc(a, (size_t)(e - b) + SKIP_ONE); + if (!buf) { return NULL; } - TSNode pprev = ts_node_prev_sibling(parent); - if (!ts_node_is_null(pprev) && is_comment_node(ts_node_type(pprev))) { - return extract_comment_text(a, pprev, source); + size_t w = 0; + for (uint32_t i = b; i < e; i++) { + char c = src[i]; + if (c >= 'A' && c <= 'Z' && i + SKIP_ONE < e && src[i + SKIP_ONE] == '<') { + uint32_t j = i + SKIP_ONE; + while (j < e && src[j] == '<') { + j++; + } + if (j - i > PAIR_LEN && j < e && (src[j] == ' ' || src[j] == '\t')) { + j++; /* X<< text >> */ + } + i = j - SKIP_ONE; + continue; + } + if (c == '>' && !(w > 0 && buf[w - SKIP_ONE] == '-')) { + continue; + } + buf[w++] = (c == '\n' || c == '\r' || c == '\t') ? ' ' : c; } - return NULL; + buf[w] = '\0'; + return buf; } -// Python-specific: docstring as first expression_statement -> string in function body. -static const char *extract_python_docstring(CBMArena *a, TSNode node, const char *source) { - TSNode body = ts_node_child_by_field_name(node, TS_FIELD("body")); - if (ts_node_is_null(body) || ts_node_named_child_count(body) == 0) { +static const char *pod_skip_spaces(const char *p) { + while (*p == ' ') { + p++; + } + return p; +} + +/* The sub a heading names: after a list bullet ("*", "1."), a `$obj->` + * receiver and a sigil, the identifier that follows. Returns its length + * (0: none). */ +static size_t pod_heading_name(CBMArena *a, const char *src, uint32_t b, uint32_t e, + const char **name) { + const char *buf = pod_plain_heading(a, src, b, e); + if (!buf) { + return 0; + } + const char *p = pod_skip_spaces(buf); + if (*p == '*') { + p++; + } else if (isdigit((unsigned char)*p)) { + while (isdigit((unsigned char)*p)) { + p++; + } + if (*p == '.') { + p++; + } + } + p = pod_skip_spaces(p); + if (*p == '$') { + const char *q = p + SKIP_ONE; + while (isalnum((unsigned char)*q) || *q == '_') { + q++; + } + q = pod_skip_spaces(q); + if (q > p + SKIP_ONE && q[0] == '-' && q[SKIP_ONE] == '>') { + p = pod_skip_spaces(q + PAIR_LEN); + } + } + if (*p == '$' || *p == '@' || *p == '%' || *p == '&') { + p++; + } + if (!(isalpha((unsigned char)*p) || *p == '_')) { + return 0; + } + const char *s = p; + while (isalnum((unsigned char)*p) || *p == '_') { + p++; + } + *name = s; + return (size_t)(p - s); +} + +static uint32_t pod_trim_end(const char *src, uint32_t sb, uint32_t eb) { + while (eb > sb && (src[eb - SKIP_ONE] == '\n' || src[eb - SKIP_ONE] == '\r' || + src[eb - SKIP_ONE] == ' ' || src[eb - SKIP_ONE] == '\t')) { + eb--; + } + return eb; +} + +static void pod_index_push(CBMArena *a, pod_index_t *ix, const pod_sec_t *sec) { + if (ix->count == ix->cap) { + int ncap = ix->cap ? ix->cap * DOC_SPAN_GROW : DOC_SPAN_INIT_CAP; + pod_sec_t *grown = (pod_sec_t *)cbm_arena_alloc(a, (size_t)ncap * sizeof(pod_sec_t)); + if (!grown) { + return; + } + if (ix->count > 0) { + memcpy(grown, ix->items, (size_t)ix->count * sizeof(pod_sec_t)); + } + ix->items = grown; + ix->cap = ncap; + } + ix->items[ix->count++] = *sec; +} + +/* Index the named =head / =item sections of one POD block. */ +static void pod_index_block(CBMExtractCtx *ctx, pod_index_t *ix, TSNode pod) { + CBMArena *a = doc_scratch(ctx); + uint32_t pb = ts_node_start_byte(pod); + uint32_t pe = ts_node_end_byte(pod); + pod_cmd_t *cmds = NULL; + int n = pod_commands(a, ctx->source, pb, pe, &cmds); + for (int ci = 0; ci < n; ci++) { + if (cmds[ci].type != POD_HEAD && cmds[ci].type != POD_ITEM) { + continue; + } + const char *nm = NULL; + size_t nl = pod_heading_name(a, ctx->source, cmds[ci].wb, cmds[ci].eb, &nm); + if (nl == 0) { + continue; + } + uint32_t se = pod_section_end(cmds, n, ci, pe); + pod_sec_t sec = {nm, nl, cmds[ci].sb, pod_trim_end(ctx->source, cmds[ci].sb, se)}; + pod_index_push(a, ix, &sec); + } +} + +/* The file's POD sections by name, built on the first Perl sub. */ +static const pod_index_t *pod_index(CBMExtractCtx *ctx) { + if (ctx->doc_pod_index) { + return (const pod_index_t *)ctx->doc_pod_index; + } + pod_index_t *ix = (pod_index_t *)cbm_arena_alloc(doc_scratch(ctx), sizeof(pod_index_t)); + if (!ix) { return NULL; } - TSNode first = ts_node_named_child(body, 0); - if (ts_node_is_null(first) || strcmp(ts_node_type(first), "expression_statement") != 0) { + memset(ix, 0, sizeof(*ix)); + TSTreeCursor cur = ts_tree_cursor_new(ctx->root); + if (ts_tree_cursor_goto_first_child(&cur)) { + do { + TSNode ch = ts_tree_cursor_current_node(&cur); + if (strcmp(ts_node_type(ch), "pod") == 0) { + pod_index_block(ctx, ix, ch); + } + } while (ts_tree_cursor_goto_next_sibling(&cur)); + } + ts_tree_cursor_delete(&cur); + ctx->doc_pod_index = ix; + return ix; +} + +static bool pod_sec_names(const pod_sec_t *s, const char *name, size_t nl) { + return s->name_len == nl && memcmp(s->name, name, nl) == 0; +} + +/* Every POD section whose =head / =item names the sub exactly, joined in + * document order; NULL when none does. */ +static const char *pod_doc_by_name(CBMExtractCtx *ctx, const char *name) { + const pod_index_t *ix = pod_index(ctx); + if (!ix || !name || !name[0]) { return NULL; } - if (ts_node_named_child_count(first) == 0) { + size_t nl = strlen(name); + size_t total = 0; + for (int i = 0; i < ix->count; i++) { + if (pod_sec_names(&ix->items[i], name, nl)) { + total += (size_t)(ix->items[i].eb - ix->items[i].sb) + PAIR_LEN; + } + } + if (total == 0) { return NULL; } - TSNode str = ts_node_named_child(first, 0); - if (ts_node_is_null(str)) { + char *buf = (char *)cbm_arena_alloc(ctx->arena, total + SKIP_ONE); + if (!buf) { return NULL; } - const char *sk = ts_node_type(str); - if (strcmp(sk, "string") == 0 || strcmp(sk, "concatenated_string") == 0) { - return extract_comment_text(a, str, source); + size_t w = 0; + for (int i = 0; i < ix->count; i++) { + const pod_sec_t *s = &ix->items[i]; + if (!pod_sec_names(s, name, nl)) { + continue; + } + if (w > 0) { + buf[w++] = '\n'; + buf[w++] = '\n'; + } + memcpy(buf + w, ctx->source + s->sb, s->eb - s->sb); + w += s->eb - s->sb; } - return NULL; + buf[w] = '\0'; + return buf; } -// Extract docstring from the node's leading comment. -static const char *extract_docstring(CBMArena *a, TSNode node, const char *source, - CBMLanguage lang) { - if (lang == CBM_LANG_GO) { - const char *doc = extract_go_type_docstring(a, node, source); - if (doc) { - return doc; +/* ── Doc entry points ── */ + +/* go/doc: a type spec's own doc, else its declaration's (the doc of a + * single-spec `type X ...` sits above the `type` keyword). */ +static const char *doc_go_type(CBMExtractCtx *ctx, TSNode node) { + const char *own = doc_for_anchor(ctx, node); + TSNode parent = doc_parent(ctx, node); + if (own || !doc_kind_is(parent, "type_declaration")) { + return own; + } + return doc_for_anchor(ctx, parent); +} + +/* Doc of a Function/Method/Class-like definition. */ +static const char *extract_docstring(CBMExtractCtx *ctx, TSNode node, const char *name) { + CBMLanguage lang = ctx->language; + if (lang == CBM_LANG_PERL) { + const char *pod = pod_doc_by_name(ctx, name); + if (pod) { + return pod; } } + if (lang == CBM_LANG_GO && + (doc_kind_is(node, "type_spec") || doc_kind_is(node, "type_alias"))) { + return doc_go_type(ctx, node); + } + /* Python's own docstring beats a comment run above the definition: the + * in-body string is the documented API, a `# TODO` or section comment is + * only a fallback for definitions without one. */ + if (lang == CBM_LANG_PYTHON) { + const char *py = extract_python_docstring(ctx->arena, node, ctx->source); + if (py) { + return py; + } + } + return doc_for_anchor(ctx, doc_anchor(ctx, node)); +} - TSNode prev = ts_node_prev_sibling(node); - if (!ts_node_is_null(prev) && is_comment_node(ts_node_type(prev))) { - return extract_comment_text(a, prev, source); +/* Doc of a Field, Variable, enum member or Macro (code languages only). */ +static const char *extract_member_docstring(CBMExtractCtx *ctx, TSNode node) { + if (!doc_lang_member_docs(ctx->language)) { + return NULL; + } + return doc_for_anchor(ctx, doc_anchor(ctx, node)); +} + +/* Go package comment: the comment group touching `package`, directives + * dropped (go/doc). */ +static const char *doc_go_package(CBMExtractCtx *ctx) { + doc_trivia_t t; + memset(&t, 0, sizeof(t)); + TSNode pkg = {0}; + bool found = false; + TSTreeCursor cur = ts_tree_cursor_new(ctx->root); + if (ts_tree_cursor_goto_first_child(&cur)) { + do { + TSNode ch = ts_tree_cursor_current_node(&cur); + const char *k = ts_node_type(ch); + if (strcmp(k, "package_clause") == 0) { + pkg = ch; + found = true; + break; + } + if (is_comment_node(k)) { + doc_span_t sp = doc_span_of(ch, ctx->source, DOC_SPAN_COMMENT); + doc_push_span(doc_scratch(ctx), &t, &sp); + } else { + t.count = 0; + } + } while (ts_tree_cursor_goto_next_sibling(&cur)); } + ts_tree_cursor_delete(&cur); + int last = t.count - SKIP_ONE; + if (!found || last < 0 || ts_node_start_point(pkg).row > t.items[last].erow + SKIP_ONE) { + return NULL; + } + int first = last; + while (first > 0 && t.items[first].srow <= t.items[first - SKIP_ONE].erow + SKIP_ONE) { + first--; + } + return doc_run_text(ctx, &t, first, last, true); +} - if (lang == CBM_LANG_PYTHON) { - return extract_python_docstring(a, node, source); +/* Rust inner docs (//! and the bang block) before the first item. */ +static const char *doc_rust_inner(CBMExtractCtx *ctx) { + doc_trivia_t t; + memset(&t, 0, sizeof(t)); + TSTreeCursor cur = ts_tree_cursor_new(ctx->root); + if (ts_tree_cursor_goto_first_child(&cur)) { + do { + TSNode ch = ts_tree_cursor_current_node(&cur); + const char *k = ts_node_type(ch); + if (is_comment_node(k)) { + doc_span_t sp = doc_span_of(ch, ctx->source, DOC_SPAN_COMMENT); + if (sp.style == DOC_CS_LINE_BANG || sp.style == DOC_CS_BLOCK_BANG) { + doc_push_span(doc_scratch(ctx), &t, &sp); + } + continue; + } + if (strcmp(k, "inner_attribute_item") != 0) { + break; /* the first item: the inner docs are over */ + } + } while (ts_tree_cursor_goto_next_sibling(&cur)); } - return NULL; + ts_tree_cursor_delete(&cur); + return t.count > 0 ? doc_run_text(ctx, &t, 0, t.count - SKIP_ONE, false) : NULL; +} + +/* A file's own doc, for its File node: the Go package comment or the Rust + * inner docs. */ +static const char *extract_module_doc(CBMExtractCtx *ctx) { + if (ctx->language == CBM_LANG_GO) { + return doc_go_package(ctx); + } + return ctx->language == CBM_LANG_RUST ? doc_rust_inner(ctx) : NULL; } static int find_jvm_modifiers(TSNode node, CBMLanguage lang, TSNode *out, int max); @@ -4124,7 +5492,7 @@ static void extract_func_def(CBMExtractCtx *ctx, TSNode node, const CBMLangSpec def.is_test = def.is_test || ctx->result->is_test_file; // Docstring - def.docstring = extract_docstring(a, node, ctx->source, ctx->language); + def.docstring = extract_docstring(ctx, node, name); // Complexity if (spec->branching_node_types && spec->branching_node_types[0]) { @@ -4248,8 +5616,8 @@ static void utf8_trim_partial_tail(char *text) { } /* Collapse `len` bytes of raw prose into a single-spaced value capped at - * MAX_COMMENT_LEN (the same 500-byte ceiling docstrings already use, which - * leaves room inside the 2 KB properties buffer that carries it). + * MAX_COMMENT_LEN (500 bytes, which leaves room inside the 2 KB properties + * buffer that carries it). * * The output buffer is FIXED at that cap, so a section body of any size costs a * bounded copy rather than a copy of the whole section. Returns NULL when @@ -4584,6 +5952,169 @@ static bool extract_sql_ddl_class_def(CBMExtractCtx *ctx, TSNode node, const cha return true; } +/* ── Python annotated instance fields (#1277) ───────────────────── + * Exported as result->field_types (never graph nodes) so the cross-file LSP + * can type `obj.x` when obj's class lives in another file. Only DECLARED + * types count: a class-body annotation, a `self.x: T = v` annotation in + * __init__, or `self.x = p` where p is an annotated __init__ parameter. An + * unannotated right-hand side is never guessed at. */ + +static void py_push_field_type(CBMExtractCtx *ctx, const char *class_qn, TSNode name_node, + const char *type_text) { + char *name = cbm_node_text(ctx->arena, name_node, ctx->source); + if (!name || !name[0] || !type_text || !type_text[0]) { + return; + } + CBMFieldType ft = {.class_qn = class_qn, .field_name = name, .type_text = type_text}; + cbm_fieldtype_push(&ctx->result->field_types, ctx->arena, ft); +} + +/* The assignment inside an expression_statement, or a null node. */ +static TSNode py_statement_assignment(TSNode stmt) { + TSNode null_node = {0}; + if (strcmp(ts_node_type(stmt), "expression_statement") != 0 || + ts_node_named_child_count(stmt) == 0) { + return null_node; + } + TSNode inner = ts_node_named_child(stmt, 0); + return strcmp(ts_node_type(inner), "assignment") == 0 ? inner : null_node; +} + +/* Annotation text of the __init__ parameter called `name`, or NULL. */ +static const char *py_init_param_annotation(CBMExtractCtx *ctx, TSNode params, const char *name) { + uint32_t n = ts_node_named_child_count(params); + for (uint32_t i = 0; i < n; i++) { + TSNode p = ts_node_named_child(params, i); + const char *pk = ts_node_type(p); + TSNode pname = {0}; + if (strcmp(pk, "typed_default_parameter") == 0) { + pname = ts_node_child_by_field_name(p, TS_FIELD("name")); + } else if (strcmp(pk, "typed_parameter") == 0 && ts_node_named_child_count(p) > 0) { + pname = ts_node_named_child(p, 0); + } else { + continue; + } + TSNode ptype = ts_node_child_by_field_name(p, TS_FIELD("type")); + if (ts_node_is_null(pname) || ts_node_is_null(ptype) || + strcmp(ts_node_type(pname), "identifier") != 0) { + continue; + } + char *pn = cbm_node_text(ctx->arena, pname, ctx->source); + if (pn && strcmp(pn, name) == 0) { + return cbm_node_text(ctx->arena, ptype, ctx->source); + } + } + return NULL; +} + +/* `self.x: T = v` / `self.x = p` inside __init__ (self = its first parameter). */ +static void py_init_field_assignment(CBMExtractCtx *ctx, const char *class_qn, TSNode assign, + TSNode params, const char *self_name) { + TSNode left = ts_node_child_by_field_name(assign, TS_FIELD("left")); + if (ts_node_is_null(left) || strcmp(ts_node_type(left), "attribute") != 0) { + return; + } + TSNode obj = ts_node_child_by_field_name(left, TS_FIELD("object")); + TSNode attr = ts_node_child_by_field_name(left, TS_FIELD("attribute")); + if (ts_node_is_null(obj) || ts_node_is_null(attr) || + strcmp(ts_node_type(obj), "identifier") != 0) { + return; + } + char *obj_name = cbm_node_text(ctx->arena, obj, ctx->source); + if (!obj_name || strcmp(obj_name, self_name) != 0) { + return; + } + TSNode ann = ts_node_child_by_field_name(assign, TS_FIELD("type")); + if (!ts_node_is_null(ann)) { + py_push_field_type(ctx, class_qn, attr, cbm_node_text(ctx->arena, ann, ctx->source)); + return; + } + TSNode right = ts_node_child_by_field_name(assign, TS_FIELD("right")); + if (ts_node_is_null(right) || strcmp(ts_node_type(right), "identifier") != 0) { + return; + } + char *rhs = cbm_node_text(ctx->arena, right, ctx->source); + if (rhs && strcmp(rhs, self_name) != 0) { + py_push_field_type(ctx, class_qn, attr, py_init_param_annotation(ctx, params, rhs)); + } +} + +/* Every `self.x` field assignment in one __init__ body, nested blocks + * included; nested functions, lambdas and classes have their own `self`. */ +static void py_extract_init_fields(CBMExtractCtx *ctx, const char *class_qn, TSNode fn) { + TSNode params = ts_node_child_by_field_name(fn, TS_FIELD("parameters")); + TSNode body = ts_node_child_by_field_name(fn, TS_FIELD("body")); + if (ts_node_is_null(params) || ts_node_is_null(body) || + ts_node_named_child_count(params) == 0) { + return; + } + TSNode self_node = ts_node_named_child(params, 0); + if (strcmp(ts_node_type(self_node), "identifier") != 0) { + return; + } + char *self_name = cbm_node_text(ctx->arena, self_node, ctx->source); + if (!self_name || !self_name[0]) { + return; + } + TSNodeStack stack; + ts_nstack_init(&stack, ctx, CBM_SZ_32); + ts_nstack_push(&stack, body); + while (stack.count > 0) { + TSNode n = ts_nstack_pop(&stack); + const char *k = ts_node_type(n); + if (strcmp(k, "function_definition") == 0 || strcmp(k, "class_definition") == 0 || + strcmp(k, "lambda") == 0) { + continue; + } + TSNode assign = py_statement_assignment(n); + if (!ts_node_is_null(assign)) { + py_init_field_assignment(ctx, class_qn, assign, params, self_name); + continue; + } + /* Reverse push keeps source order on pop. */ + for (uint32_t i = ts_node_named_child_count(n); i > 0; i--) { + ts_nstack_push(&stack, ts_node_named_child(n, i - 1)); + } + } +} + +static void extract_py_field_types(CBMExtractCtx *ctx, TSNode class_node, const char *class_qn) { + TSNode body = ts_node_child_by_field_name(class_node, TS_FIELD("body")); + if (ts_node_is_null(body)) { + return; + } + uint32_t count = ts_node_named_child_count(body); + /* Class-body annotations first: `x: T` / `x: T = v`. */ + for (uint32_t i = 0; i < count; i++) { + TSNode assign = py_statement_assignment(ts_node_named_child(body, i)); + if (ts_node_is_null(assign)) { + continue; + } + TSNode left = ts_node_child_by_field_name(assign, TS_FIELD("left")); + TSNode ann = ts_node_child_by_field_name(assign, TS_FIELD("type")); + if (!ts_node_is_null(left) && !ts_node_is_null(ann) && + strcmp(ts_node_type(left), "identifier") == 0) { + py_push_field_type(ctx, class_qn, left, cbm_node_text(ctx->arena, ann, ctx->source)); + } + } + /* Then __init__. */ + for (uint32_t i = 0; i < count; i++) { + TSNode fn = ts_node_named_child(body, i); + if (strcmp(ts_node_type(fn), "decorated_definition") == 0) { + fn = ts_node_child_by_field_name(fn, TS_FIELD("definition")); + } + if (ts_node_is_null(fn) || strcmp(ts_node_type(fn), "function_definition") != 0) { + continue; + } + TSNode fname = ts_node_child_by_field_name(fn, TS_FIELD("name")); + char *fn_name = + ts_node_is_null(fname) ? NULL : cbm_node_text(ctx->arena, fname, ctx->source); + if (fn_name && strcmp(fn_name, "__init__") == 0) { + py_extract_init_fields(ctx, class_qn, fn); + } + } +} + static void extract_class_def(CBMExtractCtx *ctx, TSNode node, const CBMLangSpec *spec) { CBMArena *a = ctx->arena; const char *kind = ts_node_type(node); @@ -4953,7 +6484,7 @@ static void extract_class_def(CBMExtractCtx *ctx, TSNode node, const CBMLangSpec def.is_exported = cbm_is_exported(name, ctx->language); def.base_classes = extract_base_classes(a, node, ctx->source, ctx->language); def.decorators = extract_decorators(a, node, ctx->source, ctx->language, spec); - def.docstring = extract_docstring(a, node, ctx->source, ctx->language); + def.docstring = extract_docstring(ctx, node, name); cbm_defs_push(&ctx->result->defs, a, def); @@ -4970,6 +6501,10 @@ static void extract_class_def(CBMExtractCtx *ctx, TSNode node, const CBMLangSpec // Extract class-level variables (field declarations) extract_class_variables(ctx, node, class_qn, spec); + if (ctx->language == CBM_LANG_PYTHON) { + extract_py_field_types(ctx, node, class_qn); + } + // C# 12 primary-constructor parameters: declared on the class line // (`class Foo(IBar bar, IBaz baz) : Base { ... }`) and bound to implicit // captured fields accessible from any instance member. Tree-sitter c-sharp @@ -5324,7 +6859,7 @@ static void push_method_def(CBMExtractCtx *ctx, TSNode child, TSNode class_node, def.route_path = prefix ? join_route_paths(a, prefix, def.route_path) : NULL; def.route_method = prefix ? def.route_method : NULL; } - def.docstring = extract_docstring(a, child, ctx->source, ctx->language); + def.docstring = extract_docstring(ctx, child, name); if (spec->branching_node_types && spec->branching_node_types[0]) { set_def_complexity(&def, child, spec); @@ -5565,6 +7100,7 @@ static void extract_rust_impl(CBMExtractCtx *ctx, TSNode node, const CBMLangSpec // MinHash fingerprint compute_fingerprint(ctx, &def, child); + def.docstring = extract_docstring(ctx, child, name); cbm_defs_push(&ctx->result->defs, a, def); } @@ -5718,6 +7254,7 @@ static void push_var_def_qn(CBMExtractCtx *ctx, const char *name, const char *qn def.start_line = ts_node_start_point(node).row + TS_LINE_OFFSET; def.end_line = ts_node_end_point(node).row + TS_LINE_OFFSET; def.is_exported = cbm_is_exported(name, ctx->language); + def.docstring = extract_member_docstring(ctx, node); cbm_defs_push(&ctx->result->defs, a, def); } @@ -5851,6 +7388,7 @@ static void extract_enum_members(CBMExtractCtx *ctx, TSNode node, const char *cl mdef.file_path = ctx->rel_path; mdef.start_line = ts_node_start_point(member).row + TS_LINE_OFFSET; mdef.end_line = ts_node_end_point(member).row + TS_LINE_OFFSET; + mdef.docstring = extract_member_docstring(ctx, member); cbm_defs_push(&ctx->result->defs, a, mdef); } } @@ -7447,6 +8985,7 @@ static void extract_class_fields(CBMExtractCtx *ctx, TSNode class_node, const ch * TS @Input()-style fields, JVM field annotations via the modifiers * child) — same extraction the class and method paths already run. */ def.decorators = extract_decorators(a, child, ctx->source, ctx->language, spec); + def.docstring = extract_member_docstring(ctx, child); cbm_defs_push(&ctx->result->defs, a, def); } @@ -7998,6 +9537,7 @@ static void extract_c_macro_def(CBMExtractCtx *ctx, TSNode node) { if (!ts_node_is_null(params)) { def.signature = cbm_node_text(a, params, ctx->source); } + def.docstring = extract_member_docstring(ctx, node); cbm_defs_push(&ctx->result->defs, a, def); } @@ -8551,6 +10091,7 @@ void cbm_extract_definitions(CBMExtractCtx *ctx) { } int mod_idx = ctx->result->defs.count; cbm_defs_push(&ctx->result->defs, a, mod); + ctx->result->module_doc = extract_module_doc(ctx); cbm_extract_definitions_without_module(ctx); diff --git a/internal/cbm/extract_imports.c b/internal/cbm/extract_imports.c index 7d416923c1..4aa0baa759 100644 --- a/internal/cbm/extract_imports.c +++ b/internal/cbm/extract_imports.c @@ -1205,7 +1205,29 @@ static void parse_wolfram_imports(CBMExtractCtx *ctx) { // require/include forms remain `expression_statement`s and are still handled by // the text fallback. Take the first qualified_name/name descendant of each // clause as the module path. -static void emit_php_use_clause(CBMExtractCtx *ctx, TSNode clause, const char *group_prefix) { +// `use function` / `use const` carry the keyword as an anonymous direct child +// of the declaration (flat form) or of the clause (grouped form +// `use A\{function f, const C, K}`). DEFAULT when neither is present. +static CBMImportKind php_use_kind(TSNode node) { + uint32_t n = ts_node_child_count(node); + for (uint32_t i = 0; i < n; i++) { + TSNode c = ts_node_child(node, i); + if (ts_node_is_named(c)) { + continue; + } + const char *k = ts_node_type(c); + if (strcmp(k, "function") == 0) { + return CBM_IMPORT_KIND_FUNCTION; + } + if (strcmp(k, "const") == 0) { + return CBM_IMPORT_KIND_CONST; + } + } + return CBM_IMPORT_KIND_DEFAULT; +} + +static void emit_php_use_clause(CBMExtractCtx *ctx, TSNode clause, const char *group_prefix, + CBMImportKind decl_kind) { CBMArena *a = ctx->arena; // The path node is the qualified_name / namespace_name / name child. TSNode path_node = clause; @@ -1230,12 +1252,14 @@ static void emit_php_use_clause(CBMExtractCtx *ctx, TSNode clause, const char *g TSNode alias = ts_node_child_by_field_name(clause, TS_FIELD("alias")); const char *local = !ts_node_is_null(alias) ? cbm_node_text(a, alias, ctx->source) : path_last(a, path); - CBMImport imp = {.local_name = local, .module_path = path}; + CBMImportKind kind = decl_kind != CBM_IMPORT_KIND_DEFAULT ? decl_kind : php_use_kind(clause); + CBMImport imp = {.local_name = local, .module_path = path, .kind = kind}; cbm_imports_push(&ctx->result->imports, a, imp); } static void emit_php_use_decl(CBMExtractCtx *ctx, TSNode decl) { CBMArena *a = ctx->arena; + const CBMImportKind decl_kind = php_use_kind(decl); // Grouped form: namespace_use_group with a leading prefix qualified_name. TSNode group = decl; if (find_first_descendant_of(decl, "namespace_use_group", &group)) { @@ -1257,7 +1281,7 @@ static void emit_php_use_decl(CBMExtractCtx *ctx, TSNode decl) { const char *ck = ts_node_type(clause); if (strcmp(ck, "namespace_use_group_clause") == 0 || strcmp(ck, "namespace_use_clause") == 0) { - emit_php_use_clause(ctx, clause, prefix); + emit_php_use_clause(ctx, clause, prefix, decl_kind); } } return; @@ -1268,13 +1292,13 @@ static void emit_php_use_decl(CBMExtractCtx *ctx, TSNode decl) { for (uint32_t i = 0; i < dc; i++) { TSNode clause = ts_node_named_child(decl, i); if (strcmp(ts_node_type(clause), "namespace_use_clause") == 0) { - emit_php_use_clause(ctx, clause, NULL); + emit_php_use_clause(ctx, clause, NULL, decl_kind); any = true; } } if (!any) { // Some grammar versions inline the path directly under the declaration. - emit_php_use_clause(ctx, decl, NULL); + emit_php_use_clause(ctx, decl, NULL, decl_kind); } } diff --git a/internal/cbm/grammar_objectscript_routine.c b/internal/cbm/grammar_objectscript_routine.c index 9006eda3ab..9294beca15 100644 --- a/internal/cbm/grammar_objectscript_routine.c +++ b/internal/cbm/grammar_objectscript_routine.c @@ -1,4 +1,16 @@ // Vendored tree-sitter grammar: objectscript_routine // Each grammar compiled as separate unit (conflicting static symbols). #include "vendored/grammars/objectscript_routine/parser.c" +/* Tree-sitter passes NULL for an empty snapshot. The scanner's byte copy is + * needed only for nonempty snapshots; keep vendored sources unchanged. */ +#define tree_sitter_objectscript_routine_external_scanner_deserialize \ + tree_sitter_objectscript_routine_external_scanner_deserialize_bytes #include "vendored/grammars/objectscript_routine/scanner.c" +#undef tree_sitter_objectscript_routine_external_scanner_deserialize +void tree_sitter_objectscript_routine_external_scanner_deserialize(void *payload, + const char *buffer, + unsigned length) { + if (length > 0) + tree_sitter_objectscript_routine_external_scanner_deserialize_bytes(payload, buffer, + length); +} diff --git a/internal/cbm/grammar_objectscript_udl.c b/internal/cbm/grammar_objectscript_udl.c index 5aea58c44a..f50ad819dd 100644 --- a/internal/cbm/grammar_objectscript_udl.c +++ b/internal/cbm/grammar_objectscript_udl.c @@ -1,4 +1,14 @@ // Vendored tree-sitter grammar: objectscript_udl // Each grammar compiled as separate unit (conflicting static symbols). #include "vendored/grammars/objectscript_udl/parser.c" +/* Tree-sitter passes NULL for an empty snapshot. The scanner's byte copy is + * needed only for nonempty snapshots; keep vendored sources unchanged. */ +#define tree_sitter_objectscript_udl_external_scanner_deserialize \ + tree_sitter_objectscript_udl_external_scanner_deserialize_bytes #include "vendored/grammars/objectscript_udl/scanner.c" +#undef tree_sitter_objectscript_udl_external_scanner_deserialize +void tree_sitter_objectscript_udl_external_scanner_deserialize(void *payload, const char *buffer, + unsigned length) { + if (length > 0) + tree_sitter_objectscript_udl_external_scanner_deserialize_bytes(payload, buffer, length); +} diff --git a/internal/cbm/lsp/c_lsp.c b/internal/cbm/lsp/c_lsp.c index 74b97d69d9..22d630020c 100644 --- a/internal/cbm/lsp/c_lsp.c +++ b/internal/cbm/lsp/c_lsp.c @@ -1025,6 +1025,53 @@ static const CBMType *c_resolve_name_to_type(CLSPContext *ctx, const char *name) return cbm_type_named(ctx->arena, name); } +/* Resolve the class of a qualified member `Scope::member` -- scope_qn is the + * dotted scope ("SecdManager", "ns.Widget") -- to the QN of ONE registered + * class, or NULL. The bare and caller-module spellings come first. A class + * declared in another file is registered under that file's module + * (".app.src.SecdManager.SecdManager"), which neither spelling reaches, + * so then every registered type whose QN ends in "." at a segment + * boundary is a candidate. More than one distinct candidate (two libraries + * each declaring `Twin`) names no single class: the call stays unresolved + * rather than bind one of them by guess. */ +static const char *c_resolve_scope_class_qn(CLSPContext *ctx, const char *scope_qn) { + if (!ctx->registry || !scope_qn || !scope_qn[0]) { + return NULL; + } + if (cbm_registry_lookup_type(ctx->registry, scope_qn)) { + return scope_qn; + } + if (ctx->module_qn) { + const char *mod_qn = cbm_arena_sprintf(ctx->arena, "%s.%s", ctx->module_qn, scope_qn); + if (cbm_registry_lookup_type(ctx->registry, mod_qn)) { + return mod_qn; + } + } + const char *last_dot = strrchr(scope_qn, '.'); + const char *short_name = last_dot ? last_dot + 1 : scope_qn; + size_t scope_len = strlen(scope_qn); + const char *found = NULL; + CBMTypeShortIter it; + cbm_registry_types_by_short_name_chain(ctx->registry, short_name, &it); + int i; + while ((i = cbm_type_short_iter_next(&it)) >= 0) { + const char *q = it.reg->types[i].qualified_name; + size_t qlen = q ? strlen(q) : 0; + if (qlen <= scope_len || q[qlen - scope_len - 1] != '.' || + strcmp(q + qlen - scope_len, scope_qn) != 0) { + continue; + } + if (found && strcmp(found, q) != 0) { + return NULL; + } + found = q; + } + return found; +} + +static const CBMRegisteredFunc *c_lookup_scope_member(CLSPContext *ctx, const char *class_qn, + const char *member_name); + // ============================================================================ // c_parse_type_node: AST type node -> CBMType // ============================================================================ @@ -1821,6 +1868,16 @@ static const CBMType *c_eval_expr_type_inner(CLSPContext *ctx, TSNode node) { } } } + + // Class::member of a class declared in another file: its signature + // carries the return type a chained or auto receiver dispatches on. + const char *scope_class = + c_resolve_scope_class_qn(ctx, cbm_arena_strndup(ctx->arena, qn, prefix_len)); + const CBMRegisteredFunc *sm = + scope_class ? c_lookup_scope_member(ctx, scope_class, dot + 1) : NULL; + if (sm && sm->signature) { + return sm->signature; + } } return cbm_type_unknown(); @@ -2860,6 +2917,15 @@ const CBMRegisteredFunc *c_lookup_member(CLSPContext *ctx, const char *type_qn, return c_lookup_member_depth(ctx, type_qn, member_name, 0); } +/* A member of a class c_resolve_scope_class_qn already pinned down: direct, + * then through its aliases and base classes. Entering below depth 0 skips the + * short-name fallback, which would re-anchor the lookup onto a same-named + * class elsewhere -- the scope names exactly this class. */ +static const CBMRegisteredFunc *c_lookup_scope_member(CLSPContext *ctx, const char *class_qn, + const char *member_name) { + return c_lookup_member_depth(ctx, class_qn, member_name, 1); +} + // True if any BASE class of type_qn (not type_qn itself) declares member_name — // i.e. a method found directly on type_qn is an OVERRIDE of an inherited method. // This mirrors the existing virtual-dispatch notion (a derived override of a base @@ -4142,6 +4208,24 @@ static void c_resolve_calls_in_node_inner(CLSPContext *ctx, TSNode node) { goto recurse; } } + // Class::method of a class declared in another file (#1153). + if (dot) { + const char *scope_class = c_resolve_scope_class_qn( + ctx, cbm_arena_strndup(ctx->arena, qn, (size_t)(dot - qn))); + const CBMRegisteredFunc *sm = NULL; + if (scope_class) { + sm = cbm_registry_lookup_method_by_types(ctx->registry, scope_class, + dot + 1, arg_types, arg_count); + if (!sm) { + sm = c_lookup_scope_member(ctx, scope_class, dot + 1); + } + } + if (sm) { + c_emit_resolved_call_at(ctx, sm->qualified_name, "lsp_scoped", 0.95f, + node); + goto recurse; + } + } c_emit_unresolved_call(ctx, qn, "scoped_not_in_registry"); } goto recurse; diff --git a/internal/cbm/lsp/go_lsp.h b/internal/cbm/lsp/go_lsp.h index 1e50bca769..7ecc60b101 100644 --- a/internal/cbm/lsp/go_lsp.h +++ b/internal/cbm/lsp/go_lsp.h @@ -104,6 +104,9 @@ typedef struct { * a decorator rebinds the function name, so the undecorated definition is * not automatically an exact callable-value target. */ const char **decorators; + /* Callable identity (#2061): signature suffix of this def's graph QN; + * qualified_name stays the base QN. NULL = none (always, until enabled). */ + const char *sig; } CBMLSPDef; // Parse source, build registry from defs + stdlib, run LSP. diff --git a/internal/cbm/lsp/py_lsp.c b/internal/cbm/lsp/py_lsp.c index 2cf66ad012..70de5dce78 100644 --- a/internal/cbm/lsp/py_lsp.c +++ b/internal/cbm/lsp/py_lsp.c @@ -4965,6 +4965,47 @@ static const char **py_split_pipe(CBMArena *arena, const char *text) { return out; } +/* Split a class def's field_defs ("name:QN|name:QN", built by the Python + * field fold in pass_lsp_cross.c) into the parallel NULL-terminated arrays a + * CBMRegisteredType carries. The types are already project QNs, so each one + * becomes a NAMED type verbatim. Without this, `obj.field.method()` on a class + * from another file lost the field's type (#1277); same-file classes get their + * fields from the per-file walk instead. Pure string work: no registry lookup + * happens here, so it is safe inside the type-registration pass. */ +static void py_split_field_defs(CBMArena *arena, const char *field_defs, const char ***names_out, + const CBMType ***types_out) { + *names_out = NULL; + *types_out = NULL; + const char **pairs = py_split_pipe(arena, field_defs); + if (!pairs) + return; + int n = 0; + while (pairs[n]) + n++; + const char **names = + (const char **)cbm_arena_alloc(arena, (size_t)(n + 1) * sizeof(const char *)); + const CBMType **types = + (const CBMType **)cbm_arena_alloc(arena, (size_t)(n + 1) * sizeof(const CBMType *)); + if (!names || !types) + return; + int kept = 0; + for (int i = 0; i < n; i++) { + char *colon = strchr(pairs[i], ':'); + if (!colon || colon == pairs[i] || !colon[1]) + continue; + *colon = '\0'; /* pairs[i] is this call's own arena copy */ + names[kept] = pairs[i]; + types[kept] = cbm_type_named(arena, colon + 1); + kept++; + } + if (kept == 0) + return; + names[kept] = NULL; + types[kept] = NULL; + *names_out = names; + *types_out = types; +} + /* Build a registry from CBMLSPDef[] supplied by the caller — covers both * the source file's own defs and cross-file referenced defs. */ static void py_register_lsp_defs(CBMArena *arena, CBMArena *idx_arena, CBMTypeRegistry *reg, @@ -4990,6 +5031,7 @@ static void py_register_lsp_defs(CBMArena *arena, CBMArena *idx_arena, CBMTypeRe if (d->method_names_str && d->method_names_str[0]) { rt.method_names = py_split_pipe(arena, d->method_names_str); } + py_split_field_defs(arena, d->field_defs, &rt.field_names, &rt.field_types); cbm_registry_add_type(reg, rt); } } diff --git a/internal/cbm/lsp/ts_lsp.c b/internal/cbm/lsp/ts_lsp.c index 7e36f97198..facda627a0 100644 --- a/internal/cbm/lsp/ts_lsp.c +++ b/internal/cbm/lsp/ts_lsp.c @@ -1998,6 +1998,22 @@ static const CBMType *ts_new_bare_class_type(TSLSPContext *ctx, const char *cnam return cbm_type_named(ctx->arena, local_qn); } +/* Type of a `new_expression`: a bare constructor name follows TS scoping + * (ts_new_bare_class_type), a dotted one is taken as spelled. NULL when the + * node carries no constructor. Shared by expression evaluation and by the + * field-initializer inference in ast_sweep_shapes, so `const t = new C()` + * and `private t = new C()` type the instance identically. */ +static const CBMType *ts_new_expression_type(TSLSPContext *ctx, TSNode node) { + TSNode ctor = ts_node_child_by_field_name(node, "constructor", TS_LSP_FIELD_LEN("constructor")); + if (ts_node_is_null(ctor)) + return NULL; + char *cname = node_text(ctx, ctor); + if (!cname) + return NULL; + return strchr(cname, '.') == NULL ? ts_new_bare_class_type(ctx, cname) + : cbm_type_named(ctx->arena, cname); +} + static const CBMRegisteredFunc *ts_lookup_namespace_call(TSLSPContext *ctx, const char *object_name, const char *method_name, TSNode args, bool *out_is_import) { @@ -2233,15 +2249,9 @@ const CBMType *ts_eval_expr_type(TSLSPContext *ctx, TSNode node) { result = lookup_member_type(ctx, recv, pname); } } else if (strcmp(kind, "new_expression") == 0) { - TSNode ctor = - ts_node_child_by_field_name(node, "constructor", TS_LSP_FIELD_LEN("constructor")); - if (!ts_node_is_null(ctor)) { - char *cname = node_text(ctx, ctor); - if (cname) { - result = strchr(cname, '.') == NULL ? ts_new_bare_class_type(ctx, cname) - : cbm_type_named(ctx->arena, cname); - } - } + const CBMType *nt = ts_new_expression_type(ctx, node); + if (nt) + result = nt; } else if (strcmp(kind, "call_expression") == 0) { TSNode fn = ts_node_child_by_field_name(node, "function", TS_LSP_FIELD_LEN("function")); fn = unwrap_await_callee(fn); @@ -5361,6 +5371,76 @@ static const CBMType *interface_method_signature_from_ast(TSLSPContext *ctx, TSN return cbm_type_func(ctx->arena, NULL, param_count ? param_types : NULL, return_types); } +/* Parsed type of a `type_annotation` field (`: T`), or unknown. */ +static const CBMType *ts_annotation_field_type(TSLSPContext *ctx, TSNode owner) { + TSNode tann = ts_node_child_by_field_name(owner, "type", TS_LSP_FIELD_LEN("type")); + if (ts_node_is_null(tann)) + return cbm_type_unknown(); + TSNode tn = + strcmp(ts_node_type(tann), "type_annotation") == 0 ? ts_node_named_child(tann, 0) : tann; + return ts_node_is_null(tn) ? cbm_type_unknown() : ts_parse_type_node(ctx, tn); +} + +/* Declared type of a class field (`public_field_definition`) or interface + * property: the annotation when present, else, for an unannotated field + * initialised with `new C()`, the type of that instance (#514). No other + * initializer is inferred: `new` names its type exactly, arbitrary + * expressions do not. */ +static const CBMType *ts_field_decl_type(TSLSPContext *ctx, TSNode m) { + if (!ts_node_is_null(ts_node_child_by_field_name(m, "type", TS_LSP_FIELD_LEN("type")))) + return ts_annotation_field_type(ctx, m); + TSNode val = ts_node_child_by_field_name(m, "value", TS_LSP_FIELD_LEN("value")); + if (ts_node_is_null(val) || strcmp(ts_node_type(val), "new_expression") != 0) + return cbm_type_unknown(); + const CBMType *nt = ts_new_expression_type(ctx, val); + return nt ? nt : cbm_type_unknown(); +} + +/* A constructor parameter is a TS parameter property, i.e. also declares a + * class field, iff it carries an accessibility modifier, `readonly` or + * `override`. */ +static bool ts_param_is_property(TSNode p) { + uint32_t pc = ts_node_child_count(p); + for (uint32_t ci = 0; ci < pc; ci++) { + const char *ck = ts_node_type(ts_node_child(p, ci)); + if (strcmp(ck, "accessibility_modifier") == 0 || strcmp(ck, "readonly") == 0 || + strcmp(ck, "override_modifier") == 0) + return true; + } + return false; +} + +/* Append the parameter properties of a `constructor` method_definition to + * the field arrays (#514): `constructor(private readonly svc: Svc)` declares + * field `svc: Svc`, the dominant NestJS injection shape. Any other member is + * ignored. Returns the new field count. */ +static int ts_collect_param_properties(TSLSPContext *ctx, TSNode m, const char **names, + const CBMType **types, int count, int cap) { + TSNode mname = ts_node_child_by_field_name(m, "name", TS_LSP_FIELD_LEN("name")); + char *mnm = ts_node_is_null(mname) ? NULL : node_text(ctx, mname); + if (!mnm || strcmp(mnm, "constructor") != 0) + return count; + TSNode params = ts_node_child_by_field_name(m, "parameters", TS_LSP_FIELD_LEN("parameters")); + uint32_t pnc = ts_node_is_null(params) ? 0 : ts_node_named_child_count(params); + for (uint32_t pi = 0; pi < pnc && count < cap; pi++) { + TSNode p = ts_node_named_child(params, pi); + const char *pk = ts_node_type(p); + if ((strcmp(pk, "required_parameter") != 0 && strcmp(pk, "optional_parameter") != 0) || + !ts_param_is_property(p)) + continue; + TSNode pat = ts_node_child_by_field_name(p, "pattern", TS_LSP_FIELD_LEN("pattern")); + if (ts_node_is_null(pat) || strcmp(ts_node_type(pat), "identifier") != 0) + continue; + char *pnm = node_text(ctx, pat); + if (!pnm) + continue; + names[count] = pnm; + types[count] = ts_annotation_field_type(ctx, p); + count++; + } + return count; +} + // AST sweep: walk class/interface bodies to collect field names+types and refine // embedded_types / type_param_names. static void ast_sweep_shapes(TSLSPContext *ctx, TSNode root, CBMTypeRegistry *reg) { @@ -5473,24 +5553,24 @@ static void ast_sweep_shapes(TSLSPContext *ctx, TSNode root, CBMTypeRegistry *re if (strcmp(mk, "public_field_definition") == 0 || strcmp(mk, "property_signature") == 0) { TSNode fname = ts_node_child_by_field_name(m, "name", TS_LSP_FIELD_LEN("name")); - TSNode ftype = ts_node_child_by_field_name(m, "type", TS_LSP_FIELD_LEN("type")); if (ts_node_is_null(fname)) continue; char *fnm = node_text(ctx, fname); if (!fnm) continue; - const CBMType *ft = cbm_type_unknown(); - if (!ts_node_is_null(ftype)) { - TSNode tch = ts_node_named_child(ftype, 0); - if (!ts_node_is_null(tch)) - ft = ts_parse_type_node(ctx, tch); - } field_names[field_count] = fnm; - field_types[field_count] = ft; + field_types[field_count] = ts_field_decl_type(ctx, m); field_count++; continue; } + // Constructor parameter properties are class fields too (#514). + if (!is_interface && strcmp(mk, "method_definition") == 0) { + field_count = + ts_collect_param_properties(ctx, m, field_names, field_types, field_count, 63); + continue; + } + // Interface method signature: `methodName(params): ReturnType;` // Or class method declaration in `.d.ts` ambient mode. if (is_interface && (strcmp(mk, "method_signature") == 0)) { diff --git a/internal/cbm/lsp/type_registry.c b/internal/cbm/lsp/type_registry.c index 4e2f02012e..06b55c391d 100644 --- a/internal/cbm/lsp/type_registry.c +++ b/internal/cbm/lsp/type_registry.c @@ -677,8 +677,28 @@ CBMRegisteredType *cbm_registry_type_for_update(CBMTypeRegistry *head, const cha for (const CBMTypeRegistry *r = head->fallback; r; r = r->fallback) { const CBMRegisteredType *base = lookup_type_self(r, qualified_name); if (base) { + CBMRegisteredType copy = *base; + /* Field refinements overwrite entries in this array. The type + * objects remain immutable, but the writable overlay must own + * the pointer array rather than mutate its shared fallback. */ + if (base->field_types) { + size_t count = 0; + /* Types are parallel to names; a named field may have no + * type, so an interior NULL does not terminate that array. */ + while (base->field_names ? base->field_names[count] != NULL + : base->field_types[count] != NULL) { + count++; + } + const CBMType **field_types = (const CBMType **)cbm_arena_alloc( + head->arena, (count + 1) * sizeof(*field_types)); + if (!field_types) { + return NULL; + } + memcpy(field_types, base->field_types, (count + 1) * sizeof(*field_types)); + copy.field_types = field_types; + } int before = head->type_count; - cbm_registry_add_type(head, *base); + cbm_registry_add_type(head, copy); if (head->type_count == before + 1) { return &head->types[before]; } diff --git a/internal/cbm/lsp/type_registry.h b/internal/cbm/lsp/type_registry.h index c752350b17..065ae25852 100644 --- a/internal/cbm/lsp/type_registry.h +++ b/internal/cbm/lsp/type_registry.h @@ -40,6 +40,9 @@ typedef struct { * remain NULL when raw cross-file provenance is ambiguous; the Rust trait * flag still prevents that method from being mistaken for inherent. */ const char *impl_trait_qn; + /* Callable identity (#2061): this overload's signature suffix; its graph + * QN is qualified_name + sig. NULL = none (always, until enabled). */ + const char *sig; } CBMRegisteredFunc; // Registered type with fields and method names. diff --git a/internal/cbm/result_compact.c b/internal/cbm/result_compact.c index 90c8c6d820..ee368896a7 100644 --- a/internal/cbm/result_compact.c +++ b/internal/cbm/result_compact.c @@ -357,6 +357,7 @@ static void cr_walk(cr_ctx_t *c, CBMFileResult *r) { cr_str(c, &r->resolved_calls.items[i].callee_qn); cr_str(c, &r->resolved_calls.items[i].strategy); cr_str(c, &r->resolved_calls.items[i].reason); + cr_str(c, &r->resolved_calls.items[i].callee_sig); } cr_array(c, (void **)&r->string_refs.items, r->string_refs.count, sizeof(CBMStringRef)); for (int i = 0; i < r->string_refs.count && r->string_refs.items; i++) { @@ -377,6 +378,12 @@ static void cr_walk(cr_ctx_t *c, CBMFileResult *r) { cr_str(c, &r->channels.items[i].transport); cr_str(c, &r->channels.items[i].enclosing_func_qn); } + cr_array(c, (void **)&r->field_types.items, r->field_types.count, sizeof(CBMFieldType)); + for (int i = 0; i < r->field_types.count && r->field_types.items; i++) { + cr_str(c, &r->field_types.items[i].class_qn); + cr_str(c, &r->field_types.items[i].field_name); + cr_str(c, &r->field_types.items[i].type_text); + } cr_str(c, &r->module_qn); cr_str(c, &r->namespace_name); cr_list(c, &r->exports); @@ -385,6 +392,7 @@ static void cr_walk(cr_ctx_t *c, CBMFileResult *r) { cr_list(c, &r->macros); cr_str(c, &r->error_msg); cr_str(c, &r->error_ranges); + cr_str(c, &r->module_doc); cr_blob(c, (const void **)&r->source, r->source ? (size_t)r->source_len + SKIP_ONE : 0); } @@ -523,6 +531,7 @@ void cbm_result_compact(CBMFileResult *result) { tmp.string_refs.cap = tmp.string_refs.count; tmp.infra_bindings.cap = tmp.infra_bindings.count; tmp.channels.cap = tmp.channels.count; + tmp.field_types.cap = tmp.field_types.count; /* A composite kept its per-unit results only so shallow-copied strings * stayed valid; every string is now a copy of its own. */ diff --git a/internal/cbm/service_patterns.c b/internal/cbm/service_patterns.c index c2425e09e1..04bd9b3a10 100644 --- a/internal/cbm/service_patterns.c +++ b/internal/cbm/service_patterns.c @@ -41,6 +41,11 @@ static const lib_pattern_t http_libraries[] = { /* JavaScript / TypeScript */ {"axios", CBM_SVC_HTTP, NULL}, + /* `import Axios from "axios"`: the capitalized default-import binding + * (also axios's own Axios / AxiosInstance types). Matching is + * case-sensitive, so without this entry `Axios.get(url)` fell through to + * the `.get` route-suffix fallback and became a route registration. */ + {"Axios", CBM_SVC_HTTP, NULL}, {"superagent", CBM_SVC_HTTP, NULL}, {"needle", CBM_SVC_HTTP, NULL}, {"node-fetch", CBM_SVC_HTTP, NULL}, @@ -752,9 +757,9 @@ static bool has_filesystem_extension(const char *path) { ext[ext_len] = '\0'; static const char *const hard_file_exts[] = { - ".cfg", ".conf", ".credentials", ".crt", ".db", ".env", - ".ini", ".key", ".pem", ".pid", ".properties", ".service", - ".sock", ".socket", ".sqlite", ".toml", NULL}; + ".cfg", ".conf", ".credentials", ".crt", ".db", ".env", ".ini", ".key", + ".log", ".md", ".pdf", ".pem", ".pid", ".properties", ".rst", ".service", + ".sock", ".socket", ".sqlite", ".toml", ".txt", NULL}; for (int i = 0; hard_file_exts[i]; i++) { if (path_ext_matches(ext, hard_file_exts[i])) { return true; @@ -865,6 +870,139 @@ bool cbm_service_pattern_is_http_route_literal(const char *literal, const char * return true; } +/* ── GraphQL operation identity (#598) ─────────────────────────── */ + +static bool gql_name_start(unsigned char c) { + return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_'; +} + +static bool gql_name_char(unsigned char c) { + return gql_name_start(c) || (c >= '0' && c <= '9'); +} + +/* Skip one lexical unit whose braces are not structure: a # comment, a "..." + * or """...""" string, or a JS ${...} template interpolation. Returns p + * unchanged when p starts none of them. */ +static const char *gql_skip_opaque(const char *p) { + if (*p == '#') { + while (*p && *p != '\n' && *p != '\r') { + p++; + } + return p; + } + if (strncmp(p, "\"\"\"", 3) == 0) { + const char *end = strstr(p + 3, "\"\"\""); + return end ? end + 3 : p + strlen(p); + } + if (*p == '"') { + p++; + while (*p && *p != '"') { + p += (*p == '\\' && p[1]) ? 2 : 1; + } + return *p ? p + 1 : p; + } + if (p[0] == '$' && p[1] == '{') { + int depth = 1; + p += 2; + while (*p && depth > 0) { + depth += (*p == '{') - (*p == '}'); + p++; + } + return p; + } + return p; +} + +/* Skip GraphQL "ignored tokens" (whitespace, commas, comments). */ +static const char *gql_skip_ignored(const char *p) { + for (;;) { + while (*p == ' ' || *p == '\t' || *p == '\n' || *p == '\r' || *p == ',') { + p++; + } + if (*p != '#') { + return p; + } + p = gql_skip_opaque(p); + } +} + +/* Copy the Name token at p (bounded). Returns false when p is not a Name. */ +static bool gql_copy_name(const char *p, char *name_buf, size_t name_sz) { + if (!gql_name_start((unsigned char)*p) || !name_buf || name_sz == 0) { + return false; + } + size_t n = 0; + while (gql_name_char((unsigned char)p[n]) && n + 1 < name_sz) { + name_buf[n] = p[n]; + n++; + } + name_buf[n] = '\0'; + return true; +} + +static const char *gql_operation_keyword(const char *tok, size_t len) { + static const char *const kws[] = {"query", "mutation", "subscription", NULL}; + for (int i = 0; kws[i]; i++) { + if (strlen(kws[i]) == len && strncmp(tok, kws[i], len) == 0) { + return kws[i]; + } + } + return NULL; +} + +bool cbm_service_pattern_graphql_operation(const char *doc, const char **op_type, char *name_buf, + size_t name_sz) { + *op_type = "operation"; + if (name_buf && name_sz > 0) { + name_buf[0] = '\0'; + } + if (!doc) { + return false; + } + int depth = 0; + bool in_fragment = false; + const char *p = doc; + while (*p) { + const char *q = gql_skip_opaque(p); + if (q != p) { + p = q; + continue; + } + unsigned char c = (unsigned char)*p; + if (c == '{' && depth == 0 && !in_fragment) { + *op_type = "query"; /* `{ ... }` shorthand is an anonymous query */ + return false; + } + if (c == '{' || c == '}') { + depth += (c == '{') ? 1 : -1; + depth = depth < 0 ? 0 : depth; + in_fragment = in_fragment && depth > 0; + p++; + continue; + } + if (depth > 0 || in_fragment || !gql_name_start(c)) { + p++; + continue; + } + const char *tok = p; + while (gql_name_char((unsigned char)*p)) { + p++; + } + size_t len = (size_t)(p - tok); + if (len == strlen("fragment") && strncmp(tok, "fragment", len) == 0) { + in_fragment = true; + continue; + } + const char *kw = gql_operation_keyword(tok, len); + if (!kw) { + return false; /* not a GraphQL document (URL, dict key, variable ...) */ + } + *op_type = kw; + return gql_copy_name(gql_skip_ignored(p), name_buf, name_sz); + } + return false; +} + /* ── Public API ────────────────────────────────────────────────── */ /* Per-worker TLS cache of cbm_service_pattern_match results. diff --git a/internal/cbm/service_patterns.h b/internal/cbm/service_patterns.h index d9a93c2519..020ab429d3 100644 --- a/internal/cbm/service_patterns.h +++ b/internal/cbm/service_patterns.h @@ -12,6 +12,7 @@ #define CBM_SERVICE_PATTERNS_H #include +#include /* Edge type returned by pattern match. */ typedef enum { @@ -77,4 +78,17 @@ const char *cbm_service_pattern_route_method(const char *callee_name); * Returns NULL if not an async pattern. */ const char *cbm_service_pattern_broker(const char *resolved_qn); +/* GraphQL operation identity of a client call argument (#598). + * Scans `doc` as a GraphQL document (skipping comments, strings, `${...}` + * template interpolations and leading fragment definitions) and reports the + * first operation: *op_type is "query", "mutation" or "subscription" ("query" + * for the `{ ... }` shorthand). Returns true and writes the operation name to + * name_buf when the operation is named. Returns false for an anonymous + * operation, or with *op_type = "operation" when `doc` is not a GraphQL + * document (NULL, a URL, a bare word such as a dict key ...). The operation + * text itself is never copied, so callers can key Routes on a bounded + * identity. */ +bool cbm_service_pattern_graphql_operation(const char *doc, const char **op_type, char *name_buf, + size_t name_sz); + #endif /* CBM_SERVICE_PATTERNS_H */ diff --git a/pkg/npm/README.md b/pkg/npm/README.md index e0acd471f1..8dcd973f12 100644 --- a/pkg/npm/README.md +++ b/pkg/npm/README.md @@ -55,6 +55,10 @@ Use `codebase-memory-cli install --plan` to inspect planned writes before applyi The source repository and full documentation are at https://github.com/ngallodev/codebase-memory-cli. +## Uninstall + +Uninstall keeps project indexes by default, including with `--yes`, `--no`, or noninteractive input. Use `--delete-indexes` to remove them explicitly; `--dry-run` previews the changes. + ## License MIT diff --git a/pkg/npm/install.js b/pkg/npm/install.js index e6abce8669..38d7e960fa 100644 --- a/pkg/npm/install.js +++ b/pkg/npm/install.js @@ -8,7 +8,7 @@ const crypto = require('crypto'); const fs = require('fs'); const path = require('path'); const os = require('os'); -const { execFileSync } = require('child_process'); +const { execFileSync, spawn } = require('child_process'); const { pipeline } = require('stream'); const REPO = 'DeusData/codebase-memory-mcp'; @@ -18,6 +18,20 @@ const MAX_REDIRECTS = 5; const DOWNLOAD_HOP_TIMEOUT_MS = 120_000; const CANDIDATE_TIMEOUT_MS = 15_000; const MAX_CHECKSUM_MANIFEST_BYTES = 1024 * 1024; +// Release archive resource limits. Keep in sync with the Go wrapper +// (pkg/go/cmd/codebase-memory-mcp/main.go) and the PyPI wrapper (_cli.py). +const MAX_RELEASE_ARCHIVE_BYTES = 256 * 1024 * 1024; +const MAX_ARCHIVE_MEMBERS = 64; +const MAX_ARCHIVE_MEMBER_BYTES = 256 * 1024 * 1024; +const MAX_ARCHIVE_EXPANDED_BYTES = 512 * 1024 * 1024; +const DEFAULT_ARCHIVE_LIMITS = Object.freeze({ + compressedBytes: MAX_RELEASE_ARCHIVE_BYTES, + members: MAX_ARCHIVE_MEMBERS, + memberBytes: MAX_ARCHIVE_MEMBER_BYTES, + expandedBytes: MAX_ARCHIVE_EXPANDED_BYTES, +}); +// Bytes of tar's own diagnostics kept for the failure message. +const MAX_TAR_STDERR_CHARS = 4096; const WINDOWS_BINARY_NAME = 'codebase-memory-cli.exe'; const UNIX_ARCHIVE_NAMES = [ 'codebase-memory-cli', @@ -90,21 +104,131 @@ function validateExactTarMemberListing(listing, expectedNames) { } } -function extractExactTarArchive( - archivePath, destPath, expectedNames, targetName, runFile = execFileSync, +function validateArchiveLimits(limits) { + if (!limits || !(limits.compressedBytes > 0) || !(limits.members > 0) || + !(limits.memberBytes > 0) || !(limits.expandedBytes > 0)) { + throw new Error('invalid archive resource safety limits'); + } +} + +function requireCompressedArchiveWithinLimit(archivePath, limits) { + const status = fs.statSync(archivePath); + if (!status.isFile()) { + throw new Error('release archive is not a regular file'); + } + if (status.size > limits.compressedBytes) { + throw new Error( + `release archive exceeds the ${limits.compressedBytes}-byte compressed safety limit`, + ); + } +} + +// Streams one member through `tar -xzOf` into a file this call creates, +// counting every byte against the per-member limit. The file is removed only +// when this call created it: a pre-existing target fails untouched. +function extractTarMemberWithLimit( + archivePath, target, memberName, limits, spawnFile = spawn, ) { + return new Promise((resolve, reject) => { + let fd; + try { + fd = fs.openSync(target, 'wx', 0o600); + } catch (err) { + reject(err); + return; + } + let written = 0; + let failure = null; + let settled = false; + let stderr = ''; + const finish = (err) => { + if (settled) return; + settled = true; + try { fs.closeSync(fd); } catch (closeError) { err = err || closeError; } + if (err) { + try { fs.unlinkSync(target); } catch (_) { /* already gone */ } + reject(err); + return; + } + resolve(written); + }; + const child = spawnFile('tar', ['-xzOf', archivePath, memberName], { + stdio: ['ignore', 'pipe', 'pipe'], + windowsHide: true, + }); + const fail = (err) => { + if (failure) return; + failure = err; + child.kill(); + }; + child.stdout.on('data', (chunk) => { + if (failure) return; + written += chunk.length; + if (written > limits.memberBytes) { + fail(new Error( + `archive member ${JSON.stringify(memberName)} exceeds the ${limits.memberBytes}-byte expanded safety limit`, + )); + return; + } + try { + let offset = 0; + while (offset < chunk.length) { + offset += fs.writeSync(fd, chunk, offset, chunk.length - offset); + } + } catch (err) { + fail(err); + } + }); + child.stderr.setEncoding('utf8'); + child.stderr.on('data', (text) => { + if (stderr.length < MAX_TAR_STDERR_CHARS) stderr += text; + }); + child.on('error', (err) => { + failure = failure || err; + // A process that never started (no tar on PATH) has no exit to wait for. + if (child.pid === undefined) finish(failure); + }); + child.on('close', (code, signal) => { + const detail = stderr.trim(); + let err = failure; + if (!err && signal) { + err = new Error( + `tar was terminated by ${signal} while extracting ${memberName}: ${detail}`, + ); + } else if (!err && code !== 0) { + err = new Error( + `tar exited with status ${code} while extracting ${memberName}: ${detail}`, + ); + } else if (!err && written === 0) { + err = new Error(`archive member ${memberName} is empty`); + } + finish(err); + }); + }); +} + +async function extractExactTarArchive( + archivePath, destPath, expectedNames, targetName, + runFile = execFileSync, limits = DEFAULT_ARCHIVE_LIMITS, spawnFile = spawn, +) { + validateArchiveLimits(limits); + requireCompressedArchiveWithinLimit(archivePath, limits); + // The exact root namespace is the member bound: its four names lie well + // within MAX_ARCHIVE_MEMBERS, which the zip path enforces by count. + if (expectedNames.length > limits.members) { + throw new Error(`archive namespace exceeds the ${limits.members}-member safety limit`); + } const listing = runFile('tar', ['-tzf', archivePath], { encoding: 'utf8', maxBuffer: MAX_CHECKSUM_MANIFEST_BYTES, windowsHide: true, }); validateExactTarMemberListing(listing, expectedNames); - // Extract only the fixed root executable. Even a malformed tar implementation - // cannot write an unvalidated companion member outside the private temp tree. - runFile('tar', ['-xzf', archivePath, '-C', destPath, targetName], { - stdio: 'inherit', - windowsHide: true, - }); + // Only the fixed root executable is ever written, through the counted + // writer; the companions are listed but never extracted. + const target = path.join(destPath, targetName); + await extractTarMemberWithLimit(archivePath, target, targetName, limits, spawnFile); + fs.chmodSync(target, 0o755); } function downloadHop(url, dest, maxBytes) { @@ -123,7 +247,11 @@ function downloadHop(url, dest, maxBytes) { const req = https.get(url, (res) => { response = res; - res.on('error', (err) => finish(err)); + let piped = false; + // Before the body is piped, a response error ends the hop directly; + // afterwards the pipeline reports it, so the partial file is discarded + // before the hop settles. + res.on('error', (err) => { if (!piped) finish(err); }); const redirectCodes = new Set([301, 302, 303, 307, 308]); if (redirectCodes.has(res.statusCode)) { const location = res.headers.location; @@ -149,6 +277,12 @@ function downloadHop(url, dest, maxBytes) { finish(new Error(`HTTP ${res.statusCode} for ${url}`)); return; } + const declared = Number(res.headers['content-length']); + if (maxBytes && Number.isFinite(declared) && declared > maxBytes) { + res.destroy(); + finish(new Error(`Download exceeds the ${maxBytes}-byte safety limit`)); + return; + } let received = 0; res.on('data', (chunk) => { @@ -158,8 +292,20 @@ function downloadHop(url, dest, maxBytes) { } }); const file = fs.createWriteStream(dest, { flags: 'w' }); + piped = true; pipeline(res, file, (err) => { - finish(err, { redirect: null }); + if (!err) { + finish(null, { redirect: null }); + return; + } + // Discard the partial file once its handle is closed (Windows refuses + // to unlink an open file), then report the failure. + const discard = () => { + try { fs.unlinkSync(dest); } catch (_) { /* nothing was written */ } + finish(err); + }; + if (file.closed) discard(); + else file.once('close', discard); }); }); req.on('error', (err) => finish(err)); @@ -457,19 +603,37 @@ function installWindowsBinaryAtomically(sourceDir, destDir, verifier = verifyCan } } -function extractZipOnWindows(archivePath, destPath, requiredNames, extractNames) { +function extractZipOnWindows( + archivePath, destPath, requiredNames, extractNames, + limits = DEFAULT_ARCHIVE_LIMITS, shell = 'powershell', +) { + validateArchiveLimits(limits); + requireCompressedArchiveWithinLimit(archivePath, limits); // -EncodedCommand is a constant program. Paths travel only through the child // environment and are consumed with -LiteralPath, so PowerShell never parses // user/TEMP path bytes as source code or wildcard syntax. const script = [ "$ErrorActionPreference = 'Stop'", 'Add-Type -AssemblyName System.IO.Compression.FileSystem', + '$maxMembers = [int]$env:CBM_NPM_MAX_MEMBERS', + '$maxMemberBytes = [long]$env:CBM_NPM_MAX_MEMBER_BYTES', + '$maxExpandedBytes = [long]$env:CBM_NPM_MAX_EXPANDED_BYTES', + '$memberCount = 0', + '$declaredExpanded = [long]0', '$zip = [System.IO.Compression.ZipFile]::OpenRead($env:CBM_NPM_ARCHIVE_PATH)', "$seen = New-Object 'System.Collections.Generic.HashSet[string]' ([System.StringComparer]::OrdinalIgnoreCase)", "$requiredNames = @($env:CBM_NPM_REQUIRED_NAMES.Split('|'))", '$targetCounts = @{}', 'foreach ($requiredName in $requiredNames) { $targetCounts[$requiredName] = 0 }', 'try { foreach ($entry in $zip.Entries) { ' + + '$memberCount = $memberCount + 1; ' + + 'if ($memberCount -gt $maxMembers) { ' + + 'throw "archive exceeds the $maxMembers-member safety limit" }; ' + + 'if ($entry.Length -gt $maxMemberBytes) { ' + + 'throw "archive member $($entry.FullName) exceeds the $maxMemberBytes-byte expanded safety limit" }; ' + + 'if ($declaredExpanded -gt ($maxExpandedBytes - $entry.Length)) { ' + + 'throw "archive exceeds the $maxExpandedBytes-byte aggregate expanded safety limit" }; ' + + '$declaredExpanded = $declaredExpanded + $entry.Length; ' + "$name = $entry.FullName.Replace('\\', '/'); " + "$directory = $name.EndsWith('/'); " + "$segmentsPath = if ($directory) { $name.TrimEnd('/') } else { $name }; " + @@ -491,27 +655,61 @@ function extractZipOnWindows(archivePath, destPath, requiredNames, extractNames) 'if ($seen.Count -ne $requiredNames.Count) { ' + 'throw "archive does not match the exact release root-file allowlist" }', "$extractNames = @($env:CBM_NPM_EXTRACT_NAMES.Split('|'))", + '$actualExpanded = [long]0', + '$buffer = New-Object byte[] 65536', '$extractZip = [System.IO.Compression.ZipFile]::OpenRead($env:CBM_NPM_ARCHIVE_PATH)', + // Copy by hand with a byte counter: ExtractToFile would write whatever the + // compressed stream yields, regardless of the declared size. 'try { foreach ($extractName in $extractNames) { ' + '$entry = @($extractZip.Entries | Where-Object { $_.FullName -ceq $extractName })[0]; ' + - '[System.IO.Compression.ZipFileExtensions]::ExtractToFile(' + - '$entry, (Join-Path $env:CBM_NPM_DEST_PATH $extractName), $false) ' + + '$targetPath = Join-Path $env:CBM_NPM_DEST_PATH $extractName; ' + + '$remaining = $maxExpandedBytes - $actualExpanded; ' + + '$copied = [long]0; ' + + '$source = $entry.Open(); ' + + '$target = [System.IO.File]::Open($targetPath, [System.IO.FileMode]::CreateNew, [System.IO.FileAccess]::Write); ' + + 'try { while ($true) { ' + + '$count = $source.Read($buffer, 0, $buffer.Length); ' + + 'if ($count -le 0) { break }; ' + + '$copied = $copied + $count; ' + + 'if ($copied -gt $maxMemberBytes) { ' + + 'throw "archive member $extractName exceeds the $maxMemberBytes-byte actual expanded safety limit" }; ' + + 'if ($copied -gt $remaining) { ' + + 'throw "archive exceeds the $maxExpandedBytes-byte aggregate actual expanded safety limit" }; ' + + '$target.Write($buffer, 0, $count) } ' + + '} catch { $target.Dispose(); ' + + 'Remove-Item -LiteralPath $targetPath -Force -ErrorAction SilentlyContinue; throw ' + + '} finally { $target.Dispose(); $source.Dispose() }; ' + + 'if ($copied -ne $entry.Length) { ' + + 'Remove-Item -LiteralPath $targetPath -Force -ErrorAction SilentlyContinue; ' + + 'throw "archive member $extractName actual size $copied does not match declared size $($entry.Length)" }; ' + + '$actualExpanded = $actualExpanded + $copied ' + '} } finally { $extractZip.Dispose() }', ].join('; '); const encoded = Buffer.from(script, 'utf16le').toString('base64'); - execFileSync('powershell', [ - '-NoLogo', '-NoProfile', '-NonInteractive', '-EncodedCommand', encoded, - ], { - env: { - ...process.env, - CBM_NPM_ARCHIVE_PATH: archivePath, - CBM_NPM_DEST_PATH: destPath, - CBM_NPM_REQUIRED_NAMES: requiredNames.join('|'), - CBM_NPM_EXTRACT_NAMES: extractNames.join('|'), - }, - stdio: 'inherit', - windowsHide: true, - }); + try { + execFileSync(shell, [ + '-NoLogo', '-NoProfile', '-NonInteractive', '-EncodedCommand', encoded, + ], { + env: { + ...process.env, + CBM_NPM_ARCHIVE_PATH: archivePath, + CBM_NPM_DEST_PATH: destPath, + CBM_NPM_REQUIRED_NAMES: requiredNames.join('|'), + CBM_NPM_EXTRACT_NAMES: extractNames.join('|'), + CBM_NPM_MAX_MEMBERS: String(limits.members), + CBM_NPM_MAX_MEMBER_BYTES: String(limits.memberBytes), + CBM_NPM_MAX_EXPANDED_BYTES: String(limits.expandedBytes), + }, + encoding: 'utf8', + stdio: ['ignore', 'inherit', 'pipe'], + windowsHide: true, + }); + } catch (err) { + // Surface the script's own reason (for example which limit was hit) + // instead of a bare "Command failed". + const detail = err && err.stderr ? String(err.stderr).trim() : ''; + throw new Error(detail ? `zip extraction failed: ${detail}` : err.message); + } } // Fetch checksums.txt and verify the archive hash. @@ -592,12 +790,12 @@ async function main() { const tmpArchive = path.join(tmpDir, `cbm.${ext}`); try { - await download(url, tmpArchive); + await download(url, tmpArchive, MAX_RELEASE_ARCHIVE_BYTES); await verifyChecksum(tmpArchive, archive); // Validate the complete archive namespace, then extract only executables. if (ext === 'tar.gz') { - extractExactTarArchive( + await extractExactTarArchive( tmpArchive, tmpDir, archiveNames, binName, ); } else { @@ -674,9 +872,13 @@ if (require.main === module || module.parent == null) { } module.exports = { + DEFAULT_ARCHIVE_LIMITS, UNIX_ARCHIVE_NAMES, + WINDOWS_ARCHIVE_NAMES, WINDOWS_BINARY_NAME, + download, extractExactTarArchive, + extractZipOnWindows, installWindowsBinaryAtomically, validateExactTarMemberListing, windowsBinaryReady, diff --git a/pkg/npm/test/archive-fixtures.js b/pkg/npm/test/archive-fixtures.js new file mode 100644 index 0000000000..253f5b2a11 --- /dev/null +++ b/pkg/npm/test/archive-fixtures.js @@ -0,0 +1,126 @@ +'use strict'; +// Small synthetic release archives for the wrapper tests. + +const { execFileSync } = require('node:child_process'); +const fs = require('node:fs'); +const path = require('node:path'); + +// The npm wrapper takes the zip path whenever Node reports win32 (see +// getPlatform and `ext` in install.js), so the tar path is unreachable on a +// Windows host and the tar cases would test nothing a user can reach there. +// They cannot even build their fixtures on the Windows runner: the first +// `tar` on its PATH is Git for Windows' GNU tar, which reads a drive-letter +// path such as C:\... as a remote host ("tar (child): Cannot connect to C: +// resolve failed"; every tar case failed that way in the first Windows run), +// while Windows' own bsdtar accepts it. Under WSL Node is a Linux binary, +// takes the tar path with POSIX paths, and these cases run there. +const TAR_PATH_SKIP = process.platform === 'win32' + ? 'the npm wrapper never takes the tar path on win32 (it installs the zip asset)' + : false; + +// Deterministic, poorly compressible bytes (xorshift32), so a fixture of a +// few hundred KiB still spans several pipe chunks after gzip. +function patternBytes(length, seed = 0x9e3779b9) { + const out = Buffer.alloc(length); + let state = seed >>> 0; + for (let index = 0; index < length; index += 1) { + state ^= state << 13; + state >>>= 0; + state ^= state >>> 17; + state ^= state << 5; + state >>>= 0; + out[index] = state & 0xff; + } + return out; +} + +// Writes entries ({ name, data }) into a staging directory under `root` and +// packs them with the system tar, the producer the release itself uses. +function writeTarGz(root, archiveName, entries) { + const stage = fs.mkdtempSync(path.join(root, 'stage-')); + for (const entry of entries) { + fs.writeFileSync(path.join(stage, entry.name), entry.data); + } + const archive = path.join(root, archiveName); + execFileSync( + 'tar', + ['-czf', archive, '-C', stage, ...entries.map((entry) => entry.name)], + { + env: { ...process.env, COPYFILE_DISABLE: '1' }, + stdio: ['ignore', 'ignore', 'inherit'], + windowsHide: true, + }, + ); + return archive; +} + +const CRC_TABLE = (() => { + const table = new Int32Array(256); + for (let n = 0; n < 256; n += 1) { + let c = n; + for (let k = 0; k < 8; k += 1) { + c = c & 1 ? 0xedb88320 ^ (c >>> 1) : c >>> 1; + } + table[n] = c; + } + return table; +})(); + +function crc32(buffer) { + let crc = -1; + for (const byte of buffer) { + crc = CRC_TABLE[(crc ^ byte) & 0xff] ^ (crc >>> 8); + } + return (crc ^ -1) >>> 0; +} + +// entries: [{ name, data, declaredSize = data.length }]; stored (no +// compression). Hand-rolled so the directory can claim a size that differs +// from the bytes the entry carries, which no zip writer would produce. +function buildZip(entries) { + const locals = []; + const centrals = []; + let offset = 0; + for (const entry of entries) { + const data = Buffer.isBuffer(entry.data) ? entry.data : Buffer.from(entry.data); + const name = Buffer.from(entry.name, 'utf8'); + const declaredSize = entry.declaredSize === undefined + ? data.length + : entry.declaredSize; + const crc = crc32(data); + + const local = Buffer.alloc(30 + name.length); + local.writeUInt32LE(0x04034b50, 0); + local.writeUInt16LE(20, 4); + local.writeUInt32LE(crc, 14); + local.writeUInt32LE(data.length, 18); + local.writeUInt32LE(declaredSize, 22); + local.writeUInt16LE(name.length, 26); + name.copy(local, 30); + + const central = Buffer.alloc(46 + name.length); + central.writeUInt32LE(0x02014b50, 0); + central.writeUInt16LE(20, 4); + central.writeUInt16LE(20, 6); + central.writeUInt32LE(crc, 16); + central.writeUInt32LE(data.length, 20); + central.writeUInt32LE(declaredSize, 24); + central.writeUInt16LE(name.length, 28); + central.writeUInt32LE(offset, 42); + name.copy(central, 46); + + locals.push(local, data); + centrals.push(central); + offset += local.length + data.length; + } + const directory = Buffer.concat(centrals); + const end = Buffer.alloc(22); + end.writeUInt32LE(0x06054b50, 0); + end.writeUInt16LE(entries.length, 8); + end.writeUInt16LE(entries.length, 10); + end.writeUInt32LE(directory.length, 12); + end.writeUInt32LE(offset, 16); + return Buffer.concat([...locals, directory, end]); +} + +module.exports = { TAR_PATH_SKIP, buildZip, patternBytes, writeTarGz }; diff --git a/pkg/npm/test/archive-limits.test.js b/pkg/npm/test/archive-limits.test.js new file mode 100644 index 0000000000..5e21e5b5ad --- /dev/null +++ b/pkg/npm/test/archive-limits.test.js @@ -0,0 +1,350 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const { EventEmitter } = require('node:events'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const { Readable } = require('node:stream'); +const test = require('node:test'); +const vm = require('node:vm'); + +const { + DEFAULT_ARCHIVE_LIMITS, + UNIX_ARCHIVE_NAMES, + WINDOWS_ARCHIVE_NAMES, + WINDOWS_BINARY_NAME, + extractExactTarArchive, + extractZipOnWindows, +} = require('../install.js'); +const { + TAR_PATH_SKIP, buildZip, patternBytes, writeTarGz, +} = require('./archive-fixtures.js'); + +const EXECUTABLE = 'codebase-memory-cli'; +// Spans several pipe chunks, so an over-limit member is cut off mid-stream. +const LARGE_EXECUTABLE = patternBytes(300 * 1024); + +function withScratch(callback) { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'cbm-npm-archive-test-')); + return Promise.resolve() + .then(() => callback(root)) + .finally(() => fs.rmSync(root, { recursive: true, force: true })); +} + +function limitsWith(overrides) { + return { ...DEFAULT_ARCHIVE_LIMITS, ...overrides }; +} + +function releaseEntries(names, executableData) { + return names.map((name) => ({ + name, + data: executableData !== undefined && (name === EXECUTABLE || name === WINDOWS_BINARY_NAME) + ? executableData + : `payload of ${name}`, + })); +} + +function makeDestination(root) { + const destination = path.join(root, 'extract'); + fs.mkdirSync(destination); + return destination; +} + +// Stands in for the system tar listing where the archive itself is damaged +// on purpose and the real listing would fail before the writer runs. +function fakeTarListing(names) { + return (command, args) => { + assert.equal(command, 'tar'); + assert.equal(args[0], '-tzf'); + return `${names.join('\n')}\n`; + }; +} + +function neverLists() { + throw new Error('the listing must not run'); +} + +test('tar extraction refuses an archive larger than the compressed limit', { skip: TAR_PATH_SKIP }, () => + withScratch(async (root) => { + const archive = writeTarGz(root, 'release.tar.gz', releaseEntries(UNIX_ARCHIVE_NAMES)); + const destination = makeDestination(root); + + await assert.rejects( + extractExactTarArchive( + archive, destination, UNIX_ARCHIVE_NAMES, EXECUTABLE, + neverLists, limitsWith({ compressedBytes: 16 }), + ), + /16-byte compressed safety limit/, + ); + assert.deepEqual(fs.readdirSync(destination), []); + })); + +test('tar extraction refuses an exact namespace wider than the member limit', { skip: TAR_PATH_SKIP }, () => + withScratch(async (root) => { + const archive = writeTarGz(root, 'release.tar.gz', releaseEntries(UNIX_ARCHIVE_NAMES)); + const destination = makeDestination(root); + + await assert.rejects( + extractExactTarArchive( + archive, destination, UNIX_ARCHIVE_NAMES, EXECUTABLE, + neverLists, limitsWith({ members: 2 }), + ), + /2-member safety limit/, + ); + assert.deepEqual(fs.readdirSync(destination), []); + })); + +test('tar extraction stops an over-limit member in the counted writer and leaves no file', { skip: TAR_PATH_SKIP }, () => + withScratch(async (root) => { + const archive = writeTarGz( + root, 'release.tar.gz', releaseEntries(UNIX_ARCHIVE_NAMES, LARGE_EXECUTABLE), + ); + const destination = makeDestination(root); + + // The real listing passes (names only); the limit trips while tar is + // still streaming the executable into the writer. + await assert.rejects( + extractExactTarArchive( + archive, destination, UNIX_ARCHIVE_NAMES, EXECUTABLE, + undefined, limitsWith({ memberBytes: 64 * 1024 }), + ), + /"codebase-memory-cli" exceeds the 65536-byte expanded safety limit/, + ); + assert.deepEqual(fs.readdirSync(destination), []); + })); + +test('tar extraction leaves a pre-existing target untouched', { skip: TAR_PATH_SKIP }, () => + withScratch(async (root) => { + const archive = writeTarGz(root, 'release.tar.gz', releaseEntries(UNIX_ARCHIVE_NAMES)); + const destination = makeDestination(root); + const target = path.join(destination, EXECUTABLE); + fs.writeFileSync(target, 'keep'); + + await assert.rejects( + extractExactTarArchive(archive, destination, UNIX_ARCHIVE_NAMES, EXECUTABLE), + /EEXIST/, + ); + assert.equal(fs.readFileSync(target, 'utf8'), 'keep'); + })); + +test('tar extraction leaves no file when tar fails mid-stream', { skip: TAR_PATH_SKIP }, () => + withScratch(async (root) => { + const complete = writeTarGz( + root, 'complete.tar.gz', releaseEntries(UNIX_ARCHIVE_NAMES, LARGE_EXECUTABLE), + ); + const bytes = fs.readFileSync(complete); + const archive = path.join(root, 'truncated.tar.gz'); + fs.writeFileSync(archive, bytes.subarray(0, Math.floor(bytes.length * 0.6))); + const destination = makeDestination(root); + + await assert.rejects( + extractExactTarArchive( + archive, destination, UNIX_ARCHIVE_NAMES, EXECUTABLE, + fakeTarListing(UNIX_ARCHIVE_NAMES), + ), + // Non-zero exit, and tar's own diagnostic text after the colon. + /tar exited with status [1-9]\d* while extracting codebase-memory-cli: \S/, + ); + assert.deepEqual(fs.readdirSync(destination), []); + })); + +test('tar extraction rejects an empty executable', { skip: TAR_PATH_SKIP }, () => + withScratch(async (root) => { + const archive = writeTarGz( + root, 'release.tar.gz', releaseEntries(UNIX_ARCHIVE_NAMES, Buffer.alloc(0)), + ); + const destination = makeDestination(root); + + await assert.rejects( + extractExactTarArchive(archive, destination, UNIX_ARCHIVE_NAMES, EXECUTABLE), + /archive member codebase-memory-cli is empty/, + ); + assert.deepEqual(fs.readdirSync(destination), []); + })); + +test('tar extraction installs the executable byte-identical with the executable bit set', { skip: TAR_PATH_SKIP }, () => + withScratch(async (root) => { + const archive = writeTarGz( + root, 'release.tar.gz', releaseEntries(UNIX_ARCHIVE_NAMES, LARGE_EXECUTABLE), + ); + const destination = makeDestination(root); + + await extractExactTarArchive(archive, destination, UNIX_ARCHIVE_NAMES, EXECUTABLE); + + const target = path.join(destination, EXECUTABLE); + assert.ok(fs.readFileSync(target).equals(LARGE_EXECUTABLE)); + if (process.platform !== 'win32') { + // Windows has no mode bits; everywhere else the file must be runnable. + assert.equal(fs.statSync(target).mode & 0o111, 0o111); + } + assert.deepEqual(fs.readdirSync(destination), [EXECUTABLE]); + })); + +function loadInstallerWithHttps(fakeHttps) { + const installPath = path.join(__dirname, '..', 'install.js'); + const source = fs.readFileSync(installPath, 'utf8'); + const sandbox = { + Buffer, + URL, + __dirname: path.dirname(installPath), + __filename: installPath, + clearTimeout, + console, + exports: {}, + // A parent keeps the postinstall entry point from running on load. + module: { exports: {}, parent: {} }, + process, + require: (specifier) => { + if (specifier === './package.json') return { version: '0.0.0-test' }; + if (specifier === 'https') return fakeHttps; + return require(specifier); + }, + setTimeout, + }; + vm.runInNewContext(source, sandbox, { filename: installPath }); + return sandbox.module.exports; +} + +// Answers every request with one HTTP 200 response shaped by `configure`. +function fakeHttpsServing(configure) { + return { + get(url, callback) { + const request = new EventEmitter(); + request.destroy = (err) => { if (err) request.emit('error', err); }; + setImmediate(() => { + const response = new Readable({ read() {} }); + response.statusCode = 200; + response.headers = {}; + configure(response); + callback(response); + }); + return request; + }, + }; +} + +test('download refuses a response whose declared length exceeds the limit', () => + withScratch(async (root) => { + const { download } = loadInstallerWithHttps(fakeHttpsServing((response) => { + response.headers['content-length'] = '9'; + response.push(null); + })); + const destination = path.join(root, 'release.tar.gz'); + + await assert.rejects( + download('https://example.invalid/release.tar.gz', destination, 8), + /8-byte safety limit/, + ); + assert.equal(fs.existsSync(destination), false); + })); + +test('download refuses a body that exceeds the limit and removes the partial file', () => + withScratch(async (root) => { + const { download } = loadInstallerWithHttps(fakeHttpsServing((response) => { + response.push(Buffer.alloc(1024, 0x78)); + response.push(null); + })); + const destination = path.join(root, 'release.tar.gz'); + + await assert.rejects( + download('https://example.invalid/release.tar.gz', destination, 8), + /8-byte safety limit/, + ); + assert.equal(fs.existsSync(destination), false); + })); + +test('download within the limit stores the body', () => + withScratch(async (root) => { + const { download } = loadInstallerWithHttps(fakeHttpsServing((response) => { + response.headers['content-length'] = '7'; + response.push('payload'); + response.push(null); + })); + const destination = path.join(root, 'release.tar.gz'); + + await download('https://example.invalid/release.tar.gz', destination, 1024); + + assert.equal(fs.readFileSync(destination, 'utf8'), 'payload'); + })); + +// The Windows path runs a constant PowerShell program through powershell.exe, +// which only a Windows host provides (pwsh is not a wrapper dependency, and +// no PowerShell is installed on the Unix wrapper hosts), so these cases run +// in the Windows wrapper leg and are skipped elsewhere. +const WINDOWS_ONLY = process.platform === 'win32' + ? false + : 'zip extraction runs through powershell.exe, which this host does not provide'; + +test('Windows zip extraction refuses an archive with more members than the limit', { skip: WINDOWS_ONLY }, () => + withScratch(async (root) => { + const archive = path.join(root, 'members.zip'); + fs.writeFileSync(archive, buildZip([ + { name: 'one', data: 'a' }, + { name: 'two', data: 'b' }, + { name: 'three', data: 'c' }, + ])); + const destination = makeDestination(root); + + assert.throws( + () => extractZipOnWindows( + archive, destination, ['one', 'two', 'three'], ['one'], + limitsWith({ members: 2 }), + ), + /2-member safety limit/, + ); + assert.deepEqual(fs.readdirSync(destination), []); + })); + +test('Windows zip extraction refuses a member whose declared size exceeds the limit', { skip: WINDOWS_ONLY }, () => + withScratch(async (root) => { + const archive = path.join(root, 'member.zip'); + fs.writeFileSync(archive, buildZip([ + { name: WINDOWS_BINARY_NAME, data: Buffer.alloc(65, 0x78) }, + ])); + const destination = makeDestination(root); + + assert.throws( + () => extractZipOnWindows( + archive, destination, [WINDOWS_BINARY_NAME], [WINDOWS_BINARY_NAME], + limitsWith({ memberBytes: 64 }), + ), + /64-byte expanded safety limit/, + ); + assert.deepEqual(fs.readdirSync(destination), []); + })); + +test('Windows zip extraction stops a member that yields more bytes than it declares', { skip: WINDOWS_ONLY }, () => + withScratch(async (root) => { + const archive = path.join(root, 'longer.zip'); + fs.writeFileSync(archive, buildZip([ + { name: WINDOWS_BINARY_NAME, data: Buffer.alloc(600, 0x78), declaredSize: 10 }, + ])); + const destination = makeDestination(root); + + assert.throws( + () => extractZipOnWindows( + archive, destination, [WINDOWS_BINARY_NAME], [WINDOWS_BINARY_NAME], + limitsWith({ memberBytes: 64 }), + ), + /64-byte actual expanded safety limit/, + ); + assert.deepEqual(fs.readdirSync(destination), []); + })); + +test('Windows zip extraction within the limits installs only the executable', { skip: WINDOWS_ONLY }, () => + withScratch(async (root) => { + const archive = path.join(root, 'release.zip'); + fs.writeFileSync(archive, buildZip(releaseEntries(WINDOWS_ARCHIVE_NAMES))); + const destination = makeDestination(root); + + extractZipOnWindows( + archive, destination, WINDOWS_ARCHIVE_NAMES, [WINDOWS_BINARY_NAME], + ); + + assert.equal( + fs.readFileSync(path.join(destination, WINDOWS_BINARY_NAME), 'utf8'), + `payload of ${WINDOWS_BINARY_NAME}`, + ); + assert.deepEqual(fs.readdirSync(destination), [WINDOWS_BINARY_NAME]); + })); diff --git a/pkg/npm/test/package-publication.test.js b/pkg/npm/test/package-publication.test.js index 909f798572..422f660744 100644 --- a/pkg/npm/test/package-publication.test.js +++ b/pkg/npm/test/package-publication.test.js @@ -13,6 +13,7 @@ const { installWindowsBinaryAtomically, validateExactTarMemberListing, } = require('../install.js'); +const { TAR_PATH_SKIP, writeTarGz } = require('./archive-fixtures.js'); function exactUnixListing(extra = []) { return [...UNIX_ARCHIVE_NAMES, ...extra].join('\n') + '\n'; @@ -33,23 +34,37 @@ test('Unix archive validation rejects traversal and unexpected members', () => { ); }); -test('Unix extraction requests only the validated root executable', () => { - const calls = []; - const runner = (command, args) => { - calls.push({ command, args: [...args] }); - return calls.length === 1 ? exactUnixListing() : Buffer.alloc(0); - }; - - extractExactTarArchive( - '/tmp/release.tar.gz', '/tmp/extract', UNIX_ARCHIVE_NAMES, - 'codebase-memory-cli', runner, - ); - - assert.deepEqual(calls[0].args, ['-tzf', '/tmp/release.tar.gz']); - assert.deepEqual( - calls[1].args, - ['-xzf', '/tmp/release.tar.gz', '-C', '/tmp/extract', 'codebase-memory-cli'], - ); +test('Unix extraction lists through the system tar and writes only the root executable', { skip: TAR_PATH_SKIP }, async () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'cbm-npm-extract-test-')); + try { + const archive = writeTarGz( + root, 'release.tar.gz', + UNIX_ARCHIVE_NAMES.map((name) => ({ name, data: `member:${name}` })), + ); + const destination = path.join(root, 'extract'); + fs.mkdirSync(destination); + const calls = []; + const runner = (command, args) => { + calls.push({ command, args: [...args] }); + return exactUnixListing(); + }; + + await extractExactTarArchive( + archive, destination, UNIX_ARCHIVE_NAMES, 'codebase-memory-cli', runner, + ); + + // The listing runner sees the namespace query only; the executable is + // written by the counted `tar -xzOf` writer, never by `tar -x` into the + // tree, so the companions never touch the disk. + assert.deepEqual(calls.map((call) => call.args), [['-tzf', archive]]); + assert.equal( + fs.readFileSync(path.join(destination, 'codebase-memory-cli'), 'utf8'), + 'member:codebase-memory-cli', + ); + assert.deepEqual(fs.readdirSync(destination), ['codebase-memory-cli']); + } finally { + fs.rmSync(root, { recursive: true, force: true }); + } }); function writeBinary(directory, tag) { diff --git a/pkg/npm/test/windows-launcher.test.js b/pkg/npm/test/windows-launcher.test.js index d003b92cbe..9611739e65 100644 --- a/pkg/npm/test/windows-launcher.test.js +++ b/pkg/npm/test/windows-launcher.test.js @@ -2,6 +2,7 @@ const assert = require('node:assert/strict'); const fs = require('node:fs'); +const { spawnSync } = require('node:child_process'); const path = require('node:path'); const test = require('node:test'); const vm = require('node:vm'); @@ -116,3 +117,53 @@ test('PowerShell install mutation runs through the downloaded binary', () => { // update path, so losing that step would silently break every update. assert.match(installer, /Move-Item[\s\S]{0,80}\$Dest[\s\S]{0,40}\$retired/); }); + + +test('PowerShell installer restores an absent target and preserves a published candidate', (t) => { + const command = process.platform === 'win32' ? 'powershell.exe' : 'pwsh'; + const probe = spawnSync(command, ['-NoProfile', '-Command', '$PSVersionTable.PSVersion.ToString()']); + if (probe.error && probe.error.code === 'ENOENT') { + t.skip('PowerShell is not installed'); + return; + } + const installerPath = path.join(__dirname, '..', '..', '..', 'install.ps1'); + const script = ` + $ErrorActionPreference = 'Stop' + $tokens = $null; $errors = $null + $ast = [System.Management.Automation.Language.Parser]::ParseFile($env:CBM_INSTALLER_TEST_PATH, [ref]$tokens, [ref]$errors) + if ($errors.Count) { throw 'installer parse failed' } + $guard = @($ast.FindAll({ param($node) + $node -is [System.Management.Automation.Language.TryStatementAst] -and + $node.Finally -and $node.Finally.Extent.Text.Contains('-not $ActivationSucceeded') + }, $true)) + if ($guard.Count -ne 1) { throw 'activation rollback guard missing' } + $rollback = [ScriptBlock]::Create(($guard[0].Finally.Statements | ForEach-Object { $_.Extent.Text }) -join [Environment]::NewLine) + $root = Join-Path ([IO.Path]::GetTempPath()) ('cbm-rollback-' + [Guid]::NewGuid().ToString('N')) + [IO.Directory]::CreateDirectory($root) | Out-Null + try { + $Dest = Join-Path $root 'codebase-memory-cli.exe' + $retired = $Dest + '.retired-test' + foreach ($candidatePresent in @($false, $true)) { + [IO.File]::WriteAllText($retired, 'previous image') + if ($candidatePresent) { [IO.File]::WriteAllText($Dest, 'failed candidate') } + $ActivationSucceeded = $false + . $rollback + if ($candidatePresent) { + if ([IO.File]::ReadAllText($Dest) -ne 'failed candidate' -or [IO.File]::ReadAllText($retired) -ne 'previous image') { throw 'partial activation image overwritten' } + Remove-Item -LiteralPath $retired + } elseif ([IO.File]::ReadAllText($Dest) -ne 'previous image' -or (Test-Path $retired)) { throw 'previous image not restored' } + Remove-Item -LiteralPath $Dest + } + [IO.File]::WriteAllText($retired, 'previous image') + [IO.File]::WriteAllText($Dest, 'accepted candidate') + $ActivationSucceeded = $true + . $rollback + if ([IO.File]::ReadAllText($Dest) -ne 'accepted candidate' -or -not (Test-Path $retired)) { throw 'successful activation rolled back' } + } finally { Remove-Item -LiteralPath $root -Recurse -Force } + `; + const result = spawnSync(command, ['-NoProfile', '-Command', script], { + env: { ...process.env, CBM_INSTALLER_TEST_PATH: installerPath }, + encoding: 'utf8', + }); + assert.equal(result.status, 0, result.stderr || result.stdout); +}); diff --git a/pkg/pypi/src/codebase_memory_cli/_cli.py b/pkg/pypi/src/codebase_memory_cli/_cli.py index db0c3f9e8d..a6cf4aaa52 100644 --- a/pkg/pypi/src/codebase_memory_cli/_cli.py +++ b/pkg/pypi/src/codebase_memory_cli/_cli.py @@ -1,5 +1,6 @@ """Downloads codebase-memory-cli on first run, then runs its native entry point.""" +import collections import errno import hashlib import json @@ -42,6 +43,22 @@ _NETWORK_TIMEOUT_SECONDS = 120 _CANDIDATE_TIMEOUT_SECONDS = 15 _MAX_CHECKSUM_MANIFEST_BYTES = 1024 * 1024 +# Release archive resource limits. Keep in sync with the Go wrapper +# (pkg/go/cmd/codebase-memory-mcp/main.go) and the npm wrapper (install.js). +_MAX_RELEASE_ARCHIVE_BYTES = 256 * 1024 * 1024 +_MAX_ARCHIVE_MEMBERS = 64 +_MAX_ARCHIVE_MEMBER_BYTES = 256 * 1024 * 1024 +_MAX_ARCHIVE_EXPANDED_BYTES = 512 * 1024 * 1024 +_ARCHIVE_COPY_CHUNK_BYTES = 64 * 1024 +_ArchiveLimits = collections.namedtuple( + "_ArchiveLimits", "compressed_bytes members member_bytes expanded_bytes" +) +_DEFAULT_ARCHIVE_LIMITS = _ArchiveLimits( + compressed_bytes=_MAX_RELEASE_ARCHIVE_BYTES, + members=_MAX_ARCHIVE_MEMBERS, + member_bytes=_MAX_ARCHIVE_MEMBER_BYTES, + expanded_bytes=_MAX_ARCHIVE_EXPANDED_BYTES, +) _REDIRECT_CODES = frozenset({301, 302, 303, 307, 308}) _RUNTIME_LOCK_NAME = ".codebase-memory-cli-runtime.lock" _RUNTIME_LOCK_WAIT_SECONDS = 45 @@ -113,27 +130,50 @@ def _download_https(url: str, dest: str, max_bytes: int = 0) -> None: with response: _validate_url_scheme(response.geturl()) + declared = (response.headers.get("Content-Length") or "").strip() + # Only plain ASCII digits are a length (str.isdigit also accepts + # characters that int() rejects); anything else is left to the + # byte counter below. + if max_bytes and re.fullmatch(r"[0-9]+", declared): + if int(declared) > max_bytes: + raise RuntimeError( + f"download exceeds the {max_bytes}-byte safety limit" + ) deadline = time.monotonic() + _NETWORK_TIMEOUT_SECONDS total = 0 - with open(dest, "wb") as out: - while True: - if time.monotonic() >= deadline: - raise TimeoutError( - f"download hop timed out: {current_url}" - ) - chunk = response.read(65536) - if not chunk: - return - total += len(chunk) - if max_bytes and total > max_bytes: - raise RuntimeError( - f"download exceeds the {max_bytes}-byte safety limit" - ) - out.write(chunk) + try: + with open(dest, "wb") as out: + while True: + if time.monotonic() >= deadline: + raise TimeoutError( + f"download hop timed out: {current_url}" + ) + chunk = response.read(65536) + if not chunk: + return + total += len(chunk) + if max_bytes and total > max_bytes: + raise RuntimeError( + f"download exceeds the {max_bytes}-byte " + f"safety limit" + ) + out.write(chunk) + except BaseException: + # Never leave a partial download behind for a later step to + # mistake for the complete asset. + _remove_quietly(dest) + raise raise RuntimeError("too many redirects") +def _remove_quietly(path) -> None: + try: + os.unlink(path) + except OSError: + pass + + def _verify_candidate(path: Path) -> None: """Require a staged native candidate to execute successfully.""" try: @@ -196,33 +236,157 @@ def _validate_archive_names(names, archive_names, casefold: bool = False): ) +def _validate_archive_limits(limits) -> None: + if ( + limits.compressed_bytes <= 0 + or limits.members <= 0 + or limits.member_bytes <= 0 + or limits.expanded_bytes <= 0 + ): + sys.exit("codebase-memory-cli: invalid archive resource safety limits") + + +def _require_compressed_archive_within_limit(archive_path: str, limits) -> None: + status = os.stat(archive_path) + if not stat.S_ISREG(status.st_mode): + sys.exit("codebase-memory-cli: release archive is not a regular file") + if status.st_size > limits.compressed_bytes: + sys.exit( + f"codebase-memory-cli: release archive exceeds the " + f"{limits.compressed_bytes}-byte compressed safety limit" + ) + + +def _require_member_count_within_limit(count: int, limits) -> None: + if count > limits.members: + sys.exit( + f"codebase-memory-cli: archive exceeds the {limits.members}-member " + f"safety limit" + ) + + +def _account_declared_archive_member( + name: str, size: int, declared_expanded: int, limits +) -> int: + """Check one member's declared size; return the new declared total.""" + if size < 0: + sys.exit( + f"codebase-memory-cli: archive member has a negative declared " + f"size: {name!r}" + ) + if size > limits.member_bytes: + sys.exit( + f"codebase-memory-cli: archive member {name!r} exceeds the " + f"{limits.member_bytes}-byte expanded safety limit" + ) + if declared_expanded > limits.expanded_bytes - size: + sys.exit( + f"codebase-memory-cli: archive exceeds the {limits.expanded_bytes}-byte " + f"aggregate expanded safety limit" + ) + return declared_expanded + size + + +def _copy_archive_member_within_limits( + source, output, name: str, declared_size: int, actual_expanded: int, limits +) -> int: + """Copy a member while counting bytes; return the new actual total. + + Reads stop one byte past the smaller of the member and remaining aggregate + limits, so a member that yields more than it declares can never fill the + disk. The caller removes the target on failure. + """ + remaining = limits.expanded_bytes - actual_expanded + copy_limit = min(limits.member_bytes, remaining) + copied = 0 + while True: + wanted = min(_ARCHIVE_COPY_CHUNK_BYTES, copy_limit + 1 - copied) + if wanted <= 0: + break + chunk = source.read(wanted) + if not chunk: + break + copied += len(chunk) + if copied > copy_limit: + break + output.write(chunk) + if copied > limits.member_bytes: + sys.exit( + f"codebase-memory-cli: archive member {name!r} exceeds the " + f"{limits.member_bytes}-byte actual expanded safety limit" + ) + if copied > remaining: + sys.exit( + f"codebase-memory-cli: archive exceeds the {limits.expanded_bytes}-byte " + f"aggregate actual expanded safety limit" + ) + if copied != declared_size: + sys.exit( + f"codebase-memory-cli: archive member {name!r} actual size {copied} " + f"does not match declared size {declared_size}" + ) + return actual_expanded + copied + + +def _write_archive_member_within_limits( + source, target: str, name: str, declared_size: int, actual_expanded: int, limits +) -> int: + try: + with open(target, "xb") as output: + return _copy_archive_member_within_limits( + source, output, name, declared_size, actual_expanded, limits + ) + except BaseException: + _remove_quietly(target) + raise + + def _safe_extract_tar( - tf, dest: str, archive_names=(), extract_names=(), ui: bool = False + tf, + dest: str, + archive_names=(), + extract_names=(), + ui: bool = False, + limits=_DEFAULT_ARCHIVE_LIMITS, ): """Validate a tar namespace, then extract only its authenticated runtime set.""" + _validate_archive_limits(limits) members = tf.getmembers() + _require_member_count_within_limit(len(members), limits) + declared_expanded = 0 for member in members: if not member.isfile(): sys.exit( f"codebase-memory-cli: refusing unsafe tar entry " f"(not a regular root file: {member.name!r})" ) + declared_expanded = _account_declared_archive_member( + member.name, member.size, declared_expanded, limits + ) _validate_archive_names([member.name for member in members], archive_names) runtime_names = list(extract_names) by_name = {member.name: member for member in members} dest_abs = os.path.abspath(dest) + actual_expanded = 0 for name in runtime_names: source = tf.extractfile(by_name[name]) if source is None: sys.exit(f"codebase-memory-cli: could not read archive member: {name}") target = os.path.join(dest_abs, name) - with source, open(target, "xb") as output: - shutil.copyfileobj(source, output) + with source: + actual_expanded = _write_archive_member_within_limits( + source, target, name, by_name[name].size, actual_expanded, limits + ) return tuple(runtime_names) def _safe_extract_zip( - zf, dest: str, archive_names=(), extract_names=(), ui: bool = False + zf, + dest: str, + archive_names=(), + extract_names=(), + ui: bool = False, + limits=_DEFAULT_ARCHIVE_LIMITS, ): """Extract a zipfile after validating its Windows-style namespace. @@ -230,9 +394,16 @@ def _safe_extract_zip( members before extraction prevents archive order from selecting the binary that a portable package-manager shim eventually executes. """ + _validate_archive_limits(limits) dest_abs = os.path.abspath(dest) names = [] - for info in zf.infolist(): + infos = zf.infolist() + _require_member_count_within_limit(len(infos), limits) + declared_expanded = 0 + for info in infos: + declared_expanded = _account_declared_archive_member( + info.filename, info.file_size, declared_expanded, limits + ) raw_name = info.filename name = raw_name.replace("\\", "/") segments_name = name[:-1] if name.endswith("/") else name @@ -266,10 +437,18 @@ def _safe_extract_zip( # Extract only the validated runtime files. This avoids relying on # platform-specific zip path rewriting and always creates regular files. runtime_names = list(extract_names) + actual_expanded = 0 for name in runtime_names: target = os.path.join(dest_abs, name) - with zf.open(name) as source, open(target, "xb") as output: - shutil.copyfileobj(source, output) + with zf.open(name) as source: + actual_expanded = _write_archive_member_within_limits( + source, + target, + name, + zf.getinfo(name).file_size, + actual_expanded, + limits, + ) return tuple(runtime_names) @@ -1179,7 +1358,7 @@ def _download(version: str) -> Path: with tempfile.TemporaryDirectory() as tmp: tmp_archive = os.path.join(tmp, f"cbm.{ext}") try: - _download_https(url, tmp_archive) + _download_https(url, tmp_archive, _MAX_RELEASE_ARCHIVE_BYTES) except (OSError, RuntimeError, urllib.error.URLError) as e: sys.exit( f"codebase-memory-cli: download failed ({e})\n" @@ -1188,6 +1367,9 @@ def _download(version: str) -> Path: ) _verify_checksum(tmp_archive, archive, version) + _require_compressed_archive_within_limit( + tmp_archive, _DEFAULT_ARCHIVE_LIMITS + ) bin_name = ( _WINDOWS_BINARY_NAME if os_name == "windows" else "codebase-memory-cli" diff --git a/pkg/pypi/tests/test_cli.py b/pkg/pypi/tests/test_cli.py index ca52bf1a79..8173f973a5 100644 --- a/pkg/pypi/tests/test_cli.py +++ b/pkg/pypi/tests/test_cli.py @@ -1,5 +1,6 @@ import errno import hashlib +import io import json import os import stat @@ -835,3 +836,324 @@ def test_windows_zip_rejects_symlink_metadata(self): False, ) + +class _FakeHttpsResponse: + """Minimal stand-in for the opener's response: headers, geturl, read.""" + + def __init__(self, body: bytes, headers=None): + self._body = body + self._offset = 0 + self.headers = dict(headers or {}) + + def __enter__(self): + return self + + def __exit__(self, *exc_info): + return False + + def geturl(self): + return "https://example.invalid/release.tar.gz" + + def read(self, size=-1): + if size is None or size < 0: + size = len(self._body) - self._offset + chunk = self._body[self._offset : self._offset + size] + self._offset += len(chunk) + return chunk + + +def _limits_with(**overrides): + return _cli._DEFAULT_ARCHIVE_LIMITS._replace(**overrides) + + +def _write_tar_gz(archive, entries): + with tarfile.open(archive, "w:gz") as tf: + for name, data in entries: + info = tarfile.TarInfo(name) + info.size = len(data) + tf.addfile(info, io.BytesIO(data)) + + +def _write_zip(archive, entries): + with zipfile.ZipFile(archive, "w") as zf: + for name, data in entries: + zf.writestr(name, data) + + +def _release_entries(names): + return [(name, f"payload of {name}".encode()) for name in names] + + +class ArchiveLimitTests(unittest.TestCase): + def test_archive_limit_errors_use_the_cli_prefix(self): + with tempfile.TemporaryDirectory() as root: + archive = Path(root) / "release.tar.gz" + archive.write_bytes(b"xx") + errors = ( + lambda: _cli._validate_archive_limits(_limits_with(members=0)), + lambda: _cli._require_compressed_archive_within_limit( + str(archive), _limits_with(compressed_bytes=1) + ), + lambda: _cli._require_member_count_within_limit( + 2, _limits_with(members=1) + ), + lambda: _cli._account_declared_archive_member( + "member", -1, 0, _limits_with() + ), + lambda: _cli._copy_archive_member_within_limits( + io.BytesIO(b"xx"), io.BytesIO(), "member", 2, 0, + _limits_with(member_bytes=1), + ), + ) + for operation in errors: + with self.subTest(operation=operation): + with self.assertRaises(SystemExit) as raised: + operation() + self.assertTrue( + str(raised.exception).startswith("codebase-memory-cli:") + ) + + URL = "https://example.invalid/release.tar.gz" + + def _download(self, response, destination, max_bytes): + with mock.patch.object(_cli._HTTPS_OPENER, "open", return_value=response): + _cli._download_https(self.URL, destination, max_bytes) + + def test_download_rejects_a_declared_length_over_the_limit(self): + with tempfile.TemporaryDirectory() as root: + destination = os.path.join(root, "release.tar.gz") + response = _FakeHttpsResponse(b"", {"Content-Length": "9"}) + with self.assertRaises(RuntimeError) as raised: + self._download(response, destination, 8) + self.assertIn("8-byte safety limit", str(raised.exception)) + self.assertFalse(os.path.exists(destination)) + + def test_download_rejects_a_body_over_the_limit_and_removes_the_partial_file( + self, + ): + with tempfile.TemporaryDirectory() as root: + destination = os.path.join(root, "release.tar.gz") + response = _FakeHttpsResponse(b"x" * 1024) + with self.assertRaises(RuntimeError) as raised: + self._download(response, destination, 8) + self.assertIn("8-byte safety limit", str(raised.exception)) + self.assertFalse(os.path.exists(destination)) + + def test_download_within_the_limit_stores_the_body(self): + with tempfile.TemporaryDirectory() as root: + destination = os.path.join(root, "release.tar.gz") + response = _FakeHttpsResponse(b"payload", {"Content-Length": "7"}) + self._download(response, destination, 1024) + with open(destination, "rb") as stored: + self.assertEqual(stored.read(), b"payload") + + def test_download_counts_bytes_when_the_declared_length_is_not_a_number(self): + # A superscript two passes str.isdigit but not int(); such values and + # plain junk must fall through to the byte counter, never crash. + for declared in ("²", "abc", "", "1e3", "-1"): + with self.subTest(declared=declared), tempfile.TemporaryDirectory() as root: + destination = os.path.join(root, "release.tar.gz") + response = _FakeHttpsResponse(b"x" * 1024, {"Content-Length": declared}) + with self.assertRaises(RuntimeError) as raised: + self._download(response, destination, 8) + self.assertIn("8-byte safety limit", str(raised.exception)) + self.assertFalse(os.path.exists(destination)) + with tempfile.TemporaryDirectory() as root: + destination = os.path.join(root, "release.tar.gz") + response = _FakeHttpsResponse(b"payload", {"Content-Length": "²"}) + self._download(response, destination, 1024) + with open(destination, "rb") as stored: + self.assertEqual(stored.read(), b"payload") + + def test_unix_tar_rejects_more_members_than_the_limit(self): + with tempfile.TemporaryDirectory() as root: + archive = Path(root) / "members.tar.gz" + _write_tar_gz(archive, [("one", b"a"), ("two", b"b"), ("three", b"c")]) + destination = Path(root) / "extract" + destination.mkdir() + with tarfile.open(archive) as tf, self.assertRaises(SystemExit) as raised: + _cli._safe_extract_tar( + tf, + str(destination), + ("one", "two", "three"), + ("one",), + limits=_limits_with(members=2), + ) + self.assertIn("2-member safety limit", str(raised.exception)) + self.assertFalse((destination / "one").exists()) + + def test_unix_tar_rejects_a_member_declared_over_the_limit(self): + with tempfile.TemporaryDirectory() as root: + archive = Path(root) / "member.tar.gz" + _write_tar_gz(archive, [("codebase-memory-cli", b"x" * 65)]) + destination = Path(root) / "extract" + destination.mkdir() + with tarfile.open(archive) as tf, self.assertRaises(SystemExit) as raised: + _cli._safe_extract_tar( + tf, + str(destination), + ("codebase-memory-cli",), + ("codebase-memory-cli",), + limits=_limits_with(member_bytes=64), + ) + self.assertIn( + "'codebase-memory-cli' exceeds the 64-byte expanded safety limit", + str(raised.exception), + ) + self.assertFalse((destination / "codebase-memory-cli").exists()) + + def test_unix_tar_rejects_a_declared_aggregate_over_the_limit(self): + with tempfile.TemporaryDirectory() as root: + archive = Path(root) / "aggregate.tar.gz" + _write_tar_gz(archive, [("one", b"a" * 40), ("two", b"b" * 40)]) + destination = Path(root) / "extract" + destination.mkdir() + with tarfile.open(archive) as tf, self.assertRaises(SystemExit) as raised: + _cli._safe_extract_tar( + tf, + str(destination), + ("one", "two"), + ("one",), + limits=_limits_with(member_bytes=64, expanded_bytes=64), + ) + self.assertIn( + "64-byte aggregate expanded safety limit", str(raised.exception) + ) + + def test_unix_tar_within_the_limits_extracts_only_the_executable(self): + with tempfile.TemporaryDirectory() as root: + archive = Path(root) / "release.tar.gz" + _write_tar_gz(archive, _release_entries(_cli._UNIX_ARCHIVE_NAMES)) + destination = Path(root) / "extract" + destination.mkdir() + with tarfile.open(archive) as tf: + extracted = _cli._safe_extract_tar( + tf, + str(destination), + _cli._UNIX_ARCHIVE_NAMES, + ("codebase-memory-cli",), + ) + self.assertEqual(extracted, ("codebase-memory-cli",)) + self.assertEqual( + (destination / "codebase-memory-cli").read_bytes(), + b"payload of codebase-memory-cli", + ) + self.assertEqual(os.listdir(destination), ["codebase-memory-cli"]) + + def test_archive_copy_stops_a_member_that_yields_more_than_the_limit(self): + source = io.BytesIO(b"x" * 100) + output = io.BytesIO() + with self.assertRaises(SystemExit) as raised: + _cli._copy_archive_member_within_limits( + source, + output, + "member", + 100, + 0, + _limits_with(member_bytes=64, expanded_bytes=1024), + ) + self.assertIn( + "'member' exceeds the 64-byte actual expanded safety limit", + str(raised.exception), + ) + # Reading stopped one byte past the limit; nothing beyond it was kept. + self.assertEqual(source.tell(), 65) + self.assertLessEqual(len(output.getvalue()), 64) + + def test_archive_copy_rejects_a_member_that_yields_more_than_it_declares(self): + source = io.BytesIO(b"x" * 100) + with self.assertRaises(SystemExit) as raised: + _cli._copy_archive_member_within_limits( + source, + io.BytesIO(), + "member", + 10, + 0, + _limits_with(member_bytes=1024, expanded_bytes=1024), + ) + self.assertIn( + "'member' actual size 100 does not match declared size 10", + str(raised.exception), + ) + + def test_archive_copy_rejects_an_aggregate_over_the_actual_limit(self): + with self.assertRaises(SystemExit) as raised: + _cli._copy_archive_member_within_limits( + io.BytesIO(b"abcd"), + io.BytesIO(), + "member", + 4, + 3, + _limits_with(member_bytes=10, expanded_bytes=5), + ) + self.assertIn( + "5-byte aggregate actual expanded safety limit", str(raised.exception) + ) + + def test_archive_member_write_removes_the_partial_file_on_failure(self): + with tempfile.TemporaryDirectory() as root: + target = os.path.join(root, "codebase-memory-cli") + with self.assertRaises(SystemExit): + _cli._write_archive_member_within_limits( + io.BytesIO(b"x" * 100), + target, + "codebase-memory-cli", + 10, + 0, + _limits_with(member_bytes=1024, expanded_bytes=1024), + ) + self.assertFalse(os.path.exists(target)) + + def test_windows_zip_rejects_more_members_than_the_limit(self): + with tempfile.TemporaryDirectory() as root: + archive = Path(root) / "members.zip" + _write_zip(archive, [("one", b"a"), ("two", b"b"), ("three", b"c")]) + destination = Path(root) / "extract" + destination.mkdir() + with zipfile.ZipFile(archive) as zf, self.assertRaises(SystemExit) as raised: + _cli._safe_extract_zip( + zf, + str(destination), + ("one", "two", "three"), + ("one",), + limits=_limits_with(members=2), + ) + self.assertIn("2-member safety limit", str(raised.exception)) + + def test_windows_zip_rejects_a_member_declared_over_the_limit(self): + with tempfile.TemporaryDirectory() as root: + archive = Path(root) / "member.zip" + _write_zip(archive, [(_cli._WINDOWS_BINARY_NAME, b"x" * 65)]) + destination = Path(root) / "extract" + destination.mkdir() + with zipfile.ZipFile(archive) as zf, self.assertRaises(SystemExit) as raised: + _cli._safe_extract_zip( + zf, + str(destination), + (_cli._WINDOWS_BINARY_NAME,), + (_cli._WINDOWS_BINARY_NAME,), + limits=_limits_with(member_bytes=64), + ) + self.assertIn("64-byte expanded safety limit", str(raised.exception)) + self.assertFalse((destination / _cli._WINDOWS_BINARY_NAME).exists()) + + def test_windows_zip_within_the_limits_extracts_only_the_executable(self): + with tempfile.TemporaryDirectory() as root: + archive = Path(root) / "release.zip" + _write_zip(archive, _release_entries(_cli._WINDOWS_ARCHIVE_NAMES)) + destination = Path(root) / "extract" + destination.mkdir() + with zipfile.ZipFile(archive) as zf: + extracted = _cli._safe_extract_zip( + zf, + str(destination), + _cli._WINDOWS_ARCHIVE_NAMES, + (_cli._WINDOWS_BINARY_NAME,), + ) + self.assertEqual(extracted, (_cli._WINDOWS_BINARY_NAME,)) + self.assertEqual( + (destination / _cli._WINDOWS_BINARY_NAME).read_bytes(), + b"payload of codebase-memory-cli.exe", + ) + self.assertEqual(os.listdir(destination), [_cli._WINDOWS_BINARY_NAME]) + diff --git a/scripts/memory-core-baseline.txt b/scripts/memory-core-baseline.txt index 96c594e1fb..94c61d994e 100644 --- a/scripts/memory-core-baseline.txt +++ b/scripts/memory-core-baseline.txt @@ -61,7 +61,7 @@ src/operations/execute.c 2 src/operations/file_outline.c 1 src/operations/index.c 56 src/operations/index_supervisor.c 8 -src/operations/project_arg.c 3 +src/operations/project_arg.c 2 src/operations/query.c 4 src/operations/read.c 6 src/operations/reliability_events.c 4 diff --git a/scripts/setup-windows.ps1 b/scripts/setup-windows.ps1 index 57ec839036..94e34afe97 100644 --- a/scripts/setup-windows.ps1 +++ b/scripts/setup-windows.ps1 @@ -1,5 +1,5 @@ # codebase-memory-cli setup script (Windows) -# Default: download pre-built native Windows binary +# Default: install the pre-built native Windows binary through install.ps1 # -FromSource: build a native Windows executable with MSYS2/Clang param( @@ -42,7 +42,7 @@ if ($Help) { Write-Host "" Write-Host "Usage: .\setup-windows.ps1 [-FromSource] [-Binary PATH] [-InstallDir PATH] [-NoPathPrompt] [-Help]" Write-Host "" - Write-Host " Default: Download pre-built Windows binary" + Write-Host " Default: Install the pre-built Windows binary through install.ps1" Write-Host " -FromSource: Build a native Windows .exe with MSYS2/Clang (no WSL)" Write-Host " -Binary PATH: Install exact existing candidate bytes (qualification/offline mode)" Write-Host " -InstallDir: Override the installation directory" @@ -130,41 +130,105 @@ if ($FromSource) { Write-Host " Remove-Item -Recurse -Force '$sourceDir'" } else { - # --- Download pre-built native Windows binary --- - Write-Host "Fetching latest release..." -ForegroundColor White - - $releaseUrl = "https://api.github.com/repos/$Repo/releases/latest" - $release = Invoke-RestMethod -Uri $releaseUrl -Headers @{ "User-Agent" = "codebase-memory-cli-setup" } - $tag = $release.tag_name + # --- Download + install through install.ps1 --- + # + # install.ps1 is the one implementation of "fetch a release and install + # it": it downloads checksums.txt next to the archive, verifies the + # archive's SHA-256 against it, validates the zip layout and only then + # runs the binary's own `install`. This script used to carry a second + # copy of that download, which did not keep up with the installer. It + # now fetches install.ps1 from the same origin and branch it is itself + # served from and hands over to it, so there is exactly one install path. + # + # CBM_DOWNLOAD_URL (the installers' download-base override, for local + # testing) also moves the installer fetch: install.ps1 is then taken from + # "$env:CBM_DOWNLOAD_URL/install.ps1". + $installerUrl = "https://raw.githubusercontent.com/$Repo/main/install.ps1" + if ($env:CBM_DOWNLOAD_URL) { + $installerUrl = $env:CBM_DOWNLOAD_URL.TrimEnd('/') + "/install.ps1" + } - if (-not $tag) { - Write-Fail "Could not determine latest release." - Write-Host " Check: https://github.com/$Repo/releases" + # Same transport rule as install.ps1: HTTPS, or plain HTTP for a loopback + # authority only (the local test fixture). No redirects are followed: both + # origins serve the installer directly. + try { $installerUri = [Uri]$installerUrl } catch { $installerUri = $null } + $loopbackHttp = ( + $installerUri -and $installerUri.IsAbsoluteUri -and + $installerUri.Scheme -eq "http" -and $installerUri.IsLoopback -and + [string]::IsNullOrEmpty($installerUri.UserInfo) + ) + if (-not $installerUri -or -not $installerUri.IsAbsoluteUri -or + ($installerUri.Scheme -ne "https" -and -not $loopbackHttp) -or + -not [string]::IsNullOrEmpty($installerUri.UserInfo)) { + Write-Fail "Refusing non-HTTPS installer URL: $installerUrl" exit 1 } - Write-Ok "Latest release: $tag" - $asset = "codebase-memory-cli-windows-amd64.zip" - $downloadUrl = "https://github.com/$Repo/releases/download/$tag/$asset" - - Write-Host "Downloading $asset..." -ForegroundColor White + # TLS 1.2+ for the fetch (older Windows PowerShell defaults to TLS 1.0, + # which GitHub rejects). TLS 1.3 only where schannel can negotiate it; see + # the note in install.ps1. + $protocols = [Net.SecurityProtocolType]::Tls12 + if ([Environment]::OSVersion.Version.Build -ge 20348 -and + ([enum]::GetNames([Net.SecurityProtocolType]) -contains 'Tls13')) { + $protocols = $protocols -bor [Net.SecurityProtocolType]::Tls13 + } + [Net.ServicePointManager]::SecurityProtocol = $protocols - # Create install directory - if (-not (Test-Path $InstallDir)) { - New-Item -ItemType Directory -Path $InstallDir -Force | Out-Null + # A fresh staging directory of its own with an owner-only DACL, the + # way install.ps1 stages its own download. The ACL is best effort: a + # filesystem that cannot carry one must not fail the install. + $stageDir = Join-Path ([System.IO.Path]::GetTempPath()) "cbm-setup-$(Get-Random)" + New-Item -ItemType Directory -Path $stageDir -Force | Out-Null + try { + $stageAcl = New-Object System.Security.AccessControl.DirectorySecurity + $stageAcl.SetAccessRuleProtection($true, $false) + $stageOwner = ([System.Security.Principal.WindowsIdentity]::GetCurrent()).User + $stageAcl.SetOwner($stageOwner) + $stageAcl.AddAccessRule((New-Object System.Security.AccessControl.FileSystemAccessRule( + $stageOwner, 'FullControl', 'ContainerInherit,ObjectInherit', 'None', 'Allow'))) + Set-Acl -Path $stageDir -AclObject $stageAcl -ErrorAction Stop + } catch { + Write-Host " note: could not harden the staging directory ACL: $($_.Exception.Message)" } - $tmpZip = Join-Path $env:TEMP $asset - Invoke-WebRequest -Uri $downloadUrl -OutFile $tmpZip -UseBasicParsing + $installer = Join-Path $stageDir "install.ps1" + $failure = $null + try { + Write-Host "Fetching install.ps1..." -ForegroundColor White + Invoke-WebRequest -Uri $installerUrl -OutFile $installer -UseBasicParsing -MaximumRedirection 0 + if (-not (Test-Path -LiteralPath $installer -PathType Leaf) -or + (Get-Item -LiteralPath $installer).Length -eq 0) { + throw "fetched an empty install.ps1 from $installerUrl" + } + Write-Ok "install.ps1 fetched" - # Extract - Expand-Archive -Path $tmpZip -DestinationPath $InstallDir -Force - Remove-Item $tmpZip -Force + Write-Host "" + Write-Host "Installing through install.ps1..." -ForegroundColor White + # The installer configures nothing (--skip-config): agent configuration + # stays this script's interactive step below. It runs in a child host + # of the same PowerShell so its exit code is unambiguous and its global + # settings do not leak into this session. The pre-seeded exit code + # keeps a child that fails to start from reading as success. + $hostExe = (Get-Process -Id $PID).Path + $global:LASTEXITCODE = 1 + & $hostExe -NoProfile -ExecutionPolicy Bypass -File $installer "--dir=$InstallDir" --skip-config + if ($LASTEXITCODE -ne 0) { + throw "install.ps1 exited with $LASTEXITCODE" + } + } catch { + $failure = "$_" + } finally { + Remove-Item -Recurse -Force $stageDir -ErrorAction SilentlyContinue + } + if ($failure) { + Write-Fail "Installation through install.ps1 failed: $failure" + exit 1 + } $binaryPath = Join-Path $InstallDir "$BinaryName.exe" if (-not (Test-Path $binaryPath)) { - Write-Fail "Binary not found at $binaryPath after extraction" + Write-Fail "Binary not found at $binaryPath after installation" exit 1 } Write-Ok "Installed to $binaryPath" diff --git a/scripts/setup.sh b/scripts/setup.sh index 16d366e850..cc15ce16c7 100755 --- a/scripts/setup.sh +++ b/scripts/setup.sh @@ -43,7 +43,7 @@ for arg in "$@"; do --help|-h) echo "Usage: $0 [--from-source]" echo "" - echo " Default: Download pre-built binary from GitHub Release" + echo " Default: Download the pre-built binary through install.sh" echo " --from-source: Clone and build from source (requires Go 1.23+ and a C compiler)" exit 0 ;; @@ -51,36 +51,6 @@ for arg in "$@"; do esac done -# --- Platform detection --- - -detect_platform() { - local os arch - os=$(uname -s) - arch=$(uname -m) - - case "$os" in - Darwin) os="darwin" ;; - Linux) os="linux" ;; - *) die "Unsupported OS: $os. Use WSL2 on Windows." ;; - esac - - case "$arch" in - arm64|aarch64) arch="arm64" ;; - x86_64|amd64) - # On macOS, uname -m returns x86_64 under Rosetta even on Apple Silicon. - # Check the actual hardware to pick the right binary. - if [ "$os" = "darwin" ] && sysctl -n hw.optional.arm64 2>/dev/null | grep -q '1'; then - arch="arm64" - else - arch="amd64" - fi - ;; - *) die "Unsupported architecture: $arch" ;; - esac - - echo "${os}-${arch}" -} - # --- Prerequisite checks --- check_download_tool() { @@ -133,46 +103,74 @@ check_git() { ok "Git found" } -# --- Download binary --- +# --- Download + install through install.sh --- +# +# install.sh is the one implementation of "fetch a release and install it": it +# downloads checksums.txt next to the archive, verifies the archive's SHA-256 +# against it, checks the archive layout, and only then runs the binary's own +# `install`. This script used to carry a second copy of that download, +# which did not keep up with the installer. It now fetches install.sh from the +# same origin and branch it is itself served from and hands over to it, so +# there is exactly one install path. +# +# CBM_DOWNLOAD_URL (the installers' download-base override, for local testing) +# also moves the installer fetch: install.sh is then taken from +# "$CBM_DOWNLOAD_URL/install.sh". + +INSTALLER_URL="https://raw.githubusercontent.com/${REPO}/main/install.sh" +if [ -n "${CBM_DOWNLOAD_URL:-}" ]; then + INSTALLER_URL="${CBM_DOWNLOAD_URL%/}/install.sh" +fi + +# Same transport rule as install.sh: HTTPS everywhere; plain HTTP only for an +# exact loopback authority (the local test fixture), with redirects disabled +# there so a fixture cannot bounce the fetch to the network. +is_loopback_http_url() { + [[ "$1" =~ ^http://(localhost|127\.0\.0\.1|\[::1\])(:[0-9]+)?([/?\#].*)?$ ]] +} -fetch() { - local url="$1" tool="$2" - if [ "$tool" = "curl" ]; then - curl -fsSL "$url" +fetch_installer() { + local url="$1" destination="$2" tool="$3" + if is_loopback_http_url "$url"; then + if [ "$tool" = "curl" ]; then + curl -fsS --noproxy '*' --proto '=http' -o "$destination" "$url" + else + wget -q --no-proxy --max-redirect=0 -O "$destination" "$url" + fi + elif [[ "$url" == https://* ]]; then + if [ "$tool" = "curl" ]; then + curl -fsSL --max-redirs 5 --proto '=https' --proto-redir '=https' \ + -o "$destination" "$url" + else + wget -q --https-only --max-redirect=5 -O "$destination" "$url" + fi else - wget -qO- "$url" + die "Refusing non-HTTPS installer URL: $url" fi } -download_binary() { - local platform="$1" tool="$2" +install_release() { + local tool="$1" echo "" - echo "${BOLD}Fetching latest release...${RESET}" - local tag - tag=$(fetch "https://api.github.com/repos/${REPO}/releases/latest" "$tool" | grep '"tag_name"' | head -1 | sed 's/.*"tag_name": *"//;s/".*//') - - if [ -z "$tag" ]; then - die "Could not determine latest release. Check https://github.com/${REPO}/releases" - fi - ok "Latest release: $tag" - - local asset="codebase-memory-cli-${platform}.tar.gz" - local url="https://github.com/${REPO}/releases/download/${tag}/${asset}" - - echo "${BOLD}Downloading ${asset}...${RESET}" + echo "${BOLD}Fetching install.sh...${RESET}" CLEANUP_DIR=$(mktemp -d) trap 'rm -rf "$CLEANUP_DIR"' EXIT - local tmpdir="$CLEANUP_DIR" + local installer="$CLEANUP_DIR/install.sh" - fetch "$url" "$tool" > "${tmpdir}/${asset}" - tar -xzf "${tmpdir}/${asset}" -C "$tmpdir" + fetch_installer "$INSTALLER_URL" "$installer" "$tool" || + die "Could not fetch install.sh from ${INSTALLER_URL}" + [ -s "$installer" ] || die "Fetched an empty install.sh from ${INSTALLER_URL}" + ok "install.sh fetched" - mkdir -p "$INSTALL_DIR" - mv "${tmpdir}/${BINARY_NAME}" "${INSTALL_DIR}/${BINARY_NAME}" - chmod +x "${INSTALL_DIR}/${BINARY_NAME}" - - ok "Installed to ${INSTALL_DIR}/${BINARY_NAME}" + echo "" + echo "${BOLD}Installing through install.sh...${RESET}" + # The installer configures nothing (--skip-config): agent configuration + # stays this script's interactive step below. stdin is detached so that, + # when this script itself arrives through `curl | bash`, nothing the + # installer runs can read the rest of this script as its input. + bash "$installer" --dir "$INSTALL_DIR" --skip-config 0 && (size_t)written < out_sz ? CLI_OK : CLI_ERR; } +/* #1180: a bare ~/.claude directory is not a Claude Code install -- other + * tools create it -- and treating it as one wrote hooks, skills and MCP + * entries for a client the user never installed. Claude Code itself leaves + * its user config .claude.json (under CLAUDE_CONFIG_DIR when set), its + * settings.json, or the claude CLI. */ +static bool cbm_claude_code_detected(const char *home_dir) { + char dir[CLI_BUF_1K]; + char path[CLI_BUF_1K]; + cbm_claude_config_dir(home_dir, dir, sizeof(dir)); + int written = snprintf(path, sizeof(path), "%s/settings.json", dir); + if (dir[0] && written > 0 && (size_t)written < sizeof(path) && cbm_file_exists(path)) { + return true; + } + cbm_claude_user_root(home_dir, dir, sizeof(dir)); + written = snprintf(path, sizeof(path), "%s/.claude.json", dir); + if (dir[0] && written > 0 && (size_t)written < sizeof(path) && cbm_file_exists(path)) { + return true; + } + return cbm_agent_cli_exists("claude", home_dir); +} + +/* OpenCode keeps its data (auth, logs, sessions; the global config itself on + * older installs) in ~/.local/share/opencode on every OS -- + * %USERPROFILE%\.local\share\opencode on Windows (opencode.ai/docs/ + * troubleshooting, "Storage"). The CLI and the Desktop app's bundled + * opencode-cli server both create it, so it identifies an install that has + * no config file, no ~/.config/opencode and no `opencode` on PATH + * (#1167, #1180). */ +static bool cbm_opencode_detected(const char *home_dir) { + char path[CLI_BUF_1K]; + cbm_opencode_config_path(home_dir, path, sizeof(path)); + if (cbm_file_exists(path) || cbm_agent_cli_exists("opencode", home_dir)) { + return true; + } + snprintf(path, sizeof(path), "%s/.config/opencode", home_dir); + if (dir_exists(path)) { + return true; + } + char env_buf[CLI_BUF_1K]; + const char *config_dir = cbm_safe_getenv("OPENCODE_CONFIG_DIR", env_buf, sizeof(env_buf), NULL); + if (config_dir && config_dir[0] && dir_exists(config_dir)) { + return true; + } + snprintf(path, sizeof(path), "%s/.local/share/opencode", home_dir); + return dir_exists(path); +} + cbm_detected_agents_t cbm_detect_agents(const char *home_dir) { cbm_detected_agents_t agents; memset(&agents, 0, sizeof(agents)); @@ -2701,8 +2795,7 @@ cbm_detected_agents_t cbm_detect_agents(const char *home_dir) { char path[CLI_BUF_1K]; - cbm_claude_config_dir(home_dir, path, sizeof(path)); - agents.claude_code = path[0] != '\0' && dir_exists(path); + agents.claude_code = cbm_claude_code_detected(home_dir); cbm_codex_config_dir(home_dir, path, sizeof(path)); agents.codex = path[0] != '\0' && dir_exists(path); @@ -2716,18 +2809,7 @@ cbm_detected_agents_t cbm_detect_agents(const char *home_dir) { cbm_zed_config_dir(home_dir, path, sizeof(path)); agents.zed = dir_exists(path); - cbm_opencode_config_path(home_dir, path, sizeof(path)); - agents.opencode = cbm_file_exists(path) || cbm_agent_cli_exists("opencode", home_dir); - if (!agents.opencode) { - snprintf(path, sizeof(path), "%s/.config/opencode", home_dir); - agents.opencode = dir_exists(path); - } - if (!agents.opencode) { - char env_buf[CLI_BUF_1K]; - const char *config_dir = - cbm_safe_getenv("OPENCODE_CONFIG_DIR", env_buf, sizeof(env_buf), NULL); - agents.opencode = config_dir && config_dir[0] && dir_exists(config_dir); - } + agents.opencode = cbm_opencode_detected(home_dir); agents.aider = cbm_agent_cli_exists("aider", home_dir); @@ -2859,6 +2941,9 @@ static const char agent_instructions_content[] = "grep/glob for structural questions, but use direct source tools for literals, configs, " "non-code files, and every reported coverage gap.\n" "\n" + "Pick the projects entry whose root_path contains your working directory; pass its name " + "or that absolute directory as project, never a repo or folder name.\n" + "\n" "## Evidence tiers\n" "- **Scout:** fast positive lookup; provisional, no negative/exhaustive claims.\n" "- **Verify (default):** task-directed evidence, relevant trace directions, exact snippets, " @@ -3181,6 +3266,8 @@ static const char aider_instructions_content[] = "4. Read exact source: `codebase-memory-cli snippet SYMBOL --json`\n" "5. Validate evidence coverage: `codebase-memory-cli coverage PATH --json`\n" "\n" + "Pick the projects entry whose root_path contains your working directory; pass its name " + "or that absolute directory as project, never a repo or folder name.\n" "First use in a repo: `codebase-memory-cli index . --json`\n" "List indexed projects: `codebase-memory-cli projects --json`\n" "\n" @@ -3336,8 +3423,11 @@ static int cbm_remove_codex_legacy_mcp(const char *config_path) { } int cbm_remove_codex_mcp(const char *config_path) { + /* #2228: only our table goes; foreign tables found between the markers + * (Codex Desktop appends below a block that ends the file) stay. */ if (!config_path || - cbm_toml_remove_managed_block(config_path, CODEX_MCP_BEGIN, CODEX_MCP_END) != 0) { + cbm_toml_remove_managed_block_owned(config_path, CODEX_MCP_BEGIN, CODEX_MCP_END, + CODEX_CMM_SECTION "\n") != 0) { return CLI_ERR; } return cbm_remove_codex_legacy_mcp(config_path) >= 0 ? CLI_OK : CLI_ERR; @@ -3804,7 +3894,8 @@ static int cbm_remove_grok_legacy_mcp(const char *config_path) { static int cbm_remove_grok_mcp_owned(const char *binary_path, const char *config_path) { (void)binary_path; if (!config_path || - cbm_toml_remove_managed_block(config_path, GROK_MCP_BEGIN, GROK_MCP_END) != 0) { + cbm_toml_remove_managed_block_owned(config_path, GROK_MCP_BEGIN, GROK_MCP_END, + GROK_CMM_SECTION "\n") != 0) { return CLI_ERR; } return cbm_remove_grok_legacy_mcp(config_path) >= 0 ? CLI_OK : CLI_ERR; @@ -4372,7 +4463,8 @@ static int cbm_upsert_kimi_context_hook(const char *config_path, const char *bin } static int cbm_remove_kimi_context_hook(const char *config_path) { - return cbm_toml_remove_managed_block(config_path, KIMI_HOOK_BEGIN, KIMI_HOOK_END) == 0 + return cbm_toml_remove_managed_block_owned(config_path, KIMI_HOOK_BEGIN, KIMI_HOOK_END, + "[[hooks]]\n") == 0 ? CLI_OK : CLI_ERR; } @@ -6849,6 +6941,11 @@ bool cbm_config_watcher_enabled(cbm_config_t *cfg) { return cbm_config_get_bool(cfg, CBM_CONFIG_WATCHER_ENABLED, true); } +/* Opt-in tree polling of non-git roots (#1948); see cli.h. */ +bool cbm_config_watch_non_git(cbm_config_t *cfg) { + return cbm_config_get_bool(cfg, CBM_CONFIG_WATCH_NON_GIT, false); +} + bool cbm_config_load_index_policy(cbm_config_t *cfg, cbm_index_resource_policy_t *policy, char *error, size_t error_size) { if (!cfg || !policy) { @@ -6890,6 +6987,8 @@ static const config_key_def_t CONFIG_KEYS[] = { {CBM_CONFIG_AUTO_WATCH, "true", "Register background git watcher on session connect"}, {CBM_CONFIG_WATCHER_ENABLED, "true", "Run the background watcher thread (auto-reindex); false to disable"}, + {CBM_CONFIG_WATCH_NON_GIT, "false", + "Also poll non-git project roots for changes (file-tree scan); off by default"}, {CBM_CONFIG_UI_LANG, "auto", "Pin graph UI language: en, zh, or auto"}, {CBM_CONFIG_UI_ENABLED, "false", "Serve the graph UI on a loopback HTTP port"}, {CBM_CONFIG_UI_PORT, "9749", "Port for the graph UI listener when enabled"}, @@ -11532,6 +11631,41 @@ static int cli_uninstall_activate(void *opaque) { return CLI_OK; } +/* A generic auto-answer is not consent to erase project indexes. An explicit + * --delete-indexes takes precedence even over --no; otherwise only a separate + * interactive answer can select deletion. This helper decides and reports, + * but all deletion remains in the final guarded uninstall activation. */ +static bool uninstall_decide_index_deletion(const char *home, bool requested, bool dry_run) { + int index_count = count_db_indexes(home); + if (index_count <= 0) { + return false; + } + printf("\nFound %d index(es):\n", index_count); + cbm_list_indexes(home); + bool delete_indexes = requested; + if (!requested) { + bool can_ask = g_auto_answer == 0 && !dry_run; +#ifdef _WIN32 + can_ask = can_ask && _isatty(_fileno(stdin)); +#else + can_ask = can_ask && isatty(fileno(stdin)); +#endif + delete_indexes = can_ask && prompt_yn("Delete these indexes? (default: no)"); + } + if (!delete_indexes) { + const char *cache_dir = get_cache_dir(home); + printf("Indexes kept in %s. To remove them, uninstall with --delete-indexes " + "or delete the project .db files there.\n", + cache_dir ? cache_dir : "the cache directory"); + return false; + } + if (dry_run) { + printf("(dry-run — indexes would be deleted)\n"); + return false; + } + return true; +} + int cbm_cmd_uninstall(int argc, char **argv) { /* `uninstall --help` used to UNINSTALL. * @@ -11546,13 +11680,15 @@ int cbm_cmd_uninstall(int argc, char **argv) { for (int i = 0; i < argc; i++) { if (argv && argv[i] && (strcmp(argv[i], "--help") == 0 || strcmp(argv[i], "-h") == 0)) { printf("Usage: codebase-memory-cli uninstall [options]\n\n" - "Removes the codebase-memory-cli binary, CLI-owned agent assets/hooks and,\n" - "with confirmation, its indexes. Existing MCP registrations and MCP-owned\n" + "Removes the codebase-memory-cli binary and CLI-owned agent assets/hooks.\n" + "Project indexes are kept by default. Existing MCP registrations and MCP-owned\n" "assets are preserved. THIS IS DESTRUCTIVE.\n\n" "Options:\n" " --dry-run Show what would be removed, change nothing\n" " --dir=PATH Uninstall from a custom install directory\n" - " -y, --yes Do not prompt for confirmation\n" + " --delete-indexes Also delete every project index (overrides --no)\n" + " -n, --no Decline prompts; keep indexes unless explicitly deleted\n" + " -y, --yes Do not prompt; keep indexes unless explicitly deleted\n" " -h, --help Show this help and exit\n\n" "Run with --dry-run first if you are unsure.\n"); return CLI_OK; @@ -11560,6 +11696,7 @@ int cbm_cmd_uninstall(int argc, char **argv) { } parse_auto_answer(argc, argv); bool dry_run = false; + bool delete_indexes_requested = false; /* An install into a custom --dir must be removable from that same dir: * without this, anyone who installed outside ~/.local/bin has no supported * uninstall path at all. Mirrors cbm_cmd_install's parsing. */ @@ -11572,6 +11709,8 @@ int cbm_cmd_uninstall(int argc, char **argv) { } if (strcmp(argv[i], "--dry-run") == 0) { dry_run = true; + } else if (strcmp(argv[i], "--delete-indexes") == 0) { + delete_indexes_requested = true; } else if (strncmp(argv[i], "--dir=", SLEN("--dir=")) == 0) { requested_bin_dir = argv[i] + SLEN("--dir="); if (!requested_bin_dir[0]) { @@ -11603,23 +11742,9 @@ int cbm_cmd_uninstall(int argc, char **argv) { agent_uninstall_failures_reset(); cbm_detected_agents_t agents = cbm_detect_agents(home); - /* Confirm index removal outside the startup lock, but defer the mutation - * until the final guarded activation. Dry-run never removes indexes. */ - bool delete_indexes = false; - int index_count = count_db_indexes(home); - if (index_count > 0) { - printf("\nFound %d index(es):\n", index_count); - cbm_list_indexes(home); - if (prompt_yn("Delete these indexes?")) { - if (dry_run) { - printf("(dry-run — indexes would be deleted)\n"); - } else { - delete_indexes = true; - } - } else { - printf("Indexes kept.\n"); - } - } + /* Decide outside the startup lock; mutate only in the final guarded + * activation. Dry-run never authorizes actual index removal. */ + bool delete_indexes = uninstall_decide_index_deletion(home, delete_indexes_requested, dry_run); char bin_path_storage[CLI_BUF_1K]; const char *bin_path = bin_path_storage; diff --git a/src/cli/cli.h b/src/cli/cli.h index 26be3cdb18..66af0a7915 100644 --- a/src/cli/cli.h +++ b/src/cli/cli.h @@ -161,11 +161,11 @@ int cbm_remove_zed_mcp_owned(const char *binary_path, const char *config_path); /* Detected coding agents on the system. */ typedef struct { - bool claude_code; /* ~/.claude/ exists */ + bool claude_code; /* settings.json, .claude.json, or claude CLI exists */ bool codex; /* $CODEX_HOME or ~/.codex exists */ bool gemini; /* Gemini settings or executable exists */ bool zed; /* platform-specific Zed config dir exists */ - bool opencode; /* opencode on PATH or config exists */ + bool opencode; /* opencode on PATH, config, or data dir exists */ bool antigravity; /* Antigravity CLI config or executable exists */ bool aider; /* aider on PATH */ bool kilocode; /* KiloCode globalStorage dir exists */ @@ -176,7 +176,7 @@ typedef struct { bool openclaw; /* ~/.openclaw/ exists */ bool kiro; /* ~/.kiro/ exists */ bool junie; /* ~/.junie/ exists */ - bool hermes; /* ~/.hermes/ or hermes CLI exists */ + bool hermes; /* Hermes home (Windows: %LOCALAPPDATA%\hermes) or CLI */ bool openhands; /* ~/.openhands/ or openhands CLI exists */ bool cline; /* ~/.cline/ or cline CLI exists */ bool warp; /* Warp footprint or oz/oz-preview/warp-cli exists */ @@ -198,6 +198,13 @@ int cbm_install_agent_configs(const char *home, const char *binary_path, bool fo int cbm_install_agent_hooks_for_testing(const char *home, const char *binary_path, bool force, bool dry_run); +#ifdef CBM_CLI_ENABLE_TEST_API +/* #1180: the Hermes home as resolved on Windows (`windows`) or elsewhere, so + * the %LOCALAPPDATA%\hermes default and its legacy ~/.hermes fallback are + * exercised on every host. */ +void cbm_hermes_home_dir_for_testing(const char *home_dir, bool windows, char *out, size_t out_sz); +#endif + #ifdef CBM_CLI_ENABLE_TEST_API /* #1558: client-selector vocabulary, exposed so a test can prove every token * resolves and an unknown one is rejected. */ @@ -407,6 +414,7 @@ bool cbm_config_load_index_policy(cbm_config_t *cfg, cbm_index_resource_policy_t #define CBM_CONFIG_AUTO_WATCH "auto_watch" #define CBM_CONFIG_UI_LANG "ui-lang" #define CBM_CONFIG_WATCHER_ENABLED "watcher_enabled" +#define CBM_CONFIG_WATCH_NON_GIT "watch_non_git" /* #1558: the graph UI's loopback listener. Stored in the UI config file rather * than the key-value store, but surfaced through `config` so it is findable. */ #define CBM_CONFIG_UI_ENABLED "ui_enabled" @@ -421,6 +429,11 @@ bool cbm_config_load_index_policy(cbm_config_t *cfg, cbm_index_resource_policy_t * the watcher. */ bool cbm_config_watcher_enabled(cbm_config_t *cfg); +/* Whether the watcher also polls NON-GIT project roots (default false, #1948). + * Non-git roots are otherwise never refreshed after their first index. Read + * once at daemon startup, like watcher_enabled. NULL-safe (NULL → false). */ +bool cbm_config_watch_non_git(cbm_config_t *cfg); + /* ── Binary activation safety ─────────────────────────────────── */ typedef void *cbm_cli_activation_lock_t; diff --git a/src/cli/config_toml_edit.c b/src/cli/config_toml_edit.c index c4625feac9..439670ee7a 100644 --- a/src/cli/config_toml_edit.c +++ b/src/cli/config_toml_edit.c @@ -149,6 +149,10 @@ typedef struct { static int toml_managed_block_conflicts(const char *existing, size_t existing_len, size_t exclude_start, size_t exclude_end, const char *block, size_t block_len); +static int toml_managed_rewrite(const char *existing, size_t existing_len, + const toml_line_t *begin_line, const toml_line_t *end_line, + const char *begin_marker, const char *end_marker, const char *owned, + size_t owned_len, int install, toml_buffer_t *output); static void toml_buffer_dispose(toml_buffer_t *buffer) { if (!buffer) { @@ -1067,15 +1071,20 @@ int cbm_toml_escape_basic_string(const char *input, char *out, size_t out_size) return TOML_EDIT_OK; } +static int toml_managed_body_valid(const char *body, const char *begin_marker, + const char *end_marker, size_t *body_len) { + return body && toml_bounded_length(body, TOML_EDIT_MAX_BYTES, body_len) == TOML_EDIT_OK && + toml_text_is_safe(body, *body_len, 1) && + !toml_contains_marker_line(body, *body_len, begin_marker, end_marker) && + toml_validate_lexical_strings(body, *body_len) == TOML_EDIT_OK; +} + int cbm_toml_upsert_managed_block(const char *file_path, const char *begin_marker, const char *end_marker, const char *block) { size_t block_len = 0U; if (!toml_valid_path(file_path) || !toml_valid_marker(begin_marker) || - !toml_valid_marker(end_marker) || strcmp(begin_marker, end_marker) == 0 || !block || - toml_bounded_length(block, TOML_EDIT_MAX_BYTES, &block_len) != TOML_EDIT_OK || - !toml_text_is_safe(block, block_len, 1) || - toml_contains_marker_line(block, block_len, begin_marker, end_marker) || - toml_validate_lexical_strings(block, block_len) != TOML_EDIT_OK) { + !toml_valid_marker(end_marker) || strcmp(begin_marker, end_marker) == 0 || + !toml_managed_body_valid(block, begin_marker, end_marker, &block_len)) { return TOML_EDIT_ERR; } @@ -1104,23 +1113,26 @@ int cbm_toml_upsert_managed_block(const char *file_path, const char *begin_marke } toml_buffer_t output = {0}; - size_t prefix_len = has_pair ? begin_line.start : existing_len; - if (has_pair && prefix_len == 0U && existing_len >= 3U && (unsigned char)existing[0] == 0xefU && - (unsigned char)existing[1] == 0xbbU && (unsigned char)existing[2] == 0xbfU) { - prefix_len = 3U; - } const char *newline = toml_newline_style(existing, existing_len); size_t payload_start = existing_len >= 3U && (unsigned char)existing[0] == 0xefU && (unsigned char)existing[1] == 0xbbU && (unsigned char)existing[2] == 0xbfU ? 3U : 0U; - if (toml_buffer_append(&output, existing, prefix_len) != TOML_EDIT_OK || - (!has_pair && existing_len > payload_start && existing[existing_len - 1] != '\n' && - toml_buffer_append_cstr(&output, newline) != TOML_EDIT_OK) || - toml_append_managed(&output, begin_marker, end_marker, block, newline) != TOML_EDIT_OK || - (has_pair && toml_buffer_append(&output, existing + end_line.full_end, - existing_len - end_line.full_end) != TOML_EDIT_OK)) { + int built = TOML_EDIT_OK; + if (has_pair) { + /* #2228: rebuild the span around the block, keeping what it holds that + * is not ours instead of discarding the whole span. */ + built = toml_managed_rewrite(existing, existing_len, &begin_line, &end_line, begin_marker, + end_marker, block, block_len, 1, &output); + } else if (toml_buffer_append(&output, existing, existing_len) != TOML_EDIT_OK || + (existing_len > payload_start && existing[existing_len - 1] != '\n' && + toml_buffer_append_cstr(&output, newline) != TOML_EDIT_OK) || + toml_append_managed(&output, begin_marker, end_marker, block, newline) != + TOML_EDIT_OK) { + built = TOML_EDIT_ERR; + } + if (built != TOML_EDIT_OK) { toml_buffer_dispose(&output); free(existing); return TOML_EDIT_ERR; @@ -1133,10 +1145,14 @@ int cbm_toml_upsert_managed_block(const char *file_path, const char *begin_marke return result; } -int cbm_toml_remove_managed_block(const char *file_path, const char *begin_marker, - const char *end_marker) { +/* owned == NULL removes the whole span (the historical contract); otherwise + * only the tables it declares go and the rest of the span stays (#2228). */ +static int toml_remove_managed(const char *file_path, const char *begin_marker, + const char *end_marker, const char *owned) { + size_t owned_len = 0U; if (!toml_valid_path(file_path) || !toml_valid_marker(begin_marker) || - !toml_valid_marker(end_marker) || strcmp(begin_marker, end_marker) == 0) { + !toml_valid_marker(end_marker) || strcmp(begin_marker, end_marker) == 0 || + (owned && !toml_managed_body_valid(owned, begin_marker, end_marker, &owned_len))) { return TOML_EDIT_ERR; } char *existing = NULL; @@ -1187,9 +1203,14 @@ int cbm_toml_remove_managed_block(const char *file_path, const char *begin_marke (unsigned char)existing[1] == 0xbbU && (unsigned char)existing[2] == 0xbfU) { prefix_len = 3U; } - if (toml_buffer_append(&output, existing, prefix_len) != TOML_EDIT_OK || - toml_buffer_append(&output, existing + end_line.full_end, - existing_len - end_line.full_end) != TOML_EDIT_OK) { + int built = owned ? toml_managed_rewrite(existing, existing_len, &begin_line, &end_line, + begin_marker, end_marker, owned, owned_len, 0, &output) + : toml_buffer_append(&output, existing, prefix_len) == TOML_EDIT_OK && + toml_buffer_append(&output, existing + end_line.full_end, + existing_len - end_line.full_end) == TOML_EDIT_OK + ? TOML_EDIT_OK + : TOML_EDIT_ERR; + if (built != TOML_EDIT_OK) { toml_buffer_dispose(&output); free(existing); return TOML_EDIT_ERR; @@ -1201,6 +1222,17 @@ int cbm_toml_remove_managed_block(const char *file_path, const char *begin_marke return result; } +int cbm_toml_remove_managed_block(const char *file_path, const char *begin_marker, + const char *end_marker) { + return toml_remove_managed(file_path, begin_marker, end_marker, NULL); +} + +int cbm_toml_remove_managed_block_owned(const char *file_path, const char *begin_marker, + const char *end_marker, const char *owned_tables) { + return owned_tables ? toml_remove_managed(file_path, begin_marker, end_marker, owned_tables) + : TOML_EDIT_ERR; +} + static int toml_hex_digit(unsigned char ch) { if (ch >= '0' && ch <= '9') { return ch - '0'; @@ -1845,6 +1877,428 @@ static int toml_managed_block_conflicts(const char *existing, size_t existing_le return multiline_state == TOML_STRING_NONE ? TOML_EDIT_OK : TOML_EDIT_ERR; } +/* ── #2228: content found between the managed markers ───────────────── + * + * The span between our markers does not only hold what we wrote. A client + * that appends tables at the end of the file (Codex Desktop does) puts them + * INSIDE the markers whenever our closing marker is the last line, and users + * add keys to our own table. Rewriting or removing the whole span deleted + * all of that. + * + * The span is read as TOML sections -- the lines before its first header + * (the lead) and one section per header -- and each is classified against + * the tables the block declares: + * owned a declared table; the k-th [[element]] of a declared array. + * Keys the block writes are ours; every other key, comment and + * blank line in it is the user's and stays in the table. + * attached a sub-table of a declared table: part of our entry, kept with + * it on install and removed with it on uninstall. + * foreign anything else: moved byte for byte, in order, to just after + * the closing marker (install) or left in place (removal). + * lead moved above the opening marker, where it keeps the table it + * belongs to. A block that itself starts with bare keys owns the + * lead, which is then replaced exactly as before. + * Whatever cannot be attributed safely is refused and the file is left + * untouched: a second copy of a declared table, an undeclared [[element]] + * of a declared array, a child of a declared array element, an unparsable + * line in an owned table, a user key colliding with one the block writes, a + * move that would hand the keys below the block to another table, or a + * result whose tables would conflict with the block. */ +#define TOML_MANAGED_MAX_TABLES 16U + +enum { + TOML_SPAN_LEAD = -1, + TOML_SPAN_FOREIGN = 0, + TOML_SPAN_OWNED = 1, + TOML_SPAN_ATTACHED = 2, +}; + +typedef struct { + size_t start; + size_t body; + size_t end; + toml_header_t header; +} toml_section_t; + +typedef struct { + const char *data; + size_t lead_end; + size_t span_end; + const char *owned; + size_t owned_len; + size_t owned_first; +} toml_managed_span_t; + +/* End of the section whose body starts at pos: the next header line outside + * a multiline string, or len. */ +static int toml_section_end(const char *data, size_t pos, size_t len, size_t *end) { + size_t cursor = pos; + toml_line_t line; + int multiline_state = TOML_STRING_NONE; + while (toml_next_line(data, len, &cursor, &line)) { + if (multiline_state == TOML_STRING_NONE) { + toml_header_t header; + if (toml_parse_header(data, &line, "", &header) != TOML_EDIT_OK) { + return TOML_EDIT_ERR; + } + int present = header.present; + toml_header_dispose(&header); + if (present) { + *end = line.start; + return TOML_EDIT_OK; + } + } + if (toml_scan_line_strings(data, &line, &multiline_state) != TOML_EDIT_OK) { + return TOML_EDIT_ERR; + } + } + *end = len; + return multiline_state == TOML_STRING_NONE ? TOML_EDIT_OK : TOML_EDIT_ERR; +} + +/* The section whose header line starts at pos. */ +static int toml_section_at(const char *data, size_t pos, size_t len, toml_section_t *section) { + memset(section, 0, sizeof(*section)); + section->start = pos; + size_t cursor = pos; + toml_line_t line; + if (!toml_next_line(data, len, &cursor, &line) || + toml_parse_header(data, &line, "", §ion->header) != TOML_EDIT_OK || + !section->header.present || + toml_section_end(data, line.full_end, len, §ion->end) != TOML_EDIT_OK) { + toml_header_dispose(§ion->header); + return TOML_EDIT_ERR; + } + section->body = line.full_end; + return TOML_EDIT_OK; +} + +static int toml_lines_significant(const char *data, size_t start, size_t end) { + size_t cursor = start; + toml_line_t line; + while (toml_next_line(data, end, &cursor, &line)) { + if (!toml_line_is_blank_or_comment(data, &line)) { + return 1; + } + } + return 0; +} + +/* Declared table `table` (1-based) of the block, or the number of declared + * tables in *count when table is 0. */ +static int toml_declared_table(const toml_managed_span_t *span, size_t table, size_t *count, + toml_section_t *out) { + size_t pos = span->owned_first; + size_t index = 0U; + while (pos < span->owned_len) { + toml_section_t declared; + if (++index > TOML_MANAGED_MAX_TABLES || + toml_section_at(span->owned, pos, span->owned_len, &declared) != TOML_EDIT_OK) { + return TOML_EDIT_ERR; + } + if (index == table) { + *out = declared; + return TOML_EDIT_OK; + } + pos = declared.end; + toml_header_dispose(&declared.header); + } + if (count) { + *count = index; + } + return table == 0U ? TOML_EDIT_OK : TOML_EDIT_ERR; +} + +/* Classify one span header. seen[] marks the declared tables the span has + * already supplied, so a second copy is refused instead of guessed at. */ +static int toml_managed_classify(const toml_managed_span_t *span, const toml_header_t *header, + unsigned char *seen, int *kind, size_t *table) { + size_t exact = 0U; + int exact_taken = 0; + size_t parent = 0U; + size_t parent_depth = 0U; + int result = TOML_EDIT_OK; + size_t pos = span->owned_first; + for (size_t index = 1U; pos < span->owned_len && result == TOML_EDIT_OK; ++index) { + toml_section_t declared; + if (toml_section_at(span->owned, pos, span->owned_len, &declared) != TOML_EDIT_OK) { + return TOML_EDIT_ERR; + } + pos = declared.end; + const toml_key_path_t *path = &declared.header.path; + if (toml_key_path_equal(path, &header->path)) { + if (declared.header.array != header->array) { + result = TOML_EDIT_ERR; + } else if (!seen[index] && exact == 0U) { + exact = index; + } else { + exact_taken = 1; + } + } else if (toml_key_path_has_prefix(&header->path, path)) { + if (declared.header.array) { + result = TOML_EDIT_ERR; + } else if (path->count > parent_depth) { + parent = index; + parent_depth = path->count; + } + } + toml_header_dispose(&declared.header); + } + if (result != TOML_EDIT_OK || (exact == 0U && exact_taken)) { + return TOML_EDIT_ERR; + } + if (exact != 0U) { + seen[exact] = 1U; + } + *kind = exact != 0U ? TOML_SPAN_OWNED : parent != 0U ? TOML_SPAN_ATTACHED : TOML_SPAN_FOREIGN; + *table = exact != 0U ? exact : parent; + return TOML_EDIT_OK; +} + +/* Sets *written when the declared table assigns key; a key that only shares + * a dotted prefix with one it assigns would redefine it and is refused. */ +static int toml_declared_writes_key(const toml_managed_span_t *span, const toml_section_t *declared, + const toml_key_path_t *key, int *written) { + *written = 0; + size_t cursor = declared->body; + toml_line_t line; + int multiline_state = TOML_STRING_NONE; + while (toml_next_line(span->owned, declared->end, &cursor, &line)) { + if (multiline_state == TOML_STRING_NONE) { + toml_assignment_t assignment; + if (toml_parse_assignment(span->owned, &line, &assignment) != TOML_EDIT_OK) { + return TOML_EDIT_ERR; + } + int equal = assignment.present && toml_key_path_equal(&assignment.key, key); + int collides = assignment.present && !equal && + (toml_key_path_has_prefix(&assignment.key, key) || + toml_key_path_has_prefix(key, &assignment.key)); + toml_assignment_dispose(&assignment); + if (collides) { + return TOML_EDIT_ERR; + } + *written |= equal; + } + if (toml_scan_line_strings(span->owned, &line, &multiline_state) != TOML_EDIT_OK) { + return TOML_EDIT_ERR; + } + } + return TOML_EDIT_OK; +} + +static size_t toml_line_first_char(const char *data, const toml_line_t *line) { + size_t pos = line->start; + while (pos < line->content_end && (data[pos] == ' ' || data[pos] == '\t')) { + pos++; + } + return pos; +} + +/* Is this line, outside any multiline string, one the user owns? Comments + * are; an assignment is unless the declared table writes its key. */ +static int toml_line_is_users(const toml_managed_span_t *span, const toml_section_t *declared, + const toml_line_t *line, int *users) { + if (span->data[toml_line_first_char(span->data, line)] == '#') { + *users = 1; + return TOML_EDIT_OK; + } + toml_assignment_t assignment; + int written = 0; + if (toml_parse_assignment(span->data, line, &assignment) != TOML_EDIT_OK || + !assignment.present || + toml_declared_writes_key(span, declared, &assignment.key, &written) != TOML_EDIT_OK) { + toml_assignment_dispose(&assignment); + return TOML_EDIT_ERR; + } + toml_assignment_dispose(&assignment); + *users = !written; + return TOML_EDIT_OK; +} + +/* The lines of an owned section the block does not write, with the blank + * lines between them; blank lines before a replaced key or at the end of the + * section go with the section. */ +static int toml_append_user_lines(const toml_managed_span_t *span, const toml_section_t *section, + size_t table, toml_buffer_t *output) { + toml_section_t declared; + if (toml_declared_table(span, table, NULL, &declared) != TOML_EDIT_OK) { + return TOML_EDIT_ERR; + } + size_t cursor = section->body; + toml_line_t line; + int multiline_state = TOML_STRING_NONE; + int keep = 0; + size_t blank_start = SIZE_MAX; + int result = TOML_EDIT_OK; + while (result == TOML_EDIT_OK && toml_next_line(span->data, section->end, &cursor, &line)) { + if (multiline_state == TOML_STRING_NONE) { + if (toml_line_first_char(span->data, &line) == line.content_end) { + blank_start = blank_start == SIZE_MAX ? line.start : blank_start; + continue; + } + result = toml_line_is_users(span, &declared, &line, &keep); + if (!keep) { + blank_start = SIZE_MAX; + } + } + if (result == TOML_EDIT_OK && keep) { + size_t from = blank_start == SIZE_MAX ? line.start : blank_start; + blank_start = SIZE_MAX; + result = toml_buffer_append(output, span->data + from, line.full_end - from); + } + if (result == TOML_EDIT_OK && + toml_scan_line_strings(span->data, &line, &multiline_state) != TOML_EDIT_OK) { + result = TOML_EDIT_ERR; + } + } + toml_header_dispose(&declared.header); + return result == TOML_EDIT_OK && multiline_state == TOML_STRING_NONE ? TOML_EDIT_OK + : TOML_EDIT_ERR; +} + +/* One pass over the span's tables in file order. Appends each section of + * want_kind (and, unless foreign, of declared table want_table): foreign and + * attached sections verbatim, an owned one as its user lines. Also reports + * how many sections are foreign and the kind of the last one. */ +static int toml_managed_span_pass(const toml_managed_span_t *span, int want_kind, size_t want_table, + toml_buffer_t *output, size_t *foreign, int *last_kind) { + unsigned char seen[TOML_MANAGED_MAX_TABLES + 1U] = {0}; + *foreign = 0U; + *last_kind = TOML_SPAN_LEAD; + size_t pos = span->lead_end; + while (pos < span->span_end) { + toml_section_t section; + int kind = TOML_SPAN_FOREIGN; + size_t table = 0U; + if (toml_section_at(span->data, pos, span->span_end, §ion) != TOML_EDIT_OK) { + return TOML_EDIT_ERR; + } + int result = toml_managed_classify(span, §ion.header, seen, &kind, &table); + toml_header_dispose(§ion.header); + pos = section.end; + *foreign += kind == TOML_SPAN_FOREIGN; + *last_kind = kind; + if (result == TOML_EDIT_OK && kind == want_kind && + (kind == TOML_SPAN_FOREIGN || table == want_table)) { + result = kind == TOML_SPAN_OWNED + ? toml_append_user_lines(span, §ion, table, output) + : toml_buffer_append(output, span->data + section.start, + section.end - section.start); + } + if (result != TOML_EDIT_OK) { + return TOML_EDIT_ERR; + } + } + return TOML_EDIT_OK; +} + +/* The declared tables of an install, each followed by what the span keeps + * for it: the user lines of its owned section, then its attached tables. */ +static int toml_append_managed_tables(const toml_managed_span_t *span, size_t tables, + const char *newline, toml_buffer_t *output) { + size_t foreign = 0U; + int last_kind = TOML_SPAN_LEAD; + if (toml_append_normalized_text(output, span->owned, span->owned_first, newline) != + TOML_EDIT_OK) { + return TOML_EDIT_ERR; + } + for (size_t table = 1U; table <= tables; ++table) { + toml_section_t declared; + if (toml_declared_table(span, table, NULL, &declared) != TOML_EDIT_OK) { + return TOML_EDIT_ERR; + } + toml_header_dispose(&declared.header); + if (toml_append_normalized_text(output, span->owned + declared.start, + declared.end - declared.start, newline) != TOML_EDIT_OK || + (output->data[output->len - 1U] != '\n' && + toml_buffer_append_cstr(output, newline) != TOML_EDIT_OK) || + toml_managed_span_pass(span, TOML_SPAN_OWNED, table, output, &foreign, &last_kind) != + TOML_EDIT_OK || + toml_managed_span_pass(span, TOML_SPAN_ATTACHED, table, output, &foreign, &last_kind) != + TOML_EDIT_OK) { + return TOML_EDIT_ERR; + } + } + return output->len == 0U || output->data[output->len - 1U] == '\n' || + toml_buffer_append_cstr(output, newline) == TOML_EDIT_OK + ? TOML_EDIT_OK + : TOML_EDIT_ERR; +} + +/* Rebuild existing with its managed span rewritten around `owned` (install) + * or removed (uninstall), keeping everything the span holds that is not + * ours. `owned` is the block on install and the declaration of our tables on + * removal. */ +static int toml_managed_rewrite(const char *existing, size_t existing_len, + const toml_line_t *begin_line, const toml_line_t *end_line, + const char *begin_marker, const char *end_marker, const char *owned, + size_t owned_len, int install, toml_buffer_t *output) { + toml_managed_span_t span = { + .data = existing, + .span_end = end_line->start, + .owned = owned, + .owned_len = owned_len, + }; + size_t tables = 0U; + size_t tail_lead_end = existing_len; + if (toml_section_end(owned, 0U, owned_len, &span.owned_first) != TOML_EDIT_OK || + toml_declared_table(&span, 0U, &tables, NULL) != TOML_EDIT_OK || + toml_section_end(existing, begin_line->full_end, span.span_end, &span.lead_end) != + TOML_EDIT_OK || + toml_section_end(existing, end_line->full_end, existing_len, &tail_lead_end) != + TOML_EDIT_OK) { + return TOML_EDIT_ERR; + } + toml_buffer_t moved = {0}; + size_t foreign = 0U; + int last_kind = TOML_SPAN_LEAD; + if (toml_managed_span_pass(&span, TOML_SPAN_FOREIGN, 0U, &moved, &foreign, &last_kind) != + TOML_EDIT_OK || + (foreign != 0U && last_kind != TOML_SPAN_FOREIGN && + toml_lines_significant(existing, end_line->full_end, tail_lead_end))) { + toml_buffer_dispose(&moved); + return TOML_EDIT_ERR; + } + + size_t prefix_len = begin_line->start; + if (prefix_len == 0U && existing_len >= 3U && (unsigned char)existing[0] == 0xefU && + (unsigned char)existing[1] == 0xbbU && (unsigned char)existing[2] == 0xbfU) { + prefix_len = 3U; + } + const char *newline = toml_newline_style(existing, existing_len); + int keep_lead = !toml_lines_significant(owned, 0U, span.owned_first); + size_t block_start = 0U; + size_t block_end = 0U; + int result = toml_buffer_append(output, existing, prefix_len) == TOML_EDIT_OK && + (!keep_lead || + toml_buffer_append(output, existing + begin_line->full_end, + span.lead_end - begin_line->full_end) == TOML_EDIT_OK) + ? TOML_EDIT_OK + : TOML_EDIT_ERR; + if (result == TOML_EDIT_OK && install) { + block_start = output->len; + result = + toml_buffer_append_cstr(output, begin_marker) == TOML_EDIT_OK && + toml_buffer_append_cstr(output, newline) == TOML_EDIT_OK && + toml_append_managed_tables(&span, tables, newline, output) == TOML_EDIT_OK && + toml_buffer_append_cstr(output, end_marker) == TOML_EDIT_OK && + toml_buffer_append_cstr(output, newline) == TOML_EDIT_OK + ? TOML_EDIT_OK + : TOML_EDIT_ERR; + block_end = output->len; + } + if (result == TOML_EDIT_OK && + (toml_buffer_append(output, moved.data, moved.len) != TOML_EDIT_OK || + toml_buffer_append(output, existing + end_line->full_end, + existing_len - end_line->full_end) != TOML_EDIT_OK || + (install && toml_managed_block_conflicts(output->data, output->len, block_start, block_end, + owned, owned_len) != TOML_EDIT_OK))) { + result = TOML_EDIT_ERR; + } + toml_buffer_dispose(&moved); + return result; +} + static int toml_finish_target_table(toml_target_table_t *current, toml_table_scan_t *result) { if (!current->active) { return TOML_EDIT_OK; diff --git a/src/cli/config_toml_edit.h b/src/cli/config_toml_edit.h index 1b0ef137f0..6b944ee194 100644 --- a/src/cli/config_toml_edit.h +++ b/src/cli/config_toml_edit.h @@ -24,11 +24,21 @@ int cbm_toml_escape_basic_string(const char *input, char *out, size_t out_size); * * Insert or replace one managed, line-delimited block. block is the body * between the two marker lines. Duplicate or unbalanced markers are rejected - * without changing the file. */ + * without changing the file. An existing span keeps what is not ours (#2228): + * keys and comments the block does not write stay in the tables it declares, + * with those tables' sub-tables; lines before the span's first table move + * above the block and any other table moves, byte for byte and in order, to + * just after it. Content whose ownership is ambiguous fails closed. */ int cbm_toml_upsert_managed_block(const char *file_path, const char *begin_marker, const char *end_marker, const char *block); +/* Removes the whole span between the markers. */ int cbm_toml_remove_managed_block(const char *file_path, const char *begin_marker, const char *end_marker); +/* Removes the markers and only the tables owned_tables declares (e.g. the + * block's header lines), with their keys and sub-tables; every other line of + * the span stays in place. Ambiguous ownership fails closed (#2228). */ +int cbm_toml_remove_managed_block_owned(const char *file_path, const char *begin_marker, + const char *end_marker, const char *owned_tables); /* Remove one pre-marker codebase-memory-mcp table only when it has the known * historical schema: one owned command basename, optional empty args, optional diff --git a/src/cli/hook_augment.c b/src/cli/hook_augment.c index 33d317d3ce..5af8f9ef08 100644 --- a/src/cli/hook_augment.c +++ b/src/cli/hook_augment.c @@ -59,8 +59,9 @@ * * Observability (#858): a fired deadline is otherwise indistinguishable from * "no matches", so the handler first write()s a pre-formatted breadcrumb to - * ~/.cache/codebase-memory-cli/logs/hook-augment-timeouts.log (fd and message - * prepared at arm time — only async-signal-safe write/_exit in the handler). */ + * /logs/hook-augment-timeouts.log (by default + * ~/.cache/codebase-memory-cli/logs; fd and message prepared at arm time — + * only async-signal-safe write/_exit in the handler). */ #ifndef _WIN32 #define HA_DEADLINE_DEFAULT_MS 2000 /* in-process budget; see ha_deadline_ms() */ #define HA_DEADLINE_MIN_MS 50 @@ -103,20 +104,31 @@ static void ha_deadline_exit(int sig) { _exit(0); } +/* Whether snprintf wrote the whole string into a buffer of `cap` bytes. */ +static bool ha_fits(int written, size_t cap) { + return written > 0 && (size_t)written < cap; +} + static void ha_open_crumb_log(int deadline_ms) { const char *override = getenv("CBM_HOOK_TIMEOUT_LOG"); /* tests + power users */ char path[CBM_SZ_1K]; + /* A path cut off at the buffer names some other file, and O_CREAT would + * create it. Such a path means no breadcrumb; the hook itself runs on. */ if (override && override[0]) { - snprintf(path, sizeof(path), "%s", override); - } else { - const char *home = getenv("HOME"); - if (!home || !home[0]) { + if (!ha_fits(snprintf(path, sizeof(path), "%s", override), sizeof(path))) { return; } + } else { + /* /logs, where every cbm component logs: the cache dir is + * CBM_CACHE_DIR, else HOME (then USERPROFILE) + the default. */ + const char *cache = cbm_resolve_cache_dir(); char dir[CBM_SZ_1K]; - snprintf(dir, sizeof(dir), "%s/.cache/codebase-memory-cli/logs", home); + if (!cache || !ha_fits(snprintf(dir, sizeof(dir), "%s/logs", cache), sizeof(dir)) || + !ha_fits(snprintf(path, sizeof(path), "%s/hook-augment-timeouts.log", dir), + sizeof(path))) { + return; + } cbm_mkdir_p_ex(dir, 0755, CBM_MKDIR_FOLLOW_OWNED); - snprintf(path, sizeof(path), "%s/hook-augment-timeouts.log", dir); } g_ha_crumb_fd = open(path, O_WRONLY | O_CREAT | O_APPEND, 0644); if (g_ha_crumb_fd < 0) { diff --git a/src/cypher/cypher.c b/src/cypher/cypher.c index af7cb566ea..2523c50597 100644 --- a/src/cypher/cypher.c +++ b/src/cypher/cypher.c @@ -12,6 +12,8 @@ #include "foundation/platform.h" #include "foundation/limits.h" #include "foundation/log.h" +#include "foundation/hash_table.h" +#include "foundation/mem_core.h" enum { CYP_BUF_16 = 16, @@ -2594,6 +2596,12 @@ static const char *resolve_condition_value(const cbm_condition_t *c, binding_t * return n->name ? n->name : ""; } +/* Set when an `=~` or inline-property pattern was refused by the regex + * wrapper's compile-size guard during the CURRENT execution; cbm_cypher_execute + * turns it into result->warning so an empty result can be told from "no such + * name". Reset at the start of every execution. */ +static _Thread_local bool g_cypher_regex_refused = false; + /* Evaluate a comparison operator between actual and expected strings. */ static bool eval_comparison_op(const char *op, const char *actual, const char *expected) { if (strcmp(op, "=") == 0) { @@ -2604,7 +2612,11 @@ static bool eval_comparison_op(const char *op, const char *actual, const char *e } if (strcmp(op, "=~") == 0) { cbm_regex_t re; - if (cbm_regcomp(&re, expected, CBM_REG_EXTENDED | CBM_REG_NOSUB) != 0) { + int comp_rc = cbm_regcomp(&re, expected, CBM_REG_EXTENDED | CBM_REG_NOSUB); + if (comp_rc != 0) { + if (comp_rc == CBM_REG_ETOOBIG) { + g_cypher_regex_refused = true; + } return false; } int rc = cbm_regexec(&re, actual, 0, NULL, 0); @@ -2881,13 +2893,17 @@ static bool check_inline_props(const cbm_node_t *n, const cbm_prop_filter_t *pro const char *actual = node_prop(n, props[i].key, store); if (looks_like_regex(props[i].value)) { cbm_regex_t re; - if (cbm_regcomp(&re, props[i].value, CBM_REG_EXTENDED | CBM_REG_NOSUB) == 0) { + int comp_rc = cbm_regcomp(&re, props[i].value, CBM_REG_EXTENDED | CBM_REG_NOSUB); + if (comp_rc == 0) { bool matched = cbm_regexec(&re, actual, 0, NULL, 0) == 0; cbm_regfree(&re); if (!matched) { return false; } } else if (strcmp(actual, props[i].value) != 0) { + if (comp_rc == CBM_REG_ETOOBIG) { + g_cypher_regex_refused = true; + } return false; } } else if (strcmp(actual, props[i].value) != 0) { @@ -2934,6 +2950,104 @@ static void rb_add_row(result_builder_t *rb, const char **values) { rb->rows[rb->row_count++] = row; } +/* ── DISTINCT while projecting (#1364) ───────────────────────────── + * + * RETURN DISTINCT used to project up to the engine row cap and only then drop + * duplicates, so a value first seen past the cap never reached the result: on + * a 150k-edge index `RETURN DISTINCT type(r)` lost its one HAS_BRANCH edge. + * The projection loops now de-duplicate as they go, so the cap counts DISTINCT + * rows. Memory stays bounded by the cap: only admitted rows are remembered; + * rows past the cap are probed (no insert) solely to decide `truncated`. */ +typedef struct { + CBMHashTable *seen; /* row key -> the same heap key; NULL = not DISTINCT */ +} rb_distinct_t; + +static void rb_distinct_init(rb_distinct_t *d, bool distinct) { + d->seen = distinct ? cbm_ht_create(CBM_SZ_64) : NULL; +} + +static void rb_distinct_free_key(const char *key, void *value, void *userdata) { + (void)key; + (void)userdata; + cbm_free(CBM_MEM_CLASS_OTHER, value); +} + +static void rb_distinct_free(rb_distinct_t *d) { + if (d->seen) { + cbm_ht_foreach(d->seen, rb_distinct_free_key, NULL); + cbm_ht_free(d->seen); + d->seen = NULL; + } +} + +/* Length-prefixed row key (":" per column), so no value content + * can make two different rows collide. NULL projects as "" (as rb_add_row). */ +static char *rb_distinct_row_key(const char **vals, int n) { + size_t total = SKIP_ONE; + for (int i = 0; i < n; i++) { + total += (vals[i] ? strlen(vals[i]) : 0) + CBM_SZ_32; + } + char *key = cbm_alloc(CBM_MEM_CLASS_OTHER, total); + if (!key) { + return NULL; + } + size_t pos = 0; + key[0] = '\0'; + for (int i = 0; i < n; i++) { + const char *v = vals[i] ? vals[i] : ""; + int w = snprintf(key + pos, total - pos, "%zu:%s", strlen(v), v); + if (w > 0) { + pos += (size_t)w; + } + } + return key; +} + +/* Admission for one projected row. Returns false for a row DISTINCT already + * holds. On true, *key_out is the key to hand to rb_distinct_keep (NULL when + * not DISTINCT, or when the key allocation failed — the row is then admitted + * without de-duplication rather than dropped). */ +static bool rb_distinct_is_new(const rb_distinct_t *d, const char **vals, int n, char **key_out) { + *key_out = NULL; + if (!d->seen) { + return true; + } + char *key = rb_distinct_row_key(vals, n); + if (!key) { + return true; + } + if (cbm_ht_has(d->seen, key)) { + cbm_free(CBM_MEM_CLASS_OTHER, key); + return false; + } + *key_out = key; + return true; +} + +static void rb_distinct_keep(rb_distinct_t *d, char *key) { + if (key) { + cbm_ht_set(d->seen, key, key); + } +} + +/* Add one projected row under a row cap, de-duplicating when DISTINCT. + * Returns false once a NEW row arrives with the cap already full: the caller + * stops there, and flags truncation when the cap is the engine budget. A + * duplicate past the cap is neither kept nor evidence of truncation. */ +static bool rb_add_row_capped(result_builder_t *rb, rb_distinct_t *d, const char **vals, int cap) { + char *key = NULL; + if (!rb_distinct_is_new(d, vals, rb->col_count, &key)) { + return true; + } + if (rb->row_count >= cap) { + cbm_free(CBM_MEM_CLASS_OTHER, key); + return false; + } + rb_distinct_keep(d, key); + rb_add_row(rb, vals); + return true; +} + /* ── Main execution ─────────────────────────────────────────────── */ /* Hard ceiling: queries returning more than this trigger an error instead of data. @@ -4483,14 +4597,20 @@ static void execute_return_star_after_with(cbm_query_t *q, binding_t *bindings, cols[i] = resolve_item_alias(&wc->items[i], name_bufs[i], sizeof(name_bufs[i])); } rb_set_columns(rb, cols, col_n); - for (int bi = 0; bi < bind_count && rb->row_count < max_rows; bi++) { + rb_distinct_t seen; + rb_distinct_init(&seen, q->ret && q->ret->distinct); + for (int bi = 0; bi < bind_count; bi++) { const char *vals[CYP_MAX_VARS]; for (int i = 0; i < col_n; i++) { cbm_node_t *vn = binding_get(&bindings[bi], cols[i]); vals[i] = vn && vn->name ? vn->name : ""; } - rb_add_row(rb, vals); + if (!rb_add_row_capped(rb, &seen, vals, max_rows)) { + g_cypher_truncated = true; + break; + } } + rb_distinct_free(&seen); } static void execute_return_star(cbm_query_t *q, binding_t *bindings, int bind_count, int max_rows, @@ -4505,19 +4625,22 @@ static void execute_return_star(cbm_query_t *q, binding_t *bindings, int bind_co int projection_cap = max_rows; bool cap_is_engine_budget = true; cbm_return_clause_t *ret = q->ret; - if (ret && ret->limit >= 0 && ret->limit <= max_rows && !ret->distinct && - ret->order_key_count == 0 && ret->skip <= 0) { - projection_cap = ret->limit; + if (ret && ret->limit >= 0 && ret->limit <= max_rows && ret->order_key_count == 0 && + ret->skip <= 0) { + projection_cap = ret->limit; /* DISTINCT: the cap counts distinct rows (#1364) */ cap_is_engine_budget = false; } - for (int bi = 0; bi < bind_count && rb->row_count < projection_cap; bi++) { + rb_distinct_t seen; + rb_distinct_init(&seen, ret && ret->distinct); + for (int bi = 0; bi < bind_count; bi++) { const char *vals[CBM_SZ_128]; project_star_row(&bindings[bi], vars, vc, vals); - rb_add_row(rb, vals); - } - if (cap_is_engine_budget && bind_count > projection_cap) { - g_cypher_truncated = true; + if (!rb_add_row_capped(rb, &seen, vals, projection_cap)) { + g_cypher_truncated = g_cypher_truncated || cap_is_engine_budget; + break; + } } + rb_distinct_free(&seen); } /* Format an aggregate value into buf based on function name */ @@ -4768,22 +4891,25 @@ static void execute_return_simple(cbm_return_clause_t *ret, binding_t *bindings, int max_rows, result_builder_t *rb) { int proj_cap = max_rows; bool cap_is_engine_budget = true; - if (ret->limit >= 0 && !ret->distinct && ret->order_key_count == 0 && ret->skip <= 0) { - proj_cap = ret->limit; + if (ret->limit >= 0 && ret->limit <= max_rows && ret->order_key_count == 0 && ret->skip <= 0) { + proj_cap = ret->limit; /* DISTINCT: the cap counts distinct rows (#1364) */ cap_is_engine_budget = false; } - for (int bi = 0; bi < bind_count && rb->row_count < proj_cap; bi++) { + rb_distinct_t seen; + rb_distinct_init(&seen, ret->distinct); + for (int bi = 0; bi < bind_count; bi++) { const char *vals[CBM_SZ_32]; char func_bufs[CBM_SZ_32][CBM_SZ_512]; for (int ci = 0; ci < ret->count; ci++) { vals[ci] = project_item(&bindings[bi], &ret->items[ci], func_bufs[ci], sizeof(func_bufs[ci])); } - rb_add_row(rb, vals); - } - if (cap_is_engine_budget && bind_count > proj_cap) { - g_cypher_truncated = true; + if (!rb_add_row_capped(rb, &seen, vals, proj_cap)) { + g_cypher_truncated = g_cypher_truncated || cap_is_engine_budget; + break; + } } + rb_distinct_free(&seen); } /* Build default 3-column headers (name, qualified_name, label) per variable */ @@ -5125,7 +5251,9 @@ static void execute_return_clause(cbm_query_t *q, cbm_return_clause_t *ret, bind } } - if (ret->distinct) { + /* The simple and star projections de-duplicate while projecting (#1364); + * only aggregate rows still need the post-pass. */ + if (ret->distinct && has_agg && !ret->star) { rb_apply_distinct(rb); } rb_apply_order_by(rb, ret); @@ -5245,7 +5373,11 @@ static int execute_single(cbm_store_t *store, cbm_query_t *q, const char *projec /* Build initial bindings with early WHERE */ int bind_cap = scan_count > max_rows ? scan_count : (max_rows > 0 ? max_rows : SKIP_ONE); - binding_t *bindings = malloc((bind_cap + SKIP_ONE) * sizeof(binding_t)); + binding_t *bindings = malloc(((size_t)bind_cap + SKIP_ONE) * sizeof(binding_t)); + if (!bindings) { + cbm_store_free_nodes(scanned, scan_count); + return CBM_NOT_FOUND; /* initial binding array refused */ + } int bind_count = 0; const char *var_name = pat0->nodes[0].variable ? pat0->nodes[0].variable : CYP_ANON_HEAD_VAR; @@ -5554,8 +5686,11 @@ int cbm_cypher_execute(cbm_store_t *store, const char *query, const char *projec g_cypher_depth_clamped = 0; g_cypher_trail_truncated = 0; g_cypher_truncated = false; + g_cypher_regex_refused = false; cypher_deadline_arm(); /* #601: start the wall-clock budget for this query */ - if (max_rows <= 0) { + /* max_rows sizes the initial binding array: non-positive means the + * ceiling, and nothing above the ceiling is ever materialized anyway. */ + if (max_rows <= 0 || max_rows > CYPHER_RESULT_CEILING) { max_rows = CYPHER_RESULT_CEILING; } @@ -5647,8 +5782,8 @@ int cbm_cypher_execute(cbm_store_t *store, const char *query, const char *projec /* Any internal ceiling that prevented exhaustive evaluation: a candidate or * traversal budget, or a variable-length range clamped to the engine cap. */ out->truncated = g_cypher_truncated || g_cypher_trail_truncated != 0; + char wbuf[CBM_SZ_512] = ""; if (g_cypher_depth_clamped > 0 || g_cypher_trail_truncated) { - char wbuf[CBM_SZ_256]; if (g_cypher_depth_clamped > 0 && g_cypher_trail_truncated) { snprintf(wbuf, sizeof(wbuf), "variable-length hop range clamped to the engine ceiling (%d) and " @@ -5663,6 +5798,16 @@ int cbm_cypher_execute(cbm_store_t *store, const char *query, const char *projec snprintf(wbuf, sizeof(wbuf), "variable-length traversal budget was exhausted — results may be partial"); } + } + /* A refused `=~` or property pattern matched nothing: say so, once per + * query, next to any traversal warning. */ + if (g_cypher_regex_refused) { + size_t used = strlen(wbuf); + snprintf(wbuf + used, sizeof(wbuf) - used, + "%sa =~ or property " CBM_REG_ETOOBIG_REASON " — the comparison matched nothing", + used ? "; " : ""); + } + if (wbuf[0]) { out->warning = heap_strdup(wbuf); } diff --git a/src/cypher/cypher.h b/src/cypher/cypher.h index 9e66b56ecc..e9bd5f74c2 100644 --- a/src/cypher/cypher.h +++ b/src/cypher/cypher.h @@ -334,7 +334,8 @@ typedef struct { } cbm_cypher_result_t; /* Execute a Cypher query against a store. - * max_rows: limit on output rows (0 = use virtual ceiling of 100k). + * max_rows: limit on output rows; 0 or any value above the 100k ceiling uses + * the ceiling. * project: project name filter (NULL = all projects). * Returns -1 on error (check out->error for message). */ int cbm_cypher_execute(cbm_store_t *store, const char *query, const char *project, int max_rows, diff --git a/src/daemon/application.c b/src/daemon/application.c index a3d9d86ec0..ec81d09690 100644 --- a/src/daemon/application.c +++ b/src/daemon/application.c @@ -1514,7 +1514,8 @@ static char *application_wire_add_notice(char *wire, const char *notice) { return rebuilt; } -static char *application_job_failure_response(const cbm_index_worker_result_t *result, +static char *application_job_failure_response(const char *args_json, + const cbm_index_worker_result_t *result, const char *log_path) { char message[1024]; if (result && result->cancellation_requested) { @@ -1526,6 +1527,44 @@ static char *application_job_failure_response(const cbm_index_worker_result_t *r (void)snprintf(message, sizeof(message), "index worker containment failed (%s); inspect log: %s", cbm_proc_outcome_str(result->outcome), log_path ? log_path : "unavailable"); + } else if (result && result->response_missing) { + /* #1300: identify a silent worker separately from a supervisor read error. */ + const char *log = log_path ? log_path + : result->worker_log[0] ? result->worker_log + : "unavailable"; + const char *reason = result->response_unreadable + ? "supervisor could not read the worker response" + : "worker exited cleanly (exit 0) without writing a response"; + (void)snprintf(message, sizeof(message), + "%s; the index was not published; last phase reached: %s; inspect log: %s", + reason, result->last_phase[0] ? result->last_phase : "unknown", log); + yyjson_mut_doc *doc = yyjson_mut_doc_new(NULL); + yyjson_mut_val *root = doc ? yyjson_mut_obj(doc) : NULL; + if (root) { + yyjson_mut_doc_set_root(doc, root); + yyjson_mut_obj_add_str(doc, root, "status", "error"); + yyjson_mut_obj_add_str(doc, root, "outcome", "clean"); + yyjson_mut_obj_add_str(doc, root, "reason", "no_response"); + yyjson_mut_obj_add_strcpy(doc, root, "last_phase", + result->last_phase[0] ? result->last_phase : "unknown"); + yyjson_mut_obj_add_strcpy(doc, root, "worker_log", log); + yyjson_mut_obj_add_strcpy(doc, root, "hint", message); + yyjson_doc *args = args_json ? yyjson_read(args_json, strlen(args_json), 0) : NULL; + const char *repo = + yyjson_get_str(yyjson_obj_get(yyjson_doc_get_root(args), "repo_path")); + if (repo) + yyjson_mut_obj_add_strcpy(doc, root, "repo_path", repo); + yyjson_doc_free(args); + cbm_operation_result_t error = + cbm_operation_result_take(cbm_operation_json_write(doc), true); + char *wire = cbm_operation_result_wire_encode(&error); + cbm_operation_result_dispose(&error); + yyjson_mut_doc_free(doc); + if (wire) + return wire; + } else if (doc) { + yyjson_mut_doc_free(doc); + } } else if (result) { (void)snprintf(message, sizeof(message), "index worker ended with %s (exit=%d, signal=%d); inspect log: %s", @@ -1817,7 +1856,7 @@ static void application_job_publish(cbm_daemon_application_job_t *job, application_job_execution_t *execution) { if (!execution->response) { execution->response = application_job_failure_response( - execution->have_last_result ? &execution->last_result : NULL, + job->args_json, execution->have_last_result ? &execution->last_result : NULL, execution->last_log[0] ? execution->last_log : NULL); } cbm_daemon_application_t *application = job->application; diff --git a/src/daemon/host.c b/src/daemon/host.c index 9554add424..5d739a32d3 100644 --- a/src/daemon/host.c +++ b/src/daemon/host.c @@ -609,6 +609,12 @@ static bool host_state_prepare(host_state_t *host, const cbm_daemon_ipc_endpoint if (watcher_enabled) { host->watch_store = cbm_store_open_memory(); host->watcher = cbm_watcher_new(host->watch_store, host_watcher_index, host); + /* #1948: non-git roots stay unwatched unless watch_non_git opts in. + * Same read-once-at-startup contract as watcher_enabled. */ + if (host->watcher && cbm_config_watch_non_git(host->runtime_config)) { + cbm_watcher_set_poll_non_git(host->watcher, true); + cbm_log_info("watcher.non_git", "mode", "tree_poll"); + } } else { cbm_log_info("watcher.disabled", "reason", "config"); } diff --git a/src/daemon/ipc.c b/src/daemon/ipc.c index d30055e4b5..58062d3873 100644 --- a/src/daemon/ipc.c +++ b/src/daemon/ipc.c @@ -7079,3 +7079,21 @@ int cbm_daemon_ipc_receive_frame(cbm_daemon_ipc_connection_t *connection, uint32 return cbm_daemon_ipc_receive_frame_bounded(connection, timeout_ms, CBM_DAEMON_MAX_FRAME_SIZE, frame_out, payload_out); } + +int cbm_daemon_ipc_receive_frame_after_failed_send(cbm_daemon_ipc_connection_t *connection, + uint32_t timeout_ms, uint32_t max_payload_length, + cbm_daemon_frame_t *frame_out, + uint8_t **payload_out) { + if (!connection) { + return cbm_daemon_ipc_receive_frame_bounded(NULL, timeout_ms, max_payload_length, frame_out, + payload_out); + } + /* The poison came from the failed send alone (nothing was read yet), and it + * guards the OUTBOUND frame boundary; the inbound one is intact. Lift it for + * this one receive, then restore it so no later send can reuse the stream. */ + atomic_store_explicit(&connection->poisoned, false, memory_order_release); + int received = cbm_daemon_ipc_receive_frame_bounded(connection, timeout_ms, max_payload_length, + frame_out, payload_out); + atomic_store_explicit(&connection->poisoned, true, memory_order_release); + return received; +} diff --git a/src/daemon/ipc_internal.h b/src/daemon/ipc_internal.h index f481b7378a..5f86d3a975 100644 --- a/src/daemon/ipc_internal.h +++ b/src/daemon/ipc_internal.h @@ -75,6 +75,22 @@ int cbm_daemon_ipc_receive_frame_bounded(cbm_daemon_ipc_connection_t *connection uint32_t timeout_ms, uint32_t max_payload_length, cbm_daemon_frame_t *frame_out, uint8_t **payload_out); +/* Bounded receive of an answer the peer queued before it closed, on a + * connection whose send just failed. A daemon at capacity or stopping answers + * from its accept loop without reading the request and closes at once; when + * that close lands before the client's first send, the send fails (EPIPE) and + * poisons the connection although the complete answer is still queued for + * reading -- POSIX keeps it readable after the peer's close (a Windows server + * close discards it, so there this fails like any other receive). A failed + * send consumes no inbound bytes, so the inbound frame boundary is intact; + * call this ONLY before anything was read from the connection. The connection + * stays poisoned for sends afterwards. Returns like + * cbm_daemon_ipc_receive_frame_bounded. */ +int cbm_daemon_ipc_receive_frame_after_failed_send(cbm_daemon_ipc_connection_t *connection, + uint32_t timeout_ms, uint32_t max_payload_length, + cbm_daemon_frame_t *frame_out, + uint8_t **payload_out); + /* Narrow crash/fault seams for publication-state and retry-state tests. They * are inert unless a test installs a hook/failure count in its own process. */ typedef enum { diff --git a/src/daemon/runtime.c b/src/daemon/runtime.c index 4ccf45f543..26f04c817a 100644 --- a/src/daemon/runtime.c +++ b/src/daemon/runtime.c @@ -76,8 +76,26 @@ static _Atomic uint32_t runtime_ephemeral_linger_timeout_seam = UINT32_MAX; void cbm_daemon_runtime_service_set_ephemeral_linger_timeout_for_testing(uint32_t timeout_ms) { atomic_store(&runtime_ephemeral_linger_timeout_seam, timeout_ms); } + +/* Connection-cap seams (flaky-ledger item 12): see runtime.h. */ +static atomic_bool runtime_hold_peer_waits_seam; +void cbm_daemon_runtime_hold_peer_waits_for_testing(bool hold) { + atomic_store(&runtime_hold_peer_waits_seam, hold); +} +static _Atomic(cbm_daemon_runtime_hello_hook_t) runtime_hello_hook_seam; +void cbm_daemon_runtime_set_hello_hook_for_testing(cbm_daemon_runtime_hello_hook_t hook) { + atomic_store(&runtime_hello_hook_seam, hook); +} +static void runtime_hello_point(cbm_daemon_runtime_hello_point_t point) { + cbm_daemon_runtime_hello_hook_t hook = atomic_load(&runtime_hello_hook_seam); + if (hook) { + hook(point); + } +} +#define RUNTIME_HELLO_POINT(point) runtime_hello_point(CBM_DAEMON_RUNTIME_HELLO_POINT_##point) #else static void runtime_note_peer_image_hash(void) {} +#define RUNTIME_HELLO_POINT(point) ((void)0) #endif #ifdef _WIN32 @@ -1989,6 +2007,29 @@ static void runtime_worker_handle_activation_shutdown(cbm_daemon_runtime_worker_ runtime_worker_finish(worker); } +/* The first frame is due within the request deadline. Under the item-12 seam a + * connection that never sends one holds its slot until it closes; stop still + * interrupts the wait like any other worker receive. */ +static uint32_t runtime_first_frame_timeout_ms(const cbm_daemon_runtime_service_t *service) { +#ifdef CBM_ENABLE_TEST_SEAMS + if (atomic_load(&runtime_hold_peer_waits_seam)) { + return CBM_DAEMON_IPC_WAIT_FOREVER; + } +#endif + return service->request_timeout_ms; +} + +/* Under the same seam an inline rejection's drain ends only on the peer's + * close, never on the 250 ms bound (the drain only waits on Windows). */ +static uint32_t runtime_reject_drain_timeout_ms(void) { +#ifdef CBM_ENABLE_TEST_SEAMS + if (atomic_load(&runtime_hold_peer_waits_seam)) { + return CBM_DAEMON_IPC_WAIT_FOREVER; + } +#endif + return RUNTIME_REJECT_DRAIN_TIMEOUT_MS; +} + static void *runtime_connection_worker(void *opaque) { cbm_daemon_runtime_worker_t *worker = opaque; cbm_daemon_runtime_service_t *service = worker->service; @@ -1998,7 +2039,7 @@ static void *runtime_connection_worker(void *opaque) { char requested_build[CBM_DAEMON_BUILD_FINGERPRINT_SIZE]; int received = cbm_daemon_ipc_receive_frame_bounded( - worker->connection, service->request_timeout_ms, + worker->connection, runtime_first_frame_timeout_ms(service), CBM_DAEMON_ACTIVATION_SHUTDOWN_REQUEST_SIZE, &frame, &payload); if (received != 1 || frame.type != CBM_DAEMON_FRAME_REQUEST) { free(payload); @@ -2274,14 +2315,19 @@ static cbm_daemon_runtime_worker_t *runtime_find_free_worker_locked( return NULL; } +/* Answers WITHOUT reading the peer's HELLO, so on POSIX (where the drain is a + * no-op) the close can land before the peer has even sent it; the client reads + * this answer after its failed send (cbm_daemon_runtime_client_connect). */ static void runtime_reject_inline(cbm_daemon_ipc_connection_t *connection, const char *message) { cbm_daemon_runtime_connect_result_t result; runtime_result_rejected(&result, message); + RUNTIME_HELLO_POINT(DAEMON_REJECTING); (void)runtime_send_hello_response(connection, &result); /* Same Windows named-pipe discard hazard as runtime_worker_finish: the * peer must get to read the rejection before the handle closes. */ - cbm_daemon_ipc_connection_drain(connection, RUNTIME_REJECT_DRAIN_TIMEOUT_MS); + cbm_daemon_ipc_connection_drain(connection, runtime_reject_drain_timeout_ms()); cbm_daemon_ipc_connection_close(connection); + RUNTIME_HELLO_POINT(DAEMON_REJECTED); } static void runtime_accept_connection(cbm_daemon_runtime_service_t *service, @@ -3075,21 +3121,39 @@ cbm_daemon_runtime_client_t *cbm_daemon_runtime_client_connect( return NULL; } cbm_daemon_ipc_connection_t *connection = cbm_daemon_ipc_connect(endpoint, timeout_ms); - if (!connection || !cbm_daemon_ipc_send_frame(connection, CBM_DAEMON_FRAME_REQUEST, - CBM_DAEMON_RUNTIME_OP_HELLO, request, - (uint32_t)sizeof(request))) { - cbm_daemon_ipc_connection_close(connection); + if (!connection) { return NULL; } + RUNTIME_HELLO_POINT(CLIENT_CONNECTED); + bool hello_sent = + cbm_daemon_ipc_send_frame(connection, CBM_DAEMON_FRAME_REQUEST, CBM_DAEMON_RUNTIME_OP_HELLO, + request, (uint32_t)sizeof(request)); + RUNTIME_HELLO_POINT(CLIENT_SENT); cbm_daemon_frame_t frame = {0}; uint8_t *payload = NULL; - int received = cbm_daemon_ipc_receive_frame_bounded( - connection, timeout_ms, CBM_DAEMON_RENDEZVOUS_RESPONSE_SIZE, &frame, &payload); + /* A daemon at capacity or stopping answers from its accept loop without + * reading this HELLO and closes at once. When that close lands first, the + * send fails (EPIPE) with the whole answer already queued here: read it, + * or the caller gets a bare transport error and bootstrap classifies by + * lock state instead of by the daemon's own "capacity" / "stopping". */ + int received = + hello_sent + ? cbm_daemon_ipc_receive_frame_bounded( + connection, timeout_ms, CBM_DAEMON_RENDEZVOUS_RESPONSE_SIZE, &frame, &payload) + : cbm_daemon_ipc_receive_frame_after_failed_send( + connection, timeout_ms, CBM_DAEMON_RENDEZVOUS_RESPONSE_SIZE, &frame, &payload); bool valid = received == 1 && frame.type == CBM_DAEMON_FRAME_RESPONSE && frame.flags == CBM_DAEMON_RUNTIME_OP_HELLO && frame.length == CBM_DAEMON_RENDEZVOUS_RESPONSE_SIZE && runtime_hello_response_decode(payload, result_out); free(payload); + if (!hello_sent && (!valid || result_out->status != CBM_DAEMON_RUNTIME_CONNECT_REJECTED)) { + /* Only a rejection can answer a HELLO the daemon never received; + * anything else stays the plain send failure it always was. */ + memset(result_out, 0, sizeof(*result_out)); + cbm_daemon_ipc_connection_close(connection); + return NULL; + } if (received != 1) { /* The kernel completed the pipe/socket connect, so a server process * exists, yet the HELLO went unanswered. Name that holder: a dead diff --git a/src/daemon/runtime.h b/src/daemon/runtime.h index a462973d76..9222d4b574 100644 --- a/src/daemon/runtime.h +++ b/src/daemon/runtime.h @@ -475,6 +475,27 @@ void cbm_daemon_runtime_set_containment_hook_for_testing( * in test time. UINT32_MAX restores the production constant; any other value * (0 = expire immediately) overrides. Process-global; reset it after use. */ void cbm_daemon_runtime_service_set_ephemeral_linger_timeout_for_testing(uint32_t timeout_ms); +/* Connection-cap seams (flaky-ledger item 12). While held, the two bounded + * waits on a peer in the connection-cap path end only on that peer: a + * connection that has not sent its first frame keeps its worker slot until it + * closes (or the service stops) instead of until the request deadline, and an + * inline rejection's drain (Windows; a no-op on POSIX) lasts until the peer + * closes instead of 250 ms. A "slow hello" becomes a state the test releases, + * not a window it races. Process-global; reset it after use. */ +void cbm_daemon_runtime_hold_peer_waits_for_testing(bool hold); +/* The four points that order a HELLO against an inline accept-loop rejection. + * The hook runs on the client thread for the CLIENT_* points and on the accept + * thread for the DAEMON_* points (never under the service mutex), so a test can + * pin either order: daemon answers after the HELLO is on the wire, or the + * daemon has answered and closed before the HELLO is sent. NULL removes it. */ +typedef enum { + CBM_DAEMON_RUNTIME_HELLO_POINT_CLIENT_CONNECTED = 0, /* transport up, HELLO not sent */ + CBM_DAEMON_RUNTIME_HELLO_POINT_CLIENT_SENT, /* HELLO send attempted */ + CBM_DAEMON_RUNTIME_HELLO_POINT_DAEMON_REJECTING, /* inline rejection not yet written */ + CBM_DAEMON_RUNTIME_HELLO_POINT_DAEMON_REJECTED, /* inline rejection written + closed */ +} cbm_daemon_runtime_hello_point_t; +typedef void (*cbm_daemon_runtime_hello_hook_t)(cbm_daemon_runtime_hello_point_t point); +void cbm_daemon_runtime_set_hello_hook_for_testing(cbm_daemon_runtime_hello_hook_t hook); #endif #endif /* CBM_DAEMON_RUNTIME_H */ diff --git a/src/discover/discover.c b/src/discover/discover.c index 4595a8b9e4..fa8e4bf2b5 100644 --- a/src/discover/discover.c +++ b/src/discover/discover.c @@ -549,8 +549,23 @@ static void file_list_add_ignored(file_list_t *fl, const char *rel_path, const c fl->ignored_count++; } +/* Nanosecond mtime from a stat result, per platform. Windows' wide_stat only + * carries whole seconds, so the value is seconds scaled to ns there. */ +#define DISCOVER_NS_PER_SEC 1000000000LL +static int64_t discover_stat_mtime_ns(const struct stat *st) { +#ifdef __APPLE__ + return ((int64_t)st->st_mtimespec.tv_sec * DISCOVER_NS_PER_SEC) + + (int64_t)st->st_mtimespec.tv_nsec; +#elif defined(_WIN32) + return (int64_t)st->st_mtime * DISCOVER_NS_PER_SEC; +#else + return ((int64_t)st->st_mtim.tv_sec * DISCOVER_NS_PER_SEC) + (int64_t)st->st_mtim.tv_nsec; +#endif +} + static void fl_add(file_list_t *fl, const char *abs_path, const char *rel_path, CBMLanguage lang, - int64_t size) { + const struct stat *st) { + int64_t size = (int64_t)st->st_size; if (fl->max_files >= 0 && fl->count >= fl->max_files) { fl->limit_exceeded = true; return; @@ -605,6 +620,7 @@ static void fl_add(file_list_t *fl, const char *abs_path, const char *rel_path, fi->rel_path = relative_copy; fi->language = lang; fi->size = size; + fi->mtime_ns = discover_stat_mtime_ns(st); } /* ── Recursive walk ─────────────────────────────── */ @@ -939,9 +955,10 @@ static void walk_dir_process_file(const char *abs_path, const char *rel_path, co const cbm_discover_opts_t *opts, const gitignore_link_t *ignore_chain, const cbm_gitignore_t *global_gi, - const cbm_gitignore_t *cbmignore, off_t size, file_list_t *out) { + const cbm_gitignore_t *cbmignore, const struct stat *st, + file_list_t *out) { const char *skip_reason = - file_skip_reason(name, rel_path, opts, ignore_chain, global_gi, cbmignore, size); + file_skip_reason(name, rel_path, opts, ignore_chain, global_gi, cbmignore, st->st_size); if (skip_reason) { /* Deliberately not indexed (#963) — record so callers can surface it. * Unsupported-language files below are NOT recorded: "no grammar for @@ -954,7 +971,7 @@ static void walk_dir_process_file(const char *abs_path, const char *rel_path, co if (lang == CBM_LANG_COUNT) { return; } - fl_add(out, abs_path, rel_path, lang, size); + fl_add(out, abs_path, rel_path, lang, st); } typedef struct { @@ -1066,7 +1083,7 @@ static void walk_dir_process_entry(cbm_dirent_t *entry, const walk_frame_t *fram } } else if (S_ISREG(st.st_mode)) { walk_dir_process_file(abs_path, rel_path, entry->name, opts, frame->ignore_chain, global_gi, - cbmignore, st.st_size, out); + cbmignore, &st, out); } } diff --git a/src/discover/discover.h b/src/discover/discover.h index 89ccca4f36..a9c2a5449b 100644 --- a/src/discover/discover.h +++ b/src/discover/discover.h @@ -147,6 +147,7 @@ typedef struct { char *rel_path; /* relative to repo root (heap-allocated) */ CBMLanguage language; /* detected language */ int64_t size; /* file size in bytes */ + int64_t mtime_ns; /* modification time, ns (whole seconds on Windows) */ } cbm_file_info_t; typedef struct { diff --git a/src/foundation/compat_regex.c b/src/foundation/compat_regex.c index 14bd815fb0..2e0e9299f9 100644 --- a/src/foundation/compat_regex.c +++ b/src/foundation/compat_regex.c @@ -3,11 +3,322 @@ * * POSIX: direct wrappers around . * Windows: vendored TRE regex library (BSD-licensed). + * + * Both backends expand bounded repetition at compile time, so cbm_regcomp + * sizes a pattern first (the compile-size guard below) and refuses one whose + * compiled form would be too large, before the platform compiler allocates + * anything. */ #include "foundation/constants.h" #include "foundation/compat_regex.h" +#include +#include +#include #include +#include +#include + +/* ── Compile-size guard (shared by both backends) ───────────────── + * + * The platform compilers expand `X{m,n}` into n copies of X, so nested + * intervals multiply: `((((a){20}){20}){20}){20}` is 160,000 expanded atoms + * from 25 bytes of pattern. A K-way alternation additionally costs about K^2 + * position-set entries on the TRE-derived backends (macOS libc and the vendored + * TRE), because every union node copies its branches' sets. + * + * Measured on macOS libc with REG_EXTENDED|REG_NOSUB: the nested shape costs + * about 0.6 KB of heap per expanded atom, 1.1 KB with submatch tracking on, + * 1.7 KB with REG_ICASE and 2.3 KB with both; a K-way alternation adds about + * 70 B x K^2 on top of its branches (twice that under REG_ICASE), and an + * alternation under an interval pays that for every copy, about 1 KB per unit + * with REG_NOSUB and 2.8 KB with REG_ICASE plus submatches. glibc costs about + * 0.45 KB per atom and expands intervals the same way. + * + * cbm_regcomp_estimate_units() counts expanded atoms in one linear pass with + * saturating arithmetic and charges a K-way alternation K^2/8 units, the + * measured ratio between the two costs. With CBM_REGEX_COMPILE_BUDGET_UNITS = + * 32768 the heaviest accepted compile is a repeated alternation at about 30 MB + * with REG_NOSUB and about 90 MB under the ICASE-plus-submatch flag set no + * call site uses; the nested shape tops out at 13^4 = 28,561 units, about + * 16 MB. `(\w{1,100}){1,20}` (2,000 units) and an alternation of 500 short + * names (31,250 units plus its letters) remain accepted. */ + +enum { REGEX_GROUP_DEPTH_MAX = 64 }; + +typedef struct { + uint64_t total; /* units of this group so far; includes `last` */ + uint64_t last; /* units of the element a following quantifier applies to */ + uint64_t branches; /* alternatives in this group: 1 + the number of '|' */ +} regex_frame_t; + +static uint64_t regex_sat_add(uint64_t a, uint64_t b) { + return a > UINT64_MAX - b ? UINT64_MAX : a + b; +} + +static uint64_t regex_sat_mul(uint64_t a, uint64_t b) { + if (a == 0 || b == 0) { + return 0; + } + return a > UINT64_MAX / b ? UINT64_MAX : a * b; +} + +static void regex_frame_atom(regex_frame_t *f, uint64_t units) { + f->total = regex_sat_add(f->total, units); + f->last = units; +} + +/* An interval applies to the last element: n copies of it replace the one. */ +static void regex_frame_repeat_last(regex_frame_t *f, uint64_t copies) { + uint64_t grown = regex_sat_mul(f->last, copies); + f->total = regex_sat_add(f->total - f->last, grown); + f->last = grown; +} + +/* Units a closed group contributes to its parent: its content plus the K^2/8 + * alternation share, never less than one atom. */ +static uint64_t regex_frame_units(const regex_frame_t *f) { + uint64_t alternation = regex_sat_mul(f->branches, f->branches) / 8; + uint64_t units = regex_sat_add(f->total, alternation); + return units ? units : 1; +} + +/* Byte-locale TRE expands a named class into disjoint ranges. Count those + * ranges even in multibyte locales, where the backend may use one class item. */ +static uint64_t regex_class_items(const char *name, size_t len, bool icase) { + char buf[64]; + if (len >= sizeof(buf)) { + return 1; /* invalid class: the backend rejects it */ + } + memcpy(buf, name, len); + buf[len] = '\0'; + wctype_t cls = wctype(buf); + uint64_t items = 0; + bool previous = false; + for (wint_t c = 0; c < 256; c++) { + bool member = + cls && (iswctype(c, cls) || + (icase && (iswctype(towlower(c), cls) || iswctype(towupper(c), cls)))); + if (member && !previous) { + items++; + } + previous = member; + } + return items ? items : 1; +} + +static size_t regex_bracket_char(const char *p, size_t i, wchar_t *out) { + mbstate_t state = {0}; + size_t len = mbrtowc(out, p + i, MB_LEN_MAX, &state); + if (len == (size_t)-1 || len == (size_t)-2 || len == 0) { + *out = (unsigned char)p[i]; + len = 1; /* malformed input is left for the backend to reject */ + } + return i + len; +} + +/* Bracket members form a union in TRE; repetition multiplies that union too. + * Count ranges, leading ']', named classes and case-folded range items, then + * apply the same conservative union charge used for ordinary alternation. */ +static size_t regex_skip_bracket(const char *p, size_t i, bool icase, uint64_t *units) { + size_t j = i + 1; + bool negate = p[j] == '^'; + if (negate) { + j++; + } + uint64_t items = 0; + if (p[j] == ']') { + items++; + j++; + } + while (p[j] && p[j] != ']') { + if (p[j] == '[' && (p[j + 1] == ':' || p[j + 1] == '=' || p[j + 1] == '.')) { + char close = p[j + 1]; + size_t k = j + 2; + while (p[k] && !(p[k] == close && p[k + 1] == ']')) { + k++; + } + items = regex_sat_add( + items, close == ':' ? regex_class_items(p + j + 2, k - j - 2, icase) : 1); + j = p[k] ? k + 2 : k; + } else { + wchar_t first; + j = regex_bracket_char(p, j, &first); + wchar_t last = first; + if (p[j] == '-' && p[j + 1] && p[j + 1] != ']') { + j = regex_bracket_char(p, j + 1, &last); + } + items = regex_sat_add(items, 1); + if (icase && last >= first) { + if (last < 256) { + wint_t previous = WEOF; + for (wint_t c = (wint_t)first; c <= (wint_t)last; c++) { + wint_t folded = + iswlower(c) ? towupper(c) : (iswupper(c) ? towlower(c) : WEOF); + if (folded != WEOF && (previous == WEOF || folded != previous + 1)) { + items = regex_sat_add(items, 1); + } + previous = folded; + } + } else { + /* ponytail: bound Unicode folding by one item per codepoint; + * count locale-specific runs if this ceiling rejects useful patterns. */ + items = regex_sat_add(items, (uint64_t)(last - first) + 1); + } + } + } + } + if (negate) { + items = regex_sat_add(items, 1); /* complement's final range */ + } + *units = regex_sat_add(items, regex_sat_mul(items, items) / 8); + return p[j] ? j + 1 : j; +} + +static bool regex_parse_digits(const char *p, size_t *i, uint64_t *value) { + bool any = false; + *value = 0; + while (p[*i] >= '0' && p[*i] <= '9') { + *value = regex_sat_add(regex_sat_mul(*value, 10), (uint64_t)(p[*i] - '0')); + (*i)++; + any = true; + } + return any; +} + +/* Parse the interval body that starts at p[i], just past the opening brace + * (`{` in extended syntax, `\{` in basic). On success stores the number of + * copies the compiler will make (the upper bound, or the lower bound for + * `{m,}`, at least 1) and the index just past the closing brace. Anything that + * is not an interval leaves the brace to be counted as a literal, which is what + * the backends that accept it do; the others reject the pattern. */ +static bool regex_parse_interval(const char *p, size_t i, bool extended, uint64_t *copies, + size_t *end) { + uint64_t lower = 0; + uint64_t upper = 0; + bool has_lower = regex_parse_digits(p, &i, &lower); + bool has_upper = false; + if (p[i] == ',') { + i++; + has_upper = regex_parse_digits(p, &i, &upper); + } else { + if (!has_lower) { + return false; + } + upper = lower; + has_upper = true; + } + if (extended) { + if (p[i] != '}') { + return false; + } + i++; + } else { + if (p[i] != '\\' || p[i + 1] != '}') { + return false; + } + i += 2; + } + uint64_t n = has_upper ? upper : lower; + if (has_upper && lower > upper) { + n = lower; /* `{5,3}` is rejected by the backends; count the larger bound */ + } + *copies = n ? n : 1; + *end = i; + return true; +} + +uint64_t cbm_regcomp_estimate_units(const char *pattern, int flags) { + const bool extended = (flags & CBM_REG_EXTENDED) != 0; + regex_frame_t frames[REGEX_GROUP_DEPTH_MAX]; + int depth = 0; + frames[0] = (regex_frame_t){0, 0, 1}; + + size_t i = 0; + while (pattern[i]) { + char c = pattern[i]; + if (c == '[') { + uint64_t units = 0; + i = regex_skip_bracket(pattern, i, (flags & CBM_REG_ICASE) != 0, &units); + regex_frame_atom(&frames[depth], units); + continue; + } + /* In extended syntax the bare characters operate and `\x` is one + * literal; in basic syntax only `\(`, `\)`, `\{`, `\|`, `\+`, `\?` and + * a bare `*` operate. */ + char op = 0; + size_t len = 1; + if (c == '\\') { + len = pattern[i + 1] ? 2 : 1; + if (!extended && len == 2 && strchr("(){|+?", pattern[i + 1])) { + op = pattern[i + 1]; + } + } else if (extended ? strchr("(){|*+?", c) != NULL : c == '*') { + op = c; + } + regex_frame_t *f = &frames[depth]; + switch (op) { + case '(': + if (depth + 1 >= REGEX_GROUP_DEPTH_MAX) { + return UINT64_MAX; + } + frames[++depth] = (regex_frame_t){0, 0, 1}; + break; + case ')': + if (depth == 0) { + regex_frame_atom(f, 1); /* unmatched: a literal where accepted */ + break; + } + depth--; + regex_frame_atom(&frames[depth], regex_frame_units(f)); + break; + case '|': + f->branches = regex_sat_add(f->branches, 1); + f->last = 0; + break; + case '{': { + uint64_t copies = 0; + size_t end = 0; + if (regex_parse_interval(pattern, i + len, extended, &copies, &end)) { + regex_frame_repeat_last(f, copies); + i = end; + continue; + } + regex_frame_atom(f, 1); + break; + } + case '*': + case '+': + case '?': + break; /* unbounded or optional repetition is not expanded */ + default: + regex_frame_atom(f, 1); + break; + } + i += len; + } + while (depth > 0) { + regex_frame_t *f = &frames[depth]; + depth--; + regex_frame_atom(&frames[depth], regex_frame_units(f)); + } + return regex_frame_units(&frames[0]); +} + +/* Refuse a pattern before the platform compiler sees it. */ +static int regex_compile_guard(const char *pattern, int flags) { + size_t len = 0; + while (len <= (size_t)CBM_REGEX_PATTERN_MAX_BYTES && pattern[len]) { + len++; + } + if (len > (size_t)CBM_REGEX_PATTERN_MAX_BYTES) { + return CBM_REG_ETOOBIG; + } + if (cbm_regcomp_estimate_units(pattern, flags) > (uint64_t)CBM_REGEX_COMPILE_BUDGET_UNITS) { + return CBM_REG_ETOOBIG; + } + return CBM_REG_OK; +} #ifdef _WIN32 @@ -31,6 +342,10 @@ static int translate_flags_tre(int flags) { } int cbm_regcomp(cbm_regex_t *r, const char *pattern, int flags) { + int refused = regex_compile_guard(pattern, flags); + if (refused != CBM_REG_OK) { + return refused; + } regex_t *re = (regex_t *)r->opaque; int rc = tre_regcomp(re, pattern, translate_flags_tre(flags)); return rc == 0 ? CBM_REG_OK : rc; @@ -88,6 +403,10 @@ static int translate_flags(int flags) { } int cbm_regcomp(cbm_regex_t *r, const char *pattern, int flags) { + int refused = regex_compile_guard(pattern, flags); + if (refused != CBM_REG_OK) { + return refused; + } regex_t *re = (regex_t *)r->opaque; int rc = regcomp(re, pattern, translate_flags(flags)); return rc == 0 ? CBM_REG_OK : rc; diff --git a/src/foundation/compat_regex.h b/src/foundation/compat_regex.h index c572459677..7545b25717 100644 --- a/src/foundation/compat_regex.h +++ b/src/foundation/compat_regex.h @@ -11,6 +11,7 @@ #include "foundation/constants.h" #include +#include /* ── Flags ────────────────────────────────────────────────────── */ @@ -23,6 +24,29 @@ #define CBM_REG_OK 0 #define CBM_REG_NOMATCH (-1) +/* cbm_regcomp refused the pattern before compiling it: longer than + * CBM_REGEX_PATTERN_MAX_BYTES, or its compiled form would exceed + * CBM_REGEX_COMPILE_BUDGET_UNITS (see cbm_regcomp_estimate_units). Well above + * every platform compile error code, which are small positive integers. */ +#define CBM_REG_ETOOBIG 100 +/* Reason text for a CBM_REG_ETOOBIG refusal, for user-facing error messages. */ +#define CBM_REG_ETOOBIG_REASON \ + "regex pattern too large to compile: reduce nested {m,n} repetition or alternation" + +/* ── Compile-size guard ───────────────────────────────────────── */ + +enum { + /* Longest pattern cbm_regcomp hands to the platform compiler. Bounds only + * the sizing pass: a pattern without repetition costs one unit per atom, + * so the unit budget below already bounds its compiled size. Equal to the + * budget, so a realistic pattern is only ever refused for its cost. */ + CBM_REGEX_PATTERN_MAX_BYTES = CBM_SZ_32K, + /* Largest estimated compiled size, in units of one expanded atom, that + * cbm_regcomp accepts. Bounds what bounded repetition and alternation can + * expand a pattern into. compat_regex.c documents the measured bytes per + * unit and what the budget admits. */ + CBM_REGEX_COMPILE_BUDGET_UNITS = CBM_SZ_32K +}; /* ── Types ────────────────────────────────────────────────────── */ @@ -40,9 +64,21 @@ typedef struct { /* ── Functions ────────────────────────────────────────────────── */ -/* Compile a regular expression. Returns CBM_REG_OK on success, non-zero on error. */ +/* Compile a regular expression. Returns CBM_REG_OK on success, non-zero on error: + * CBM_REG_ETOOBIG when the pattern is refused before compilation (nothing to + * free), otherwise the platform compiler's error code. */ int cbm_regcomp(cbm_regex_t *r, const char *pattern, int flags); +/* Estimated size of the compiled form of pattern, in units of one expanded + * atom: an atom costs 1, concatenation and alternation add, a group costs its + * content, and an interval `{m}`, `{m,}`, `{m,n}` multiplies the element it + * applies to by the number of copies the compiler makes (the upper bound; m for + * `{m,}`; at least 1), so nested intervals multiply. `*`, `+` and `?` do not + * multiply. A K-way alternation adds K*K/8 for its position sets. Arithmetic + * saturates at UINT64_MAX; a group nesting deeper than the pass tracks returns + * UINT64_MAX. Honors CBM_REG_EXTENDED (basic syntax otherwise). */ +uint64_t cbm_regcomp_estimate_units(const char *pattern, int flags); + /* Execute compiled regex against str. nmatch/matches may be 0/NULL. * eflags: 0 or combination of platform-specific exec flags. * Returns CBM_REG_OK on match, CBM_REG_NOMATCH on no match. */ diff --git a/src/foundation/subprocess.c b/src/foundation/subprocess.c index 96d122e36d..c0cea3fc4c 100644 --- a/src/foundation/subprocess.c +++ b/src/foundation/subprocess.c @@ -30,6 +30,9 @@ #include #endif extern char **environ; +#ifdef __linux__ +#include /* SYS_close_range: raw syscall, see cbm_posix_close_fds_from */ +#endif #include #include #include @@ -1069,6 +1072,64 @@ static void cbm_posix_reset_child_signals(void) { (void)sigprocmask(SIG_SETMASK, &empty, NULL); } +/* Close every descriptor >= lowfd in the fork+exec child (#1484). + * + * The portable answer is a close() per possible descriptor, which costs + * O(RLIMIT_NOFILE) syscalls: at nofile=524288 that is ~0.8 s between fork and + * exec for EVERY spawn, independent of how many descriptors are actually open. + * The kernel can do the same in one call: + * - Linux >= 5.9: close_range(lowfd, ~0U, 0). Issued as a raw syscall, not + * through the libc wrapper: the static musl release build has no wrapper + * and the glibc wrapper only exists from 2.34, while the syscall number is + * 436 on the supported x86-64 and AArch64 targets. Other architectures + * use their header's number, or the loop if no number is available. + * An older kernel answers + * ENOSYS (or EPERM under a seccomp filter that predates it) and we fall back. + * - FreeBSD/OpenBSD/NetBSD/DragonFly: closefrom(lowfd). + * - otherwise (and on fallback): the bounded close() loop. + * macOS never reaches this on its primary path: posix_spawn's + * CLOEXEC_DEFAULT does the closing (see cbm_posix_spawn_apple). + * + * Runs between fork and exec, so it is async-signal-safe: raw syscalls and + * close() only, no allocation, no stdio. Returns the strategy that did the + * work so the test seam can assert which one ran. */ +#if defined(__linux__) && !defined(SYS_close_range) && (defined(__x86_64__) || defined(__aarch64__)) +#define SYS_close_range 436 +#endif + +#ifdef CBM_ENABLE_TEST_SEAMS +static bool g_force_close_range_enosys = false; +#endif + +static cbm_fd_close_strategy_t cbm_posix_close_fds_from(int lowfd, long max_fd) { +#if defined(__linux__) && defined(SYS_close_range) + bool range_unavailable = false; +#ifdef CBM_ENABLE_TEST_SEAMS + range_unavailable = g_force_close_range_enosys; +#endif + if (!range_unavailable && syscall(SYS_close_range, (unsigned int)lowfd, ~0U, 0U) == 0) { + return CBM_FD_CLOSE_RANGE; + } +#elif defined(__FreeBSD__) || defined(__OpenBSD__) || defined(__NetBSD__) || defined(__DragonFly__) + (void)max_fd; + closefrom(lowfd); + return CBM_FD_CLOSEFROM; +#endif + for (int fd = lowfd; fd < max_fd; fd++) { + (void)close(fd); + } + return CBM_FD_CLOSE_LOOP; +} + +#ifdef CBM_ENABLE_TEST_SEAMS +void cbm_subprocess_force_close_range_enosys_for_testing(bool force) { + g_force_close_range_enosys = force; +} +cbm_fd_close_strategy_t cbm_subprocess_close_fds_from_for_testing(int lowfd, long max_fd) { + return cbm_posix_close_fds_from(lowfd, max_fd); +} +#endif + /* fork+exec child setup. On Apple this runs ONLY for the exec-failure * fallback (see cbm_posix_spawn_apple), which preserves the documented * "bogus binary => child exits 127" contract across platforms. */ @@ -1090,9 +1151,7 @@ static void cbm_posix_child_exec(cbm_subprocess_t *process, int input, int outpu if (output > STDERR_FILENO) { (void)close(output); } - for (int fd = STDERR_FILENO + 1; fd < max_fd; fd++) { - (void)close(fd); - } + (void)cbm_posix_close_fds_from(STDERR_FILENO + 1, max_fd); if (process->envp) { environ = process->envp; /* execvp passes environ to the new image */ } diff --git a/src/foundation/subprocess.h b/src/foundation/subprocess.h index 333509fb60..22981b501e 100644 --- a/src/foundation/subprocess.h +++ b/src/foundation/subprocess.h @@ -193,4 +193,21 @@ void cbm_subprocess_force_spawn_eagain_for_testing(int attempts); int cbm_subprocess_pending_spawn_eagain_for_testing(void); #endif +/* How the POSIX fork+exec child closed its inherited descriptors (#1484). */ +typedef enum { + CBM_FD_CLOSE_RANGE = 1, /* Linux close_range(2): one syscall */ + CBM_FD_CLOSEFROM, /* BSD closefrom(3): one call */ + CBM_FD_CLOSE_LOOP /* close() per descriptor up to _SC_OPEN_MAX: O(RLIMIT_NOFILE) */ +} cbm_fd_close_strategy_t; + +#if defined(CBM_ENABLE_TEST_SEAMS) && !defined(_WIN32) +/* Run the child's close-inherited-descriptors step in the CALLING process for + * descriptors >= lowfd (callers pick a lowfd above everything they need) and + * report which strategy did the work. Test builds only. */ +cbm_fd_close_strategy_t cbm_subprocess_close_fds_from_for_testing(int lowfd, long max_fd); +/* Make the close_range fast path behave as if the kernel lacked it (ENOSYS), + * so the fallback loop is exercised deterministically. Test builds only. */ +void cbm_subprocess_force_close_range_enosys_for_testing(bool force); +#endif + #endif /* CBM_SUBPROCESS_H */ diff --git a/src/foundation/workspace.c b/src/foundation/workspace.c index ec6f8c1d75..37fe9cbbe6 100644 --- a/src/foundation/workspace.c +++ b/src/foundation/workspace.c @@ -6,6 +6,7 @@ #include "foundation/compat.h" #include "foundation/compat_fs.h" +#include "foundation/constants.h" #include "foundation/platform.h" #include "foundation/sha256.h" #ifdef _WIN32 @@ -307,6 +308,22 @@ static bool ws_is_windows_user_programs_tree(const char *path) { return true; } +/* strncmp that treats "/" and "\" as the same character. On Windows a + * canonical path arrives with backslashes from the resolver and with forward + * slashes from callers that normalize, and both spell one directory. Stops at + * the end of either string like strncmp does. */ +static bool ws_prefix_equal(const char *a, const char *b, size_t len) { + for (size_t i = 0; i < len; i++) { + if (a[i] != b[i] && !(ws_is_sep(a[i]) && ws_is_sep(b[i]))) { + return false; + } + if (a[i] == '\0') { + break; + } + } + return true; +} + /* True when b is a or lives under a. Compares on a separator boundary so * "/a/bc" is not treated as living under "/a/b". */ static bool ws_is_ancestor_or_equal(const char *a, const char *b) { @@ -317,7 +334,7 @@ static bool ws_is_ancestor_or_equal(const char *a, const char *b) { while (la > 1 && ws_is_sep(a[la - 1])) { la--; } - if (strncmp(a, b, la) != 0) { + if (!ws_prefix_equal(a, b, la)) { return false; } return b[la] == '\0' || ws_is_sep(b[la]); @@ -668,14 +685,34 @@ bool cbm_workspace_root_allowed(const char *canonical_path, const char *home_dir /* ── Environment helpers ──────────────────────────────────────────────────── */ /* Callers should not each re-derive these; a caller that resolved the home - * directory differently would classify the same path differently. */ + * directory differently would classify the same path differently. + * + * The value is the CANONICAL form of $HOME (then %USERPROFILE%), through the + * same resolver callers apply to the path they classify. The policy compares + * the two byte for byte, and a home reached through a link — "/home" kept on + * another volume, a macOS firmlink, a HOME that is itself a link — differs + * from its resolved form, so the raw value would never match and the home + * rule would not fire. A home that cannot be resolved (it does not exist) + * falls back to the raw value, which is all the comparison could use anyway. + * + * Storage is per thread and recomputed on every call, as cbm_resolve_cache_dir + * does: no lock, a changed environment is seen at once, and the pointer stays + * valid until the next call on the same thread. */ const char *cbm_workspace_home_dir(void) { - const char *home = getenv("HOME"); - if (home && home[0]) { - return home; + static CBM_TLS char resolved[CBM_SZ_4K]; + char raw[CBM_SZ_4K]; + const char *home = cbm_safe_getenv("HOME", raw, sizeof(raw), NULL); + if (!home || !home[0]) { + home = cbm_safe_getenv("USERPROFILE", raw, sizeof(raw), NULL); + } + if (!home || !home[0]) { + return NULL; + } + if (!cbm_canonical_path(home, resolved, sizeof(resolved))) { + (void)snprintf(resolved, sizeof(resolved), "%s", home); } - home = getenv("USERPROFILE"); - return (home && home[0]) ? home : NULL; + cbm_normalize_path_sep(resolved); + return resolved; } const char *cbm_workspace_cache_dir(void) { diff --git a/src/main.c b/src/main.c index fbb702ea95..6979666a66 100644 --- a/src/main.c +++ b/src/main.c @@ -1217,7 +1217,7 @@ static void print_help(void) { printf(" [--clients=] Run " "'install --clients' to list tokens\n"); printf(" codebase-memory-cli install-hooks [--dry-run] [--plan] [--clients=]\n"); - printf(" codebase-memory-cli uninstall [-y|-n] [--dry-run]\n"); + printf(" codebase-memory-cli uninstall [-y|-n] [--dry-run] [--delete-indexes]\n"); printf(" codebase-memory-cli update [-y|-n]\n"); printf(" codebase-memory-cli config \n"); printf(" codebase-memory-cli daemon \n"); diff --git a/src/operations/adr.c b/src/operations/adr.c index f645bfd8ba..d8b6384bf9 100644 --- a/src/operations/adr.c +++ b/src/operations/adr.c @@ -1,3 +1,4 @@ +#include "operations/json_args.h" #include "operations/result_wire.h" #include "operations/adr.h" #include "operations/project_arg.h" diff --git a/src/operations/changes.c b/src/operations/changes.c index 87c0271a1e..90c873d458 100644 --- a/src/operations/changes.c +++ b/src/operations/changes.c @@ -1,7 +1,8 @@ +#include "foundation/mem_core.h" +#include "operations/json_args.h" #include "operations/output_budget.h" #include "operations/result_wire.h" #include "operations/changes.h" -#include "operations/json_args.h" #include "foundation/compat_fs.h" #include "foundation/constants.h" @@ -87,13 +88,7 @@ static char *changes_project_arg(const char *args) { } static int changes_int_arg(const char *args, const char *name, int fallback) { - yyjson_doc *doc = changes_args_doc(args); - yyjson_val *root = doc ? yyjson_doc_get_root(doc) : NULL; - yyjson_val *value = yyjson_is_obj(root) ? yyjson_obj_get(root, name) : NULL; - int result = value && yyjson_is_int(value) ? (int)yyjson_get_sint(value) : fallback; - if (doc) - yyjson_doc_free(doc); - return result; + return cbm_json_int_arg(args, name, fallback); } static int changes_clamp_depth(int depth, const char *tool) { @@ -804,6 +799,89 @@ static char *detect_budget_floor(bool legacy_json, bool engine_saturated, size_t return json; } +static bool detect_git_first_line(const cbm_operation_runtime_t *runtime, const char *command, + char *out, size_t out_size) { + out[0] = '\0'; + char output_path[CBM_SZ_2K] = {0}; + cbm_proc_result_t result = {0}; + int run = cbm_operation_run_shell_command(runtime, command, output_path, &result); + bool ok = run == 0 && result.exit_code == 0 && !result.cancellation_requested && + !cbm_operation_runtime_cancelled(runtime); + FILE *fp = ok ? cbm_fopen(output_path, "rb") : NULL; + bool read = fp && fgets(out, (int)out_size, fp) != NULL; + if (fp) { + (void)fclose(fp); + } + if (output_path[0]) { + (void)cbm_unlink(output_path); + } + size_t length = read ? strlen(out) : 0; + /* A line that filled the buffer without its newline was truncated. */ + bool complete = length > 0 && out[length - 1] == '\n'; + while (length > 0 && (out[length - 1] == '\n' || out[length - 1] == '\r')) { + out[--length] = '\0'; + } + ok = ok && complete && length > 0 && out[0] != '-' && cbm_validate_shell_arg(out) && + changes_validate_windows_cmd_interpolation_arg(out); + if (!ok) { + out[0] = '\0'; + } + return ok; +} + +/* #1357: without base_branch, diff against the repository's default branch + * rather than a literal "main" that trunk/master/develop repositories do not + * have. Order: origin/HEAD (what the clone calls the default), a local main, + * a local master, then the current branch's upstream. main and master come + * before the upstream on purpose: a pushed feature branch tracks its own + * remote twin, and diffing against that would silently shrink the impact + * analysis to unpushed commits. Nothing resolves -> false, out = "main", and + * the caller reports that ref as the one it tried. */ +static bool detect_default_base(const cbm_operation_runtime_t *runtime, const char *root_path, + char *out, size_t out_size) { +#ifdef _WIN32 +#define DETECT_GIT_Q "\"" +#define DETECT_GIT_NULL "2>NUL" +#else +#define DETECT_GIT_Q "'" +#define DETECT_GIT_NULL "2>/dev/null" +#endif + char command[CBM_SZ_2K]; + char line[CBM_SZ_1K]; + snprintf(command, sizeof(command), + "git -C " DETECT_GIT_Q "%s" DETECT_GIT_Q + " symbolic-ref -q --short refs/remotes/origin/HEAD " DETECT_GIT_NULL, + root_path); + if (detect_git_first_line(runtime, command, line, sizeof(line))) { + snprintf(out, out_size, "%s", line); + return true; + } + static const char *const local_defaults[] = {"main", "master"}; + for (size_t i = 0; i < sizeof(local_defaults) / sizeof(local_defaults[0]); i++) { + snprintf(command, sizeof(command), + "git -C " DETECT_GIT_Q "%s" DETECT_GIT_Q " rev-parse -q --verify " DETECT_GIT_Q + "refs/heads/%s^{commit}" DETECT_GIT_Q " " DETECT_GIT_NULL, + root_path, local_defaults[i]); + if (detect_git_first_line(runtime, command, line, sizeof(line))) { + snprintf(out, out_size, "%s", local_defaults[i]); + return true; + } + } + snprintf(command, sizeof(command), + "git -C " DETECT_GIT_Q "%s" DETECT_GIT_Q + " rev-parse -q --abbrev-ref --symbolic-full-name " DETECT_GIT_Q + "@{upstream}" DETECT_GIT_Q " " DETECT_GIT_NULL, + root_path); + if (detect_git_first_line(runtime, command, line, sizeof(line))) { + snprintf(out, out_size, "%s", line); + return true; + } +#undef DETECT_GIT_Q +#undef DETECT_GIT_NULL + snprintf(out, out_size, "%s", "main"); + return false; +} + cbm_operation_result_t cbm_changes_operation_execute(const char *args, const cbm_operation_runtime_t *runtime) { char *project = changes_project_arg(args); @@ -829,9 +907,8 @@ cbm_operation_result_t cbm_changes_operation_execute(const char *args, } cbm_operation_arg_free(since); /* no-op after the swap (since is NULL); frees it otherwise */ - if (!base_branch) { - base_branch = cbm_mem_strdup(CBM_MEM_CLASS_OPERATION_ARG, "main"); - } + bool base_defaulted = base_branch == NULL; + bool default_base_found = true; cbm_operation_result_t result = {0}; cbm_store_t *store = NULL; @@ -859,8 +936,8 @@ cbm_operation_result_t cbm_changes_operation_execute(const char *args, * ""...HEAD`; a value starting with '-' would be read by git as an * option rather than a ref (e.g. `--output=` writes the diff to an * arbitrary file). A real git ref never begins with '-'. */ - if (!cbm_validate_shell_arg(base_branch) || base_branch[0] == '-' || - !changes_validate_windows_cmd_interpolation_arg(base_branch)) { + if (base_branch && (!cbm_validate_shell_arg(base_branch) || base_branch[0] == '-' || + !changes_validate_windows_cmd_interpolation_arg(base_branch))) { result = changes_error("base_branch contains invalid characters"); goto done; } @@ -878,6 +955,17 @@ cbm_operation_result_t cbm_changes_operation_execute(const char *args, goto done; } + if (base_defaulted) { + char default_base[CBM_SZ_1K]; + default_base_found = + detect_default_base(runtime, root_path, default_base, sizeof(default_base)); + base_branch = cbm_mem_strdup(CBM_MEM_CLASS_OPERATION_ARG, default_base); + if (!base_branch) { + result = changes_error("out of memory resolving default base"); + goto done; + } + } + /* Every detect snapshot and cursor is generation-bound. Validate the * metadata immediately after store resolution so fresh requests and cursor * replays fail identically before Git work or cursor minting. */ @@ -950,9 +1038,16 @@ cbm_operation_result_t cbm_changes_operation_execute(const char *args, int run = git.run; changes_git_finish(&git); result = changes_error( - cancelled ? "detect_changes cancelled for this request" - : run != 0 ? "git revision resolution failed: the contained command could not complete" - : "git revision resolution failed: base_branch or HEAD is not a commit"); + cancelled ? "detect_changes cancelled for this request" + : run != 0 + ? "git revision resolution failed: the contained command could not complete" + : (!base_defaulted + ? "git revision resolution failed: base_branch or HEAD is not a commit" + : !default_base_found ? "git revision resolution failed: no default base found " + "(tried origin/HEAD, main, master and the upstream; " + "fell back to \"main\"); pass base_branch" + : "git revision resolution failed: default base or HEAD " + "is not a commit; pass base_branch")); goto done; } changes_git_finish(&git); diff --git a/src/operations/compare.c b/src/operations/compare.c index e6b08c3277..769f43df21 100644 --- a/src/operations/compare.c +++ b/src/operations/compare.c @@ -157,7 +157,9 @@ static bool compare_parse_bounded_integer(yyjson_val *root, const char *key, int *error_message = "limit values must be integers"; return false; } - parsed = yyjson_get_int(value); + parsed = yyjson_is_uint(value) && yyjson_get_uint(value) > (uint64_t)maximum + ? maximum + 1 + : yyjson_get_sint(value); } if (parsed < 1 || parsed > maximum) { *error_message = strcmp(key, "limit") == 0 ? "limit must be between 1 and 1000" diff --git a/src/operations/coverage.c b/src/operations/coverage.c index 9f655e5e51..b43fe76407 100644 --- a/src/operations/coverage.c +++ b/src/operations/coverage.c @@ -1,7 +1,7 @@ +#include "operations/json_args.h" #include "operations/result_wire.h" #include "operations/operation.h" #include "operations/store_host.h" -#include "operations/json_args.h" #include "foundation/compat_fs.h" #include "foundation/platform.h" @@ -50,13 +50,7 @@ static char *op_string_arg(const char *args, const char *name) { } static int op_int_arg(const char *args, const char *name, int fallback) { - yyjson_doc *doc = args ? yyjson_read(args, strlen(args), 0) : NULL; - yyjson_val *root = doc ? yyjson_doc_get_root(doc) : NULL; - yyjson_val *value = yyjson_is_obj(root) ? yyjson_obj_get(root, name) : NULL; - int result = value && yyjson_is_int(value) ? (int)yyjson_get_sint(value) : fallback; - if (doc) - yyjson_doc_free(doc); - return result; + return cbm_json_int_arg(args, name, fallback); } static cbm_operation_result_t op_json_result(yyjson_mut_doc *doc, bool error) { diff --git a/src/operations/index.c b/src/operations/index.c index fc312fa09d..6542c5b653 100644 --- a/src/operations/index.c +++ b/src/operations/index.c @@ -1,3 +1,4 @@ +#include "operations/json_args.h" #include "operations/index.h" #include "cli/cli.h" #include "foundation/index_policy.h" @@ -5,7 +6,6 @@ #include "operations/cross_repo.h" #include "operations/index_supervisor.h" #include "operations/project_arg.h" -#include "operations/json_args.h" #include "operations/read.h" #include "foundation/compat_fs.h" @@ -351,7 +351,7 @@ static char *index_repo_path_from_project(const char *args_json) { cbm_store_close(store); } } - cbm_free(CBM_MEM_CLASS_OPERATION_ARG, project); + cbm_operation_arg_free(project); return root_path; } @@ -599,7 +599,11 @@ static bool index_write_log(const char *project, const cbm_file_error_t *errs, i char path[CBM_SZ_1K]; const char *override = getenv("CBM_INDEX_LOG"); if (override && override[0]) { - snprintf(path, sizeof(path), "%s", override); + int written = snprintf(path, sizeof(path), "%s", override); + if (written <= 0 || (size_t)written >= sizeof(path)) { + cbm_log_warn("index.logfile_path_too_long", "source", "CBM_INDEX_LOG"); + return false; + } } else { const char *cache = cbm_resolve_cache_dir(); if (!cache) diff --git a/src/operations/index_supervisor.c b/src/operations/index_supervisor.c index 8145c8c3ce..0c6c2663eb 100644 --- a/src/operations/index_supervisor.c +++ b/src/operations/index_supervisor.c @@ -359,6 +359,7 @@ typedef enum { WORKER_RESPONSE_READ_OK = 0, WORKER_RESPONSE_READ_ERROR, WORKER_RESPONSE_READ_TOO_LARGE, + WORKER_RESPONSE_READ_EMPTY, /* the pre-created file was never written (#1300) */ } worker_response_read_status_t; /* Read at most one daemon-application payload. The worker is already reaped, @@ -384,6 +385,14 @@ static char *slurp_worker_response(const char *path, worker_response_read_status (void)fclose(f); return NULL; } + if (n == 0) { + /* The supervisor creates the response file before spawning, so an + * empty file is a worker that never wrote its result. Reading it as + * "" made that a successful run with an empty response (#1300). */ + *status_out = WORKER_RESPONSE_READ_EMPTY; + (void)fclose(f); + return NULL; + } if (fseek(f, 0, SEEK_SET) != 0) { (void)fclose(f); return NULL; @@ -405,6 +414,67 @@ static char *slurp_worker_response(const char *path, worker_response_read_status return buf; } +enum { + WORKER_LOG_TAIL_BYTES = 64 * 1024, /* last-phase scan window: bounded, never the whole log */ + WORKER_LOG_LINE_CAP = 1024, +}; + +/* Copy the event name of one structured log line ("level=... msg= ..." + * or {"level":...,"event":"",...}) into out. False for any other line + * (plain stderr text, a truncated fragment). */ +static bool worker_log_line_event(const char *line, char *out, size_t out_size) { + static const char text_key[] = " msg="; + static const char json_key[] = "\"event\":\""; + const char *start = NULL; + char stop = ' '; + if (strncmp(line, "level=", strlen("level=")) == 0) { + const char *msg = strstr(line, text_key); + start = msg ? msg + strlen(text_key) : NULL; + } else if (strncmp(line, "{\"level\":", strlen("{\"level\":")) == 0) { + const char *event = strstr(line, json_key); + start = event ? event + strlen(json_key) : NULL; + stop = '"'; + } + if (!start) { + return false; + } + size_t length = 0; + while (start[length] && start[length] != stop && start[length] != '\n' && + start[length] != '\r') { + length++; + } + if (length == 0 || length >= out_size) { + return false; + } + memcpy(out, start, length); + out[length] = '\0'; + return true; +} + +/* #1300: the last structured event the worker logged before it exited — the + * phase it reached. Scans only the log's final WORKER_LOG_TAIL_BYTES. */ +static void worker_log_last_phase(const char *log_path, char *out, size_t out_size) { + (void)snprintf(out, out_size, "%s", "unknown"); + FILE *log = cbm_fopen(log_path, "rb"); + if (!log) { + return; + } + if (fseek(log, 0, SEEK_END) == 0) { + long size = ftell(log); + long from = size > WORKER_LOG_TAIL_BYTES ? size - WORKER_LOG_TAIL_BYTES : 0; + if (size >= 0 && fseek(log, from, SEEK_SET) == 0) { + char line[WORKER_LOG_LINE_CAP]; + char event[CBM_SZ_128]; + while (fgets(line, sizeof(line), log)) { + if (worker_log_line_event(line, event, sizeof(event))) { + (void)snprintf(out, out_size, "%s", event); + } + } + } + } + (void)fclose(log); +} + enum { INDEX_WORKER_PATH_CAP = CBM_SZ_4K, INDEX_WORKER_ARGV_CAP = 17, @@ -634,6 +704,13 @@ static void worker_terminal_log(cbm_index_worker_handle_t *handle) { } else if (handle->result.cancellation_requested) { cbm_log_warn("index.supervisor.worker_cancelled", "outcome", cbm_proc_outcome_str(handle->result.outcome), "log", handle->log_path); + } else if (handle->result.response_missing) { + /* #1300: keep and name the log — it is the only record of the run. */ + cbm_log_error("index.supervisor.no_response", "reason", + handle->result.response_unreadable ? "response_read_error" + : "worker_did_not_write", + "exit_code", exit_text, "last_phase", handle->result.last_phase, "log", + handle->log_path); } else if (handle->result.outcome == CBM_PROC_CLEAN && !cbm_profile_active) { (void)cbm_unlink(handle->log_path); } else if (handle->result.outcome == CBM_PROC_CLEAN) { @@ -828,6 +905,17 @@ cbm_index_worker_poll_t cbm_index_worker_poll(cbm_index_worker_handle_t *handle, handle->result.response_rejected = true; handle->result.outcome = CBM_PROC_EXIT_NONZERO; handle->result.exit_code = -1; + } else if (!handle->result.response) { + /* #1300: a clean exit that wrote no response. The outcome stays + * CLEAN (the disposition is FALLBACK: an explicit error, no + * in-process fallback and no recovery re-run of a worker that did + * not crash); the log and the last phase it reached are kept. */ + handle->result.response_missing = true; + handle->result.response_unreadable = response_status == WORKER_RESPONSE_READ_ERROR; + worker_log_last_phase(handle->log_path, handle->result.last_phase, + sizeof(handle->result.last_phase)); + (void)snprintf(handle->result.worker_log, sizeof(handle->result.worker_log), "%s", + handle->log_path); } } (void)cbm_unlink(handle->response_path); diff --git a/src/operations/index_supervisor.h b/src/operations/index_supervisor.h index 6137c3b446..c023136c0c 100644 --- a/src/operations/index_supervisor.h +++ b/src/operations/index_supervisor.h @@ -154,6 +154,15 @@ typedef struct { bool forced; bool tree_quiesced; bool supervision_failed; + /* #1300: CLEAN exit with no usable response. A contained failure, never a + * success: response stays NULL, the worker log is retained and named, and + * last_phase is the last structured event ("unknown" when absent). + * response_unreadable distinguishes supervisor read errors from an empty + * response file the worker never wrote. */ + bool response_missing; + bool response_unreadable; /* supervisor could not read a non-empty response */ + char last_phase[128]; + char worker_log[4096]; bool response_rejected; /* clean worker exceeded the bounded response protocol */ char *response; /* worker result only after a contained, uncancelled CLEAN exit; * borrowed for async polls, caller-owned from the sync wrapper */ diff --git a/src/operations/json_args.c b/src/operations/json_args.c index 9d0ab93cb8..2d26d6443b 100644 --- a/src/operations/json_args.c +++ b/src/operations/json_args.c @@ -1,5 +1,8 @@ #include "operations/json_args.h" +#include "foundation/mem_core.h" #include "yyjson/yyjson.h" +#include +#include #include void cbm_operation_arg_free(char *value) { @@ -28,7 +31,16 @@ int cbm_json_int_arg(const char *args_json, const char *key, int fallback) { yyjson_doc *doc = yyjson_read(json, strlen(json), 0); yyjson_val *root = doc ? yyjson_doc_get_root(doc) : NULL; yyjson_val *value = yyjson_is_obj(root) ? yyjson_obj_get(root, key) : NULL; - int out = yyjson_is_int(value) ? (int)yyjson_get_sint(value) : fallback; + int out = fallback; + if (yyjson_is_sint(value)) { + int64_t parsed = yyjson_get_sint(value); + if (parsed >= INT_MIN && parsed <= INT_MAX) + out = (int)parsed; + } else if (yyjson_is_uint(value)) { + uint64_t parsed = yyjson_get_uint(value); + if (parsed <= (uint64_t)INT_MAX) + out = (int)parsed; + } if (doc) yyjson_doc_free(doc); return out; diff --git a/src/operations/json_args.h b/src/operations/json_args.h index 0f6f6ac802..7571070978 100644 --- a/src/operations/json_args.h +++ b/src/operations/json_args.h @@ -5,7 +5,7 @@ #include "foundation/mem_core.h" /* Returned strings belong to CBM_MEM_CLASS_OPERATION_ARG; release with - * cbm_free(CBM_MEM_CLASS_OPERATION_ARG, value). */ + * cbm_operation_arg_free(value). */ char *cbm_json_string_arg(const char *args_json, const char *key); void cbm_operation_arg_free(char *value); diff --git a/src/operations/mutation.c b/src/operations/mutation.c index 6d3b84252e..f5cfea38db 100644 --- a/src/operations/mutation.c +++ b/src/operations/mutation.c @@ -1,3 +1,4 @@ +#include "operations/json_args.h" #include "operations/mutation.h" #include "operations/project_arg.h" #include "operations/index.h" @@ -44,27 +45,27 @@ static cbm_operation_result_t execute_delete_project(const char *args_json, const cbm_operation_runtime_t *runtime) { char *project = cbm_operation_project_arg(args_json); if (!project || !project[0]) { - cbm_free(CBM_MEM_CLASS_OPERATION_ARG, project); + cbm_operation_arg_free(project); return cbm_operation_result_copy("{\"error\":\"project is required\"}", true); } if (!cbm_validate_project_name(project)) { cbm_operation_result_t result = mutation_json_result(project, "invalid_project", "invalid project name", true); - cbm_free(CBM_MEM_CLASS_OPERATION_ARG, project); + cbm_operation_arg_free(project); return result; } if (!runtime || !runtime->mutation_begin || !runtime->mutation_end) { cbm_operation_result_t result = mutation_json_result( project, "coordination_unavailable", "project mutation coordination is unavailable for this execution path", true); - cbm_free(CBM_MEM_CLASS_OPERATION_ARG, project); + cbm_operation_arg_free(project); return result; } if (!runtime->mutation_begin(runtime->mutation_context, project)) { cbm_operation_result_t result = mutation_json_result( project, "busy", "project operation cancelled or blocked by an active index or mutation", true); - cbm_free(CBM_MEM_CLASS_OPERATION_ARG, project); + cbm_operation_arg_free(project); return result; } @@ -116,7 +117,7 @@ static cbm_operation_result_t execute_delete_project(const char *args_json, cbm_mem_collect(); runtime->mutation_end(runtime->mutation_context, project); cbm_operation_result_t result = mutation_json_result(project, status, error_detail, is_error); - cbm_free(CBM_MEM_CLASS_OPERATION_ARG, project); + cbm_operation_arg_free(project); return result; } diff --git a/src/operations/operation.h b/src/operations/operation.h index 50437dcd12..18f3f4bab8 100644 --- a/src/operations/operation.h +++ b/src/operations/operation.h @@ -79,6 +79,7 @@ typedef cbm_operation_result_t (*cbm_operation_index_status_fn)(void *context, c typedef void (*cbm_operation_project_invalidate_fn)(void *context, const char *project); typedef struct cbm_operation_runtime { + const char *search_scratch_dir; /* optional private parent; NULL uses cbm_tmpdir() */ cbm_operation_cancelled_fn cancelled; void *cancelled_context; cbm_operation_command_allowed_fn command_allowed; diff --git a/src/operations/output_budget.c b/src/operations/output_budget.c index d71a5f130b..b04e9679f7 100644 --- a/src/operations/output_budget.c +++ b/src/operations/output_budget.c @@ -1,8 +1,10 @@ #include "operations/output_budget.h" +#include "operations/json_args.h" #include "yyjson/yyjson.h" #include +#include #include int cbm_output_budget_tokens(const char *args_json, int fallback) { @@ -10,8 +12,10 @@ int cbm_output_budget_tokens(const char *args_json, int fallback) { yyjson_doc *doc = yyjson_read(json, strlen(json), 0); yyjson_val *root = doc ? yyjson_doc_get_root(doc) : NULL; yyjson_val *value = yyjson_is_obj(root) ? yyjson_obj_get(root, "max_output_tokens") : NULL; - bool present = value && yyjson_is_int(value); - int tokens = present ? (int)yyjson_get_sint(value) : fallback; + bool present = (yyjson_is_sint(value) && yyjson_get_sint(value) >= INT_MIN && + yyjson_get_sint(value) <= INT_MAX) || + (yyjson_is_uint(value) && yyjson_get_uint(value) <= (uint64_t)INT_MAX); + int tokens = cbm_json_int_arg(args_json, "max_output_tokens", fallback); if (doc) { yyjson_doc_free(doc); } diff --git a/src/operations/project_arg.c b/src/operations/project_arg.c index 626bda35d5..040176dd40 100644 --- a/src/operations/project_arg.c +++ b/src/operations/project_arg.c @@ -1,9 +1,12 @@ +#include "operations/json_args.h" #include "operations/project_arg.h" #include "foundation/compat_fs.h" #include "foundation/constants.h" #include "foundation/mem_core.h" #include "foundation/platform.h" +#include "foundation/workspace.h" +#include "store/store.h" #include "foundation/str_util.h" #include "pipeline/pipeline.h" #include "yyjson/yyjson.h" @@ -18,8 +21,60 @@ static char *project_arg_strdup(const char *text) { } static bool project_arg_is_db_file(const char *name, size_t len) { - return name && len >= 4U && strcmp(name + len - 3U, ".db") == 0 && name[0] != '_' && - strncmp(name, ":memory:", 8U) != 0; + (void)len; + return cbm_is_project_index_db(name); +} + +static char *project_arg_path_owner(const char *path, char *derived) { + const char *cache = cbm_resolve_cache_dir(); + char db_path[CBM_SZ_4K]; + if (!cache || + snprintf(db_path, sizeof(db_path), "%s/%s.db", cache, derived) >= (int)sizeof(db_path) || + cbm_file_exists(db_path)) + return derived; + cbm_dir_t *dir = cbm_opendir(cache); + if (!dir) + return derived; + char *owner = NULL; + size_t depth = 0; + bool ambiguous = false; + cbm_dirent_t *entry; + while ((entry = cbm_readdir(dir)) != NULL) { + if (!cbm_is_project_index_db(entry->name) || + snprintf(db_path, sizeof(db_path), "%s/%s", cache, entry->name) >= (int)sizeof(db_path)) + continue; + cbm_store_t *store = cbm_store_open_path_query(db_path); + cbm_project_t *projects = NULL; + int count = 0; + if (store && cbm_store_list_projects(store, &projects, &count) == CBM_STORE_OK && + count == 1) { + char root[CBM_SZ_4K]; + if (projects[0].root_path && + cbm_canonical_path(projects[0].root_path, root, sizeof(root))) { + cbm_normalize_path_sep(root); + if (cbm_path_within_root(root, path)) { + size_t n = strlen(root); + if (n > depth) { + cbm_operation_arg_free(owner); + owner = project_arg_strdup(projects[0].name); + depth = n; + ambiguous = false; + } else if (n == depth) { + ambiguous = true; + } + } + } + } + cbm_store_free_projects(projects, count); + cbm_store_close(store); + } + cbm_closedir(dir); + if (!owner || ambiguous) { + cbm_operation_arg_free(owner); + return derived; + } + cbm_operation_arg_free(derived); + return owner; } static char *project_arg_normalize(char *project) { @@ -29,18 +84,12 @@ static char *project_arg_normalize(char *project) { bool non_ascii = false; for (const unsigned char *p = (const unsigned char *)project; *p; p++) non_ascii |= *p >= 0x80; - char *encoded = non_ascii ? cbm_project_name_sanitize(project) : NULL; + char *raw_encoded = non_ascii ? cbm_project_name_sanitize(project) : NULL; + char *encoded = raw_encoded ? project_arg_strdup(raw_encoded) : NULL; + free(raw_encoded); bool usable = encoded && strcmp(encoded, "root") != 0 && cbm_validate_project_name(encoded); - if (usable) { - char *copy = project_arg_strdup(encoded); - free(encoded); - if (!copy) - return project; /* keep the supplied project on allocation failure */ - cbm_free(CBM_MEM_CLASS_OPERATION_ARG, project); - return copy; - } - free(encoded); - return project; + cbm_operation_arg_free(usable ? project : encoded); + return usable ? encoded : project; } char real[CBM_SZ_4K]; @@ -48,18 +97,17 @@ static char *project_arg_normalize(char *project) { cbm_normalize_path_sep(real); char *canonical = project_arg_strdup(real); if (canonical) { - cbm_free(CBM_MEM_CLASS_OPERATION_ARG, project); + cbm_operation_arg_free(project); project = canonical; } } - char *normalized = cbm_project_name_from_path(project); + char *raw_normalized = cbm_project_name_from_path(project); + char *normalized = raw_normalized ? project_arg_strdup(raw_normalized) : NULL; + free(raw_normalized); if (normalized) { - char *copy = project_arg_strdup(normalized); - free(normalized); - if (!copy) - return project; /* keep the supplied project on allocation failure */ - cbm_free(CBM_MEM_CLASS_OPERATION_ARG, project); - return copy; + normalized = project_arg_path_owner(project, normalized); + cbm_operation_arg_free(project); + return normalized; } return project; } diff --git a/src/operations/query.c b/src/operations/query.c index ff99eeb382..8ca291421b 100644 --- a/src/operations/query.c +++ b/src/operations/query.c @@ -1,5 +1,5 @@ -#include "operations/result_wire.h" #include "operations/json_args.h" +#include "operations/result_wire.h" #include "operations/query.h" #include "operations/store_host.h" @@ -53,13 +53,7 @@ static char *project_arg(const char *args_json) { } static int int_arg(const char *args_json, const char *name, int fallback) { - yyjson_doc *doc = read_args(args_json); - yyjson_val *root = doc ? yyjson_doc_get_root(doc) : NULL; - yyjson_val *value = yyjson_is_obj(root) ? yyjson_obj_get(root, name) : NULL; - int result = value && yyjson_is_int(value) ? (int)yyjson_get_sint(value) : fallback; - if (doc) - yyjson_doc_free(doc); - return result; + return cbm_json_int_arg(args_json, name, fallback); } static cbm_operation_result_t error_result(const char *message) { diff --git a/src/operations/read.c b/src/operations/read.c index 05ced6c340..3674cdfe18 100644 --- a/src/operations/read.c +++ b/src/operations/read.c @@ -1,3 +1,6 @@ +#include "operations/project_arg.h" +#include "foundation/str_util.h" +#include "operations/json_args.h" #include "operations/result_wire.h" #include "operations/read.h" #include "operations/compact_out.h" @@ -15,7 +18,6 @@ #include "operations/file_outline.h" #include "operations/compare.h" #include "operations/store_host.h" -#include "operations/json_args.h" #include "foundation/platform.h" #include "foundation/compat_fs.h" @@ -37,14 +39,7 @@ static char *copy_string(const char *text); static int json_int_arg(const char *args_json, const char *name, int fallback) { - yyjson_doc *doc = args_json ? yyjson_read(args_json, strlen(args_json), 0) : NULL; - yyjson_val *root = doc ? yyjson_doc_get_root(doc) : NULL; - yyjson_val *value = yyjson_is_obj(root) ? yyjson_obj_get(root, name) : NULL; - int result = value && yyjson_is_int(value) ? (int)yyjson_get_sint(value) : fallback; - if (doc) { - yyjson_doc_free(doc); - } - return result; + return cbm_json_int_arg(args_json, name, fallback); } static bool json_bool_arg(const char *args_json, const char *name, bool fallback) { @@ -100,14 +95,7 @@ static cbm_operation_result_t json_error(const char *message, const char *hint) } static bool project_db_file(const char *name) { - if (!name) { - return false; - } - size_t len = strlen(name); - if (len <= OP_DB_EXT_LEN || strcmp(name + len - OP_DB_EXT_LEN, ".db") != 0) { - return false; - } - return name[0] != '_' && strncmp(name, ":memory:", strlen(":memory:")) != 0; + return cbm_is_project_index_db(name); } static char *copy_string(const char *text) { @@ -671,9 +659,9 @@ bool cbm_read_operation_supported(cbm_operation_id_t operation) { operation == CBM_OPERATION_COMPARE || operation == CBM_OPERATION_INGEST_TRACES; } -cbm_operation_result_t cbm_read_operation_execute(cbm_operation_id_t operation, - const char *args_json, - const cbm_operation_runtime_t *runtime) { +static cbm_operation_result_t read_operation_execute(cbm_operation_id_t operation, + const char *args_json, + const cbm_operation_runtime_t *runtime) { (void)runtime; if (operation == CBM_OPERATION_PROJECTS) { return execute_projects(args_json); @@ -719,3 +707,30 @@ cbm_operation_result_t cbm_read_operation_execute(cbm_operation_id_t operation, } return cbm_operation_result_copy("native read operation not implemented", true); } + +cbm_operation_result_t cbm_read_operation_execute(cbm_operation_id_t operation, + const char *args_json, + const cbm_operation_runtime_t *runtime) { + char *project = cbm_operation_project_arg(args_json); + if (!project) + return read_operation_execute(operation, args_json, runtime); + const char *json = args_json ? args_json : "{}"; + yyjson_doc *doc = yyjson_read(json, strlen(json), 0); + yyjson_mut_doc *copy = doc ? yyjson_doc_mut_copy(doc, NULL) : NULL; + if (doc) + yyjson_doc_free(doc); + yyjson_mut_val *root = copy ? yyjson_mut_doc_get_root(copy) : NULL; + bool ok = + yyjson_mut_is_obj(root) && (yyjson_mut_obj_remove_key(root, "project"), + yyjson_mut_obj_add_strcpy(copy, root, "project", project)); + cbm_operation_arg_free(project); + char *normalized = ok ? cbm_operation_json_write(copy) : NULL; + if (copy) + yyjson_mut_doc_free(copy); + if (!normalized) + return json_error("project argument normalization failed", NULL); + cbm_operation_result_t owned = cbm_operation_result_take(normalized, false); + cbm_operation_result_t result = read_operation_execute(operation, normalized, runtime); + cbm_operation_result_dispose(&owned); + return result; +} diff --git a/src/operations/schema.c b/src/operations/schema.c index caeb66d78f..9f7708c8b5 100644 --- a/src/operations/schema.c +++ b/src/operations/schema.c @@ -1,7 +1,7 @@ +#include "operations/json_args.h" #include "operations/result_wire.h" #include "operations/schema.h" #include "operations/store_host.h" -#include "operations/json_args.h" #include "store/store.h" #include "yyjson/yyjson.h" @@ -49,14 +49,7 @@ static bool project_has_adr(cbm_store_t *store, const char *project, const char } static int int_arg(const char *args_json, const char *name, int fallback) { - yyjson_doc *doc = - yyjson_read(args_json ? args_json : "{}", strlen(args_json ? args_json : "{}"), 0); - yyjson_val *root = doc ? yyjson_doc_get_root(doc) : NULL; - yyjson_val *value = yyjson_is_obj(root) ? yyjson_obj_get(root, name) : NULL; - int result = value && yyjson_is_int(value) ? (int)yyjson_get_sint(value) : fallback; - if (doc) - yyjson_doc_free(doc); - return result; + return cbm_json_int_arg(args_json, name, fallback); } static bool diagnostics_full(const char *args_json) { diff --git a/src/operations/search.c b/src/operations/search.c index b201cfaa9d..7ce43a01c6 100644 --- a/src/operations/search.c +++ b/src/operations/search.c @@ -1,5 +1,6 @@ -#include "operations/result_wire.h" +#include "callable_sig.h" #include "operations/json_args.h" +#include "operations/result_wire.h" #include "operations/search.h" #include "operations/compact_out.h" #include "operations/output_budget.h" @@ -67,13 +68,7 @@ static char *string_arg(const char *args, const char *name) { } static int int_arg(const char *args, const char *name, int fallback) { - yyjson_doc *doc = read_args(args); - yyjson_val *root = doc ? yyjson_doc_get_root(doc) : NULL; - yyjson_val *value = yyjson_is_obj(root) ? yyjson_obj_get(root, name) : NULL; - int result = value && yyjson_is_int(value) ? (int)yyjson_get_sint(value) : fallback; - if (doc) - yyjson_doc_free(doc); - return result; + return cbm_json_int_arg(args, name, fallback); } static bool bool_arg(const char *args, const char *name) { @@ -509,9 +504,14 @@ static int result_qn_cmp(const void *left, const void *right) { return strcmp(aq, bq); } -static size_t prefix_length(const char *qn) { - const char *last = qn ? strrchr(qn, '.') : NULL; - return last ? (size_t)(last - qn) : 0U; +static size_t prefix_length(const char *qn, const char *name) { + size_t last = 0; + size_t base_len = cbm_qn_callable_base_len_named(qn, name); + for (size_t i = 0; i < base_len; i++) { + if (qn[i] == '.') + last = i; + } + return last; } static void sg_toon_property_cell(cbm_sb_t *sb, yyjson_val *v) { @@ -566,7 +566,7 @@ static void emit_search_results_grouped_tree(cbm_sb_t *sb, const cbm_search_outp const cbm_search_result_t *sr = &out->results[i]; const char *qn = sr->node.qualified_name ? sr->node.qualified_name : ""; const char *file = sr->node.file_path ? sr->node.file_path : ""; - size_t plen = prefix_length(qn); + size_t plen = prefix_length(qn, sr->node.name); bool same_group = previous_prefix && strlen(previous_prefix) == plen && memcmp(previous_prefix, qn, plen) == 0 && previous_file && strcmp(previous_file, file) == 0; @@ -738,7 +738,7 @@ static void emit_structural(yyjson_mut_doc *doc, yyjson_mut_val *root, cbm_searc cbm_search_result_t *search = &output->results[i]; const char *qn = search->node.qualified_name ? search->node.qualified_name : ""; const char *file = search->node.file_path ? search->node.file_path : ""; - size_t prefix = prefix_length(qn); + size_t prefix = prefix_length(qn, search->node.name); char key[2048]; (void)snprintf(key, sizeof(key), "%.*s|%s", (int)prefix, qn, file); if (!rows || strcmp(key, current) != 0) { @@ -969,10 +969,9 @@ cbm_operation_result_t cbm_search_operation_execute(const char *args) { } store = open_indexed_project(project, &open_status); if (!store) { - result = - open_status == CBM_STORE_OPEN_CORRUPT - ? error_result(CBM_STORE_CORRUPT_MESSAGE, CBM_STORE_CORRUPT_HINT) - : error_result("project not indexed", "Run 'codebase-memory-cli index .' first."); + result = open_status == CBM_STORE_OPEN_CORRUPT + ? error_result(CBM_STORE_CORRUPT_MESSAGE, CBM_STORE_CORRUPT_HINT) + : cbm_operation_result_take(cbm_store_host_error(project), true); goto done; } @@ -1021,7 +1020,7 @@ cbm_operation_result_t cbm_search_operation_execute(const char *args) { .min_degree = min_degree, .max_degree = max_degree}; if (!semantic_only && cbm_store_search(store, ¶ms, &output) != CBM_STORE_OK) { - result = error_result("graph search failed", NULL); + result = error_result(cbm_store_error(store), NULL); goto done; } @@ -1048,20 +1047,10 @@ cbm_operation_result_t cbm_search_operation_execute(const char *args) { yyjson_mut_obj_add_str(doc, root, "fields_hint", "Core fields are already included and were not duplicated as " "extra property columns."); - if (output.total == 0 && !semantic_only) { - if (name_pattern && label) - yyjson_mut_obj_add_str( - doc, root, "hint", - "No results. Remove the label filter or broaden name_pattern."); - else if (name_pattern) - yyjson_mut_obj_add_str( - doc, root, "hint", - "No nodes match this pattern. Check spelling or broaden the regex."); - else if (label) - yyjson_mut_obj_add_str( - doc, root, "hint", - "No nodes have this label. Use architecture/schema discovery to " - "inspect available labels."); + if (output.total == 0 && !semantic_only && structural) { + yyjson_mut_obj_add_str( + doc, root, "hint", + "No results match the current filters; broaden or remove filters and retry."); } if (semantic_present) { emit_semantic(doc, root, &semantic_page); diff --git a/src/operations/snippet.c b/src/operations/snippet.c index 90cc260ad0..364fd0af5b 100644 --- a/src/operations/snippet.c +++ b/src/operations/snippet.c @@ -1,6 +1,6 @@ +#include "operations/json_args.h" #include "operations/output_budget.h" #include "operations/result_wire.h" -#include "operations/json_args.h" #include "operations/operation.h" #include "operations/store_host.h" @@ -45,13 +45,7 @@ static char *string_arg(const char *args, const char *name) { } static int int_arg(const char *args, const char *name, int fallback) { - yyjson_doc *doc = args ? yyjson_read(args, strlen(args), 0) : NULL; - yyjson_val *root = doc ? yyjson_doc_get_root(doc) : NULL; - yyjson_val *value = yyjson_is_obj(root) ? yyjson_obj_get(root, name) : NULL; - int result = value && yyjson_is_int(value) ? (int)yyjson_get_sint(value) : fallback; - if (doc) - yyjson_doc_free(doc); - return result; + return cbm_json_int_arg(args, name, fallback); } static bool bool_arg(const char *args, const char *name) { @@ -608,13 +602,26 @@ cbm_operation_result_t cbm_snippet_operation_execute(const char *args) { cbm_node_t *matches = NULL; int count = 0; - (void)cbm_store_find_nodes_by_qn_suffix(store, project, qualified_name, &matches, &count); + const char *method = "base"; + cbm_store_find_nodes_by_qn_base(store, project, qualified_name, false, &matches, &count); + if (count == 0) { + cbm_store_free_nodes(matches, count); + matches = NULL; + method = "suffix"; + cbm_store_find_nodes_by_qn_suffix(store, project, qualified_name, &matches, &count); + } + if (count == 0) { + cbm_store_free_nodes(matches, count); + matches = NULL; + method = "base_suffix"; + cbm_store_find_nodes_by_qn_base(store, project, qualified_name, true, &matches, &count); + } if (count > 0) { bool ambiguous = false; int selected = resolved_node(matches, count, &ambiguous); cbm_operation_result_t result = ambiguous ? ambiguous_result(qualified_name, matches, count) : node_result(store, project, &matches[selected], - "suffix", include_neighbors, args); + method, include_neighbors, args); cbm_store_free_nodes(matches, count); cbm_store_close(store); cbm_operation_arg_free(project); diff --git a/src/operations/source_search.c b/src/operations/source_search.c index fe728d5468..66c986c6e0 100644 --- a/src/operations/source_search.c +++ b/src/operations/source_search.c @@ -1,5 +1,6 @@ -#include "operations/result_wire.h" +#include "foundation/mem_core.h" #include "operations/json_args.h" +#include "operations/result_wire.h" #include "operations/source_search.h" #include "foundation/compat.h" @@ -98,13 +99,7 @@ static char *source_project_arg(const char *args) { } static int source_int_arg(const char *args, const char *name, int fallback) { - yyjson_doc *doc = source_args_doc(args); - yyjson_val *root = doc ? yyjson_doc_get_root(doc) : NULL; - yyjson_val *value = yyjson_is_obj(root) ? yyjson_obj_get(root, name) : NULL; - int result = value && yyjson_is_int(value) ? (int)yyjson_get_sint(value) : fallback; - if (doc) - yyjson_doc_free(doc); - return result; + return cbm_json_int_arg(args, name, fallback); } static bool source_bool_arg(const char *args, const char *name, bool fallback) { @@ -551,8 +546,12 @@ static void attach_result_source(yyjson_mut_doc *doc, yyjson_mut_val *item, sear } } } else if (context_lines > 0 && r->match_count > 0) { - int ctx_start = r->match_lines[0] - context_lines; - int ctx_end = r->match_lines[r->match_count - SOURCE_SKIP_ONE] + context_lines; + int ctx_lines = context_lines > SOURCE_MAX_SOURCE_MAX_LINES ? SOURCE_MAX_SOURCE_MAX_LINES + : context_lines; + int first_match = r->match_lines[0]; + int last_match = r->match_lines[r->match_count - SOURCE_SKIP_ONE]; + int ctx_start = first_match > ctx_lines ? first_match - ctx_lines : SOURCE_SKIP_ONE; + int ctx_end = last_match > INT_MAX - ctx_lines ? INT_MAX : last_match + ctx_lines; if (ctx_start < SOURCE_SKIP_ONE) { ctx_start = SOURCE_SKIP_ONE; } @@ -1751,14 +1750,14 @@ static void search_scratch_close(search_scratch_t *scratch) { /* Open the scratch directory, write `pattern` to the grep -f file, and leave the * file list open for write_scoped_filelist. Returns true on success; on failure * everything already created is removed before returning. */ -static bool search_scratch_open(search_scratch_t *scratch, const char *pattern) { +static bool search_scratch_open(search_scratch_t *scratch, const char *parent, + const char *pattern) { scratch->dir[0] = '\0'; scratch->pattern_path[0] = '\0'; scratch->filelist_path[0] = '\0'; scratch->filelist = NULL; - int written = - snprintf(scratch->dir, sizeof(scratch->dir), "%s/cbm-search-XXXXXX", cbm_tmpdir()); + int written = snprintf(scratch->dir, sizeof(scratch->dir), "%s/cbm-search-XXXXXX", parent); if (written <= 0 || (size_t)written >= sizeof(scratch->dir) || !cbm_mkdtemp(scratch->dir)) { scratch->dir[0] = '\0'; return false; @@ -1787,11 +1786,13 @@ static bool search_scratch_open(search_scratch_t *scratch, const char *pattern) } /* Compile a path filter regex. Returns true if compiled successfully. */ -static bool compile_path_filter(const char *filter, cbm_regex_t *re) { +static bool compile_path_filter(const char *filter, cbm_regex_t *re, int *rc) { + *rc = CBM_REG_OK; if (!filter || !filter[0]) { return false; } - return cbm_regcomp(re, filter, CBM_REG_EXTENDED | CBM_REG_NOSUB) == CBM_REG_OK; + *rc = cbm_regcomp(re, filter, CBM_REG_EXTENDED | CBM_REG_NOSUB); + return *rc == CBM_REG_OK; } static char *search_code_timeout_payload(void) { @@ -2007,6 +2008,10 @@ cbm_operation_result_t cbm_source_search_operation_execute(const char *args, * left exactly as this operation has always produced it. */ size_t byte_budget = cbm_output_budget_bytes(cbm_output_budget_tokens(args, 0)); int context_lines = source_int_arg(args, "context", 0); + if (context_lines < 0) + context_lines = 0; + else if (context_lines > SOURCE_MAX_SOURCE_MAX_LINES) + context_lines = SOURCE_MAX_SOURCE_MAX_LINES; bool use_regex = source_bool_arg(args, "regex", false); uint64_t search_t0 = cbm_now_ms(); search_metrics_t metrics = {0}; @@ -2019,11 +2024,14 @@ cbm_operation_result_t cbm_source_search_operation_execute(const char *args, cbm_operation_arg_free(mode_str); cbm_regex_t path_regex; - bool has_path_filter = compile_path_filter(path_filter, &path_regex); + int path_filter_rc = CBM_REG_OK; + bool has_path_filter = compile_path_filter(path_filter, &path_regex, &path_filter_rc); cbm_operation_arg_free(path_filter); path_filter = NULL; if (!pattern) { + if (has_path_filter) + cbm_regfree(&path_regex); cbm_operation_arg_free(project); cbm_operation_arg_free(file_pattern); return source_error("pattern is required"); @@ -2031,6 +2039,8 @@ cbm_operation_result_t cbm_source_search_operation_execute(const char *args, /* Project is required */ if (!project) { + if (has_path_filter) + cbm_regfree(&path_regex); cbm_operation_arg_free(pattern); cbm_operation_arg_free(file_pattern); return source_project_error(NULL, CBM_STORE_OPEN_NOT_FOUND); @@ -2042,6 +2052,8 @@ cbm_operation_result_t cbm_source_search_operation_execute(const char *args, if (!store) { cbm_operation_result_t error = source_project_error(project, open_status); cbm_operation_arg_free(pattern); + if (has_path_filter) + cbm_regfree(&path_regex); cbm_operation_arg_free(project); cbm_operation_arg_free(file_pattern); return error; @@ -2064,21 +2076,33 @@ cbm_operation_result_t cbm_source_search_operation_execute(const char *args, * otherwise report as an empty result set — indistinguishable from a * legitimate no-match. Validate the user's regex up front and return an * explicit error so callers can tell "broken pattern" from "no matches". */ - if (use_regex) { + const char *regex_error = NULL; + if (path_filter_rc != CBM_REG_OK) { + regex_error = path_filter_rc == CBM_REG_ETOOBIG + ? "path_filter: " CBM_REG_ETOOBIG_REASON + : "invalid path_filter regex: check for unbalanced (), [], or {}"; + } else if (use_regex) { cbm_regex_t probe; - if (cbm_regcomp(&probe, pattern, CBM_REG_EXTENDED | CBM_REG_NOSUB) != CBM_REG_OK) { - if (has_path_filter) { - cbm_regfree(&path_regex); - } - cbm_store_close(store); - free(root_path); - cbm_operation_arg_free(pattern); - cbm_operation_arg_free(project); - cbm_operation_arg_free(file_pattern); - return source_error( - "invalid regex pattern (regex=true): check for unbalanced (), [], or {}"); + int probe_rc = cbm_regcomp(&probe, pattern, CBM_REG_EXTENDED | CBM_REG_NOSUB); + if (probe_rc != CBM_REG_OK) { + regex_error = + probe_rc == CBM_REG_ETOOBIG + ? CBM_REG_ETOOBIG_REASON " (regex=true)" + : "invalid regex pattern (regex=true): check for unbalanced (), [], or {}"; + } else { + cbm_regfree(&probe); } - cbm_regfree(&probe); + } + if (regex_error) { + if (has_path_filter) { + cbm_regfree(&path_regex); + } + free(root_path); + cbm_operation_arg_free(pattern); + cbm_operation_arg_free(project); + cbm_operation_arg_free(file_pattern); + cbm_store_close(store); + return source_error(regex_error); } /* ── Phase 0.5: Multi-word → regex conversion ───────────── */ @@ -2088,7 +2112,6 @@ cbm_operation_result_t cbm_source_search_operation_execute(const char *args, if (!use_regex && strchr(pattern, ' ')) { size_t plen = strlen(pattern); /* Worst case: every char is a space → ".*" between each char */ - /* Same class as the pattern it replaces: released via cbm_operation_arg_free(). */ char *regex_pat = cbm_alloc(CBM_MEM_CLASS_OPERATION_ARG, plen * 3 + 1); if (regex_pat) { char *dst = regex_pat; @@ -2128,7 +2151,9 @@ cbm_operation_result_t cbm_source_search_operation_execute(const char *args, : scan_started_ms + scan_budget_ms; bool scan_deadline_latched = false; search_scratch_t scratch; - if (!search_scratch_open(&scratch, pattern)) { + const char *scratch_parent = + runtime && runtime->search_scratch_dir ? runtime->search_scratch_dir : cbm_tmpdir(); + if (!search_scratch_open(&scratch, scratch_parent, pattern)) { bool scan_cancelled = cbm_operation_runtime_cancelled(runtime); bool scan_timed_out = cbm_now_ms() >= scan_deadline_ms; char errmsg[CBM_SZ_256]; @@ -2137,6 +2162,8 @@ cbm_operation_result_t cbm_source_search_operation_execute(const char *args, cbm_store_close(store); free(root_path); cbm_operation_arg_free(pattern); + if (has_path_filter) + cbm_regfree(&path_regex); cbm_operation_arg_free(project); cbm_operation_arg_free(file_pattern); if (scan_cancelled) { diff --git a/src/operations/store_host.c b/src/operations/store_host.c index 581cd014f7..a51539dcab 100644 --- a/src/operations/store_host.c +++ b/src/operations/store_host.c @@ -72,11 +72,8 @@ static const char *project_db_path(const char *project, char *buf, size_t bufsz) } bool cbm_store_host_is_project_db_file(const char *name, size_t len) { - if (!name || len < 4 || strcmp(name + len - 3, ".db") != 0) - return false; - if (name[0] == '_' || strncmp(name, ":memory:", 8) == 0) - return false; - return true; + (void)len; + return cbm_is_project_index_db(name); } bool cbm_store_host_db_internal_project_name(const char *full_path, char *name_out, size_t name_sz, diff --git a/src/operations/tool_catalog.c b/src/operations/tool_catalog.c index 39c694b3a4..53de1d688b 100644 --- a/src/operations/tool_catalog.c +++ b/src/operations/tool_catalog.c @@ -301,7 +301,8 @@ static const tool_def_t TOOLS[] = { "filter on result file paths (e.g. ^src/ or \\\\.(go|ts)$)\"},\"mode\":{\"type\":\"string\"," "\"enum\":[\"compact\",\"full\",\"files\"],\"default\":\"compact\",\"description\":\"compact: " "signatures+metadata (default). full: with source. files: just file list.\"}," - "\"context\":{\"type\":\"integer\",\"description\":\"Lines of context around each match " + "\"context\":{\"type\":\"integer\",\"default\":0,\"minimum\":0,\"maximum\":200," + "\"description\":\"Lines of context around each match " "(like grep -C). Only used in compact mode.\"}," "\"regex\":{\"type\":\"boolean\",\"default\":false},\"debug\":{\"type\":\"boolean\"," "\"default\":false,\"description\":\"Include scope_ms, scan_ms, and enrich_ms phase timing " @@ -418,7 +419,8 @@ static const tool_def_t TOOLS[] = { "\"description\":\"Per-symbol impacted rows shown (nearest hops first). impacted_total is " "always exact and the impacted_modules rollup always complete regardless.\"}," "\"base_branch\":{\"type\":" - "\"string\",\"default\":\"main\"},\"since\":{\"type\":\"string\",\"description\":" + "\"string\",\"description\":\"Default: origin/HEAD, else main, else master, else the " + "upstream.\"},\"since\":{\"type\":\"string\",\"description\":" "\"Git ref or tag to compare from (e.g. HEAD~5, v0.5.0). Diffs ...HEAD and includes " "staged, unstaged, and untracked working-tree files.\"}," "\"format\":{\"type\":\"string\",\"enum\":[\"tree\",\"json\"],\"default\":\"tree\"}," diff --git a/src/operations/trace.c b/src/operations/trace.c index 2674333d1f..d137113243 100644 --- a/src/operations/trace.c +++ b/src/operations/trace.c @@ -1,6 +1,7 @@ +#include "callable_sig.h" +#include "operations/json_args.h" #include "operations/output_budget.h" #include "operations/result_wire.h" -#include "operations/json_args.h" #include "operations/trace.h" #include "operations/store_host.h" @@ -57,13 +58,7 @@ static char *string_arg(const char *args, const char *name) { } static int int_arg(const char *args, const char *name, int fallback) { - yyjson_doc *doc = args_doc(args); - yyjson_val *root = doc ? yyjson_doc_get_root(doc) : NULL; - yyjson_val *value = yyjson_is_obj(root) ? yyjson_obj_get(root, name) : NULL; - int result = value && yyjson_is_int(value) ? (int)yyjson_get_sint(value) : fallback; - if (doc) - yyjson_doc_free(doc); - return result; + return cbm_json_int_arg(args, name, fallback); } static bool bool_arg(const char *args, const char *name) { @@ -620,9 +615,14 @@ static bool edge_evidence(const cbm_edge_info_t *edge, char class_buffer[32], do return true; } -static size_t qn_prefix_length(const char *qualified_name) { - const char *last = qualified_name ? strrchr(qualified_name, '.') : NULL; - return last ? (size_t)(last - qualified_name) : 0U; +static size_t qn_prefix_length(const char *qualified_name, const char *name) { + size_t last = 0; + size_t base_len = cbm_qn_callable_base_len_named(qualified_name, name); + for (size_t i = 0; i < base_len; i++) { + if (qualified_name[i] == '.') + last = i; + } + return last; } static yyjson_mut_val *leg_json(yyjson_mut_doc *doc, const cbm_traverse_result_t *result, @@ -653,7 +653,7 @@ static yyjson_mut_val *leg_json(yyjson_mut_doc *doc, const cbm_traverse_result_t for (int i = 0; i < result->visited_count; ++i) { const cbm_node_hop_t *hop = &result->visited[i]; const char *qn = hop->node.qualified_name ? hop->node.qualified_name : ""; - size_t prefix = qn_prefix_length(qn); + size_t prefix = qn_prefix_length(qn, hop->node.name); if (prefix >= sizeof(group_name)) prefix = 0U; if (!have_group || strlen(group_name) != prefix || strncmp(group_name, qn, prefix) != 0) { @@ -772,6 +772,11 @@ cbm_operation_result_t cbm_trace_operation_execute(const char *args) { node_count = 1; } } + if (node_count == 0) { + cbm_store_free_nodes(nodes, 0); + nodes = NULL; + cbm_store_find_nodes_by_qn_base(store, project, function, false, &nodes, &node_count); + } if (node_count == 0) { result = error_result( "function not found", diff --git a/src/pipeline/laravel_routing.c b/src/pipeline/laravel_routing.c new file mode 100644 index 0000000000..44acf8e288 --- /dev/null +++ b/src/pipeline/laravel_routing.c @@ -0,0 +1,403 @@ +/* + * laravel_routing.c — Laravel 11+ framework route mounts (#1146). + * + * See laravel_routing.h for the contract. bootstrap/app.php is parsed with + * the same PHP grammar the extractor uses; only named `withRouting(...)` + * arguments whose values are plain literals (or `__DIR__`/`dirname(__DIR__)` + * concatenations and `base_path()` calls over literals) are trusted. + */ +#include "pipeline/laravel_routing.h" + +#include "cbm.h" +#include "lang_specs.h" +#include "service_patterns.h" +#include "tree_sitter/api.h" +#include "foundation/compat_fs.h" +#include "foundation/constants.h" +#include "foundation/mem_core.h" + +#include +#include +#include +#include + +enum { + LR_PATH_MAX = CBM_SZ_512, /* repo-relative path buffer */ + LR_KEY_MAX = CBM_SZ_32, /* named-argument key */ + LR_PREFIX_MAX = CBM_SZ_128, /* apiPrefix literal */ + LR_SEG_MAX = CBM_SZ_64, /* path segments tracked while normalising */ + LR_SOURCE_MAX = CBM_SZ_256 * CBM_SZ_1K, /* bootstrap/app.php read cap */ +}; + +static const char LR_BOOTSTRAP_REL[] = "bootstrap/app.php"; + +/* Copy a node's source text into buf. False when it does not fit. */ +static bool lr_node_text(const char *src, TSNode node, char *buf, size_t buf_sz) { + if (ts_node_is_null(node)) { + return false; + } + uint32_t start = ts_node_start_byte(node); + uint32_t end = ts_node_end_byte(node); + size_t len = end > start ? (size_t)(end - start) : 0; + if (len >= buf_sz) { + return false; + } + memcpy(buf, src + start, len); + buf[len] = '\0'; + return true; +} + +static bool lr_node_is(TSNode node, const char *type) { + return !ts_node_is_null(node) && strcmp(ts_node_type(node), type) == 0; +} + +static bool lr_node_text_is(const char *src, TSNode node, const char *text) { + char buf[LR_KEY_MAX]; + return lr_node_text(src, node, buf, sizeof(buf)) && strcmp(buf, text) == 0; +} + +/* A plain string literal: '...' or "..." whose only children are + * string_content (no interpolation, no escapes). Writes the unquoted value. */ +static bool lr_string_literal(const char *src, TSNode node, char *buf, size_t buf_sz) { + if (!lr_node_is(node, "string") && !lr_node_is(node, "encapsed_string")) { + return false; + } + size_t pos = 0; + buf[0] = '\0'; + uint32_t nc = ts_node_named_child_count(node); + for (uint32_t i = 0; i < nc; i++) { + TSNode part = ts_node_named_child(node, i); + if (!lr_node_is(part, "string_content") || + !lr_node_text(src, part, buf + pos, buf_sz - pos)) { + return false; + } + pos += strlen(buf + pos); + } + return true; +} + +/* The value expression of a call argument: its first named child that is + * neither the `name:` label nor a comment. */ +static TSNode lr_arg_value(TSNode arg, TSNode label) { + uint32_t nc = ts_node_named_child_count(arg); + for (uint32_t i = 0; i < nc; i++) { + TSNode child = ts_node_named_child(arg, i); + if ((ts_node_is_null(label) || !ts_node_eq(child, label)) && + !lr_node_is(child, "comment")) { + return child; + } + } + TSNode none = {0}; + return none; +} + +/* The single literal argument of `fn('...')` when the callee is `fn`. */ +static bool lr_call_literal_arg(const char *src, TSNode call, const char *fn, char *buf, + size_t buf_sz) { + if (!lr_node_is(call, "function_call_expression") || + !lr_node_text_is(src, ts_node_child_by_field_name(call, TS_FIELD("function")), fn)) { + return false; + } + TSNode args = ts_node_child_by_field_name(call, TS_FIELD("arguments")); + if (ts_node_is_null(args) || ts_node_named_child_count(args) != SKIP_ONE) { + return false; + } + TSNode arg = ts_node_named_child(args, 0); + TSNode val = lr_arg_value(arg, ts_node_child_by_field_name(arg, TS_FIELD("name"))); + return lr_string_literal(src, val, buf, buf_sz); +} + +/* `dirname(__DIR__)` — the application base directory. */ +static bool lr_is_dirname_dir(const char *src, TSNode node) { + if (!lr_node_is(node, "function_call_expression") || + !lr_node_text_is(src, ts_node_child_by_field_name(node, TS_FIELD("function")), "dirname")) { + return false; + } + TSNode args = ts_node_child_by_field_name(node, TS_FIELD("arguments")); + if (ts_node_is_null(args) || ts_node_named_child_count(args) != SKIP_ONE) { + return false; + } + TSNode arg = ts_node_named_child(args, 0); + TSNode val = lr_arg_value(arg, ts_node_child_by_field_name(arg, TS_FIELD("name"))); + return lr_node_is(val, "name") && lr_node_text_is(src, val, "__DIR__"); +} + +/* Resolve "." and ".." segments of a '/'-joined path in place. False when the + * path climbs above the repository root. */ +static bool lr_normalize(char *path) { + size_t seg_start[LR_SEG_MAX]; + int depth = 0; + size_t out = 0; + const char *p = path; + while (*p) { + const char *slash = strchr(p, '/'); + size_t len = slash ? (size_t)(slash - p) : strlen(p); + if (len == PAIR_LEN && p[0] == '.' && p[SKIP_ONE] == '.') { + if (depth == 0) { + return false; + } + out = seg_start[--depth]; + } else if (len > 0 && !(len == SKIP_ONE && p[0] == '.')) { + if (depth == LR_SEG_MAX) { + return false; + } + size_t at = out > 0 ? out + SKIP_ONE : 0; + seg_start[depth++] = out; + if (out > 0) { + path[out] = '/'; + } + memmove(path + at, p, len); + out = at + len; + } + p += len + (slash ? SKIP_ONE : 0); + } + path[out] = '\0'; + return true; +} + +/* Resolve one `api:` path expression of the app rooted at app_dir to a + * normalised repo-relative path: + * __DIR__ . '/../routes/api.php' (relative to bootstrap/) + * dirname(__DIR__) . '/routes/api.php' (relative to the app base) + * base_path('routes/api.php') (relative to the app base) */ +static bool lr_route_file_path(const char *src, TSNode expr, const char *app_dir, char *out, + size_t out_sz) { + char lit[LR_PATH_MAX]; + const char *base_suffix = NULL; + if (lr_node_is(expr, "binary_expression")) { + TSNode op = ts_node_child_by_field_name(expr, TS_FIELD("operator")); + TSNode left = ts_node_child_by_field_name(expr, TS_FIELD("left")); + if (!lr_node_text_is(src, op, ".") || + !lr_string_literal(src, ts_node_child_by_field_name(expr, TS_FIELD("right")), lit, + sizeof(lit))) { + return false; + } + if (lr_node_is(left, "name") && lr_node_text_is(src, left, "__DIR__")) { + base_suffix = "bootstrap"; + } else if (lr_is_dirname_dir(src, left)) { + base_suffix = ""; + } else { + return false; + } + } else if (lr_call_literal_arg(src, expr, "base_path", lit, sizeof(lit))) { + base_suffix = ""; + } else { + return false; + } + int n = snprintf(out, out_sz, "%s/%s/%s", app_dir, base_suffix, lit); + return n > 0 && (size_t)n < out_sz && lr_normalize(out); +} + +/* Does the `api:` value (one path or an array of paths) name rel_path? */ +static bool lr_api_names_file(const char *src, TSNode value, const char *app_dir, + const char *rel_path) { + char resolved[LR_PATH_MAX]; + if (!lr_node_is(value, "array_creation_expression")) { + return lr_route_file_path(src, value, app_dir, resolved, sizeof(resolved)) && + strcmp(resolved, rel_path) == 0; + } + uint32_t nc = ts_node_named_child_count(value); + for (uint32_t i = 0; i < nc; i++) { + TSNode elem = ts_node_named_child(value, i); + uint32_t ec = ts_node_named_child_count(elem); + if (!lr_node_is(elem, "array_element_initializer") || ec == 0) { + continue; + } + TSNode path_expr = ts_node_named_child(elem, ec - SKIP_ONE); /* value of `k => v` too */ + if (lr_route_file_path(src, path_expr, app_dir, resolved, sizeof(resolved)) && + strcmp(resolved, rel_path) == 0) { + return true; + } + } + return false; +} + +/* Evaluate the named arguments of a withRouting(...) call. */ +static bool lr_eval_with_routing(const char *src, TSNode call, const char *app_dir, + const char *rel_path, char *out, size_t out_sz) { + TSNode args = ts_node_child_by_field_name(call, TS_FIELD("arguments")); + char prefix[LR_PREFIX_MAX] = "api"; /* ApplicationBuilder::withRouting default */ + bool prefix_known = true; + bool mounted = false; + uint32_t nc = ts_node_is_null(args) ? 0 : ts_node_named_child_count(args); + for (uint32_t i = 0; i < nc; i++) { + TSNode arg = ts_node_named_child(args, i); + if (lr_node_is(arg, "comment")) { + continue; + } + TSNode label = ts_node_child_by_field_name(arg, TS_FIELD("name")); + char key[LR_KEY_MAX]; + /* Positional arguments would need the parameter order modelled, and a + * `using:` callback replaces the web/api registration entirely. */ + if (!lr_node_is(arg, "argument") || ts_node_is_null(label) || + !lr_node_text(src, label, key, sizeof(key)) || strcmp(key, "using") == 0) { + return false; + } + TSNode value = lr_arg_value(arg, label); + if (strcmp(key, "api") == 0) { + mounted = lr_api_names_file(src, value, app_dir, rel_path); + } else if (strcmp(key, "apiPrefix") == 0) { + prefix_known = lr_string_literal(src, value, prefix, sizeof(prefix)); + } + } + if (!mounted || !prefix_known) { + return false; + } + /* Laravel trims the group prefix of '/' on both ends. */ + const char *start = prefix; + size_t len = strlen(prefix); + while (*start == '/') { + start++; + len--; + } + while (len > 0 && start[len - SKIP_ONE] == '/') { + len--; + } + int n = snprintf(out, out_sz, "/%.*s", (int)len, start); + return len > 0 && n > 0 && (size_t)n < out_sz; +} + +/* First `->withRouting(...)` call in the tree (iterative pre-order walk). */ +static TSNode lr_find_with_routing(const char *src, TSNode root) { + TSNode found = {0}; + TSTreeCursor cur = ts_tree_cursor_new(root); + bool more = true; + while (more) { + TSNode node = ts_tree_cursor_current_node(&cur); + if ((lr_node_is(node, "member_call_expression") || + lr_node_is(node, "nullsafe_member_call_expression")) && + lr_node_text_is(src, ts_node_child_by_field_name(node, TS_FIELD("name")), + "withRouting")) { + found = node; + break; + } + if (ts_tree_cursor_goto_first_child(&cur)) { + continue; + } + while (!ts_tree_cursor_goto_next_sibling(&cur)) { + if (!ts_tree_cursor_goto_parent(&cur)) { + more = false; + break; + } + } + } + ts_tree_cursor_delete(&cur); + return found; +} + +static bool lr_mount_from_bootstrap(const char *src, uint32_t len, const char *app_dir, + const char *rel_path, char *out, size_t out_sz) { + const TSLanguage *php = cbm_ts_language(CBM_LANG_PHP); + TSParser *parser = php ? ts_parser_new() : NULL; + if (!parser) { + return false; + } + bool mounted = false; + TSTree *tree = + ts_parser_set_language(parser, php) ? ts_parser_parse_string(parser, NULL, src, len) : NULL; + if (tree) { + TSNode call = lr_find_with_routing(src, ts_tree_root_node(tree)); + mounted = !ts_node_is_null(call) && + lr_eval_with_routing(src, call, app_dir, rel_path, out, out_sz); + ts_tree_delete(tree); + } + ts_parser_delete(parser); + return mounted; +} + +/* Read //bootstrap/app.php. NULL when absent or unreadable. */ +static char *lr_read_bootstrap(const char *repo_path, const char *app_dir, uint32_t *out_len) { + char path[CBM_SZ_4K]; + int n = snprintf(path, sizeof(path), "%s/%s%s%s", repo_path, app_dir, app_dir[0] ? "/" : "", + LR_BOOTSTRAP_REL); + if (n <= 0 || (size_t)n >= sizeof(path)) { + return NULL; + } + FILE *f = cbm_fopen(path, "rb"); + if (!f) { + return NULL; + } + (void)fseek(f, 0, SEEK_END); + long size = ftell(f); + (void)fseek(f, 0, SEEK_SET); + char *buf = (size > 0 && size <= (long)LR_SOURCE_MAX) + ? cbm_alloc(CBM_MEM_CLASS_EXTRACT, (size_t)size + SKIP_ONE) + : NULL; + if (buf) { + size_t nread = fread(buf, SKIP_ONE, (size_t)size, f); + buf[nread] = '\0'; + *out_len = (uint32_t)nread; + } + (void)fclose(f); + return buf; +} + +bool cbm_laravel_api_mount(const char *repo_path, const char *rel_path, char *out, size_t out_sz) { + if (!out || out_sz == 0) { + return false; + } + out[0] = '\0'; + size_t rel_len = rel_path ? strlen(rel_path) : 0; + if (!repo_path || rel_len < sizeof(".php") || rel_len >= LR_PATH_MAX || + strcmp(rel_path + rel_len - (sizeof(".php") - SKIP_ONE), ".php") != 0) { + return false; + } + /* Walk the file's ancestor directories nearest-first; the nearest one + * holding bootstrap/app.php is the Laravel app that owns the file. */ + char app_dir[LR_PATH_MAX]; + memcpy(app_dir, rel_path, rel_len + SKIP_ONE); + char *slash = strrchr(app_dir, '/'); + for (;;) { + if (slash) { + *slash = '\0'; + } else { + app_dir[0] = '\0'; + } + uint32_t len = 0; + char *src = lr_read_bootstrap(repo_path, app_dir, &len); + if (src) { + bool mounted = lr_mount_from_bootstrap(src, len, app_dir, rel_path, out, out_sz); + cbm_free(CBM_MEM_CLASS_EXTRACT, src); + if (!mounted) { + out[0] = '\0'; + } + return mounted; + } + if (!app_dir[0]) { + return false; + } + slash = strrchr(app_dir, '/'); + } +} + +void cbm_laravel_file_route_mount(const char *repo_path, const char *rel_path, CBMLanguage lang, + const CBMFileResult *result, char *out, size_t out_sz) { + if (!out || out_sz == 0) { + return; + } + out[0] = '\0'; + if (lang != CBM_LANG_PHP || !result) { + return; + } + for (int i = 0; i < result->calls.count; i++) { + const CBMCall *call = &result->calls.items[i]; + if (call->callee_name && call->first_string_arg && call->first_string_arg[0] == '/' && + cbm_service_pattern_route_method(call->callee_name) != NULL) { + (void)cbm_laravel_api_mount(repo_path, rel_path, out, out_sz); + return; + } + } +} + +const char *cbm_laravel_mount_route(const char *mount, const char *path, char *buf, size_t buf_sz) { + if (!mount || !mount[0] || !path || !buf || buf_sz == 0) { + return path; + } + const char *rel = path; + while (*rel == '/') { + rel++; + } + int n = + rel[0] ? snprintf(buf, buf_sz, "%s/%s", mount, rel) : snprintf(buf, buf_sz, "%s", mount); + return (n > 0 && (size_t)n < buf_sz) ? buf : path; +} diff --git a/src/pipeline/laravel_routing.h b/src/pipeline/laravel_routing.h new file mode 100644 index 0000000000..66e2d6a751 --- /dev/null +++ b/src/pipeline/laravel_routing.h @@ -0,0 +1,48 @@ +#ifndef CBM_LARAVEL_ROUTING_H +#define CBM_LARAVEL_ROUTING_H + +/* + * laravel_routing.h — Laravel 11+ framework route mounts (#1146). + * + * Laravel 11+ registers route files in bootstrap/app.php: + * + * Application::configure(basePath: dirname(__DIR__)) + * ->withRouting(web: __DIR__.'/../routes/web.php', + * api: __DIR__.'/../routes/api.php', + * apiPrefix: 'api') + * + * and mounts every file passed as `api:` under `apiPrefix` (default 'api'); + * the `web:` file gets no prefix. The prefix is declared in a DIFFERENT file + * than the routes, so per-file extraction cannot compose it the way it does + * the in-file `Route::prefix()->group()` chain (#952). The route passes ask + * here once per PHP file that registers routes and prepend the mount to every + * Route path minted from that file. + * + * Only what bootstrap/app.php states as source fact is used: the nearest + * ancestor bootstrap/app.php must name the file under `api:`, and a given + * `apiPrefix:` must be a string literal. A non-literal prefix, positional + * arguments or a `using:` callback leave the paths untouched — nothing is + * inferred from file names. + */ + +#include "cbm.h" +#include +#include + +/* Write the mount ("/api", "/v1/api") under which the Laravel app that owns + * rel_path serves its routes into out and return true. Returns false with + * out = "" when rel_path is not mounted under a known, non-empty prefix. */ +bool cbm_laravel_api_mount(const char *repo_path, const char *rel_path, char *out, size_t out_sz); + +/* Per-file entry for the route passes: probes cbm_laravel_api_mount only for + * a PHP file that registers at least one path-shaped route; out = "" for + * every other file, so no other language pays a lookup. */ +void cbm_laravel_file_route_mount(const char *repo_path, const char *rel_path, CBMLanguage lang, + const CBMFileResult *result, char *out, size_t out_sz); + +/* Compose mount + route path: "/api" + "/users/me" -> "/api/users/me", + * "/api" + "/" -> "/api". Returns path unchanged when mount is empty or the + * composed path does not fit buf. */ +const char *cbm_laravel_mount_route(const char *mount, const char *path, char *buf, size_t buf_sz); + +#endif /* CBM_LARAVEL_ROUTING_H */ diff --git a/src/pipeline/lsp_resolve.h b/src/pipeline/lsp_resolve.h index 666f9fd552..0e92150661 100644 --- a/src/pipeline/lsp_resolve.h +++ b/src/pipeline/lsp_resolve.h @@ -22,6 +22,7 @@ #define CBM_PIPELINE_LSP_RESOLVE_H #include "cbm.h" +#include "callable_sig.h" /* cbm_qn_callable_base_len — leaf splitters skip the suffix */ #include "graph_buffer/graph_buffer.h" #include "foundation/constants.h" @@ -64,7 +65,10 @@ static inline const char *cbm_lsp_bare_segment(const char *name) { return name; } const char *seg = name; - for (const char *p = name; *p; p++) { + /* Scan the base only: a callable identity suffix (#2061) belongs to the + * leaf and may itself contain ':' (Swift labels) or '>' (generics). */ + const char *end = name + cbm_qn_callable_base_len(name); + for (const char *p = name; p < end; p++) { /* '.' (dotted QN / Java-style member) and ':' (C++ `::`, last colon * wins) are member/scope separators. '>' is only a separator when it * closes the `->` arrow (preceded by '-'); a bare '>' closes a template @@ -86,7 +90,15 @@ static inline const char *cbm_pipeline_qn_class_method_tail(const char *qn) { if (!qn) { return NULL; } - const char *last = strrchr(qn, '.'); + /* The last '.' of the BASE QN: a callable identity suffix (#2061) never + * contains '.', so this is the historical strrchr for every QN. */ + const char *last = NULL; + size_t base_len = cbm_qn_callable_base_len(qn); + for (size_t i = 0; i < base_len; i++) { + if (qn[i] == '.') { + last = qn + i; + } + } if (!last || last == qn) { return NULL; } diff --git a/src/pipeline/pass_calls.c b/src/pipeline/pass_calls.c index f068ad4112..6689196820 100644 --- a/src/pipeline/pass_calls.c +++ b/src/pipeline/pass_calls.c @@ -19,6 +19,7 @@ enum { PC_RING = 4, PC_RING_MASK = 3, PC_SIG_SCAN = 15, PC_REGEX_GRP = 2 }; #include #include "pipeline/pipeline_internal.h" #include "pipeline/lsp_resolve.h" +#include "pipeline/laravel_routing.h" #include "graph_buffer/graph_buffer.h" #include "foundation/log.h" #include "foundation/compat.h" @@ -374,23 +375,29 @@ bool cbm_pipeline_http_client_call_url(const cbm_gbuf_t *gbuf, const char *proje return n > 0 && (size_t)n < out_sz; } -/* Handle a route registration call: create Route node + HANDLES edge. */ +/* Handle a route registration call: create Route node + HANDLES edge. + * route_mount is the framework mount of the registering file ("/api" for a + * Laravel 11+ `withRouting(api: ...)` file, #1146) or "". */ static void handle_route_registration(cbm_pipeline_ctx_t *ctx, const CBMCall *call, const cbm_gbuf_node_t *source_node, const char *module_qn, - const char **imp_keys, const char **imp_vals, int imp_count) { + const char **imp_keys, const char **imp_vals, int imp_count, + const char *route_mount) { const char *method = cbm_service_pattern_route_method(call->callee_name); + char mounted[CBM_SZ_256]; + const char *path = + cbm_laravel_mount_route(route_mount, call->first_string_arg, mounted, sizeof(mounted)); char route_qn[CBM_ROUTE_QN_SIZE]; char cpath[CBM_SZ_256]; snprintf(route_qn, sizeof(route_qn), "__route__%s__%s", method ? method : "ANY", - cbm_route_canon_path(call->first_string_arg, cpath, sizeof(cpath))); + cbm_route_canon_path(path, cpath, sizeof(cpath))); char route_props[CBM_SZ_256]; snprintf(route_props, sizeof(route_props), "{\"method\":\"%s\"}", method ? method : "ANY"); - int64_t route_id = cbm_gbuf_upsert_node(ctx->gbuf, "Route", call->first_string_arg, route_qn, - "", 0, 0, route_props); + int64_t route_id = + cbm_gbuf_upsert_node(ctx->gbuf, "Route", path, route_qn, "", 0, 0, route_props); char esc_cn[CBM_SZ_256]; /* sliced source text: escape quotes/newlines */ char esc_fa[CBM_SZ_256]; cbm_json_escape(esc_cn, sizeof(esc_cn), call->callee_name); - cbm_json_escape(esc_fa, sizeof(esc_fa), call->first_string_arg); + cbm_json_escape(esc_fa, sizeof(esc_fa), path); char props[CBM_SZ_512]; snprintf(props, sizeof(props), "{\"callee\":\"%s\",\"url_path\":\"%s\",\"via\":\"route_registration\"}", esc_cn, @@ -588,10 +595,11 @@ static void emit_classified_edge(cbm_pipeline_ctx_t *ctx, const CBMCall *call, const cbm_gbuf_node_t *source, const cbm_gbuf_node_t *target, const cbm_resolution_t *res, const char *module_qn, const char **imp_keys, const char **imp_vals, int imp_count, - bool suppress_plain_calls) { + bool suppress_plain_calls, const char *route_mount) { cbm_svc_kind_t svc = cbm_service_pattern_match(res->qualified_name); if (svc == CBM_SVC_ROUTE_REG && call->first_string_arg && call->first_string_arg[0] == '/') { - handle_route_registration(ctx, call, source, module_qn, imp_keys, imp_vals, imp_count); + handle_route_registration(ctx, call, source, module_qn, imp_keys, imp_vals, imp_count, + route_mount); return; } if (svc == CBM_SVC_HTTP || svc == CBM_SVC_ASYNC) { @@ -646,7 +654,7 @@ static const cbm_gbuf_node_t *calls_find_source(cbm_pipeline_ctx_t *ctx, const c static int resolve_single_call(cbm_pipeline_ctx_t *ctx, CBMCall *call, const CBMFileResult *result, const char *rel, const char *module_qn, const char **imp_keys, const char **imp_vals, int imp_count, CBMLanguage lang, - const CBMImportArray *imports) { + const CBMImportArray *imports, const char *route_mount) { const CBMResolvedCallArray *lsp_calls = &result->resolved_calls; const cbm_gbuf_node_t *source_node = calls_find_source(ctx, rel, call->enclosing_func_qn); if (!source_node) { @@ -675,7 +683,7 @@ static int resolve_single_call(cbm_pipeline_ctx_t *ctx, CBMCall *call, const CBM res.strategy = lsp->strategy; res.candidate_count = 1; emit_classified_edge(ctx, call, source_node, target_node, &res, module_qn, imp_keys, - imp_vals, imp_count, false); + imp_vals, imp_count, false, route_mount); return SKIP_ONE; } } @@ -767,7 +775,7 @@ static int resolve_single_call(cbm_pipeline_ctx_t *ctx, CBMCall *call, const CBM if (cbm_service_pattern_route_method(call->callee_name) != NULL && call->first_string_arg && call->first_string_arg[0] == '/') { handle_route_registration(ctx, call, source_node, module_qn, imp_keys, imp_vals, - imp_count); + imp_count, route_mount); return SKIP_ONE; } cbm_svc_kind_t esvc = cbm_service_pattern_match(call->callee_name); @@ -885,10 +893,23 @@ static int resolve_single_call(cbm_pipeline_ctx_t *ctx, CBMCall *call, const CBM * Store.commit() call to a JS function named commit (or a Bash main * to a Python main). Drop that weak cross-language edge. */ if (cbm_suppress_cross_language_suffix_match(lang, target_node->file_path, res.strategy)) { + /* Refusing the binding leaves the callee unresolved, and an + * unresolved route-registration callee is still a registration: + * `Route::get('/x', ...)` beside a JS `get`, `app.get('/x', h)` + * beside a Python `get`. Give it the same Route the #952 + * unresolved-callee fallback above mints, and nothing else. Dropping + * the whole call lost every GET registration in a mixed-language repo + * while POST (no `post` to collide with) survived. */ + if (cbm_service_pattern_route_method(call->callee_name) != NULL && call->first_string_arg && + call->first_string_arg[0] == '/') { + handle_route_registration(ctx, call, source_node, module_qn, imp_keys, imp_vals, + imp_count, route_mount); + return SKIP_ONE; + } return 0; } emit_classified_edge(ctx, call, source_node, target_node, &res, module_qn, imp_keys, imp_vals, - imp_count, drop_plain_call); + imp_count, drop_plain_call, route_mount); return SKIP_ONE; } @@ -1039,6 +1060,11 @@ int cbm_pipeline_pass_calls(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *file char *module_qn = cbm_pipeline_fqn_module_dir(ctx->project_name, rel, pc_module_is_dir(files[i].language)); + /* Framework mount of this file's routes (Laravel 11+ withRouting, #1146). */ + char route_mount[CBM_SZ_128]; + cbm_laravel_file_route_mount(ctx->repo_path, rel, files[i].language, result, route_mount, + sizeof(route_mount)); + /* Resolve each call */ for (int c = 0; c < result->calls.count; c++) { CBMCall *call = &result->calls.items[c]; @@ -1047,7 +1073,7 @@ int cbm_pipeline_pass_calls(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *file } total_calls++; if (resolve_single_call(ctx, call, result, rel, module_qn, imp_keys, imp_vals, - imp_count, files[i].language, &result->imports)) { + imp_count, files[i].language, &result->imports, route_mount)) { resolved++; } else { unresolved++; diff --git a/src/pipeline/pass_definitions.c b/src/pipeline/pass_definitions.c index 804ffba659..453894785f 100644 --- a/src/pipeline/pass_definitions.c +++ b/src/pipeline/pass_definitions.c @@ -22,6 +22,7 @@ enum { PD_JSON_FIELD_OVERHEAD = 6 }; #include "pipeline/pipeline_internal.h" #include "graph_buffer/graph_buffer.h" #include "foundation/log.h" +#include "foundation/mem_core.h" /* cbm_alloc/cbm_free: the grown properties buffer */ #include "foundation/compat.h" #include "foundation/compat_fs.h" #include "foundation/limits.h" @@ -247,8 +248,9 @@ static void append_json_str_array(char *buf, size_t bufsize, size_t *pos, const *pos = p; } -/* Build properties JSON for a definition node. */ -static void build_def_props(char *buf, size_t bufsize, const CBMDefinition *def) { +/* Build properties JSON for a definition node. Shared with pass_parallel.c + * (declared in pipeline_internal.h) so the two passes cannot drift. */ +void cbm_def_props_build(char *buf, size_t bufsize, const CBMDefinition *def) { /* The complexity/loop/recursion metrics are only meaningful for executable * units (Function/Method). Emitting them on the millions of Macro/Field/ * Variable/Class/Enum nodes — where they are always zero — bloats every @@ -321,16 +323,73 @@ static void build_def_props(char *buf, size_t bufsize, const CBMDefinition *def) } } -/* Process one definition: create node, register, DEFINES + DEFINES_METHOD edges. */ -static void process_def(cbm_pipeline_ctx_t *ctx, const CBMDefinition *def, const char *rel) { +/* A def's properties buffer: CBM_SZ_2K for every other field plus the whole + * serialized docstring field, which has no length cap (a field that does not + * fit is dropped whole). Returns `stack` for a def without a docstring, or + * when the larger buffer cannot be allocated. Shared with pass_parallel.c. */ +char *cbm_def_props_buf(const CBMDefinition *def, char *stack, size_t *size) { + if (!def->docstring || !def->docstring[0]) { + return stack; + } + size_t need = + *size + strlen("docstring") + def_json_escaped_len(def->docstring) + PD_JSON_FIELD_OVERHEAD; + char *buf = cbm_alloc(CBM_MEM_CLASS_GBUF_STRING, need); + if (!buf) { + return stack; + } + *size = need; + return buf; +} + +/* Add a file's own doc (Go package comment, Rust inner docs) to its File + * node as "docstring". Shared with pass_parallel.c. */ +void cbm_def_file_doc(const cbm_gbuf_node_t *file_node, const char *doc) { + if (!file_node || !doc || !doc[0]) { + return; + } + const char *old = file_node->properties_json ? file_node->properties_json : "{}"; + size_t olen = strlen(old); + if (olen < PAIR_LEN || old[olen - SKIP_ONE] != '}') { + return; /* not a JSON object -- leave it untouched */ + } + size_t cap = olen + strlen("docstring") + def_json_escaped_len(doc) + PD_JSON_FIELD_OVERHEAD + + PD_ESC_SPACE + SKIP_ONE; + char *neu = cbm_alloc(CBM_MEM_CLASS_GBUF_STRING, cap); + if (!neu) { + return; + } + size_t pos = olen - SKIP_ONE; /* without the closing brace */ + memcpy(neu, old, pos); + neu[pos] = '\0'; + append_json_string(neu, cap, &pos, "docstring", doc); + if (olen == PAIR_LEN && pos > PAIR_LEN) { /* "{}": drop the leading comma */ + memmove(neu + SKIP_ONE, neu + PAIR_LEN, pos - SKIP_ONE); + pos--; + } + neu[pos++] = '}'; + neu[pos] = '\0'; + (void)cbm_gbuf_node_set_properties_json((cbm_gbuf_node_t *)file_node, neu); + cbm_free(CBM_MEM_CLASS_GBUF_STRING, neu); +} + +/* Process one definition: create node, register, DEFINES + DEFINES_METHOD edges. + * `file_doc`, the file's own doc (passed with its first def only), goes on the + * File node. */ +static void process_def(cbm_pipeline_ctx_t *ctx, const CBMDefinition *def, const char *rel, + const char *file_doc) { if (!def->qualified_name || !def->name) { return; } - char props[CBM_SZ_2K]; - build_def_props(props, sizeof(props), def); + char stack[CBM_SZ_2K]; + size_t props_size = sizeof(stack); + char *props = cbm_def_props_buf(def, stack, &props_size); + cbm_def_props_build(props, props_size, def); int64_t node_id = cbm_gbuf_upsert_node( ctx->gbuf, def->label ? def->label : "Function", def->name, def->qualified_name, def->file_path ? def->file_path : rel, (int)def->start_line, (int)def->end_line, props); + if (props != stack) { + cbm_free(CBM_MEM_CLASS_GBUF_STRING, props); + } /* Registry membership is defined ONCE by cbm_label_is_registry_symbol * (helpers.c): callables + type-like containers (INHERITS/IMPLEMENTS/method/ * field resolution), Variable/Field (READS/WRITES resolution), and Table/View @@ -341,6 +400,7 @@ static void process_def(cbm_pipeline_ctx_t *ctx, const CBMDefinition *def, const } char *file_qn = cbm_pipeline_fqn_compute(ctx->project_name, rel, "__file__"); const cbm_gbuf_node_t *file_node = cbm_gbuf_find_by_qn(ctx->gbuf, file_qn); + cbm_def_file_doc(file_node, file_doc); if (file_node && node_id > 0) { cbm_gbuf_insert_edge(ctx->gbuf, file_node->id, node_id, "DEFINES", "{}"); } @@ -860,8 +920,10 @@ int cbm_pipeline_pass_definitions(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t } /* Create nodes for each definition */ + const char *file_doc = result->module_doc; /* goes with the first def */ for (int d = 0; d < result->defs.count; d++) { - process_def(ctx, &result->defs.items[d], rel); + process_def(ctx, &result->defs.items[d], rel, file_doc); + file_doc = NULL; total_defs++; } diff --git a/src/pipeline/pass_ensemble_routing.c b/src/pipeline/pass_ensemble_routing.c index 0aa574839d..1a6ca09809 100644 --- a/src/pipeline/pass_ensemble_routing.c +++ b/src/pipeline/pass_ensemble_routing.c @@ -6,6 +6,7 @@ #include "foundation/compat_fs.h" #include "foundation/constants.h" #include "foundation/str_util.h" +#include "callable_sig.h" #include #include @@ -425,7 +426,14 @@ static void collect_prod_defs(cbm_pipeline_ctx_t *ctx, ens_prod_def_t ***defs_ou char class_qn[CBM_SZ_256]; class_qn[0] = '\0'; if (xd->qualified_name) { - const char *dot = strrchr(xd->qualified_name, '.'); + /* Owner = base QN minus its leaf (#2061 suffixes hold no '.'). */ + const char *dot = NULL; + size_t base_len = cbm_qn_callable_base_len_named(xd->qualified_name, xd->name); + for (size_t i = 0; i < base_len; i++) { + if (xd->qualified_name[i] == '.') { + dot = xd->qualified_name + i; + } + } if (dot) { int len = (int)(dot - xd->qualified_name); if (len > 0 && len < CBM_SZ_256) { diff --git a/src/pipeline/pass_lsp_cross.c b/src/pipeline/pass_lsp_cross.c index 3b74cc1ebb..8f5c1efe32 100644 --- a/src/pipeline/pass_lsp_cross.c +++ b/src/pipeline/pass_lsp_cross.c @@ -19,6 +19,7 @@ #include "result_spill.h" #include "pipeline/pipeline_internal.h" #include "pipeline/lsp_resolve.h" +#include "callable_sig.h" #include "lsp/go_lsp.h" #include "lsp/c_lsp.h" #include "lsp/py_lsp.h" @@ -273,12 +274,20 @@ static const char *pxc_join_base_qns(CBMArena *arena, const char *const *bases, static bool pxc_is_jvm_lang(CBMLanguage lang); +/* Leaf of the BASE QN: a callable identity suffix (#2061) never contains + * '.', so this equals the historical strrchr split for every QN. */ static const char *pxc_last_component(const char *qn) { if (!qn) { return NULL; } - const char *dot = strrchr(qn, '.'); - return dot ? dot + 1 : qn; + const char *leaf = qn; + size_t base_len = cbm_qn_callable_base_len(qn); + for (size_t i = 0; i < base_len; i++) { + if (qn[i] == '.') { + leaf = qn + i + 1; + } + } + return leaf; } /* Every return is arena-owned: the fallback spelling is a copy, never the @@ -377,7 +386,9 @@ static const char *pxc_qn_leaf(const char *name) { return NULL; } const char *leaf = name; - for (const char *p = name; *p; p++) { + /* Base only: a callable identity suffix (#2061) may carry ':' labels. */ + const char *end = name + cbm_qn_callable_base_len(name); + for (const char *p = name; p < end; p++) { if (*p == '.' || *p == ':' || *p == '/' || *p == '\\') { leaf = p + 1; } @@ -529,6 +540,132 @@ static void pxc_fold_go_struct_fields(CBMArena *arena, const CBMFileResult *resu } } +/* Python annotation text -> the one type name it declares, or NULL. Strips a + * string-literal forward reference and unwraps `Optional[X]`, `X | None` and + * `None | X`; anything else with brackets, commas or a real union (generics, + * Callable, Union[A, B]) names no single receiver type and is dropped. */ +static const char *pxc_py_annotation_type_name(CBMArena *arena, const char *text) { + if (!text) { + return NULL; + } + while (*text == ' ') { + text++; + } + size_t n = strlen(text); + while (n > 0 && text[n - 1] == ' ') { + n--; + } + if (n >= 2 && (text[0] == '"' || text[0] == '\'') && text[n - 1] == text[0]) { + text++; + n -= 2; + } + static const char *const optional_prefixes[] = {"Optional[", "typing.Optional["}; + for (size_t i = 0; i < sizeof(optional_prefixes) / sizeof(optional_prefixes[0]); i++) { + size_t plen = strlen(optional_prefixes[i]); + if (n > plen + 1 && strncmp(text, optional_prefixes[i], plen) == 0 && text[n - 1] == ']') { + text += plen; + n -= plen + 1; + break; + } + } + static const char none_tail[] = " | None"; + static const char none_head[] = "None | "; + const size_t none_len = sizeof(none_tail) - 1; + if (n > none_len && strncmp(text + n - none_len, none_tail, none_len) == 0) { + n -= none_len; + } else if (n > none_len && strncmp(text, none_head, none_len) == 0) { + text += none_len; + n -= none_len; + } + if (n == 0) { + return NULL; + } + for (size_t i = 0; i < n; i++) { + char c = text[i]; + bool ok = (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || + c == '_' || c == '.'; + if (!ok) { + return NULL; + } + } + return cbm_arena_strndup(arena, text, n); +} + +/* Python: fold the file's annotated instance fields (result->field_types, + * #1277) into their class defs' field_defs as "name:QN|name:QN", the format + * py_register_lsp_defs reads. Each annotation resolves through the same + * registry + import map, and the same weak-strategy and type-like vetoes, as + * the file's base classes (pxc_resolve_base_qn): a field whose type is not a + * confidently-known project type is left out rather than guessed. Linear in + * the file: one hash of the file's class defs, one pass over its fields. */ +typedef struct { + int owner; /* index into defs[], or -1 */ + const char *type_qn; +} pxc_py_field_t; + +static void pxc_fold_py_field_types(CBMArena *arena, const CBMFileResult *result, CBMLSPDef *defs, + int start, int end, const cbm_registry_t *reg, + const char *module_qn, const char **imp_keys, + const char **imp_vals, int imp_count) { + int nf = result ? result->field_types.count : 0; + if (!arena || nf == 0 || !defs || start >= end || !reg) { + return; + } + CBMHashTable *by_qn = cbm_ht_create((uint32_t)(end - start)); + pxc_py_field_t *fields = (pxc_py_field_t *)cbm_arena_alloc(arena, (size_t)nf * sizeof(*fields)); + /* Per class def: exact buffer size, then the write cursor into it. */ + size_t *lens = (size_t *)cbm_arena_alloc(arena, (size_t)(end - start) * sizeof(size_t)); + size_t *used = (size_t *)cbm_arena_alloc(arena, (size_t)(end - start) * sizeof(size_t)); + if (!by_qn || !fields || !lens || !used) { + cbm_ht_free(by_qn); /* NULL-safe */ + return; + } + memset(lens, 0, (size_t)(end - start) * sizeof(size_t)); + memset(used, 0, (size_t)(end - start) * sizeof(size_t)); + for (int si = start; si < end; si++) { + if (defs[si].label && strcmp(defs[si].label, "Class") == 0 && defs[si].qualified_name) { + cbm_ht_set(by_qn, defs[si].qualified_name, &defs[si]); + } + } + for (int f = 0; f < nf; f++) { + const CBMFieldType *ft = &result->field_types.items[f]; + fields[f].owner = -1; + fields[f].type_qn = NULL; + CBMLSPDef *owner = ft->class_qn ? (CBMLSPDef *)cbm_ht_get(by_qn, ft->class_qn) : NULL; + const char *name = owner ? pxc_py_annotation_type_name(arena, ft->type_text) : NULL; + const char *qn = + name ? pxc_resolve_base_qn(reg, name, module_qn, imp_keys, imp_vals, imp_count) : NULL; + if (!qn || !ft->field_name || !ft->field_name[0]) { + continue; + } + fields[f].owner = (int)(owner - defs); + fields[f].type_qn = qn; + /* "name:QN" plus one byte for the '|' or the NUL after it. */ + lens[fields[f].owner - start] += strlen(ft->field_name) + 1 + strlen(qn) + 1; + } + cbm_ht_free(by_qn); + for (int si = start; si < end; si++) { + char *buf = lens[si - start] ? (char *)cbm_arena_alloc(arena, lens[si - start]) : NULL; + if (buf) { + buf[0] = '\0'; + } + defs[si].field_defs = buf; + } + for (int f = 0; f < nf; f++) { + char *buf = fields[f].owner >= 0 ? (char *)defs[fields[f].owner].field_defs : NULL; + if (!buf) { + continue; + } + int slot = fields[f].owner - start; + int wrote = + snprintf(buf + used[slot], lens[slot] - used[slot], "%s%s:%s", used[slot] ? "|" : "", + result->field_types.items[f].field_name, fields[f].type_qn); + if (wrote > 0) { + used[slot] += (size_t)wrote; + } + } +} + /* Carry one Rust type-level impl independently of any method definition. * `impl Trait for Type {}` is semantically meaningful even when the block is * empty (the trait may provide defaults), so attaching the relation only to @@ -634,6 +771,10 @@ CBMLSPDef *cbm_pxc_collect_all_defs(const cbm_pipeline_ctx_t *ctx, CBMArena *are idx++; } } + if (files[fi].language == CBM_LANG_PYTHON) { + pxc_fold_py_field_types(arena, fr, defs, file_start, idx, base_reg, def_modules[fi], + imp_keys, imp_vals, imp_count); + } cbm_pxc_free_import_map(imp_keys, imp_vals, imp_count); /* NULL-safe */ if (files[fi].language == CBM_LANG_GO) { pxc_fold_go_struct_fields(arena, fr, defs, file_start, idx); diff --git a/src/pipeline/pass_parallel.c b/src/pipeline/pass_parallel.c index 08f137422f..4e12441b7e 100644 --- a/src/pipeline/pass_parallel.c +++ b/src/pipeline/pass_parallel.c @@ -15,12 +15,7 @@ enum { PP_RING = 4, PP_RING_MASK = 3, - PP_JSON_MARGIN = 10, - PP_ESC_MARGIN = 3, PP_ESC_SPACE = 2, - /* Fixed bytes around a serialized JSON field: ,"key":"value" / ,"key":[...] - * -> comma + 2 key quotes + colon + 2 value quotes (resp. brackets). */ - PP_JSON_FIELD_OVERHEAD = 6, PP_ARGS_MARGIN = 20, /* ,"line": -> comma + key (7) + colon + up to 10 digits + NUL. */ PP_LINE_MARGIN = 24, @@ -74,6 +69,7 @@ enum { PP_CSHARP_M_PREFIX_LEN = 2 }; #include "foundation/platform.h" /* cbm_resolve_cache_dir */ #include "pipeline/pass_lsp_cross.h" /* cbm_pxc_* helpers for fused cross-file LSP */ #include "pipeline/lsp_resolve.h" +#include "pipeline/laravel_routing.h" #include "lsp/rust_cargo.h" #include "helpers.h" /* cbm_kind_in_set_free_cache — per-worker-thread cache teardown */ #include "pipeline/worker_pool.h" @@ -344,212 +340,6 @@ static const char *itoa_log(int val) { return bufs[i]; } -/* Append a JSON-escaped string value to buf at position *pos. */ -/* Escape one character for JSON. Returns bytes written (1 or 2). */ -static int json_escape_char(char *buf, size_t avail, char ch) { - char esc = 0; - switch (ch) { - case '"': - esc = '"'; - break; - case '\\': - esc = '\\'; - break; - case '\n': - esc = 'n'; - break; - case '\r': - esc = 'r'; - break; - case '\t': - esc = 't'; - break; - default: - if (avail >= SKIP_ONE) { - /* Any other raw control byte (e.g. form feed) is invalid inside a - * JSON string — degrade to a space. */ - buf[0] = ((unsigned char)ch < 0x20) ? ' ' : ch; - } - return SKIP_ONE; - } - if (avail >= PP_ESC_SPACE) { - buf[0] = '\\'; - buf[SKIP_ONE] = esc; - } - return PP_ESC_SPACE; -} - -/* Escaped length of a string under json_escape_char's rules: escaped - * characters expand to 2 bytes, everything else stays 1. */ -static size_t pp_json_escaped_len(const char *s) { - size_t n = 0; - for (; *s; s++) { - switch (*s) { - case '"': - case '\\': - case '\n': - case '\r': - case '\t': - n += PP_ESC_SPACE; - break; - default: - n += SKIP_ONE; - } - } - return n; -} - -/* Appends are ATOMIC: a field is emitted only if the WHOLE serialized form - * fits (with PP_ESC_SPACE bytes reserved for the closing '}' + NUL). Cutting a - * field mid-value produced unterminated strings/arrays — malformed properties - * JSON that aborts every json_extract()-based consumer downstream (seen on the - * Linux kernel: 50-param functions truncated at the 2 KB cap). Dropping an - * oversized optional field whole keeps the JSON valid. Twin of - * pass_definitions.c — keep both in sync. */ -static void append_json_string(char *buf, size_t bufsize, size_t *pos, const char *key, - const char *val) { - if (!val || val[0] == '\0') { - return; - } - size_t required = strlen(key) + pp_json_escaped_len(val) + PP_JSON_FIELD_OVERHEAD; - if (*pos + required + PP_ESC_SPACE > bufsize) { - return; /* whole field would not fit — skip it atomically */ - } - size_t p = *pos; - int w = snprintf(buf + p, bufsize - p, ",\"%s\":\"", key); - if (w <= 0 || (size_t)w >= bufsize - p) { - return; - } - p += (size_t)w; - for (const char *s = val; *s && p < bufsize - PP_ESC_MARGIN; s++) { - int n = json_escape_char(buf + p, bufsize - p - PP_ESC_SPACE, *s); - p += (size_t)n; - } - if (p < bufsize - SKIP_ONE) { - buf[p++] = '"'; - } - buf[p] = '\0'; - *pos = p; -} - -/* Append a JSON array of strings: ,"key":["a","b","c"]. Atomic like - * append_json_string: emitted only if the whole array fits. */ -static void append_json_str_array(char *buf, size_t bufsize, size_t *pos, const char *key, - const char **arr) { - if (!arr || !arr[0] || *pos >= bufsize - PP_JSON_MARGIN) { - return; - } - /* ,"key":[ + per item "" + separating commas + ] */ - size_t required = strlen(key) + PP_JSON_FIELD_OVERHEAD; - for (int i = 0; arr[i]; i++) { - required += pp_json_escaped_len(arr[i]) + PP_ESC_SPACE + (i > 0 ? SKIP_ONE : 0); - } - if (*pos + required + PP_ESC_SPACE > bufsize) { - return; /* whole array would not fit — skip it atomically */ - } - size_t p = *pos; - int n = snprintf(buf + p, bufsize - p, ",\"%s\":[", key); - if (n <= 0 || p + (size_t)n >= bufsize - PP_ESC_SPACE) { - return; - } - p += (size_t)n; - for (int i = 0; arr[i]; i++) { - if (i > 0 && p < bufsize - SKIP_ONE) { - buf[p++] = ','; - } - if (p < bufsize - SKIP_ONE) { - buf[p++] = '"'; - } - /* Full escaping (not just quote/backslash): items like C param types - * sliced from multi-line declarations carry raw \n/\t bytes, which are - * invalid inside JSON strings. */ - for (const char *s = arr[i]; *s && p < bufsize - PP_ESC_SPACE; s++) { - p += (size_t)json_escape_char(buf + p, bufsize - p - PP_ESC_SPACE, *s); - } - if (p < bufsize - SKIP_ONE) { - buf[p++] = '"'; - } - } - if (p < bufsize - SKIP_ONE) { - buf[p++] = ']'; - } - buf[p] = '\0'; - *pos = p; -} - -static void build_def_props(char *buf, size_t bufsize, const CBMDefinition *def) { - /* Complexity/loop/recursion metrics are meaningful only for Function/Method. - * Gate the block so the millions of Macro/Field/Variable/Class/Enum nodes - * keep a lean properties blob (lossless — those fields are always zero for - * non-functions). Cuts RAM, gbuf-merge copy and dump volume. Mirrors - * pass_definitions.c::build_def_props — keep both in sync. */ - const bool is_fn = - def->label && (strcmp(def->label, "Function") == 0 || strcmp(def->label, "Method") == 0); - int n; - if (is_fn) { - n = snprintf(buf, bufsize, - "{\"complexity\":%d,\"cognitive\":%d,\"loop_count\":%d,\"loop_depth\":%d," - "\"self_recursive\":%s,\"param_count\":%d,\"max_access_depth\":%d," - "\"linear_scan_in_loop\":%d,\"alloc_in_loop\":%d,\"recursion_in_loop\":%s," - "\"unguarded_recursion\":%s," - "\"lines\":%d,\"is_exported\":%s,\"is_test\":%s,\"is_entry_point\":%s", - def->complexity, def->cognitive, def->loop_count, def->loop_depth, - def->is_recursive ? "true" : "false", def->param_count, def->max_access_depth, - def->linear_scan_in_loop, def->alloc_in_loop, - def->recursion_in_loop ? "true" : "false", - def->unguarded_recursion ? "true" : "false", def->lines, - def->is_exported ? "true" : "false", def->is_test ? "true" : "false", - def->is_entry_point ? "true" : "false"); - } else { - n = snprintf(buf, bufsize, - "{\"complexity\":%d,\"lines\":%d,\"is_exported\":%s,\"is_test\":%s," - "\"is_entry_point\":%s", - def->complexity, def->lines, def->is_exported ? "true" : "false", - def->is_test ? "true" : "false", def->is_entry_point ? "true" : "false"); - } - if (n <= 0 || (size_t)n >= bufsize) { - buf[0] = '\0'; - return; - } - size_t pos = (size_t)n; - append_json_string(buf, bufsize, &pos, "docstring", def->docstring); - append_json_string(buf, bufsize, &pos, "signature", def->signature); - append_json_string(buf, bufsize, &pos, "return_type", def->return_type); - append_json_string(buf, bufsize, &pos, "parent_class", def->parent_class); - append_json_str_array(buf, bufsize, &pos, "decorators", def->decorators); - append_json_str_array(buf, bufsize, &pos, "base_classes", def->base_classes); - append_json_str_array(buf, bufsize, &pos, "param_names", def->param_names); - append_json_str_array(buf, bufsize, &pos, "param_types", def->param_types); - append_json_string(buf, bufsize, &pos, "route_path", def->route_path); - append_json_string(buf, bufsize, &pos, "route_method", def->route_method); - append_json_string(buf, bufsize, &pos, "http_client", def->http_client); - append_json_string(buf, bufsize, &pos, "http_base_url", def->http_base_url); - - /* MinHash fingerprint — append if present and buffer has room. - * Hex-encoded K=64 uint32 = 512 chars + key/quotes ≈ 520 chars. */ - if (def->fingerprint && def->fingerprint_k > 0 && - pos + CBM_MINHASH_HEX_LEN + CBM_MINHASH_JSON_OVERHEAD < bufsize) { - char fp_hex[CBM_MINHASH_HEX_BUF]; - cbm_minhash_to_hex((const cbm_minhash_t *)def->fingerprint, fp_hex, sizeof(fp_hex)); - append_json_string(buf, bufsize, &pos, "fp", fp_hex); - } - - /* AST structural profile — append if present and buffer has room. */ - if (def->structural_profile && pos + CBM_AST_PROFILE_BUF < bufsize) { - append_json_string(buf, bufsize, &pos, "sp", def->structural_profile); - } - - /* Body tokens — raw identifiers from function body AST for semantic search. */ - if (def->body_tokens && pos + CBM_SZ_512 < bufsize) { - append_json_string(buf, bufsize, &pos, "bt", def->body_tokens); - } - - if (pos < bufsize - SKIP_ONE) { - buf[pos] = '}'; - buf[pos + SKIP_ONE] = '\0'; - } -} - /* True for languages whose module QN derives from the CONTAINING DIRECTORY * (Java/Go package). MUST match cbm_lang_module_is_dir() (internal/cbm/helpers.c) * and pxc_module_is_dir() (pass_lsp_cross.c) so same-module callee resolution @@ -700,12 +490,17 @@ enum { PP_OVERSIZED_WARN_MAX = 32 }; /* Insert one definition node (and its route if present) into the local gbuf. */ static void insert_def_into_gbuf(extract_worker_state_t *ws, const cbm_file_info_t *fi, CBMDefinition *def) { - char props[CBM_SZ_2K]; - build_def_props(props, sizeof(props), def); + char stack[CBM_SZ_2K]; + size_t props_size = sizeof(stack); + char *props = cbm_def_props_buf(def, stack, &props_size); + cbm_def_props_build(props, props_size, def); int64_t func_id = cbm_gbuf_upsert_node(ws->local_gbuf, def->label ? def->label : "Function", def->name, def->qualified_name, def->file_path ? def->file_path : fi->rel_path, (int)def->start_line, (int)def->end_line, props); + if (props != stack) { + cbm_free(CBM_MEM_CLASS_GBUF_STRING, props); + } ws->nodes_created++; if (def->route_path && def->route_path[0] != '\0') { const char *rm = def->route_method ? def->route_method : "ANY"; @@ -1826,6 +1621,7 @@ int cbm_build_registry_from_cache(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t const cbm_gbuf_node_t *file_node = cbm_gbuf_find_by_qn(ctx->gbuf, file_qn); int64_t file_node_id = file_node ? file_node->id : 0; free(file_qn); + cbm_def_file_doc(file_node, result->module_doc); for (int d = 0; d < result->defs.count; d++) { defines_edges += register_and_link_def(ctx, &result->defs.items[d], file_node_id, ®_entries); @@ -2209,13 +2005,18 @@ static void emit_normal_calls_edge(cbm_gbuf_t *gbuf, const cbm_gbuf_node_t *sour } /* Classify a resolved call by library identity and emit the appropriate edge. */ -/* Create Route node + CALLS + HANDLES edges for a route registration call. */ +/* Create Route node + CALLS + HANDLES edges for a route registration call. + * route_mount is the framework mount of the registering file ("/api" for a + * Laravel 11+ `withRouting(api: ...)` file, #1146) or "". */ static void emit_route_registration(cbm_gbuf_t *gbuf, const cbm_gbuf_node_t *source, const CBMCall *call, const char *route_path, const char *handler_ref, const char *module_qn, const cbm_registry_t *registry, const cbm_gbuf_t *main_gbuf, - const char **ik, const char **iv, int ic) { + const char **ik, const char **iv, int ic, + const char *route_mount) { const char *method = cbm_service_pattern_route_method(call->callee_name); + char mounted[CBM_SZ_256]; + route_path = cbm_laravel_mount_route(route_mount, route_path, mounted, sizeof(mounted)); char rqn[CBM_ROUTE_QN_SIZE]; char cpath[CBM_SZ_256]; snprintf(rqn, sizeof(rqn), "__route__%s__%s", method ? method : "ANY", @@ -2441,40 +2242,43 @@ static void emit_grpc_edge(cbm_gbuf_t *gbuf, const cbm_gbuf_node_t *source, cons cbm_gbuf_insert_edge(gbuf, source->id, route_id, "GRPC_CALLS", props); } -/* Emit GRAPHQL_CALLS edge. Extract operation from first string arg if available. */ +/* Emit GRAPHQL_CALLS edge. The Route is keyed by the operation NAME only + * (#598), never by the operation text: + * named operation -> __graphql__ name "" + * anonymous query/mutation/ -> __graphql____anonymous name "(anonymous )" + * subscription (incl. the `{ ... }` query shorthand) + * no GraphQL document -> __graphql__operation__anonymous name "(anonymous operation)" + * (URL, variable, missing string argument) + * Keys are bounded by the GraphQL Name grammar and deterministic. */ static void emit_graphql_edge(cbm_gbuf_t *gbuf, const cbm_gbuf_node_t *source, const CBMCall *call, const cbm_resolution_t *res) { - const char *op = call->first_string_arg; - if (!op || !op[0]) { - op = call->callee_name; - } - /* Try to extract a query/mutation name from the operation string */ - char op_name[CBM_SZ_256]; - snprintf(op_name, sizeof(op_name), "%s", op); - /* Trim leading whitespace and "query "/"mutation " prefix */ - const char *p = op_name; - while (*p == ' ' || *p == '\t' || *p == '\n') { - p++; - } - if (strncmp(p, "query ", CBM_SZ_6) == 0) { - p += CBM_SZ_6; - } else if (strncmp(p, "mutation ", CBM_SZ_8) == 0) { - p += CBM_SZ_8; - } + const char *op_type = "operation"; + char op_name[CBM_SZ_128]; + bool named = cbm_service_pattern_graphql_operation(call->first_string_arg, &op_type, op_name, + sizeof(op_name)); - char route_qn[CBM_SZ_512]; - snprintf(route_qn, sizeof(route_qn), "__graphql__%s", p); + char route_name[CBM_SZ_256]; + char route_qn[CBM_SZ_256]; + if (named) { + snprintf(route_name, sizeof(route_name), "%s", op_name); + snprintf(route_qn, sizeof(route_qn), "__graphql__%s", op_name); + } else { + snprintf(route_name, sizeof(route_name), "(anonymous %s)", op_type); + snprintf(route_qn, sizeof(route_qn), "__graphql__%s__anonymous", op_type); + } - int64_t route_id = - cbm_gbuf_upsert_node(gbuf, "Route", p, route_qn, "", 0, 0, "{\"source\":\"graphql\"}"); + int64_t route_id = cbm_gbuf_upsert_node(gbuf, "Route", route_name, route_qn, "", 0, 0, + "{\"source\":\"graphql\"}"); char esc_c[CBM_SZ_256]; char esc_op[CBM_SZ_512]; cbm_json_escape(esc_c, sizeof(esc_c), call->callee_name); - cbm_json_escape(esc_op, sizeof(esc_op), p); + cbm_json_escape(esc_op, sizeof(esc_op), route_name); char props[CBM_SZ_1K]; - snprintf(props, sizeof(props), "{\"callee\":\"%s\",\"operation\":\"%s\",\"confidence\":%.2f}", - esc_c, esc_op, res->confidence); + snprintf(props, sizeof(props), + "{\"callee\":\"%s\",\"operation\":\"%s\",\"operation_type\":\"%s\"," + "\"confidence\":%.2f}", + esc_c, esc_op, op_type, res->confidence); cbm_gbuf_insert_edge(gbuf, source->id, route_id, "GRAPHQL_CALLS", props); } @@ -2526,7 +2330,7 @@ static void emit_service_edge(cbm_gbuf_t *gbuf, const cbm_gbuf_node_t *source, const cbm_resolution_t *res, const char *module_qn, const cbm_registry_t *registry, const cbm_gbuf_t *main_gbuf, const char **imp_keys, const char **imp_vals, int imp_count, - bool suppress_plain_calls) { + bool suppress_plain_calls, const char *route_mount) { cbm_svc_kind_t svc = cbm_service_pattern_match(res->qualified_name); const char *arg = call->first_string_arg; @@ -2553,7 +2357,8 @@ static void emit_service_edge(cbm_gbuf_t *gbuf, const cbm_gbuf_node_t *source, const char *route_path = find_route_path_in_args(call, &handler_ref); if (route_path) { emit_route_registration(gbuf, source, call, route_path, handler_ref, module_qn, - registry, main_gbuf, imp_keys, imp_vals, imp_count); + registry, main_gbuf, imp_keys, imp_vals, imp_count, + route_mount); return; } /* No path found — fall through to normal CALLS edge */ @@ -2561,9 +2366,13 @@ static void emit_service_edge(cbm_gbuf_t *gbuf, const cbm_gbuf_node_t *source, bool has_url = (arg && arg[0] != '\0' && (arg[0] == '/' || strstr(arg, "://") != NULL)); bool has_topic = (arg && arg[0] != '\0' && svc == CBM_SVC_ASYNC && strlen(arg) > PP_ESC_SPACE); + /* Set when this call already has its HTTP_CALLS edge from the service + * patterns (typed with the verb its callee names). */ + bool http_edge_emitted = false; if ((svc == CBM_SVC_HTTP || svc == CBM_SVC_ASYNC) && (has_url || has_topic)) { emit_http_async_service_edge(gbuf, source, call, res, svc, arg); + http_edge_emitted = svc == CBM_SVC_HTTP; } else if (svc == CBM_SVC_GRPC) { emit_grpc_edge(gbuf, source, call, res); } else if (svc == CBM_SVC_GRAPHQL) { @@ -2576,7 +2385,43 @@ static void emit_service_edge(cbm_gbuf_t *gbuf, const cbm_gbuf_node_t *source, emit_normal_calls_edge(gbuf, source, target, call, res); } - detect_url_in_args(gbuf, source, call); + /* The arg-URL heuristic is for calls no service pattern knows (a local + * fetch wrapper). On a classified HTTP call it only minted a method-less + * __route__ANY__ twin of the typed edge, which cross-repo matching then + * bound to server handlers of any method. */ + if (!http_edge_emitted) { + detect_url_in_args(gbuf, source, call); + } +} + +/* The #725 guard refuses a suffix_match binding across a language boundary, + * which leaves the callee unresolved. An unresolved callee the route + * classifier recognises is still a registration: `Route::get('/x', ...)` + * beside a JS `get`, `app.get('/x', h)` beside a Python `get`. Emit exactly + * the Route + CALLS + HANDLES that the unresolved-callee (callee_suffix) + * fallback in resolve_file_calls emits for it, classified by callee name the + * same way emit_service_edge classifies that fallback, and nothing else: no + * CALLS edge to the refused target and no URL-argument scan. Dropping the + * whole call lost every GET registration in a mixed-language repo while POST + * (no `post` to collide with) survived. Mirrors pass_calls.c. */ +static void emit_xlang_refused_route(cbm_gbuf_t *gbuf, const cbm_gbuf_node_t *source, + const CBMCall *call, const char *module_qn, + const cbm_registry_t *registry, const cbm_gbuf_t *main_gbuf, + const char **imp_keys, const char **imp_vals, int imp_count, + const char *route_mount) { + if (cbm_service_pattern_route_method(call->callee_name) == NULL) { + return; + } + cbm_svc_kind_t svc = cbm_service_pattern_match(call->callee_name); + if (svc != CBM_SVC_NONE && svc != CBM_SVC_ROUTE_REG) { + return; + } + const char *handler_ref = NULL; + const char *route_path = find_route_path_in_args(call, &handler_ref); + if (route_path) { + emit_route_registration(gbuf, source, call, route_path, handler_ref, module_qn, registry, + main_gbuf, imp_keys, imp_vals, imp_count, route_mount); + } } /* Find the source node for an edge: enclosing function or file node. */ @@ -2827,6 +2672,10 @@ static const CBMResolvedCall *lsp_idx_lookup(const CBMHashTable *index, const CB static void resolve_file_calls(resolve_ctx_t *rc, resolve_worker_state_t *ws, CBMFileResult *result, const char *rel, const char *module_qn, const char **imp_keys, const char **imp_vals, int imp_count, CBMLanguage lang) { + /* Framework mount of this file's routes (Laravel 11+ withRouting, #1146). */ + char route_mount[CBM_SZ_128]; + cbm_laravel_file_route_mount(rc->repo_path, rel, lang, result, route_mount, + sizeof(route_mount)); /* Two occurrence-aware indexes preserve the authoritative matcher's * primary ordering without restoring its O(calls × resolutions) scan: * exact caller+leaf+span first, then the legacy caller+leaf fallback. @@ -3025,7 +2874,7 @@ static void resolve_file_calls(resolve_ctx_t *rc, resolve_worker_state_t *ws, CB /* Dynamic-language weak-member suppression (#592/#606/#1276). The * receiver-aware guard must NOT drop this call here: doing so would also * skip the #523 callee-name service bypass below, emit_service_edge's - * route/gRPC/config branches, and its unconditional detect_url_in_args + * route/gRPC/config branches, and its detect_url_in_args * (which classifies verb-suffix HTTP clients like api.patch('/x')). * Instead, defer to the emit path and suppress ONLY the plain-CALLS * fall-through (emit_normal_calls_edge), so every service edge stays @@ -3096,7 +2945,7 @@ static void resolve_file_calls(resolve_ctx_t *rc, resolve_worker_state_t *ws, CB .strategy = "service_pattern"}; emit_service_edge(ws->local_edge_buf, source_node, source_node, call, &svc_res, module_qn, rc->registry, rc->main_gbuf, imp_keys, imp_vals, - imp_count, false); + imp_count, false, route_mount); continue; } } @@ -3113,7 +2962,7 @@ static void resolve_file_calls(resolve_ctx_t *rc, resolve_worker_state_t *ws, CB * self-call, so it keeps only the route/service edges. */ emit_service_edge(ws->local_edge_buf, source_node, source_node, call, &fake_res, module_qn, rc->registry, rc->main_gbuf, imp_keys, imp_vals, - imp_count, rust_external); + imp_count, rust_external, route_mount); } else if (cbm_service_pattern_is_global_fetch(call->callee_name)) { /* Native `fetch()` (#856): only the global API once resolution * has failed to find a local/imported `fetch`. Call the low-level @@ -3147,7 +2996,10 @@ static void resolve_file_calls(resolve_ctx_t *rc, resolve_worker_state_t *ws, CB if (target_node && source_node->id != target_node->id && cbm_suppress_cross_language_suffix_match(lang, target_node->file_path, res.strategy)) { /* #725: same guard as pass_calls.c — do not emit a suffix_match - * CALLS edge across a language boundary. */ + * CALLS edge across a language boundary. A route registration + * behind the refused binding still gets its Route. */ + emit_xlang_refused_route(ws->local_edge_buf, source_node, call, module_qn, rc->registry, + rc->main_gbuf, imp_keys, imp_vals, imp_count, route_mount); continue; } if (!target_node || source_node->id == target_node->id) { @@ -3165,7 +3017,7 @@ static void resolve_file_calls(resolve_ctx_t *rc, resolve_worker_state_t *ws, CB if (url_or_topic) { emit_service_edge(ws->local_edge_buf, source_node, NULL, call, &res, module_qn, rc->registry, rc->main_gbuf, imp_keys, imp_vals, imp_count, - false); + false, route_mount); ws->calls_resolved++; } } @@ -3174,7 +3026,7 @@ static void resolve_file_calls(resolve_ctx_t *rc, resolve_worker_state_t *ws, CB _rc_t0 = extract_now_ns(); emit_service_edge(ws->local_edge_buf, source_node, target_node, call, &res, module_qn, rc->registry, rc->main_gbuf, imp_keys, imp_vals, imp_count, - drop_plain_call); + drop_plain_call, route_mount); atomic_fetch_add_explicit(&rc->time_ns_rc_emit, extract_now_ns() - _rc_t0, memory_order_relaxed); ws->calls_resolved++; diff --git a/src/pipeline/pass_pkgmap.c b/src/pipeline/pass_pkgmap.c index 953c593ad6..2f200a5626 100644 --- a/src/pipeline/pass_pkgmap.c +++ b/src/pipeline/pass_pkgmap.c @@ -440,9 +440,12 @@ static void parse_pyproject_toml(const char *source, int source_len, const char } } -/* Extract PSR-4 autoload entries from composer.json root. */ -static void extract_psr4(yyjson_val *root, const char *dir, cbm_pkg_entries_t *entries) { - yyjson_val *autoload = yyjson_obj_get(root, "autoload"); +/* Extract PSR-4 entries from one composer.json autoload section ("autoload" + * or "autoload-dev": composer registers both, so test namespaces such as + * Tests\ map to their directory as well). */ +static void extract_psr4(yyjson_val *root, const char *section, const char *dir, + cbm_pkg_entries_t *entries) { + yyjson_val *autoload = yyjson_obj_get(root, section); if (!yyjson_is_obj(autoload)) { return; } @@ -454,22 +457,35 @@ static void extract_psr4(yyjson_val *root, const char *dir, cbm_pkg_entries_t *e yyjson_obj_iter iter = yyjson_obj_iter_with(psr4); while ((key = yyjson_obj_iter_next(&iter)) != NULL) { yyjson_val *val = yyjson_obj_iter_get_val(key); - if (!yyjson_is_str(key) || !yyjson_is_str(val)) { + if (!yyjson_is_str(key)) { continue; } const char *ns_prefix = yyjson_get_str(key); - const char *ns_dir = yyjson_get_str(val); - char ns_entry[PKGMAP_PATH_BUF]; - if (dir[0]) { - snprintf(ns_entry, sizeof(ns_entry), "%s/%s", dir, ns_dir); - } else { - snprintf(ns_entry, sizeof(ns_entry), "%s", ns_dir); - } - size_t nelen = strlen(ns_entry); - if (nelen > 0 && ns_entry[nelen - SKIP_ONE] == '/') { - ns_entry[nelen - SKIP_ONE] = '\0'; + size_t count = yyjson_is_arr(val) ? yyjson_arr_size(val) : 1; + for (size_t i = 0; i < count; i++) { + yyjson_val *path = yyjson_is_arr(val) ? yyjson_arr_get(val, i) : val; + if (!yyjson_is_str(path)) { + continue; + } + const char *ns_dir = yyjson_get_str(path); + while (ns_dir[0] == '.' && ns_dir[SKIP_ONE] == '/') { + ns_dir += PAIR_LEN; /* "./src/" names src/ */ + } + if (strcmp(ns_dir, ".") == 0) { + ns_dir = ""; /* "." names the manifest's own directory */ + } + char ns_entry[PKGMAP_PATH_BUF]; + if (dir[0]) { + snprintf(ns_entry, sizeof(ns_entry), "%s/%s", dir, ns_dir); + } else { + snprintf(ns_entry, sizeof(ns_entry), "%s", ns_dir); + } + size_t nelen = strlen(ns_entry); + if (nelen > 0 && ns_entry[nelen - SKIP_ONE] == '/') { + ns_entry[nelen - SKIP_ONE] = '\0'; + } + pkg_entries_push(entries, strdup(ns_prefix), strdup(ns_entry)); } - pkg_entries_push(entries, strdup(ns_prefix), strdup(ns_entry)); } } @@ -497,7 +513,8 @@ static void parse_composer_json(const char *source, int source_len, const char * } } - extract_psr4(root, dir, entries); + extract_psr4(root, "autoload", dir, entries); + extract_psr4(root, "autoload-dev", dir, entries); free(dir); yyjson_doc_free(doc); @@ -987,6 +1004,23 @@ bool cbm_pkgmap_try_parse(const char *basename, const char *rel_path, const char /* ── Merge: per-worker entries → hash table ────────────────────── */ +/* A composer.json `autoload.psr-4` key: a namespace prefix ending in '\\'. */ +static bool pkgmap_is_psr4_prefix(const char *pkg_name) { + size_t n = pkg_name ? strlen(pkg_name) : 0; + return n > 0 && pkg_name[n - SKIP_ONE] == '\\'; +} + +/* PSR-4 values own an ordered directory list; other package values remain QNs. */ +typedef struct psr4_dir { + struct psr4_dir *next; + char path[]; +} psr4_dir_t; + +static const char *pkgmap_first_path(CBMHashTable *map, const char *key) { + void *value = cbm_ht_get(map, key); + return value && pkgmap_is_psr4_prefix(key) ? ((psr4_dir_t *)value)->path : value; +} + CBMHashTable *cbm_pkgmap_build(cbm_pkg_entries_t *worker_entries, int worker_count, const char *project_name) { /* Count total entries */ @@ -1004,22 +1038,43 @@ CBMHashTable *cbm_pkgmap_build(cbm_pkg_entries_t *worker_entries, int worker_cou for (int w = 0; w < worker_count; w++) { cbm_pkg_entries_t *we = &worker_entries[w]; for (int i = 0; i < we->count; i++) { - /* Convert entry_rel to QN: project.dir.parts */ - char *qn = cbm_pipeline_fqn_module(project_name, we->items[i].entry_rel); - if (!qn) { + const bool psr4 = pkgmap_is_psr4_prefix(we->items[i].pkg_name); + psr4_dir_t *dirs = psr4 ? cbm_ht_get(map, we->items[i].pkg_name) : NULL; + /* Other package mappings retain their existing first-wins rule. */ + if (!psr4 && cbm_ht_has(map, we->items[i].pkg_name)) { continue; } - - /* Check for duplicate — first wins */ - if (cbm_ht_has(map, we->items[i].pkg_name)) { - free(qn); + char *key = dirs ? NULL : strdup(we->items[i].pkg_name); + if (!dirs && !key) { continue; } - - /* Transfer ownership: key = strdup'd pkg_name, value = qn */ - char *key = strdup(we->items[i].pkg_name); - cbm_ht_set(map, key, qn); - merged++; + const char *entry_rel = we->items[i].entry_rel; + void *value = NULL; + if (psr4) { + size_t len = strlen(entry_rel); + psr4_dir_t *node = cbm_alloc(CBM_MEM_CLASS_SEMANTIC, sizeof(*node) + len + 1); + if (node) { + node->next = NULL; + memcpy(node->path, entry_rel, len + 1); + } + value = node; + } else { + value = cbm_pipeline_fqn_module(project_name, entry_rel); + } + if (!value) { + free(key); + continue; + } + if (dirs) { + /* ponytail: linear append for small lists; cache a tail if large lists matter. */ + while (dirs->next) { + dirs = dirs->next; + } + dirs->next = value; + } else { + cbm_ht_set(map, key, value); + merged++; + } } } @@ -1274,8 +1329,17 @@ CBMHashTable *cbm_pkgmap_build_from_repo(const char *repo_path, const cbm_file_i static void pkgmap_free_entry(const char *key, void *value, void *userdata) { (void)userdata; + if (pkgmap_is_psr4_prefix(key)) { + psr4_dir_t *dir = value; + while (dir) { + psr4_dir_t *next = dir->next; + cbm_free(CBM_MEM_CLASS_SEMANTIC, dir); + dir = next; + } + } else { + free(value); + } free((void *)key); - free(value); } void cbm_pkgmap_free(CBMHashTable *pkgmap) { @@ -1300,7 +1364,7 @@ static char *resolve_slash_prefix(CBMHashTable *map, const char *module_path) { continue; } *slash = '\0'; - const char *base_qn = (const char *)cbm_ht_get(map, buf); + const char *base_qn = pkgmap_first_path(map, buf); if (!base_qn) { continue; } @@ -1333,7 +1397,7 @@ static char *resolve_dot_prefix(CBMHashTable *map, const char *module_path, continue; } *dot = '\0'; - const char *base_qn = (const char *)cbm_ht_get(map, buf); + const char *base_qn = pkgmap_first_path(map, buf); if (!base_qn) { continue; } @@ -1355,7 +1419,8 @@ static char *resolve_dot_prefix(CBMHashTable *map, const char *module_path, } /* Try backslash-based prefix matching (PHP PSR-4: App\\Controllers\\Foo). - * Returns heap QN or NULL. */ + * Generic module resolution uses the first mapped directory; class imports + * search every directory through resolve_php_psr4_class. Returns heap QN or NULL. */ static char *resolve_backslash_prefix(CBMHashTable *map, const char *module_path, const char *project_name) { char *buf = strdup(module_path); @@ -1369,13 +1434,14 @@ static char *resolve_backslash_prefix(CBMHashTable *map, const char *module_path *bs = '\0'; char prefix[PKGMAP_PATH_BUF]; snprintf(prefix, sizeof(prefix), "%s\\", buf); - const char *base_dir = (const char *)cbm_ht_get(map, prefix); + const char *base_dir = pkgmap_first_path(map, prefix); if (!base_dir) { continue; } const char *subpath = module_path + (size_t)(bs - buf) + SKIP_ONE; char path_result[PKGMAP_PATH_BUF]; - snprintf(path_result, sizeof(path_result), "%s/%s", base_dir, subpath); + snprintf(path_result, sizeof(path_result), "%s%s%s", base_dir, base_dir[0] ? "/" : "", + subpath); for (char *c = path_result; *c; c++) { if (*c == '\\') { *c = '/'; @@ -1428,7 +1494,7 @@ char *cbm_pipeline_resolve_module(const cbm_pipeline_ctx_t *ctx, const char *sou } /* 3. Exact lookup */ - const char *mapped_qn = (const char *)cbm_ht_get(pkgmap, module_path); + const char *mapped_qn = pkgmap_first_path(pkgmap, module_path); if (mapped_qn) { return strdup(mapped_qn); } @@ -2027,6 +2093,94 @@ static const cbm_gbuf_node_t *resolve_sibling_file(const cbm_pipeline_ctx_t *ctx return found; } +/* ── PHP PSR-4 class imports (#1186) ──────────────────────────────── */ + +typedef enum { + PSR4_NOT_APPLICABLE = 0, /* no composer psr-4 prefix covers the import */ + PSR4_RESOLVED, /* the class file exists: *out is its File node */ + PSR4_UNRESOLVED, /* a prefix covers it but no class file exists */ +} psr4_outcome_t; + +/* The File node at exactly `rel_path`, or NULL. Looked up through the name + * index (File nodes are named by basename) and matched on the full path, so + * a same-named file in another directory never qualifies. */ +static const cbm_gbuf_node_t *psr4_file_node(const cbm_pipeline_ctx_t *ctx, const char *rel_path) { + const cbm_gbuf_node_t **hits = NULL; + int hit_count = 0; + if (cbm_gbuf_find_by_name(ctx->gbuf, path_leaf(rel_path), &hits, &hit_count) != 0 || !hits) { + return NULL; + } + for (int i = 0; i < hit_count; i++) { + const cbm_gbuf_node_t *n = hits[i]; + if (n && n->label && strcmp(n->label, "File") == 0 && n->file_path && + strcmp(n->file_path, rel_path) == 0) { + return n; + } + } + return NULL; +} + +/* Resolve a PHP class import `use A\B\C;` the way composer's PSR-4 autoloader + * does: for every autoload.psr-4 prefix covering the name, longest first, the + * class lives in exactly /.php with sub-namespaces as + * subdirectories; the first existing file wins. When a prefix covers the name + * but no such file exists the import is UNRESOLVED: it must never fall through + * to the namespace bucket, which binds it to whichever file of the namespace + * came first. `use function` / `use const` name namespace members, not class + * files, so they are not applicable. */ +static psr4_outcome_t resolve_php_psr4_class(const cbm_pipeline_ctx_t *ctx, + const char *source_file_qn, const CBMImport *imp, + const cbm_gbuf_node_t **out) { + *out = NULL; + CBMHashTable *pkgmap = cbm_pipeline_get_pkgmap(); + if (!pkgmap || imp->kind != CBM_IMPORT_KIND_DEFAULT) { + return PSR4_NOT_APPLICABLE; + } + const char *name = imp->module_path; + if (name[0] == '\\') { + name++; /* fully qualified `use \App\X;` */ + } + char key[PKGMAP_PATH_BUF]; + size_t name_len = strlen(name); + if (name_len == 0 || name_len >= sizeof(key) || !strchr(name, '\\')) { + return PSR4_NOT_APPLICABLE; + } + bool covered = false; + /* Right to left: each '\' ends a candidate prefix, longest first. */ + for (size_t cut = name_len - SKIP_ONE; cut > 0; cut--) { + if (name[cut - SKIP_ONE] != '\\') { + continue; + } + memcpy(key, name, cut); + key[cut] = '\0'; + const psr4_dir_t *dirs = cbm_ht_get(pkgmap, key); + if (!dirs) { + continue; + } + covered = true; + for (const psr4_dir_t *dir = dirs; dir; dir = dir->next) { + char rel[PKGMAP_PATH_BUF]; + int w = snprintf(rel, sizeof(rel), "%s%s%s.php", dir->path, dir->path[0] ? "/" : "", + name + cut); + if (w <= 0 || (size_t)w >= sizeof(rel)) { + continue; + } + for (char *c = rel; *c; c++) { + if (*c == '\\') { + *c = '/'; + } + } + const cbm_gbuf_node_t *file = psr4_file_node(ctx, rel); + if (file && (!source_file_qn || !file->qualified_name || + strcmp(file->qualified_name, source_file_qn) != 0)) { + *out = file; + return PSR4_RESOLVED; + } + } + } + return covered ? PSR4_UNRESOLVED : PSR4_NOT_APPLICABLE; +} + const cbm_gbuf_node_t *cbm_pipeline_resolve_import_node(const cbm_pipeline_ctx_t *ctx, const char *source_rel, const char *source_file_qn, @@ -2044,6 +2198,23 @@ const cbm_gbuf_node_t *cbm_pipeline_resolve_import_node(const cbm_pipeline_ctx_t return header_target; } + /* PHP class imports covered by a composer psr-4 prefix name exactly one + * file; when it is absent the import stays unresolved (#1186). Only PHP + * importers spell class paths with backslashes, so the unresolved + * short-circuit is scoped to them and every other importer keeps falling + * through to the strategies below. */ + if (cbm_language_for_filename(source_rel) == CBM_LANG_PHP) { + const cbm_gbuf_node_t *psr4_target = NULL; + switch (resolve_php_psr4_class(ctx, source_file_qn, imp, &psr4_target)) { + case PSR4_RESOLVED: + return psr4_target; + case PSR4_UNRESOLVED: + return NULL; + case PSR4_NOT_APPLICABLE: + break; + } + } + /* Strategy 1: module-path resolution → existing node (Python/TS/Go). * No label filter here: directory-module languages (Go/Java packages) * legitimately resolve straight to a Folder node -- that's the intended, diff --git a/src/pipeline/pass_semantic.c b/src/pipeline/pass_semantic.c index 48dab00831..597d629ba4 100644 --- a/src/pipeline/pass_semantic.c +++ b/src/pipeline/pass_semantic.c @@ -23,6 +23,7 @@ #include "foundation/compat_fs.h" #include "foundation/limits.h" #include "cbm.h" +#include "callable_sig.h" #include #include @@ -356,7 +357,14 @@ static void go_index_add(go_method_index_t *ix, const char *name, int type_index static void go_index_visit_node(const cbm_gbuf_node_t *node, void *userdata) { go_method_index_t *ix = userdata; const char *qn = node->qualified_name; - const char *dot = qn ? strrchr(qn, '.') : NULL; + /* Last '.' of the base QN (a callable identity suffix never holds one). */ + const char *dot = NULL; + size_t base_len = cbm_qn_callable_base_len_named(qn, node->name); + for (size_t i = 0; qn && i < base_len; i++) { + if (qn[i] == '.') { + dot = qn + i; + } + } if (!dot || dot == qn || !dot[SKIP_ONE]) { return; } diff --git a/src/pipeline/path_alias.c b/src/pipeline/path_alias.c index 8fae754c15..ef999e14de 100644 --- a/src/pipeline/path_alias.c +++ b/src/pipeline/path_alias.c @@ -433,6 +433,15 @@ static void find_alias_files(const char *abs_dir, const char *rel_dir, alias_con if (cbm_pipeline_relpath_is_excluded(child_rel, excluded_dirs, excluded_count)) { continue; } + /* A link is not followed, as discovery does not follow one. is_dir is + * also true for a junction or a directory symbolic link on Windows, + * and either can lead out of the repository. A child whose metadata + * cannot be read is skipped the same way. */ + cbm_path_info_t child_info; + if (cbm_path_info_utf8(child_abs, &child_info) != CBM_PATH_INFO_OK || + child_info.is_symlink) { + continue; + } find_alias_files(child_abs, child_rel, out, count, max_count, depth + 1, excluded_dirs, excluded_count); } diff --git a/src/pipeline/pipeline.c b/src/pipeline/pipeline.c index 77ad945206..fa35ce7600 100644 --- a/src/pipeline/pipeline.c +++ b/src/pipeline/pipeline.c @@ -1150,7 +1150,8 @@ static void log_result_census(const char *tag, CBMFileResult **cache, int file_c (size_t)r->string_refs.count * sizeof(CBMStringRef) + (size_t)r->impl_traits.count * sizeof(CBMImplTrait) + (size_t)r->infra_bindings.count * sizeof(CBMInfraBinding) + - (size_t)r->channels.count * sizeof(CBMChannel); + (size_t)r->channels.count * sizeof(CBMChannel) + + (size_t)r->field_types.count * sizeof(CBMFieldType); cap_other += (size_t)r->imports.cap * sizeof(CBMImport) + (size_t)r->resolved_calls.cap * sizeof(CBMResolvedCall) + (size_t)r->throws.cap * sizeof(CBMThrow) + @@ -1159,7 +1160,8 @@ static void log_result_census(const char *tag, CBMFileResult **cache, int file_c (size_t)r->string_refs.cap * sizeof(CBMStringRef) + (size_t)r->impl_traits.cap * sizeof(CBMImplTrait) + (size_t)r->infra_bindings.cap * sizeof(CBMInfraBinding) + - (size_t)r->channels.cap * sizeof(CBMChannel); + (size_t)r->channels.cap * sizeof(CBMChannel) + + (size_t)r->field_types.cap * sizeof(CBMFieldType); n_defs += (size_t)r->defs.count; n_calls += (size_t)r->calls.count; n_usages += (size_t)r->usages.count; @@ -2287,10 +2289,15 @@ int cbm_pipeline_publish_staged(char *stage_path, const cbm_pipeline_generation_ free(stage_path); return CBM_PIPELINE_PERSIST_FAILED; } - bool ok = cbm_store_exec(store, "PRAGMA synchronous=FULL;") == CBM_STORE_OK; - ok = ok && cbm_store_delete_file_hashes(store, generation->project) == CBM_STORE_OK && - cbm_store_upsert_file_hash_batch(store, generation->manifest, - generation->manifest_count) == CBM_STORE_OK; + /* No synchronous=FULL for these writes (#1419): the stage is private until + * the atomic rename and a crash discards it, so an fsync per WAL commit + * protects nothing here. The store's NORMAL level is SQLite's + * corruption-safe setting under WAL, and cbm_store_seal_for_atomic_publish() + * raises this connection to FULL for the checkpoint that makes the + * published file durable. */ + bool ok = cbm_store_delete_file_hashes(store, generation->project) == CBM_STORE_OK && + cbm_store_upsert_file_hash_batch(store, generation->manifest, + generation->manifest_count) == CBM_STORE_OK; /* LSP surfaces belong to the generation: written inside the same staging * store, before the atomic rename, so graph and surface data can never * publish separately. The delete guards the incremental path, whose diff --git a/src/pipeline/pipeline_incremental.c b/src/pipeline/pipeline_incremental.c index 154518658d..99195cc2c0 100644 --- a/src/pipeline/pipeline_incremental.c +++ b/src/pipeline/pipeline_incremental.c @@ -102,6 +102,25 @@ static const char *itoa_buf(int v) { return buf[idx]; } +/* #1300: the purge of stale per-file nodes can run for a long time on a large + * persisted graph (reported: hundreds of files against ~268k stored hashes), + * and it logged nothing between incremental.edge_snapshot and + * incremental.purge — a stall there and a clean exit left no trace of where + * the run was. Progress is counts only: a ceiling stride keeps it to at most + * INCR_PURGE_PROGRESS_STEPS lines plus the final one, never one per file. */ +enum { INCR_PURGE_PROGRESS_STEPS = 20 }; + +static void incr_purge_progress(int done, int total, struct timespec start) { + int stride = (total + INCR_PURGE_PROGRESS_STEPS - SKIP_ONE) / INCR_PURGE_PROGRESS_STEPS; + if (stride < SKIP_ONE) { + stride = SKIP_ONE; + } + if (done % stride == 0 || done == total) { + cbm_log_info("incremental.purge.progress", "files_done", itoa_buf(done), "files_total", + itoa_buf(total), "elapsed_ms", itoa_buf((int)elapsed_ms(start))); + } +} + static const char *incr_mode_name(int mode) { switch (mode) { case CBM_MODE_FULL: @@ -2688,12 +2707,17 @@ int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_fil /* Step 2: Purge stale nodes */ cbm_clock_gettime(CLOCK_MONOTONIC, &t); + int purge_total = ci + deleted_count; + cbm_log_info("incremental.purge.start", "changed", itoa_buf(ci), "deleted", + itoa_buf(deleted_count)); for (int i = 0; i < ci; i++) { cbm_gbuf_delete_by_file(existing, changed_files[i].rel_path); + incr_purge_progress(i + SKIP_ONE, purge_total, t); } for (int i = 0; i < deleted_count; i++) { cbm_gbuf_delete_by_file(existing, deleted[i]); free(deleted[i]); + incr_purge_progress(ci + i + SKIP_ONE, purge_total, t); } free(deleted); cbm_log_info("incremental.purge", "elapsed_ms", itoa_buf((int)elapsed_ms(t))); diff --git a/src/pipeline/pipeline_internal.h b/src/pipeline/pipeline_internal.h index 324bddddb5..912b5e56c9 100644 --- a/src/pipeline/pipeline_internal.h +++ b/src/pipeline/pipeline_internal.h @@ -47,6 +47,22 @@ * out_sz >= strlen(in) + 1 always suffices. Returns out. */ const char *cbm_route_canon_path(const char *in, char *out, size_t out_sz); +/* ── Definition properties JSON (shared by pass_definitions.c and the + * parallel extraction pass) ──────────────────────────────────── */ + +/* Serialize one definition's properties JSON, dropping oversized fields whole. + * Implemented in pass_definitions.c. */ +void cbm_def_props_build(char *buf, size_t bufsize, const CBMDefinition *def); + +/* A def's properties buffer: CBM_SZ_2K plus the whole serialized docstring + * field. Returns `stack` for a def without a docstring; the caller frees a + * returned larger buffer with cbm_free(CBM_MEM_CLASS_GBUF_STRING, ...). */ +char *cbm_def_props_buf(const CBMDefinition *def, char *stack, size_t *size); + +/* Add a file's own doc (Go package comment, Rust inner docs) to its File node + * as "docstring". */ +void cbm_def_file_doc(const cbm_gbuf_node_t *file_node, const char *doc); + /* True when a graph node is a structural directory container (Folder/Project) * rather than a code node. In a directory-based-module language (Java/Go, see * cbm_lang_module_is_dir) a file's module QN equals its directory QN, so an diff --git a/src/pipeline/registry.c b/src/pipeline/registry.c index 1f5335c10c..ff56a41ac4 100644 --- a/src/pipeline/registry.c +++ b/src/pipeline/registry.c @@ -30,6 +30,7 @@ enum { REG_MAX_CANDIDATES = 256 }; #include "foundation/hash_table.h" #include "foundation/dyn_array.h" #include "foundation/platform.h" +#include "callable_sig.h" /* cbm_qn_callable_base_len_named: overloads share the name key */ #include #include @@ -959,15 +960,24 @@ void cbm_registry_add(cbm_registry_t *r, const char *name, const char *qualified * their dotted form would only widen the bucket the scorer walks. * * `name` is NULL or empty only for callers that have no symbol name to - * give; those have the derived key and nothing else. */ + * give; those have the derived key and nothing else. + * + * A signature-qualified callable (#2061) is indexed by its bare `name` in + * place of the derived key, so every overload shares one bucket; any other + * QN keeps its historical last-segment key. */ const char *derived = simple_name(qualified_name); - index_under_name(r, derived, owned_qn); + const char *primary = + name && cbm_qn_callable_base_len_named(owned_qn, name) < strlen(owned_qn) ? name : derived; + index_under_name(r, primary, owned_qn); /* '#' is a QN fence, and extract_defs.c's rust_cfg_qualified_name is the * only thing in the tree that mints one today. A grammar that starts * minting a '#' opts into this second key by doing so, whatever it means by * the fence: its symbols become reachable under the passed name as well, - * and they share that name's bucket with everything else filed under it. */ - if (name && name[0] && strchr(derived, '#') && strcmp(name, derived) != 0) { + * and they share that name's bucket with everything else filed under it. + * The passed name is compared with the key already written: a + * signature-qualified callable is filed under `name` already, and its + * capped suffix may carry a '#' of its own. */ + if (name && name[0] && strchr(derived, '#') && strcmp(name, primary) != 0) { index_under_name(r, name, owned_qn); } } diff --git a/src/store/store.c b/src/store/store.c index fad3932193..7631e9c00b 100644 --- a/src/store/store.c +++ b/src/store/store.c @@ -78,6 +78,8 @@ enum { #include "foundation/compat.h" #include "foundation/log.h" #include "foundation/compat_regex.h" +#include "callable_sig.h" /* cbm_qn_callable_base_len: base-match tier */ +#include "foundation/mem_core.h" /* cbm_alloc: pattern buffers */ #include "foundation/str_util.h" #define XXH_INLINE_ALL @@ -539,6 +541,16 @@ int64_t cbm_store_resolve_mmap_size(void) { return (int64_t)parsed; } +/* #1419: page cache for read-write on-disk connections, in SQLite's negative + * KiB form. sqlite_writer.c gives every index 64 KiB pages, so SQLite's + * default 2000 KiB holds only ~31 of them, while a delta's purge and patch + * dirty pages across the node and edge tables and all of their indexes. The + * transaction then spills dirty pages into the WAL and rewrites them over and + * over (5.2 GiB of WAL writes for a one-line change on a 294 MB django index). + * 64 MiB = 1024 pages, the budget bulk mode already used; the cache only grows + * as pages are touched. Query connections keep the default. */ +static const char ST_WRITE_CACHE_SQL[] = "PRAGMA cache_size = -65536;"; + /* Configure connection pragmas. * in_memory — :memory: DB (synchronous OFF, no journal file). * read_only — query-only connection opened SQLITE_OPEN_READONLY. Runs @@ -594,8 +606,9 @@ static int configure_pragmas(cbm_store_t *s, bool in_memory, bool read_only) { * cbm_store_checkpoint's SIGBUS note), so without a size limit the -wal * file only ever grows; journal_size_limit truncates it back to N bytes * on the next successful - * reset. N is far above the healthy WAL (~4 MiB under the default - * 1000-page autocheckpoint), so normal indexing never triggers + * reset. N is far above the healthy WAL (~64 MiB: the default + * 1000-page autocheckpoint at the 64 KiB page size sqlite_writer.c + * gives every index), so normal indexing never triggers * truncate/regrow churn — it only fires after abnormal growth. * Shared/live paths do NOT use a TRUNCATE checkpoint: truncating the WAL * to zero can raise SIGBUS in a sibling process that has the DB mmap'd @@ -604,6 +617,10 @@ static int configure_pragmas(cbm_store_t *s, bool in_memory, bool read_only) { if (rc != CBM_STORE_OK) { return rc; } + rc = exec_sql(s, ST_WRITE_CACHE_SQL); + if (rc != CBM_STORE_OK) { + return rc; + } char mmap_sql[ST_BUF_64]; snprintf(mmap_sql, sizeof(mmap_sql), "PRAGMA mmap_size = %lld;", (long long)cbm_store_resolve_mmap_size()); @@ -719,9 +736,12 @@ static void sqlite_regexp(sqlite3_context *ctx, int argc, sqlite3_value **argv) sqlite3_result_error_nomem(ctx); return; } - if (cbm_regcomp(re, pattern, CBM_REG_EXTENDED | CBM_REG_NOSUB) != 0) { + int rc = cbm_regcomp(re, pattern, CBM_REG_EXTENDED | CBM_REG_NOSUB); + if (rc != 0) { free(re); - sqlite3_result_error(ctx, "invalid regex", CBM_NOT_FOUND); + sqlite3_result_error(ctx, + rc == CBM_REG_ETOOBIG ? CBM_REG_ETOOBIG_REASON : "invalid regex", + CBM_NOT_FOUND); return; } sqlite3_set_auxdata(ctx, 0, re, regex_free_cb); @@ -747,9 +767,12 @@ static void sqlite_iregexp(sqlite3_context *ctx, int argc, sqlite3_value **argv) sqlite3_result_error_nomem(ctx); return; } - if (cbm_regcomp(re, pattern, CBM_REG_EXTENDED | CBM_REG_NOSUB | CBM_REG_ICASE) != 0) { + int rc = cbm_regcomp(re, pattern, CBM_REG_EXTENDED | CBM_REG_NOSUB | CBM_REG_ICASE); + if (rc != 0) { free(re); - sqlite3_result_error(ctx, "invalid regex", CBM_NOT_FOUND); + sqlite3_result_error(ctx, + rc == CBM_REG_ETOOBIG ? CBM_REG_ETOOBIG_REASON : "invalid regex", + CBM_NOT_FOUND); return; } sqlite3_set_auxdata(ctx, 0, re, regex_free_cb); @@ -1811,12 +1834,12 @@ int cbm_store_begin_bulk(cbm_store_t *s) { * because the in-memory rollback journal is lost on crash. * WAL mode is crash-safe: uncommitted WAL entries are simply discarded * on the next open. Performance is preserved via synchronous=OFF and a - * larger cache, which are safe with WAL. */ + * 64 MiB cache, which are safe with WAL. */ int rc = exec_sql(s, "PRAGMA synchronous = OFF;"); if (rc != CBM_STORE_OK) { return rc; } - return exec_sql(s, "PRAGMA cache_size = -65536;"); /* CBM_SZ_64 MB */ + return exec_sql(s, ST_WRITE_CACHE_SQL); } int cbm_store_end_bulk(cbm_store_t *s) { @@ -1824,7 +1847,8 @@ int cbm_store_end_bulk(cbm_store_t *s) { if (rc != CBM_STORE_OK) { return rc; } - return exec_sql(s, "PRAGMA cache_size = -2000;"); /* default ~2 MB */ + /* Back to the read-write cache, not SQLite's 2000 KiB default (#1419). */ + return exec_sql(s, ST_WRITE_CACHE_SQL); } int cbm_store_drop_indexes(cbm_store_t *s) { @@ -4389,29 +4413,95 @@ int cbm_store_find_nodes_by_file_overlap(cbm_store_t *s, const char *project, co /* ── FindNodesByQNSuffix ───────────────────────────────────────── */ +/* Row filter on the raw qualified_name + name columns (NULL keeps every row). */ +typedef bool (*store_qn_keep_fn)(const char *qn, const char *name, const void *arg); + +/* Step `stmt` (columns as scan_node reads them) into a node array, keeping the + * rows `keep` accepts. Finalizes `stmt`. */ +static int store_collect_nodes(cbm_store_t *s, sqlite3_stmt *stmt, store_qn_keep_fn keep, + const void *keep_arg, cbm_node_t **out, int *count) { + int cap = ST_INIT_CAP_8; + int n = 0; + cbm_node_t *nodes = malloc(cap * sizeof(cbm_node_t)); + int scan_rc8; + while ((scan_rc8 = sqlite3_step(stmt)) == SQLITE_ROW) { + if (keep && !keep((const char *)sqlite3_column_text(stmt, CBM_SZ_4), + (const char *)sqlite3_column_text(stmt, CBM_SZ_3), keep_arg)) { + continue; + } + if (n >= cap) { + cap *= ST_GROWTH; + nodes = safe_realloc(nodes, cap * sizeof(cbm_node_t)); + } + memset(&nodes[n], 0, sizeof(cbm_node_t)); + scan_node(stmt, &nodes[n]); + n++; + } + if (scan_rc8 != SQLITE_DONE) { /* SCANCHK:8:stmt */ + store_set_error_sqlite(s, "row scan aborted"); + sqlite3_finalize(stmt); + cbm_store_free_nodes(nodes, n); + *out = NULL; + *count = 0; + return CBM_STORE_ERR; + } + sqlite3_finalize(stmt); + *out = nodes; + *count = n; + return CBM_STORE_OK; +} + +/* "%." + text + tail with LIKE's wildcards ('%', '_') and the escape + * character escaped, for `LIKE ? ESCAPE '\'`. Heap-owned. The caller's text + * is matched literally at any length: a fixed 512-byte buffer used to + * truncate a long suffix into a pattern that matched nothing, and an + * unescaped '_' in `my_func` also matched `myXfunc`. */ +static char *store_like_dot_suffix(const char *text, const char *tail) { + size_t n = strlen(text); + size_t tail_len = strlen(tail); + char *pat = cbm_alloc(CBM_MEM_CLASS_STORE, (2 * n) + tail_len + ST_COL_3); /* "%." + NUL */ + if (!pat) { + return NULL; + } + size_t k = 0; + pat[k++] = '%'; + pat[k++] = '.'; + for (size_t i = 0; i < n; i++) { + if (text[i] == '%' || text[i] == '_' || text[i] == '\\') { + pat[k++] = '\\'; + } + pat[k++] = text[i]; + } + memcpy(pat + k, tail, tail_len + SKIP_ONE); + return pat; +} + int cbm_store_find_nodes_by_qn_suffix(cbm_store_t *s, const char *project, const char *suffix, cbm_node_t **out, int *count) { *out = NULL; *count = 0; - if (!s || !s->db) { + if (!s || !s->db || !suffix) { return CBM_STORE_ERR; } /* Match QNs ending with ".suffix" or exactly equal to suffix */ - char like_pattern[CBM_SZ_512]; - snprintf(like_pattern, sizeof(like_pattern), "%%.%s", suffix); + char *like_pattern = store_like_dot_suffix(suffix, ""); + if (!like_pattern) { + return CBM_STORE_ERR; + } const char *sql_with_project = "SELECT id, project, label, name, qualified_name, file_path, " "start_line, end_line, properties FROM nodes " - "WHERE project = ?1 AND (qualified_name LIKE ?2 OR qualified_name = ?3)"; + "WHERE project = ?1 AND (qualified_name LIKE ?2 ESCAPE '\\' OR qualified_name = ?3)"; const char *sql_any = "SELECT id, project, label, name, qualified_name, file_path, " "start_line, end_line, properties FROM nodes " - "WHERE (qualified_name LIKE ?1 OR qualified_name = ?2)"; + "WHERE (qualified_name LIKE ?1 ESCAPE '\\' OR qualified_name = ?2)"; sqlite3_stmt *stmt = NULL; int rc = sqlite3_prepare_v2(s->db, project ? sql_with_project : sql_any, CBM_NOT_FOUND, &stmt, NULL); if (rc != SQLITE_OK) { + cbm_free(CBM_MEM_CLASS_STORE, like_pattern); store_set_error_sqlite(s, "qn_suffix prepare"); return CBM_STORE_ERR; } @@ -4424,32 +4514,130 @@ int cbm_store_find_nodes_by_qn_suffix(cbm_store_t *s, const char *project, const bind_text(stmt, SKIP_ONE, like_pattern); bind_text(stmt, ST_COL_2, suffix); } + rc = store_collect_nodes(s, stmt, NULL, NULL, out, count); + cbm_free(CBM_MEM_CLASS_STORE, like_pattern); + return rc; +} - int cap = ST_INIT_CAP_8; - int n = 0; - cbm_node_t *nodes = malloc(cap * sizeof(cbm_node_t)); - int scan_rc8; - while ((scan_rc8 = sqlite3_step(stmt)) == SQLITE_ROW) { - if (n >= cap) { - cap *= ST_GROWTH; - nodes = safe_realloc(nodes, cap * sizeof(cbm_node_t)); +/* ── FindNodesByQNBase ─────────────────────────────────────────── */ + +typedef struct { + const char *base; + size_t len; + bool suffix; +} store_qn_base_arg_t; + +/* ASCII case-insensitive equality: SQLite's LIKE folds exactly ASCII. */ +static bool store_ascii_ieq(const char *a, const char *b, size_t n) { + for (size_t i = 0; i < n; i++) { + unsigned char x = (unsigned char)a[i]; + unsigned char y = (unsigned char)b[i]; + if (x >= 'A' && x <= 'Z') { + x = (unsigned char)(x - 'A' + 'a'); + } + if (y >= 'A' && y <= 'Z') { + y = (unsigned char)(y - 'A' + 'a'); + } + if (x != y) { + return false; } - memset(&nodes[n], 0, sizeof(cbm_node_t)); - scan_node(stmt, &nodes[n]); - n++; } - if (scan_rc8 != SQLITE_DONE) { /* SCANCHK:8:stmt */ - store_set_error_sqlite(s, "row scan aborted"); - sqlite3_finalize(stmt); - cbm_store_free_nodes(nodes, n); - *out = NULL; - *count = 0; + return true; +} + +/* Keep a row whose QN carries a callable identity suffix (#2061) over `base`: + * its base QN equals `base` (exact) or ends with "." + `base` (suffix mode, + * ASCII case-insensitive like the LIKE suffix tier). An unsuffixed QN never + * qualifies, so this tier finds nothing until a language mints suffixes. */ +static bool store_qn_base_keep(const char *qn, const char *name, const void *argp) { + const store_qn_base_arg_t *arg = argp; + if (!qn || !name) { + return false; + } + size_t base_len = cbm_qn_callable_base_len_named(qn, name); + if (qn[base_len] == '\0') { + return false; + } + if (!arg->suffix) { + return base_len == arg->len && memcmp(qn, arg->base, arg->len) == 0; + } + return base_len > arg->len && qn[base_len - arg->len - SKIP_ONE] == '.' && + store_ascii_ieq(qn + base_len - arg->len, arg->base, arg->len); +} + +/* Exact mode reads the qualified_name index range [base "(", base "="): a + * suffix starts with '(' (0x28) or '<' (0x3C). Suffix mode needs the leading + * wildcard, like the suffix tier it follows. Both bounds are heap-owned. */ +static bool store_qn_base_bounds(const char *base, bool suffix_match, char **lo, char **hi) { + if (suffix_match) { + *lo = store_like_dot_suffix(base, "(%"); + *hi = store_like_dot_suffix(base, "<%"); + } else { + size_t n = strlen(base); + *lo = cbm_alloc(CBM_MEM_CLASS_STORE, n + ST_COL_2); + *hi = cbm_alloc(CBM_MEM_CLASS_STORE, n + ST_COL_2); + if (*lo && *hi) { + memcpy(*lo, base, n); + memcpy(*hi, base, n); + (*lo)[n] = '('; + (*hi)[n] = '='; + (*lo)[n + SKIP_ONE] = '\0'; + (*hi)[n + SKIP_ONE] = '\0'; + } + } + if (*lo && *hi) { + return true; + } + cbm_free(CBM_MEM_CLASS_STORE, *lo); + cbm_free(CBM_MEM_CLASS_STORE, *hi); + return false; +} + +int cbm_store_find_nodes_by_qn_base(cbm_store_t *s, const char *project, const char *base, + bool suffix_match, cbm_node_t **out, int *count) { + *out = NULL; + *count = 0; + if (!s || !s->db || !base || !base[0]) { + return CBM_STORE_ERR; + } + static const char *const sql[2][2] = { + /* [suffix_match][has project] */ + {"SELECT id, project, label, name, qualified_name, file_path, start_line, end_line, " + "properties FROM nodes WHERE qualified_name >= ?1 AND qualified_name < ?2", + "SELECT id, project, label, name, qualified_name, file_path, start_line, end_line, " + "properties FROM nodes WHERE project = ?1 AND qualified_name >= ?2 AND " + "qualified_name < ?3"}, + {"SELECT id, project, label, name, qualified_name, file_path, start_line, end_line, " + "properties FROM nodes WHERE (qualified_name LIKE ?1 ESCAPE '\\' OR " + "qualified_name LIKE ?2 ESCAPE '\\')", + "SELECT id, project, label, name, qualified_name, file_path, start_line, end_line, " + "properties FROM nodes WHERE project = ?1 AND (qualified_name LIKE ?2 ESCAPE '\\' OR " + "qualified_name LIKE ?3 ESCAPE '\\')"}, + }; + char *lo = NULL; + char *hi = NULL; + if (!store_qn_base_bounds(base, suffix_match, &lo, &hi)) { return CBM_STORE_ERR; } - sqlite3_finalize(stmt); - *out = nodes; - *count = n; - return CBM_STORE_OK; + sqlite3_stmt *stmt = NULL; + if (sqlite3_prepare_v2(s->db, sql[suffix_match ? 1 : 0][project ? 1 : 0], CBM_NOT_FOUND, &stmt, + NULL) != SQLITE_OK) { + cbm_free(CBM_MEM_CLASS_STORE, lo); + cbm_free(CBM_MEM_CLASS_STORE, hi); + store_set_error_sqlite(s, "qn_base prepare"); + return CBM_STORE_ERR; + } + int col = SKIP_ONE; + if (project) { + bind_text(stmt, col++, project); + } + bind_text(stmt, col++, lo); + bind_text(stmt, col, hi); + store_qn_base_arg_t arg = {base, strlen(base), suffix_match}; + int rc = store_collect_nodes(s, stmt, store_qn_base_keep, &arg, out, count); + cbm_free(CBM_MEM_CLASS_STORE, lo); + cbm_free(CBM_MEM_CLASS_STORE, hi); + return rc; } /* ── NodeDegree ────────────────────────────────────────────────── */ diff --git a/src/store/store.h b/src/store/store.h index b5321483a9..79bd0b99b1 100644 --- a/src/store/store.h +++ b/src/store/store.h @@ -105,6 +105,13 @@ int cbm_store_find_nodes_by_file_overlap(cbm_store_t *s, const char *project, co int cbm_store_find_nodes_by_qn_suffix(cbm_store_t *s, const char *project, const char *suffix, cbm_node_t **out, int *count); +/* Find callables whose signature-qualified QN (#2061) has `base` as its base + * QN (exact), or a base ending with "." + `base` (suffix_match). QNs without + * a callable identity suffix never match: this is the tier that lets a bare + * QN name every overload. */ +int cbm_store_find_nodes_by_qn_base(cbm_store_t *s, const char *project, const char *base, + bool suffix_match, cbm_node_t **out, int *count); + /* Get CALLS degree of a node (inbound and outbound). */ void cbm_store_node_degree(cbm_store_t *s, int64_t node_id, int *in_deg, int *out_deg); @@ -427,11 +434,12 @@ int cbm_store_rollback(cbm_store_t *s); /* ── Bulk write optimization ────────────────────────────────────── */ -/* Tune pragmas for bulk write throughput (synchronous=OFF, large cache). +/* Tune pragmas for bulk write throughput (synchronous=OFF, 64 MiB cache). * WAL journal mode is preserved throughout for crash safety. */ int cbm_store_begin_bulk(cbm_store_t *s); -/* Restore normal pragmas (synchronous=NORMAL, default cache) after bulk writes. */ +/* Restore normal pragmas (synchronous=NORMAL, the 64 MiB read-write cache) + * after bulk writes. */ int cbm_store_end_bulk(cbm_store_t *s); /* Drop user indexes for faster bulk inserts. */ diff --git a/src/watcher/watcher.c b/src/watcher/watcher.c index 3c0194fac3..b84824c101 100644 --- a/src/watcher/watcher.c +++ b/src/watcher/watcher.c @@ -2,7 +2,10 @@ * watcher.c — Git-based file change watcher. * * Strategy: git status + HEAD tracking (the most reliable approach). - * For non-git projects, the watcher skips polling (no fsnotify/dirmtime yet). + * Non-git projects are NOT polled by default. With the opt-in + * cbm_watcher_set_poll_non_git(w, true) (config key `watch_non_git`, #1948) + * they are polled with a tree signature instead: the indexer's own discovery + * walk (same ignore rules), folded over (relative path, size, mtime). * * * Per-project state tracks: @@ -32,6 +35,7 @@ #include "foundation/str_util.h" #include "foundation/subprocess.h" #include "pipeline/artifact.h" /* CBM_ARTIFACT_DIR: the indexer's own output directory */ +#include "discover/discover.h" /* cbm_discover: the non-git tree signature (#1948) */ #ifdef _WIN32 #include "foundation/win_utf8.h" #define WIN32_LEAN_AND_MEAN @@ -61,7 +65,8 @@ typedef struct { cbm_subprocess_t *active_git; /* Room for Git's 64-hex SHA-256 object ID plus newline/NUL while reading. */ char last_head[CBM_SZ_128]; /* git HEAD hash (committed baseline) */ - bool is_git; /* false → skip polling */ + bool is_git; /* false → tree-polled if tree_poll, else skipped */ + bool tree_poll; /* non-git root polled by tree signature (#1948) */ bool baseline_done; /* true after first poll */ int missing_root_count; /* consecutive polls where root was missing (ENOENT/ENOTDIR) */ uint64_t first_missing_ms; /* cbm_now_ms() of the streak's first miss (0 = no streak) */ @@ -73,8 +78,14 @@ typedef struct { * are committed only after a SUCCESSFUL reindex (busy-skips and failed * runs retry); check_changes stages its observations in the pending_* * fields. 0 = clean tree. */ - uint64_t last_dirty_sig; /* committed dirty-state signature */ - uint64_t pending_dirty_sig; /* observed at check time */ + uint64_t last_dirty_sig; /* committed dirty-state signature */ + uint64_t pending_dirty_sig; /* observed at check time */ + /* Non-git tree signature (#1948), same commit discipline as the dirty + * signature. 0 = unknown: the baseline never adopts the tree as already + * indexed, so the first poll reindexes once (at-least-once, like a tree + * that is already dirty at a git baseline). Signatures are never 0. */ + uint64_t last_tree_sig; + uint64_t pending_tree_sig; char pending_head[CBM_SZ_128]; /* HEAD observed at check time */ /* Consecutive hard index failures (index_fn < 0). A hard error is * usually persistent — a poisoned coordination endpoint, an unreadable @@ -106,6 +117,9 @@ struct cbm_watcher { cbm_watcher_project_pruned_fn project_pruned; void *mutation_context; atomic_int stopped; + /* Opt-in (#1948): poll non-git roots by tree signature. Default false — + * non-git roots are not watched. Read at each project's baseline. */ + atomic_bool poll_non_git; /* Deferred-free list: freed after the next poll_once. */ project_state_t **pending_free; int pending_free_count; @@ -851,6 +865,46 @@ static watcher_git_status_t git_file_count(cbm_watcher_t *w, project_state_t *st return WATCHER_GIT_OK; } +/* ── Non-git tree signature (#1948) ─────────────────────────────── */ + +static int tree_entry_cmp(const void *a, const void *b) { + const cbm_file_info_t *fa = a; + const cbm_file_info_t *fb = b; + return strcmp(fa->rel_path, fb->rel_path); +} + +/* Signature of a non-git tree: FNV-1a over every file the INDEXER would + * discover (cbm_discover, full mode — the same skip lists, .gitignore stack + * and .cbmignore), folding each (relative path, size, mtime) in path order so + * readdir order cannot change the value. Adding, deleting, renaming or editing + * an indexable file yields a new signature; touching anything discovery skips + * does not. That includes everything cbm writes itself: the .codebase-memory + * artifact directory is a built-in skip and the cache directory is pruned by + * absolute path, so a successful reindex can never look like a new change + * (the runaway history in init_baseline and #1953). Never returns 0 in + * *sig_out, which stays reserved for "unknown". */ +static bool tree_signature(const char *root_path, uint64_t *sig_out, int *count_out) { + cbm_discover_opts_t opts = {.mode = CBM_MODE_FULL}; + cbm_file_info_t *files = NULL; + int count = 0; + if (cbm_discover(root_path, &opts, &files, &count) != 0) { + return false; + } + if (count > 1) { + qsort(files, (size_t)count, sizeof(*files), tree_entry_cmp); + } + uint64_t h = SIG_FNV_OFFSET; + for (int i = 0; i < count; i++) { + h = sig_fold(h, files[i].rel_path, strlen(files[i].rel_path) + SKIP_ONE); + h = sig_fold(h, &files[i].size, sizeof(files[i].size)); + h = sig_fold(h, &files[i].mtime_ns, sizeof(files[i].mtime_ns)); + } + cbm_discover_free(files, count); + *sig_out = h ? h : 1; + *count_out = count; + return true; +} + /* ── Project state lifecycle ────────────────────────────────────── */ static void state_free(project_state_t *s); @@ -1042,9 +1096,16 @@ cbm_watcher_t *cbm_watcher_new(cbm_store_t *store, cbm_index_fn index_fn, void * cbm_mutex_init(&w->projects_lock); cbm_mutex_init(&w->coordination_lock); atomic_init(&w->stopped, 0); + atomic_init(&w->poll_non_git, false); return w; } +void cbm_watcher_set_poll_non_git(cbm_watcher_t *w, bool enabled) { + if (w) { + atomic_store_explicit(&w->poll_non_git, enabled, memory_order_release); + } +} + void cbm_watcher_free(cbm_watcher_t *w) { if (!w) { return; @@ -1238,8 +1299,10 @@ static bool init_baseline(cbm_watcher_t *w, project_state_t *s) { * A directory is only really git-managed here if it carries its own .git, * or the ancestor repository actually tracks something inside it. A * genuine sub-package of a monorepo passes the second test; a scratch or - * gitignored folder sitting under a repo fails both and is polled as a - * plain directory instead. */ + * gitignored folder sitting under a repo fails both and is treated as a + * non-git root: NOT polled by default, or polled by its own tree signature + * when the watch_non_git opt-in is on (#1948). Either way the ancestor's + * dirty state never reaches it. */ if (s->is_git && !git_has_own_dot_git(s->root_path)) { bool tracked = false; watcher_git_status_t tracked_status = git_tracks_anything_here(w, s, &tracked); @@ -1282,6 +1345,14 @@ static bool init_baseline(cbm_watcher_t *w, project_state_t *s) { s->interval_ms = cbm_watcher_poll_interval_ms(s->file_count); cbm_log_info("watcher.baseline", "project", s->project_name, "strategy", "git", "files", s->file_count > 0 ? "yes" : "0"); + } else if (atomic_load_explicit(&w->poll_non_git, memory_order_acquire)) { + /* #1948 opt-in. last_tree_sig stays 0 ("unknown"): nothing records + * which tree state the DB holds (edits made while the daemon was down + * are exactly the silent-staleness case), so the first poll reindexes + * once and the signature gates every poll after it. The interval is + * sized from the file count of each signature walk. */ + s->tree_poll = true; + cbm_log_info("watcher.baseline", "project", s->project_name, "strategy", "tree"); } else { cbm_log_info("watcher.baseline", "project", s->project_name, "strategy", "none"); } @@ -1295,13 +1366,31 @@ static bool init_baseline(cbm_watcher_t *w, project_state_t *s) { * poll_project commits them only after a SUCCESSFUL reindex so that * busy-skips and failed runs retry instead of silently losing the change * (#937). Observations are staged in the pending_* fields. */ +/* Non-git leg of check_changes (#1948). A failed walk reports "no verdict" + * (false) so the committed baseline is kept and a later poll retries. */ +static bool check_tree_changes(project_state_t *s, bool *changed_out) { + uint64_t sig = 0; + int file_count = 0; + if (!tree_signature(s->root_path, &sig, &file_count)) { + cbm_log_warn("watcher.tree_walk_failed", "project", s->project_name, "path", s->root_path); + /* Retry at the regular cadence, not on every run-loop tick. */ + s->next_poll_ns = now_ns() + ((int64_t)s->interval_ms * US_PER_MS); + return false; + } + s->pending_tree_sig = sig; + s->file_count = file_count; + s->interval_ms = cbm_watcher_poll_interval_ms(file_count); + *changed_out = sig != s->last_tree_sig; + return true; +} + static bool check_changes(cbm_watcher_t *w, project_state_t *s, bool *changed_out) { if (!changed_out) { return false; } *changed_out = false; if (!s->is_git) { - return true; + return s->tree_poll ? check_tree_changes(s, changed_out) : true; } bool changed = false; @@ -1416,6 +1505,28 @@ static void prune_missing_project(cbm_watcher_t *w, project_state_t *s) { free(root_path); } +/* After a SUCCESSFUL reindex, commit the baselines OBSERVED AT CHECK TIME — + * the state whose reindex just succeeded. A commit/edit landing during the + * reindex is deliberately not absorbed: the next poll sees it as a new delta + * (at-least-once, never lost). */ +static void commit_baselines(cbm_watcher_t *w, project_state_t *s) { + if (!s->is_git) { + /* Tree strategy (#1948): the walk already refreshed the file count. */ + s->last_tree_sig = s->pending_tree_sig; + return; + } + if (s->pending_head[0] != '\0') { + snprintf(s->last_head, sizeof(s->last_head), "%s", s->pending_head); + } + s->last_dirty_sig = s->pending_dirty_sig; + /* Refresh file count for interval */ + int file_count = 0; + if (git_file_count(w, s, &file_count) == WATCHER_GIT_OK) { + s->file_count = file_count; + s->interval_ms = cbm_watcher_poll_interval_ms(s->file_count); + } +} + static void poll_project(const char *key, void *val, void *ud) { (void)key; poll_ctx_t *ctx = ud; @@ -1468,8 +1579,9 @@ static void poll_project(const char *key, void *val, void *ud) { return; } - /* Skip non-git projects */ - if (!s->is_git) { + /* Non-git projects are skipped unless the watch_non_git opt-in turned on + * tree polling for them at baseline (#1948). */ + if (!s->is_git && !s->tree_poll) { return; } @@ -1495,26 +1607,14 @@ static void poll_project(const char *key, void *val, void *ud) { * not admit an index callback after that ownership boundary. */ return; } - cbm_log_info("watcher.changed", "project", s->project_name, "strategy", "git"); + cbm_log_info("watcher.changed", "project", s->project_name, "strategy", + s->is_git ? "git" : "tree"); if (ctx->w->index_fn) { int rc = ctx->w->index_fn(s->project_name, s->root_path, ctx->w->user_data); if (rc == 0) { ctx->reindexed++; s->index_failures = 0; - /* Commit the baselines OBSERVED AT CHECK TIME — the state whose - * reindex just succeeded. A commit/edit landing during the - * reindex is deliberately not absorbed: the next poll sees it - * as a new delta (at-least-once, never lost). */ - if (s->pending_head[0] != '\0') { - snprintf(s->last_head, sizeof(s->last_head), "%s", s->pending_head); - } - s->last_dirty_sig = s->pending_dirty_sig; - /* Refresh file count for interval */ - int file_count = 0; - if (git_file_count(ctx->w, s, &file_count) == WATCHER_GIT_OK) { - s->file_count = file_count; - s->interval_ms = cbm_watcher_poll_interval_ms(s->file_count); - } + commit_baselines(ctx->w, s); } else if (rc > 0) { /* Busy-skip keeps the uncommitted baseline and any prior failure streak. */ cbm_log_info("watcher.index.retry", "project", s->project_name); @@ -1657,6 +1757,17 @@ int cbm_watcher_run(cbm_watcher_t *w, int base_interval_ms) { } #if defined(CBM_ENABLE_TEST_SEAMS) && CBM_ENABLE_TEST_SEAMS +int64_t cbm_watcher_test_next_poll_ns(cbm_watcher_t *w, const char *project_name) { + if (!w || !project_name) { + return -1; + } + cbm_mutex_lock(&w->projects_lock); + project_state_t *s = cbm_ht_get(w->projects, project_name); + int64_t result = s ? s->next_poll_ns : -1; + cbm_mutex_unlock(&w->projects_lock); + return result; +} + int cbm_watcher_test_pending_free_count(cbm_watcher_t *w) { if (!w) { return -1; diff --git a/src/watcher/watcher.h b/src/watcher/watcher.h index 2b33d78f49..17feb308ed 100644 --- a/src/watcher/watcher.h +++ b/src/watcher/watcher.h @@ -4,6 +4,7 @@ * Polls indexed projects for git changes (HEAD movement or dirty working tree) * and triggers re-indexing via a callback. Uses adaptive polling intervals * based on project size (5s base + 1s per 500 files, capped at 60s). + * Non-git roots are not polled unless cbm_watcher_set_poll_non_git() opts in. * * Depends on: foundation, store (for project metadata) */ @@ -59,6 +60,17 @@ void cbm_watcher_set_project_mutation_guard(cbm_watcher_t *w, cbm_watcher_project_mutation_end_fn end, cbm_watcher_project_pruned_fn pruned, void *context); +/* Opt in to polling NON-GIT project roots (#1948; default off — non-git roots + * are not watched). When on, a non-git root is polled on the same adaptive + * cadence by a tree signature: the indexer's own discovery walk (same skip + * lists, .gitignore and .cbmignore rules), folded over each file's (relative + * path, size, mtime). A changed signature triggers index_fn; the first poll + * after baseline reindexes once, since nothing records which tree state the + * index holds. Paths discovery skips — including cbm's own .codebase-memory + * artifact directory and cache directory — never change the signature. + * Read at each project's baseline, so set it before registering projects. */ +void cbm_watcher_set_poll_non_git(cbm_watcher_t *w, bool enabled); + /* ── Watch list management ──────────────────────────────────────── */ /* Add a project to the watch list. root_path is copied. Returns true only when @@ -116,6 +128,8 @@ int cbm_watcher_index_failure_count(cbm_watcher_t *w, const char *project_name); * otherwise unobservable — cbm_watcher_free frees the same list, so a poll * that stopped draining would neither crash nor leak. Test builds only. */ int cbm_watcher_test_pending_free_count(cbm_watcher_t *w); +/* Single-threaded test inspection of touch/backoff scheduling. */ +int64_t cbm_watcher_test_next_poll_ns(cbm_watcher_t *w, const char *project_name); #endif /* Return the adaptive poll interval (ms) for a given file count. */ diff --git a/tests/test_c_lsp.c b/tests/test_c_lsp.c index 933cacc36f..d890911506 100644 --- a/tests/test_c_lsp.c +++ b/tests/test_c_lsp.c @@ -15350,6 +15350,10 @@ TEST(registry_overlay_chain_iterates_and_copies_on_write) { memset(&t, 0, sizeof(t)); t.qualified_name = "pkg.T"; t.short_name = "T"; + const char *field_names[] = {"untyped", "typed", NULL}; + const CBMType *field_types[] = {NULL, cbm_type_named(&arena, "pkg.Value"), NULL}; + t.field_names = field_names; + t.field_types = field_types; cbm_registry_add_type(&base, t); cbm_registry_finalize(&base); base.read_only = true; @@ -15414,6 +15418,10 @@ TEST(registry_overlay_chain_iterates_and_copies_on_write) { CBMRegisteredType *wt = cbm_registry_type_for_update(&overlay, "pkg.T"); ASSERT_NOT_NULL(wt); ASSERT_EQ(overlay.type_count, 1); + /* A missing type for an earlier named field must not truncate the copy. */ + bool field_slots_preserved = wt->field_types && wt->field_types != field_types && + wt->field_types[0] == NULL && + wt->field_types[1] == field_types[1] && wt->field_types[2] == NULL; cbm_registry_all_types_chain(&overlay, &ti); seen = 0; while (cbm_type_short_iter_next(&ti) >= 0) { @@ -15430,6 +15438,7 @@ TEST(registry_overlay_chain_iterates_and_copies_on_write) { cbm_arena_destroy(&scratch); cbm_arena_destroy(&arena); + ASSERT_TRUE(field_slots_preserved); PASS(); } @@ -15804,6 +15813,111 @@ TEST(seal_py_shared_registry_readonly_fields) { PASS(); } +/* #1277: the production resolver receives a writable overlay, so refining an + * existing field must detach its pointer array from the sealed fallback. */ +TEST(seal_py_overlay_existing_field_keeps_shared_type) { + CBMArena arena; + cbm_arena_init(&arena); + CBMLSPDef defs[4] = {0}; + defs[0].qualified_name = "test.mod.Foo"; + defs[0].short_name = "Foo"; + defs[0].label = "Class"; + defs[0].def_module_qn = "test.mod"; + defs[0].field_defs = "x:test.contracts.Contract"; + defs[1].qualified_name = "test.mod.Foo.m"; + defs[1].short_name = "m"; + defs[1].label = "Method"; + defs[1].receiver_type = "test.mod.Foo"; + defs[1].def_module_qn = "test.mod"; + defs[2].qualified_name = "test.contracts.Contract"; + defs[2].short_name = "Contract"; + defs[2].label = "Class"; + defs[2].def_module_qn = "test.contracts"; + defs[3].qualified_name = "test.contracts.Contract.process_batch"; + defs[3].short_name = "process_batch"; + defs[3].label = "Method"; + defs[3].receiver_type = "test.contracts.Contract"; + defs[3].def_module_qn = "test.contracts"; + for (int i = 0; i < 4; i++) { + defs[i].lang = CBM_LANG_PYTHON; + } + CBMTypeRegistry *reg = cbm_py_build_cross_registry(&arena, defs, 4); + ASSERT_NOT_NULL(reg); + ASSERT_TRUE(reg->read_only); + const CBMRegisteredType *base = cbm_registry_lookup_type(reg, "test.mod.Foo"); + ASSERT_NOT_NULL(base); + ASSERT_NOT_NULL(base->field_names); + ASSERT_NOT_NULL(base->field_types); + const char **names_before = base->field_names; + const CBMType **types_before = base->field_types; + const CBMType *type_before = base->field_types[0]; + ASSERT_NOT_NULL(type_before); + ASSERT_NULL(base->field_types[1]); + + CBMArena scratch; + cbm_arena_init(&scratch); + CBMTypeRegistry overlay; + cbm_registry_init(&overlay, &scratch); + overlay.fallback = reg; + const char *writer = "class Foo:\n" + " def m(self):\n" + " self.x = 1\n"; + CBMResolvedCallArray out = {0}; + cbm_run_py_lsp_cross_with_registry(&scratch, writer, (int)strlen(writer), "test.mod", &overlay, + NULL, NULL, 0, NULL, &out, NULL); + + /* Observe while scratch is live, then release both arenas before any + * failure assertion so the regression also fails cleanly under LSan. */ + bool shared_names_unchanged = base->field_names == names_before; + bool shared_types_unchanged = base->field_types == types_before; + bool shared_element_unchanged = base->field_types[0] == type_before; + bool shared_terminator_unchanged = base->field_types[1] == NULL; + const CBMRegisteredType *refined = cbm_registry_lookup_type(&overlay, "test.mod.Foo"); + bool local_refined = refined && refined != base && refined->field_types && + refined->field_types != types_before && refined->field_types[0] && + refined->field_types[0] != type_before && + refined->field_types[0]->kind == CBM_TYPE_BUILTIN && + refined->field_types[1] == NULL; + cbm_arena_destroy(&scratch); + + /* Never read a corrupted shared pointer after scratch destruction. When + * intact, a later file must still see the original Contract field. */ + int hits = 0; + if (shared_names_unchanged && shared_types_unchanged && shared_element_unchanged && + shared_terminator_unchanged && local_refined) { + cbm_arena_init(&scratch); + cbm_registry_init(&overlay, &scratch); + overlay.fallback = reg; + const char *reader = "from mod import Foo\n\n" + "def run(foo: Foo):\n" + " foo.x.process_batch()\n"; + const char *import_names[] = {"Foo"}; + const char *import_qns[] = {"test.mod.Foo"}; + memset(&out, 0, sizeof(out)); + cbm_run_py_lsp_cross_with_registry(&scratch, reader, (int)strlen(reader), "test.reader", + &overlay, import_names, import_qns, 1, NULL, &out, NULL); + for (int i = 0; i < out.count; i++) { + const CBMResolvedCall *call = &out.items[i]; + if (call->caller_qn && call->callee_qn && + strcmp(call->caller_qn, "test.reader.run") == 0 && + strcmp(call->callee_qn, "test.contracts.Contract.process_batch") == 0 && + call->kind == CBM_RESOLVED_INVOCATION && + call->confidence >= CBM_LSP_CONFIDENCE_FLOOR) { + hits++; + } + } + cbm_arena_destroy(&scratch); + } + cbm_arena_destroy(&arena); + ASSERT_TRUE(shared_names_unchanged); + ASSERT_TRUE(shared_types_unchanged); + ASSERT_TRUE(shared_element_unchanged); + ASSERT_TRUE(shared_terminator_unchanged); + ASSERT_TRUE(local_refined); + ASSERT_EQ(hits, 1); + PASS(); +} + TEST(seal_cs_shared_registry_readonly) { CBMArena arena; cbm_arena_init(&arena); @@ -16402,6 +16516,7 @@ SUITE(c_lsp) { RUN_TEST(registry_overlay_chain_iterates_and_copies_on_write); RUN_TEST(clsp_method_return_refinement_is_copy_on_write); RUN_TEST(seal_py_shared_registry_readonly_fields); + RUN_TEST(seal_py_overlay_existing_field_keeps_shared_type); RUN_TEST(seal_cs_shared_registry_readonly); RUN_TEST(seal_ts_shared_registry_readonly); RUN_TEST(seal_go_shared_registry_readonly); diff --git a/tests/test_callable_sig.c b/tests/test_callable_sig.c new file mode 100644 index 0000000000..ad0315032a --- /dev/null +++ b/tests/test_callable_sig.c @@ -0,0 +1,348 @@ +/* + * test_callable_sig.c — golden tables for the signature-qualified callable + * identity builder and its inverse (#2061, plumbing: no language enabled). + */ +#include "test_framework.h" +#include "callable_sig.h" +#include "lang_specs.h" +#include "foundation/arena.h" +#include "pipeline/pipeline.h" +#include "pipeline/lsp_resolve.h" +#include "tree_sitter/api.h" +#include + +typedef struct { + CBMLanguage lang; + CBMCallableIdentity mode; + const char *source; + const char *kind; /* node kind of the callable */ + int nth; /* 0-based occurrence of `kind` in preorder */ + const char *expected; +} sig_case_t; + +static bool find_nth(TSNode n, const char *kind, int *left, TSNode *out) { + if (strcmp(ts_node_type(n), kind) == 0) { + if (*left == 0) { + *out = n; + return true; + } + (*left)--; + } + uint32_t count = ts_node_child_count(n); + for (uint32_t i = 0; i < count; i++) { + if (find_nth(ts_node_child(n, i), kind, left, out)) { + return true; + } + } + return false; +} + +/* Parse `source`, build the nth `kind` node's suffix in `mode`; the returned + * string is arena-owned (NULL when absent). */ +static const char *sig_of(CBMArena *a, CBMLanguage lang, CBMCallableIdentity mode, + const char *source, const char *kind, int nth) { + TSParser *parser = ts_parser_new(); + ts_parser_set_language(parser, cbm_ts_language(lang)); + TSTree *tree = ts_parser_parse_string(parser, NULL, source, (uint32_t)strlen(source)); + TSNode node = {0}; + int left = nth; + const char *sig = NULL; + if (tree && find_nth(ts_tree_root_node(tree), kind, &left, &node)) { + sig = cbm_callable_sig_mode(a, node, source, lang, mode); + } + ts_tree_delete(tree); + ts_parser_delete(parser); + return sig; +} + +static const sig_case_t k_cases[] = { + /* Java: generics kept, qualifiers and annotations dropped, varargs = []. */ + {CBM_LANG_JAVA, CBM_CALLABLE_ID_TYPED, + "class A { > void work(final java.util.List xs, int[] a, " + "String... rest) {} }", + "method_declaration", 0, "(List,int[],String[])"}, + {CBM_LANG_JAVA, CBM_CALLABLE_ID_TYPED, "class A { void f(@Ann Map m, int a[]){} }", + "method_declaration", 0, "(Map,int[])"}, + {CBM_LANG_JAVA, CBM_CALLABLE_ID_TYPED, + "class A { void f(\n @Ann Map< K ,\n V > m, /* c */ int a [ ]\n) {} }", "method_declaration", + 0, "(Map,int[])"}, + {CBM_LANG_JAVA, CBM_CALLABLE_ID_TYPED, "class A { A(int x){} void g(){} }", + "constructor_declaration", 0, "(int)"}, + {CBM_LANG_JAVA, CBM_CALLABLE_ID_TYPED, "class A { A(int x){} void g(){} }", + "method_declaration", 0, "()"}, + /* C#: generic arity, ref/out, params inlined by the grammar, tuple names + * dropped, operators and indexers. */ + {CBM_LANG_CSHARP, CBM_CALLABLE_ID_TYPED, + "class A { public void Work(ref int x, out string y, params object[] rest, int? n = 3, " + "List> m, (int a, string b) t, this Foo z) {} }", + "method_declaration", 0, + "(ref int,out string,object[],int?,List>,(int,string),Foo)"}, + {CBM_LANG_CSHARP, CBM_CALLABLE_ID_TYPED, + "class A { public static A operator +(A a, A b) => a; }", "operator_declaration", 0, "(A,A)"}, + {CBM_LANG_CSHARP, CBM_CALLABLE_ID_TYPED, "class A { int this[int i] { get => 0; } }", + "indexer_declaration", 0, "(int)"}, + /* C++: template kinds, declarator markers, top-level cv dropped, arrays + * decay, function pointers, C variadic, packs, cvref, operators. */ + {CBM_LANG_CPP, CBM_CALLABLE_ID_TYPED, + "struct S { template void work(const std::vector & v, " + "int (*cb)(int), unsigned long x, ...) const && noexcept; };", + "template_declaration", 0, + "(const vector&,int(*)(int),unsigned long,~)const&&"}, + {CBM_LANG_CPP, CBM_CALLABLE_ID_TYPED, "struct S { S& operator=(S&&) &; };", "field_declaration", + 0, "(S&&)&"}, + {CBM_LANG_CPP, CBM_CALLABLE_ID_TYPED, "struct S { bool operator()(int) volatile; };", + "field_declaration", 0, "(int)volatile"}, + {CBM_LANG_CPP, CBM_CALLABLE_ID_TYPED, + "struct S { template void emplace(Args&&... args); };", "template_declaration", + 0, "(Args&&~)"}, + {CBM_LANG_CPP, CBM_CALLABLE_ID_TYPED, "void n::S::arr(int a[10], char *const p, int q) {}", + "function_definition", 0, "(int*,char*,int)"}, + {CBM_LANG_CPP, CBM_CALLABLE_ID_TYPED, + "void w(\n const int a , // trailing\n char const * /* c */ const p\n);", + "declaration", 0, "(int,const char*)"}, + {CBM_LANG_CPP, CBM_CALLABLE_ID_TYPED, "void v(void);", "declaration", 0, "()"}, + {CBM_LANG_CPP, CBM_CALLABLE_ID_TYPED, "void f(int (*(*g)(int x))(double), void h(char *s));", + "declaration", 0, "(int(*(*)(int))(double),void(*)(char*))"}, + {CBM_LANG_CPP, CBM_CALLABLE_ID_TYPED, + "template struct S { void f(int); };\ntemplate void S::f(int x) {}", + "function_definition", 0, "(int)"}, + /* Kotlin: extension receiver, vararg, function types, nullable, + * qualified types, suspend. */ + {CBM_LANG_KOTLIN, CBM_CALLABLE_ID_TYPED, + "class A {\n fun > String.work(vararg xs: Int, f: (Int) -> Unit, " + "m: Map>? = null, @Ann q: kotlin.collections.List<*>) {}\n}", + "function_declaration", 0, "(this:String,Int~,(Int)=>Unit,Map>?,List<*>)"}, + {CBM_LANG_KOTLIN, CBM_CALLABLE_ID_TYPED, "fun top(x: Int) = x", "function_declaration", 0, + "(Int)"}, + {CBM_LANG_KOTLIN, CBM_CALLABLE_ID_TYPED, "class A { constructor(x: Int) {} }", + "secondary_constructor", 0, "(Int)"}, + /* Swift: labels + types (the #2061 Alamofire shape), variadic, inout, + * attributes dropped, qualified types, init/subscript/operators. */ + {CBM_LANG_SWIFT, CBM_CALLABLE_ID_LABELED_TYPED, + "class Session {\n func upload(_ data: Data, to url: URLConvertible, method: " + "HTTPMethod = .post, headers: [String: String]? = nil, handler: @escaping (Int) -> Void, " + "xs: Int..., io: inout Swift.Int, t: (a: Int, String)) -> UploadRequest { }\n}", + "function_declaration", 0, + "(_:Data,to:URLConvertible,method:HTTPMethod,headers:[String:String]?,handler:(Int)=>Void," + "xs:Int~,io:inout Int,t:(a:Int,String))"}, + {CBM_LANG_SWIFT, CBM_CALLABLE_ID_LABELED_TYPED, + "class Session {\n func upload(_ data: Data, with request: URLRequest) {}\n}", + "function_declaration", 0, "(_:Data,with:URLRequest)"}, + {CBM_LANG_SWIFT, CBM_CALLABLE_ID_LABELED_TYPED, "class C { init(frame: CGRect) {} }", + "init_declaration", 0, "(frame:CGRect)"}, + {CBM_LANG_SWIFT, CBM_CALLABLE_ID_LABELED_TYPED, + "class C { subscript(index i: Int) -> Int { return 0 } }", "subscript_declaration", 0, + "(index:Int)"}, + {CBM_LANG_SWIFT, CBM_CALLABLE_ID_LABELED_TYPED, + "class C { static func + (l: C, r: C) -> C { l } }", "function_declaration", 0, "(l:C,r:C)"}, + {CBM_LANG_SWIFT, CBM_CALLABLE_ID_LABELED_TYPED, "func extra() {}", "function_declaration", 0, + "()"}, + /* Scala: every clause flattened, by-name and repeated parameters. */ + {CBM_LANG_SCALA, CBM_CALLABLE_ID_TYPED, + "class A {\n def work[T <: AnyRef](xs: List[T], n: => Int, rest: String*)(implicit ord: " + "Ordering[T]): Unit = {}\n}", + "function_definition", 0, "(List[T],=>Int,String*,Ordering[T])"}, + {CBM_LANG_SCALA, CBM_CALLABLE_ID_TYPED, + "class A {\n def f(g: (Int, String) => scala.collection.immutable.Map[String, Int]): Unit\n}", + "function_declaration", 0, "((Int,String)=>Map[String,Int])"}, + /* Objective-C: the selector's keyword labels. */ + {CBM_LANG_OBJC, CBM_CALLABLE_ID_LABELED, + "@implementation Foo\n- (instancetype)initWithFrame:(CGRect)frame style:(NSInteger)style " + "{ return self; }\n@end", + "method_definition", 0, "(_:style:)"}, + {CBM_LANG_OBJC, CBM_CALLABLE_ID_LABELED, "@implementation Foo\n- (void)run { }\n@end", + "method_definition", 0, "()"}, + {CBM_LANG_OBJC, CBM_CALLABLE_ID_LABELED, + "@implementation Foo\n- (void)setX:(int)x :(int)y {}\n@end", "method_definition", 0, "(_:_:)"}, + {CBM_LANG_OBJC, CBM_CALLABLE_ID_LABELED, + "@implementation Foo\n+ (id)make:(int)a, ... { return nil; }\n@end", "method_definition", 0, + "(_:~)"}, + /* ARITY (dynamic tier-2 languages), shown on a Swift declaration. */ + {CBM_LANG_SWIFT, CBM_CALLABLE_ID_ARITY, "class C { func upload(_ data: Data, to url: URL) {} }", + "function_declaration", 0, "(2)"}, +}; + +TEST(callable_sig_golden_table) { + for (size_t i = 0; i < sizeof(k_cases) / sizeof(k_cases[0]); i++) { + const sig_case_t *tc = &k_cases[i]; + CBMArena a; + cbm_arena_init(&a); + const char *sig = sig_of(&a, tc->lang, tc->mode, tc->source, tc->kind, tc->nth); + if (!sig || strcmp(sig, tc->expected) != 0) { + printf(" case %zu: got %s, want %s\n", i, sig ? sig : "(null)", tc->expected); + } + ASSERT_NOT_NULL(sig); + ASSERT_STR_EQ(sig, tc->expected); + ASSERT_TRUE(strlen(sig) <= CBM_CALLABLE_SIG_MAX); + /* Contract: no separator a leaf splitter uses, and the inverse + * recovers the base from a real-shaped QN. */ + ASSERT_NULL(strchr(sig, '.')); + ASSERT_NULL(strstr(sig, "::")); + ASSERT_NULL(strstr(sig, "->")); + char *qn = cbm_arena_sprintf(&a, "proj.pkg.Cls.name%s", sig); + ASSERT_EQ(cbm_qn_callable_base_len(qn), strlen("proj.pkg.Cls.name")); + cbm_arena_destroy(&a); + } + PASS(); +} + +/* Long signatures cap at CBM_CALLABLE_SIG_MAX with a hash of the FULL + * suffix: deterministic, still distinct, still invertible. */ +TEST(callable_sig_cap_keeps_identity) { + char src_a[4096]; + char src_b[4096]; + size_t n = (size_t)snprintf(src_a, sizeof(src_a), "class A { void f("); + for (int i = 0; i < 30; i++) { + n += (size_t)snprintf(src_a + n, sizeof(src_a) - n, "%sVeryLongParameterTypeName%d p%d", + i ? ", " : "", i, i); + } + memcpy(src_b, src_a, n); + snprintf(src_a + n, sizeof(src_a) - n, ", int last) {} }"); + snprintf(src_b + n, sizeof(src_b) - n, ", long last) {} }"); + CBMArena a; + cbm_arena_init(&a); + const char *sa = + sig_of(&a, CBM_LANG_JAVA, CBM_CALLABLE_ID_TYPED, src_a, "method_declaration", 0); + const char *sa2 = + sig_of(&a, CBM_LANG_JAVA, CBM_CALLABLE_ID_TYPED, src_a, "method_declaration", 0); + const char *sb = + sig_of(&a, CBM_LANG_JAVA, CBM_CALLABLE_ID_TYPED, src_b, "method_declaration", 0); + ASSERT_NOT_NULL(sa); + ASSERT_NOT_NULL(sb); + ASSERT_TRUE(strlen(sa) <= CBM_CALLABLE_SIG_MAX); + ASSERT_TRUE(strlen(sb) <= CBM_CALLABLE_SIG_MAX); + ASSERT_STR_EQ(sa, sa2); + ASSERT_STR_NEQ(sa, sb); /* they differ only past the cap: the hash tells */ + ASSERT_TRUE(strncmp(sa, "(VeryLongParameterTypeName0,", 28) == 0); + const char *hash = strchr(sa, '#'); + ASSERT_NOT_NULL(hash); + ASSERT_EQ(strlen(hash), (size_t)18); /* "#" + 16 hex + ")" */ + char *qn = cbm_arena_sprintf(&a, "p.A.f%s", sa); + ASSERT_EQ(cbm_qn_callable_base_len(qn), strlen("p.A.f")); + cbm_arena_destroy(&a); + PASS(); +} + +/* PR1 is plumbing: no language mints a suffix yet. */ +TEST(callable_sig_every_language_is_none) { + CBMArena a; + cbm_arena_init(&a); + const char *src = "class A { void f(int x) {} }"; + for (int lang = 0; lang < CBM_LANG_COUNT; lang++) { + ASSERT_EQ(cbm_callable_identity((CBMLanguage)lang), CBM_CALLABLE_ID_NONE); + } + TSParser *parser = ts_parser_new(); + ts_parser_set_language(parser, cbm_ts_language(CBM_LANG_JAVA)); + TSTree *tree = ts_parser_parse_string(parser, NULL, src, (uint32_t)strlen(src)); + TSNode m = {0}; + int left = 0; + ASSERT_TRUE(find_nth(ts_tree_root_node(tree), "method_declaration", &left, &m)); + ASSERT_NULL(cbm_callable_sig(&a, m, src, CBM_LANG_JAVA)); + ts_tree_delete(tree); + ts_parser_delete(parser); + cbm_arena_destroy(&a); + PASS(); +} + +/* The inverse is the identity on every historical QN shape. */ +TEST(callable_sig_base_len_unsuffixed_is_full) { + static const char *const qns[] = { + "proj.pkg.Cls.method", + "proj.S.operator()", + "proj.S.operator<", + "proj.S.operator<=>", + "proj.S.operator->", + "proj.S.operator>>", + "proj.S.operator&&", + "proj.app/(auth)", + "proj.Makefile.$(OBJ)", + "proj.f.(anon)", + "", + "proj.x.list", + "proj.x.make_const", + "a::b::c", + "proj.~Dtor", + "(x)", + "proj.Makefile.foo$(EXT)", + "proj.lib/(group)/page", + "proj.C.+", + "proj.A.unary_!", + }; + for (size_t i = 0; i < sizeof(qns) / sizeof(qns[0]); i++) { + ASSERT_EQ(cbm_qn_callable_base_len(qns[i]), strlen(qns[i])); + } + ASSERT_EQ(cbm_qn_callable_base_len(NULL), (size_t)0); + PASS(); +} + +TEST(callable_sig_base_len_suffixed) { + static const struct { + const char *qn; + const char *base; + } rows[] = { + {"p.S.work(int,String)", "p.S.work"}, + {"p.S.work()", "p.S.work"}, + {"p.S.operator()()", "p.S.operator()"}, + {"p.S.operator()(int)const", "p.S.operator()"}, + {"p.S.operator<(T)", "p.S.operator<"}, + {"p.S.operator<=>(S)", "p.S.operator<=>"}, + {"p.S.operator->()", "p.S.operator->"}, + {"p.S.f(const vector&,int(*)(int))const&&", "p.S.f"}, + {"p.C.Work(ref int)", "p.C.Work"}, + {"p.Session.upload(_:Data,to:URL)", "p.Session.upload"}, + {"p.K.work(this:String,(Int)=>Unit)", "p.K.work"}, + {"p.O.initWithFrame(_:style:)", "p.O.initWithFrame"}, + {"p.C.+(l:C,r:C)", "p.C.+"}, + {"p.A.unary_!()", "p.A.unary_!"}, + {"p.S.operator/(S)", "p.S.operator/"}, + {"p.S.~S()", "p.S.~S"}, + {"p.J.f(VeryLong0,#0123456789abcdef)", "p.J.f"}, + }; + for (size_t i = 0; i < sizeof(rows) / sizeof(rows[0]); i++) { + ASSERT_EQ(cbm_qn_callable_base_len(rows[i].qn), strlen(rows[i].base)); + } + PASS(); +} + +/* The leaf splitters look only at the base: Swift labels carry ':' and a + * suffix may carry '>' — neither is a separator. */ +TEST(callable_sig_leaf_splitters_skip_suffix) { + const char *swift = "p.Session.upload(_:Data,to:URL)"; + ASSERT_STR_EQ(cbm_lsp_bare_segment(swift), "upload(_:Data,to:URL)"); + ASSERT_STR_EQ(cbm_pipeline_qn_class_method_tail(swift), "Session.upload(_:Data,to:URL)"); + /* Unsuffixed spellings are unchanged. */ + ASSERT_STR_EQ(cbm_lsp_bare_segment("Math::square"), "square"); + ASSERT_STR_EQ(cbm_lsp_bare_segment("p->run"), "run"); + ASSERT_STR_EQ(cbm_lsp_bare_segment("identity"), "identity"); + ASSERT_STR_EQ(cbm_pipeline_qn_class_method_tail("p.q.Cls.m"), "Cls.m"); + PASS(); +} + +/* The registry's by-name bucket is the base leaf, so every overload of a + * signature-qualified callable is found by its bare name; an unsuffixed QN + * keeps its historical key (last '.' or "::" segment). */ +TEST(callable_sig_registry_by_name_uses_base_leaf) { + cbm_registry_t *r = cbm_registry_new(); + cbm_registry_add(r, "upload", "p.Session.upload(_:Data,to:URL)", "Method"); + cbm_registry_add(r, "upload", "p.Session.upload(_:Data,with:URLRequest)", "Method"); + cbm_registry_add(r, "square", "p.Math::square", "Function"); + const char **out = NULL; + int count = 0; + cbm_registry_find_by_name(r, "upload", &out, &count); + ASSERT_EQ(count, 2); + count = 0; + cbm_registry_find_by_name(r, "square", &out, &count); + ASSERT_EQ(count, 1); + cbm_registry_free(r); + PASS(); +} + +SUITE(callable_sig) { + RUN_TEST(callable_sig_golden_table); + RUN_TEST(callable_sig_cap_keeps_identity); + RUN_TEST(callable_sig_every_language_is_none); + RUN_TEST(callable_sig_base_len_unsuffixed_is_full); + RUN_TEST(callable_sig_base_len_suffixed); + RUN_TEST(callable_sig_leaf_splitters_skip_suffix); + RUN_TEST(callable_sig_registry_by_name_uses_base_leaf); +} diff --git a/tests/test_cli.c b/tests/test_cli.c index 593c945609..70a1501767 100644 --- a/tests/test_cli.c +++ b/tests/test_cli.c @@ -41,6 +41,10 @@ #include #include #include +#ifdef _WIN32 +#include +#include "../src/foundation/win_utf8.h" +#endif #ifndef _WIN32 #include #include @@ -3119,8 +3123,8 @@ TEST(cli_uninstall_quiesces_active_cohort_before_removing_binary_and_index) { }; cbm_cli_activation_ops_t ops = cli_activation_fake_ops(&fake); cbm_cli_set_activation_ops_for_test(&ops); - char *argv[] = {"--yes"}; - int rc = cli_test_cmd_uninstall(1, argv); + char *argv[] = {"--yes", "--delete-indexes"}; + int rc = cli_test_cmd_uninstall(2, argv); cbm_cli_set_activation_ops_for_test(NULL); cbm_set_auto_answer_for_test(0); @@ -3142,6 +3146,24 @@ TEST(cli_uninstall_quiesces_active_cohort_before_removing_binary_and_index) { PASS(); } +static bool cli_uninstall_test_absent(const char *path) { +#ifdef _WIN32 + wchar_t *wide = cbm_path_to_wide(path); + if (!wide) { + return false; + } + DWORD attributes = GetFileAttributesW(wide); + DWORD error = GetLastError(); + free(wide); + return attributes == INVALID_FILE_ATTRIBUTES && + (error == ERROR_FILE_NOT_FOUND || error == ERROR_PATH_NOT_FOUND); +#else + struct stat st; + errno = 0; + return lstat(path, &st) != 0 && errno == ENOENT; +#endif +} + TEST(cli_uninstall_leaves_foreign_cache_running_while_removing_cli_files) { char tmpdir[256]; snprintf(tmpdir, sizeof(tmpdir), "/tmp/cli-foreign-cache-XXXXXX"); @@ -3187,10 +3209,10 @@ TEST(cli_uninstall_leaves_foreign_cache_running_while_removing_cli_files) { int saved = capture ? dup(STDERR_FILENO) : -1; bool capturing = saved >= 0 && dup2(fileno(capture), STDERR_FILENO) >= 0; cbm_cli_set_activation_runtime_parent_for_test(runtime); - char *args[] = {"--yes"}; + char *args[] = {"--yes", "--delete-indexes"}; /* Call the production command directly; Windows' convenience dispatcher * substitutes fake activation ops and cannot exercise ownership checks. */ - int rc = owned && capturing ? cbm_cmd_uninstall(1, args) : -1; + int rc = owned && capturing ? cbm_cmd_uninstall(2, args) : -1; cbm_cli_set_activation_runtime_parent_for_test(NULL); cbm_set_auto_answer_for_test(0); fflush(stderr); @@ -3204,9 +3226,8 @@ TEST(cli_uninstall_leaves_foreign_cache_running_while_removing_cli_files) { (void)fread(output, 1, sizeof(output) - 1, capture); fclose(capture); } - struct stat removed_status; - bool binary_removed = lstat(binary, &removed_status) != 0 && errno == ENOENT; - bool index_removed = lstat(index, &removed_status) != 0 && errno == ENOENT; + bool binary_removed = cli_uninstall_test_absent(binary); + bool index_removed = cli_uninstall_test_absent(index); if (lease) (void)cbm_version_cohort_lease_release(&lease); if (owner) @@ -3260,8 +3281,8 @@ TEST(cli_uninstall_preserves_binary_and_index_when_cohort_does_not_drain) { }; cbm_cli_activation_ops_t ops = cli_activation_fake_ops(&fake); cbm_cli_set_activation_ops_for_test(&ops); - char *argv[] = {"--yes"}; - int rc = cli_test_cmd_uninstall(1, argv); + char *argv[] = {"--yes", "--delete-indexes"}; + int rc = cli_test_cmd_uninstall(2, argv); cbm_cli_set_activation_ops_for_test(NULL); cbm_set_auto_answer_for_test(0); @@ -3282,6 +3303,350 @@ TEST(cli_uninstall_preserves_binary_and_index_when_cohort_does_not_drain) { PASS(); } +/* Uninstall requires a separate choice before deleting project data. Keep + * fixture state and activation coordination independent of the developer's + * installation, and validate setup before absence can count as deletion. */ +enum { UNINSTALL_INDEX_COUNT = 3, UNINSTALL_FILES_PER_INDEX = 6 }; + +typedef struct { + char tmpdir[256]; + char cache_dir[512]; + char bin_target[640]; + char internal_store[640]; + char index_paths[UNINSTALL_INDEX_COUNT * UNINSTALL_FILES_PER_INDEX][640]; + char index_bodies[UNINSTALL_INDEX_COUNT * UNINSTALL_FILES_PER_INDEX][64]; + char *old_home; + char *old_cache; + char *old_path; + bool created; +} cli_uninstall_index_fixture_t; + +static bool cli_uninstall_index_fixture_intact(const cli_uninstall_index_fixture_t *fx) { + for (int i = 0; i < UNINSTALL_INDEX_COUNT * UNINSTALL_FILES_PER_INDEX; i++) { + const char *body = read_test_file(fx->index_paths[i]); + if (!body || strcmp(body, fx->index_bodies[i]) != 0) { + return false; + } + } + return true; +} + +static bool cli_uninstall_test_file_is(const char *path, const char *expected) { + const char *body = read_test_file(path); + return body && strcmp(body, expected) == 0; +} + +static bool cli_uninstall_index_fixture_setup(cli_uninstall_index_fixture_t *fx, const char *tag) { + memset(fx, 0, sizeof(*fx)); + snprintf(fx->tmpdir, sizeof(fx->tmpdir), "/tmp/cli-uninstall-%s-XXXXXX", tag); + if (!cbm_mkdtemp(fx->tmpdir)) { + return false; + } + fx->created = true; + cli_activation_save_env(&fx->old_home, &fx->old_cache); + fx->old_path = save_test_env("PATH"); + cbm_setenv("HOME", fx->tmpdir, 1); + cbm_setenv("PATH", fx->tmpdir, 1); + snprintf(fx->cache_dir, sizeof(fx->cache_dir), "%s/cache", fx->tmpdir); + cbm_setenv("CBM_CACHE_DIR", fx->cache_dir, 1); + const char *home = getenv("HOME"); + const char *path = getenv("PATH"); + const char *cache = getenv("CBM_CACHE_DIR"); + if (!home || !path || !cache || strcmp(home, fx->tmpdir) != 0 || + strcmp(path, fx->tmpdir) != 0 || strcmp(cache, fx->cache_dir) != 0) { + return false; + } + test_mkdirp(fx->cache_dir); + static const char *const suffixes[UNINSTALL_FILES_PER_INDEX] = { + ".db", ".db-wal", ".db-shm", ".db-journal", ".db.tmp", ".db.tmp-wal"}; + for (int i = 0; i < UNINSTALL_INDEX_COUNT; i++) { + for (int j = 0; j < UNINSTALL_FILES_PER_INDEX; j++) { + int k = i * UNINSTALL_FILES_PER_INDEX + j; + snprintf(fx->index_paths[k], sizeof(fx->index_paths[k]), "%s/project-%d%s", + fx->cache_dir, i, suffixes[j]); + snprintf(fx->index_bodies[k], sizeof(fx->index_bodies[k]), "index %d file %d %s", i, j, + tag); + write_test_file(fx->index_paths[k], fx->index_bodies[k]); + } + } + snprintf(fx->internal_store, sizeof(fx->internal_store), "%s/_config.db", fx->cache_dir); + write_test_file(fx->internal_store, "internal store sentinel"); + char bin_dir[512]; + snprintf(bin_dir, sizeof(bin_dir), "%s/.local/bin", fx->tmpdir); + test_mkdirp(bin_dir); +#ifdef _WIN32 + snprintf(fx->bin_target, sizeof(fx->bin_target), "%s/codebase-memory-cli.exe", bin_dir); +#else + snprintf(fx->bin_target, sizeof(fx->bin_target), "%s/codebase-memory-cli", bin_dir); +#endif + write_test_file(fx->bin_target, "uninstall fixture binary"); + return cli_uninstall_index_fixture_intact(fx) && + cli_uninstall_test_file_is(fx->internal_store, "internal store sentinel") && + cli_uninstall_test_file_is(fx->bin_target, "uninstall fixture binary"); +} + +static void cli_uninstall_index_fixture_teardown(cli_uninstall_index_fixture_t *fx) { + if (fx->created) { + cli_activation_restore_env(fx->old_home, fx->old_cache); + restore_test_env("PATH", fx->old_path); + test_rmdir_r(fx->tmpdir); + } +} + +static int cli_uninstall_index_fixture_run(char **argv, int argc, int *reservations) { + cli_activation_fake_t fake = {.mutation_reserve_result = 1}; + cbm_cli_activation_ops_t ops = cli_activation_fake_ops(&fake); + cbm_set_auto_answer_for_test(0); + cbm_cli_set_activation_ops_for_test(&ops); + int rc = cli_test_cmd_uninstall(argc, argv); + cbm_cli_set_activation_ops_for_test(NULL); + cbm_set_auto_answer_for_test(0); + if (reservations) { + *reservations = fake.mutation_reserve_count; + } + return rc; +} + +/* These local redirect helpers also exercise the Windows CRT descriptor path; + * unlike the older POSIX capture fixture, neither test is compiled out there. */ +static int cli_uninstall_redirect_stream(FILE *stream, FILE *replacement) { + if (stream != stdin && fflush(stream) != 0) { + return -1; + } +#ifdef _WIN32 + int target = _fileno(stream); + int saved = _dup(target); + if (saved >= 0 && _dup2(_fileno(replacement), target) != 0) { + _close(saved); + return -1; + } +#else + int target = fileno(stream); + int saved = dup(target); + if (saved >= 0 && dup2(fileno(replacement), target) < 0) { + close(saved); + return -1; + } +#endif + clearerr(stream); + return saved; +} + +static bool cli_uninstall_restore_stream(FILE *stream, int saved) { + if (saved < 0) { + return false; + } + bool flushed = stream == stdin || fflush(stream) == 0; +#ifdef _WIN32 + bool restored = _dup2(saved, _fileno(stream)) == 0; + _close(saved); +#else + bool restored = dup2(saved, fileno(stream)) >= 0; + close(saved); +#endif + clearerr(stream); + return flushed && restored; +} + +static bool cli_uninstall_capture_text(FILE *file, char *out, size_t capacity) { + if (fseek(file, 0, SEEK_SET) != 0) { + return false; + } + size_t n = fread(out, 1, capacity - 1, file); + out[n] = '\0'; + return !ferror(file); +} + +TEST(cli_uninstall_auto_answers_keep_indexes_without_delete_indexes) { + char *answers[] = {"--yes", "-y", "--no", "-n"}; + for (size_t i = 0; i < sizeof(answers) / sizeof(answers[0]); i++) { + cli_uninstall_index_fixture_t fx; + if (!cli_uninstall_index_fixture_setup(&fx, "keep")) { + cli_uninstall_index_fixture_teardown(&fx); + FAIL("uninstall fixture setup failed"); + } + char *argv[] = {answers[i]}; + int rc = cli_uninstall_index_fixture_run(argv, 1, NULL); + bool kept = cli_uninstall_index_fixture_intact(&fx); + bool binary_gone = cli_uninstall_test_absent(fx.bin_target); + bool internal_kept = + cli_uninstall_test_file_is(fx.internal_store, "internal store sentinel"); + cli_uninstall_index_fixture_teardown(&fx); + ASSERT_EQ(rc, 0); + ASSERT_TRUE(kept); + ASSERT_TRUE(binary_gone); + ASSERT_TRUE(internal_kept); + } + PASS(); +} + +TEST(cli_uninstall_delete_indexes_is_explicit_and_overrides_no) { + char *argvs[][2] = {{"--delete-indexes", NULL}, + {"--yes", "--delete-indexes"}, + {"--no", "--delete-indexes"}, + {"--delete-indexes", "--no"}}; + for (size_t i = 0; i < sizeof(argvs) / sizeof(argvs[0]); i++) { + cli_uninstall_index_fixture_t fx; + if (!cli_uninstall_index_fixture_setup(&fx, "delete")) { + cli_uninstall_index_fixture_teardown(&fx); + FAIL("uninstall fixture setup failed"); + } + int reservations = 0; + int rc = cli_uninstall_index_fixture_run(argvs[i], i == 0 ? 1 : 2, &reservations); + bool absent = true; + for (int k = 0; k < UNINSTALL_INDEX_COUNT * UNINSTALL_FILES_PER_INDEX; k++) { + absent = cli_uninstall_test_absent(fx.index_paths[k]) && absent; + } + bool binary_gone = cli_uninstall_test_absent(fx.bin_target); + bool internal_kept = + cli_uninstall_test_file_is(fx.internal_store, "internal store sentinel"); + cli_uninstall_index_fixture_teardown(&fx); + ASSERT_EQ(rc, 0); + ASSERT_EQ(reservations, 1); + ASSERT_TRUE(absent); + ASSERT_TRUE(binary_gone); + ASSERT_TRUE(internal_kept); + } + PASS(); +} + +TEST(cli_uninstall_absence_check_preserves_dangling_symlink_entries) { +#ifdef _WIN32 + SKIP_PLATFORM("POSIX symlink fixture; Windows uses non-following Win32 attributes"); +#else + char dir[256]; + snprintf(dir, sizeof(dir), "%s/cbm-uninstall-absence-XXXXXX", cbm_tmpdir()); + ASSERT_NOT_NULL(cbm_mkdtemp(dir)); + char link[512]; + snprintf(link, sizeof(link), "%s/dangling", dir); + bool created = symlink("missing-target", link) == 0; + bool retained = created && !cli_uninstall_test_absent(link); + bool removed = created && cbm_unlink(link) == 0 && cli_uninstall_test_absent(link); + th_rmtree(dir); + ASSERT_TRUE(retained && removed); + PASS(); +#endif +} + +static int cli_uninstall_dry_run_case(bool requested) { + cli_uninstall_index_fixture_t fx; + if (!cli_uninstall_index_fixture_setup(&fx, "dry")) { + cli_uninstall_index_fixture_teardown(&fx); + FAIL("uninstall fixture setup failed"); + } + FILE *output = tmpfile(); + int saved = output ? cli_uninstall_redirect_stream(stdout, output) : -1; + int rc = -1; + int reservations = -1; + if (saved >= 0) { + char *argv[] = {"--dry-run", "--no", "--delete-indexes"}; + rc = cli_uninstall_index_fixture_run(argv, requested ? 3 : 2, &reservations); + } + bool restored = saved >= 0 && cli_uninstall_restore_stream(stdout, saved); + char text[8192] = ""; + bool captured = output && cli_uninstall_capture_text(output, text, sizeof(text)); + if (output) { + fclose(output); + } + bool kept = cli_uninstall_index_fixture_intact(&fx) && + cli_uninstall_test_file_is(fx.bin_target, "uninstall fixture binary") && + cli_uninstall_test_file_is(fx.internal_store, "internal store sentinel"); + bool named = strstr(text, requested ? "indexes would be deleted" : "Indexes kept in") != NULL && + !strstr(text, "Delete these indexes?"); + cli_uninstall_index_fixture_teardown(&fx); + ASSERT_TRUE(restored && captured); + ASSERT_EQ(rc, 0); + ASSERT_EQ(reservations, 0); + ASSERT_TRUE(kept); + ASSERT_TRUE(named); + PASS(); +} + +TEST(cli_uninstall_delete_indexes_dry_run_keeps_files) { + return cli_uninstall_dry_run_case(true); +} + +TEST(cli_uninstall_dry_run_without_delete_indexes_keeps_files) { + return cli_uninstall_dry_run_case(false); +} + +TEST(cli_uninstall_non_tty_affirmative_input_keeps_indexes) { + cli_uninstall_index_fixture_t fx; + if (!cli_uninstall_index_fixture_setup(&fx, "notty")) { + cli_uninstall_index_fixture_teardown(&fx); + FAIL("uninstall fixture setup failed"); + } + FILE *input = tmpfile(); + FILE *output = tmpfile(); + FILE *errors = tmpfile(); + bool input_ready = + input && fputs("y\n", input) >= 0 && fflush(input) == 0 && fseek(input, 0, SEEK_SET) == 0; + int saved_in = input_ready ? cli_uninstall_redirect_stream(stdin, input) : -1; + int saved_out = output ? cli_uninstall_redirect_stream(stdout, output) : -1; + int saved_err = errors ? cli_uninstall_redirect_stream(stderr, errors) : -1; + int rc = -1; + if (saved_in >= 0 && saved_out >= 0 && saved_err >= 0 && fseek(stdin, 0, SEEK_SET) == 0) { + char *argv[] = {"uninstall"}; + rc = cli_uninstall_index_fixture_run(argv, 1, NULL); + } + bool restored_err = saved_err >= 0 && cli_uninstall_restore_stream(stderr, saved_err); + bool restored_out = saved_out >= 0 && cli_uninstall_restore_stream(stdout, saved_out); + bool restored_in = saved_in >= 0 && cli_uninstall_restore_stream(stdin, saved_in); + char out_text[8192] = ""; + char err_text[8192] = ""; + bool captured = output && errors && + cli_uninstall_capture_text(output, out_text, sizeof(out_text)) && + cli_uninstall_capture_text(errors, err_text, sizeof(err_text)); + bool unread = input && fgetc(input) == 'y'; + if (input) { + fclose(input); + } + if (output) { + fclose(output); + } + if (errors) { + fclose(errors); + } + bool kept = cli_uninstall_index_fixture_intact(&fx) && + cli_uninstall_test_file_is(fx.internal_store, "internal store sentinel"); + bool binary_gone = cli_uninstall_test_absent(fx.bin_target); + bool explained = strstr(out_text, "--delete-indexes") && strstr(out_text, fx.cache_dir) && + !strstr(out_text, "Delete these indexes?") && + !strstr(err_text, "requires a terminal"); + cli_uninstall_index_fixture_teardown(&fx); + ASSERT_TRUE(restored_in && restored_out && restored_err && captured); + ASSERT_EQ(rc, 0); + ASSERT_TRUE(unread); + ASSERT_TRUE(kept); + ASSERT_TRUE(binary_gone); + ASSERT_TRUE(explained); + PASS(); +} + +TEST(cli_uninstall_delete_indexes_help_and_invalid_options_preserve_files) { + char *argvs[][3] = {{"--yes", "--delete-indexes", "--help"}, + {"--delete-indexes", "--unknown", NULL}, + {"--delete-indexes", "--dir=", NULL}, + {"--delete-indexes", "--dir", NULL}}; + for (size_t i = 0; i < sizeof(argvs) / sizeof(argvs[0]); i++) { + cli_uninstall_index_fixture_t fx; + if (!cli_uninstall_index_fixture_setup(&fx, "early")) { + cli_uninstall_index_fixture_teardown(&fx); + FAIL("uninstall fixture setup failed"); + } + int reservations = -1; + int rc = cli_uninstall_index_fixture_run(argvs[i], i == 0 ? 3 : 2, &reservations); + bool kept = cli_uninstall_index_fixture_intact(&fx) && + cli_uninstall_test_file_is(fx.bin_target, "uninstall fixture binary") && + cli_uninstall_test_file_is(fx.internal_store, "internal store sentinel"); + cli_uninstall_index_fixture_teardown(&fx); + ASSERT_EQ(rc, i == 0 ? 0 : 1); + ASSERT_EQ(reservations, 0); + ASSERT_TRUE(kept); + } + PASS(); +} + #ifndef _WIN32 /* #1954: an agent config that cannot be cleaned must not hold the executable * and the indexes hostage. The fixture's ~/.cursor/mcp.json is a dangling @@ -3332,8 +3697,8 @@ TEST(cli_uninstall_removes_binary_and_index_when_agent_config_cleanup_fails) { cli_fd_capture_t err_capture; cli_fd_capture_begin(&out_capture, stdout, STDOUT_FILENO); cli_fd_capture_begin(&err_capture, stderr, STDERR_FILENO); - char *argv[] = {"--yes"}; - int rc = cli_test_cmd_uninstall(1, argv); + char *argv[] = {"--yes", "--delete-indexes"}; + int rc = cli_test_cmd_uninstall(2, argv); char *err_text = cli_fd_capture_end(&err_capture); char *out_text = cli_fd_capture_end(&out_capture); cbm_cli_set_activation_ops_for_test(NULL); @@ -3419,8 +3784,8 @@ TEST(cli_uninstall_cleans_user_owned_symlinked_config) { cli_fd_capture_t err_capture; cli_fd_capture_begin(&out_capture, stdout, STDOUT_FILENO); cli_fd_capture_begin(&err_capture, stderr, STDERR_FILENO); - char *argv[] = {"--yes"}; - int rc = cli_test_cmd_uninstall(1, argv); + char *argv[] = {"--yes", "--delete-indexes"}; + int rc = cli_test_cmd_uninstall(2, argv); char *err_text = cli_fd_capture_end(&err_capture); char *out_text = cli_fd_capture_end(&out_capture); cbm_cli_set_activation_ops_for_test(NULL); @@ -5251,6 +5616,11 @@ TEST(cli_detect_agents_finds_claude) { char dir[512]; snprintf(dir, sizeof(dir), "%s/.claude", tmpdir); test_mkdirp(dir); + /* #1180: a bare directory is not evidence; the settings file Claude Code + * reads is. */ + char settings[640]; + snprintf(settings, sizeof(settings), "%s/settings.json", dir); + write_test_file(settings, "{}\n"); /* Unset CLAUDE_CONFIG_DIR so detection is exercised against home_dir/.claude * and the runner's real env (which may set it) does not leak in. */ @@ -5280,6 +5650,10 @@ TEST(cli_detect_agents_finds_claude_via_env) { char ccd[512]; snprintf(ccd, sizeof(ccd), "%s/custom-claude", tmpdir); test_mkdirp(ccd); + /* #1180: Claude Code keeps .claude.json inside CLAUDE_CONFIG_DIR. */ + char user_config[640]; + snprintf(user_config, sizeof(user_config), "%s/.claude.json", ccd); + write_test_file(user_config, "{}\n"); const char *saved_ccd = getenv("CLAUDE_CONFIG_DIR"); char *saved_ccd_copy = saved_ccd ? strdup(saved_ccd) : NULL; @@ -5300,6 +5674,255 @@ TEST(cli_detect_agents_finds_claude_via_env) { PASS(); } +/* #1167/#1180: detection runs against a synthetic home with every env var that + * can redirect Claude/OpenCode/Hermes lookups cleared and PATH reduced to one + * empty directory, so the host's own installs never leak into the verdict. */ +typedef struct { + char *path; + char *claude_config_dir; + char *opencode_config; + char *opencode_config_dir; + char *hermes_home; +} client_detect_env_t; + +static client_detect_env_t client_detect_env_isolate(const char *path_dir) { + client_detect_env_t saved = { + save_test_env("PATH"), save_test_env("CLAUDE_CONFIG_DIR"), save_test_env("OPENCODE_CONFIG"), + save_test_env("OPENCODE_CONFIG_DIR"), save_test_env("HERMES_HOME")}; + cbm_setenv("PATH", path_dir, 1); + cbm_unsetenv("CLAUDE_CONFIG_DIR"); + cbm_unsetenv("OPENCODE_CONFIG"); + cbm_unsetenv("OPENCODE_CONFIG_DIR"); + cbm_unsetenv("HERMES_HOME"); + return saved; +} + +static void client_detect_env_restore(client_detect_env_t *saved) { + restore_test_env("PATH", saved->path); + restore_test_env("CLAUDE_CONFIG_DIR", saved->claude_config_dir); + restore_test_env("OPENCODE_CONFIG", saved->opencode_config); + restore_test_env("OPENCODE_CONFIG_DIR", saved->opencode_config_dir); + restore_test_env("HERMES_HOME", saved->hermes_home); +} + +/* Fresh home plus an empty bin dir used as the whole PATH. */ +static bool client_detect_fixture(char *home, size_t home_sz, char *bin, size_t bin_sz) { + snprintf(home, home_sz, "/tmp/cli-client-detect-XXXXXX"); + if (!cbm_mkdtemp(home)) { + return false; + } + snprintf(bin, bin_sz, "%s/bin", home); + test_mkdirp(bin); + return true; +} + +/* #1180: a bare ~/.claude directory no longer counts as Claude Code. Fixtures + * that model a Claude Code user add the user config Claude Code itself writes + * (.claude.json in the home, or in CLAUDE_CONFIG_DIR when that is set). */ +static void test_mark_claude_code_installed(const char *user_root) { + char path[768]; + snprintf(path, sizeof(path), "%s/.claude.json", user_root); + write_test_file(path, "{}\n"); +} + +/* #1180: a user with only an (empty) ~/.claude folder and no Claude Code CLI + * was reported as "Detected agents: Claude-Code" and got hooks, skills and + * MCP entries written for a client they never installed. */ +TEST(cli_detect_claude_empty_dir_not_detected_issue1180) { + char home[256]; + char bin[320]; + if (!client_detect_fixture(home, sizeof(home), bin, sizeof(bin))) + FAIL("cbm_mkdtemp failed"); + char dir[512]; + snprintf(dir, sizeof(dir), "%s/.claude", home); + test_mkdirp(dir); + + client_detect_env_t saved = client_detect_env_isolate(bin); + cbm_detected_agents_t agents = cbm_detect_agents(home); + client_detect_env_restore(&saved); + test_rmdir_r(home); + if (agents.claude_code) + FAIL("an empty ~/.claude directory must not count as an installed Claude Code"); + PASS(); +} + +TEST(cli_detect_claude_empty_config_dir_env_not_detected_issue1180) { + char home[256]; + char bin[320]; + if (!client_detect_fixture(home, sizeof(home), bin, sizeof(bin))) + FAIL("cbm_mkdtemp failed"); + char ccd[512]; + snprintf(ccd, sizeof(ccd), "%s/custom-claude", home); + test_mkdirp(ccd); + + client_detect_env_t saved = client_detect_env_isolate(bin); + cbm_setenv("CLAUDE_CONFIG_DIR", ccd, 1); + cbm_detected_agents_t agents = cbm_detect_agents(home); + client_detect_env_restore(&saved); + test_rmdir_r(home); + if (agents.claude_code) + FAIL("an empty CLAUDE_CONFIG_DIR must not count as an installed Claude Code"); + PASS(); +} + +/* The evidence that DOES mean Claude Code is installed, one at a time: its + * settings file, its user config ~/.claude.json, or the claude CLI. */ +TEST(cli_detect_claude_real_install_detected_issue1180) { + enum { EVIDENCE_SETTINGS, EVIDENCE_USER_CONFIG, EVIDENCE_CLI, EVIDENCE_COUNT }; + static const char *const names[EVIDENCE_COUNT] = {"settings.json", ".claude.json", + "claude CLI"}; + for (int kind = 0; kind < EVIDENCE_COUNT; kind++) { + char home[256]; + char bin[320]; + if (!client_detect_fixture(home, sizeof(home), bin, sizeof(bin))) + FAIL("cbm_mkdtemp failed"); + char path[640]; + if (kind == EVIDENCE_SETTINGS) { + snprintf(path, sizeof(path), "%s/.claude", home); + test_mkdirp(path); + snprintf(path, sizeof(path), "%s/.claude/settings.json", home); + write_test_file(path, "{}\n"); + } else if (kind == EVIDENCE_USER_CONFIG) { + snprintf(path, sizeof(path), "%s/.claude.json", home); + write_test_file(path, "{}\n"); + } else { + snprintf(path, sizeof(path), "%s/claude", bin); + write_test_file(path, "#!/bin/sh\nexit 0\n"); + chmod(path, 0755); + } + client_detect_env_t saved = client_detect_env_isolate(bin); + cbm_detected_agents_t agents = cbm_detect_agents(home); + client_detect_env_restore(&saved); + test_rmdir_r(home); + if (!agents.claude_code) { + printf(" evidence: %s\n", names[kind]); + FAIL("real Claude Code evidence must be detected"); + } + } + PASS(); +} + +/* #1180: OpenCode keeps its data in ~/.local/share/opencode on every OS + * (opencode.ai/docs/troubleshooting, "Storage"; Windows: + * %USERPROFILE%\.local\share\opencode), and the Desktop app's bundled + * opencode-cli server creates it too. With no config file, no + * ~/.config/opencode and no opencode on PATH it is the remaining evidence. */ +TEST(cli_detect_opencode_desktop_storage_issue1180) { + char home[256]; + char bin[320]; + if (!client_detect_fixture(home, sizeof(home), bin, sizeof(bin))) + FAIL("cbm_mkdtemp failed"); + char dir[512]; + snprintf(dir, sizeof(dir), "%s/.local/share/opencode", home); + test_mkdirp(dir); + + client_detect_env_t saved = client_detect_env_isolate(bin); + cbm_detected_agents_t agents = cbm_detect_agents(home); + client_detect_env_restore(&saved); + test_rmdir_r(home); + if (!agents.opencode) + FAIL("OpenCode's data dir ~/.local/share/opencode must detect OpenCode"); + PASS(); +} + +/* #1180: native Windows Hermes (install.ps1 CLI, MSIX package, Desktop app) + * defaults HERMES_HOME to %LOCALAPPDATA%\hermes and reads a legacy ~/.hermes + * only while that directory is absent (hermes-agent windows-native.md "Data + * layout"; apps/desktop/electron/data-paths.mjs). cbm probed and wrote + * ~/.hermes only. Elsewhere the home stays ~/.hermes. */ +TEST(cli_hermes_home_windows_localappdata_issue1180) { + char home[256]; + char bin[320]; + if (!client_detect_fixture(home, sizeof(home), bin, sizeof(bin))) + FAIL("cbm_mkdtemp failed"); + char native[512]; + char legacy[512]; + snprintf(native, sizeof(native), "%s/AppData/Local/hermes", home); + snprintf(legacy, sizeof(legacy), "%s/.hermes", home); + + client_detect_env_t saved = client_detect_env_isolate(bin); + char fresh_windows[512]; + char fresh_posix[512]; + char legacy_windows[512]; + char both_windows[512]; + char both_posix[512]; + cbm_hermes_home_dir_for_testing(home, true, fresh_windows, sizeof(fresh_windows)); + cbm_hermes_home_dir_for_testing(home, false, fresh_posix, sizeof(fresh_posix)); + test_mkdirp(legacy); + cbm_hermes_home_dir_for_testing(home, true, legacy_windows, sizeof(legacy_windows)); + test_mkdirp(native); + cbm_hermes_home_dir_for_testing(home, true, both_windows, sizeof(both_windows)); + cbm_hermes_home_dir_for_testing(home, false, both_posix, sizeof(both_posix)); + client_detect_env_restore(&saved); + test_rmdir_r(home); + + bool ok = strcmp(fresh_windows, native) == 0 && strcmp(fresh_posix, legacy) == 0 && + strcmp(legacy_windows, legacy) == 0 && strcmp(both_windows, native) == 0 && + strcmp(both_posix, legacy) == 0; + if (!ok) { + printf(" fresh: windows=%s posix=%s\n legacy-only windows=%s\n" + " both: windows=%s posix=%s\n", + fresh_windows, fresh_posix, legacy_windows, both_windows, both_posix); + FAIL("Windows Hermes home must be AppData/Local/hermes unless only ~/.hermes exists"); + } + PASS(); +} + +/* #1180 end to end on the compiled-in platform: a Hermes Desktop home in + * AppData/Local/hermes is detected and receives config.yaml + the skill on + * Windows, and is not evidence anywhere else. */ +TEST(cli_detect_hermes_desktop_home_issue1180) { + char home[256]; + char bin[320]; + if (!client_detect_fixture(home, sizeof(home), bin, sizeof(bin))) + FAIL("cbm_mkdtemp failed"); + char dir[512]; + snprintf(dir, sizeof(dir), "%s/AppData/Local/hermes", home); + test_mkdirp(dir); + + client_detect_env_t saved = client_detect_env_isolate(bin); + cbm_detected_agents_t agents = cbm_detect_agents(home); + char *json = cbm_build_install_plan_json(home, "/usr/local/bin/codebase-memory-mcp"); + client_detect_env_restore(&saved); +#ifdef _WIN32 + bool ok = agents.hermes && json && strstr(json, "/AppData/Local/hermes/config.yaml") && + strstr(json, "/AppData/Local/hermes/skills/codebase-memory/SKILL.md") && + !strstr(json, "/.hermes/config.yaml"); +#else + bool ok = !agents.hermes && json && !strstr(json, "/AppData/Local/hermes/"); +#endif + free(json); + test_rmdir_r(home); + if (!ok) + FAIL("the Hermes home must follow the platform's documented default"); + PASS(); +} + +/* Control: look-alike neighbours of every new probe must not detect anything. */ +TEST(cli_detect_client_non_install_control_issue1167_1180) { + char home[256]; + char bin[320]; + if (!client_detect_fixture(home, sizeof(home), bin, sizeof(bin))) + FAIL("cbm_mkdtemp failed"); + char path[640]; + snprintf(path, sizeof(path), "%s/.local/share/opencode-other", home); + test_mkdirp(path); + snprintf(path, sizeof(path), "%s/.claude-other", home); + test_mkdirp(path); + snprintf(path, sizeof(path), "%s/claude.json", home); + write_test_file(path, "{}\n"); + snprintf(path, sizeof(path), "%s/AppData/Local/hermes-other", home); + test_mkdirp(path); + + client_detect_env_t saved = client_detect_env_isolate(bin); + cbm_detected_agents_t agents = cbm_detect_agents(home); + client_detect_env_restore(&saved); + test_rmdir_r(home); + if (agents.claude_code || agents.opencode || agents.hermes) + FAIL("unrelated files next to the probed locations must not detect a client"); + PASS(); +} + TEST(cli_detect_agents_finds_codex) { char tmpdir[256]; snprintf(tmpdir, sizeof(tmpdir), "/tmp/cli-detect-XXXXXX"); @@ -6326,6 +6949,7 @@ TEST(cli_dry_run_predicts_refused_hook_script_issue1387) { snprintf(hooks_dir, sizeof(hooks_dir), "%s/.claude/hooks", tmpdir); if (!cbm_mkdir_p(hooks_dir, 0755)) FAIL("mkdir hooks_dir failed"); + test_mark_claude_code_installed(tmpdir); /* A gate script that is NOT ours: a manual install pointing at another * binary. The real install refuses to rewrite it (TEXT_UNOWNED). */ char gate_path[768]; @@ -7212,6 +7836,7 @@ TEST(cli_devin_does_not_duplicate_owned_claude_session_start) { snprintf(claude_settings, sizeof(claude_settings), "%s/settings.json", claude_dir); snprintf(devin_config, sizeof(devin_config), "%s/config.json", devin_dir); test_mkdirp(claude_dir); + test_mark_claude_code_installed(tmpdir); test_mkdirp(devin_dir); write_test_file(devin_config, "{}\n"); @@ -8127,7 +8752,12 @@ TEST(cli_fresh_cli_only_yaml_and_toml_agents_create_parent_dirs) { cbm_install_agent_hooks_for_testing(tmpdir, "/usr/local/bin/codebase-memory-cli", false, false); char path[768]; +#ifdef _WIN32 + /* #1180: native Windows Hermes defaults to %LOCALAPPDATA%\hermes. */ + snprintf(path, sizeof(path), "%s/AppData/Local/hermes/config.yaml", tmpdir); +#else snprintf(path, sizeof(path), "%s/.hermes/config.yaml", tmpdir); +#endif bool installed = test_file_contains_all( path, (const char *const[]){"pre_llm_call:", "hook-augment", "--dialect hermes"}, 3); snprintf(path, sizeof(path), "%s/.config/goose/.goosehints", tmpdir); @@ -8795,6 +9425,7 @@ TEST(cli_uninstall_preserves_hook_script_with_modified_binary) { claude_dir); #endif test_mkdirp(claude_dir); + test_mark_claude_code_installed(tmpdir); char *saved_home = save_test_env("HOME"); char *saved_path = save_test_env("PATH"); @@ -9145,6 +9776,7 @@ TEST(cli_claude_lifecycle_hooks_delegate_to_augmenter) { char config_dir[512]; snprintf(config_dir, sizeof(config_dir), "%s/.claude", tmpdir); test_mkdirp(config_dir); + test_mark_claude_code_installed(tmpdir); char *saved_path = save_test_env("PATH"); char *saved_claude = save_test_env("CLAUDE_CONFIG_DIR"); @@ -9609,6 +10241,7 @@ TEST(cli_claude_hook_scripts_shell_quote_binary_path) { char config_dir[512]; snprintf(config_dir, sizeof(config_dir), "%s/.claude", tmpdir); test_mkdirp(config_dir); + test_mark_claude_code_installed(tmpdir); char copilot_dir[512]; snprintf(copilot_dir, sizeof(copilot_dir), "%s/.copilot", tmpdir); test_mkdirp(copilot_dir); @@ -9682,6 +10315,7 @@ TEST(cli_claude_hook_commands_use_exec_form_with_custom_config_dir) { snprintf(config_dir, sizeof(config_dir), "%s/custom claude;$(touch cbm-hook-path-pwned)", tmpdir); test_mkdirp(config_dir); + test_mark_claude_code_installed(config_dir); char *saved_path = save_test_env("PATH"); char *saved_claude = save_test_env("CLAUDE_CONFIG_DIR"); cbm_setenv("PATH", tmpdir, 1); @@ -10719,6 +11353,7 @@ TEST(cli_upgrade_migrates_released_claude_hook_scripts) { snprintf(subagent_path, sizeof(subagent_path), "%s/cbm-subagent-reminder", hooks_dir); snprintf(settings_path, sizeof(settings_path), "%s/.claude/settings.json", tmpdir); test_mkdirp(hooks_dir); + test_mark_claude_code_installed(tmpdir); char legacy_gate[8192]; ASSERT_TRUE(test_build_released_gate_hook_script("/opt/codebase-memory-mcp", legacy_gate, @@ -10775,6 +11410,7 @@ TEST(cli_upgrade_preserves_near_legacy_claude_hook_script) { snprintf(gate_path, sizeof(gate_path), "%s/cbm-code-discovery-gate", hooks_dir); snprintf(settings_path, sizeof(settings_path), "%s/.claude/settings.json", tmpdir); test_mkdirp(hooks_dir); + test_mark_claude_code_installed(tmpdir); const char *modified_legacy = "#!/usr/bin/env bash\n" "# codebase-memory-mcp search augmenter (Claude Code PreToolUse).\n" @@ -10866,6 +11502,7 @@ TEST(cli_claude_hook_script_collisions_are_not_registered) { snprintf(session, sizeof(session), "%s/cbm-session-reminder", hooks_dir); snprintf(settings, sizeof(settings), "%s/.claude/settings.json", tmpdir); test_mkdirp(hooks_dir); + test_mark_claude_code_installed(tmpdir); write_test_file(victim, "victim-owned\n"); ASSERT_EQ(symlink(victim, gate), 0); write_test_file(session, "#!/bin/sh\necho user-owned\n"); @@ -10908,6 +11545,7 @@ TEST(cli_uninstall_removes_claude_hook_scripts) { char config_dir[512]; snprintf(config_dir, sizeof(config_dir), "%s/.claude", tmpdir); test_mkdirp(config_dir); + test_mark_claude_code_installed(tmpdir); char *saved_home = save_test_env("HOME"); char *saved_path = save_test_env("PATH"); @@ -10978,6 +11616,7 @@ TEST(cli_uninstall_preserves_modified_claude_hook_script) { char config_dir[512]; snprintf(config_dir, sizeof(config_dir), "%s/.claude", tmpdir); test_mkdirp(config_dir); + test_mark_claude_code_installed(tmpdir); char *saved_home = save_test_env("HOME"); char *saved_path = save_test_env("PATH"); @@ -11416,6 +12055,33 @@ TEST(cli_agent_instructions_content) { PASS(); } +/* #1690: an agent in a checkout whose directory name differs from the repo + * name guessed the project from the repo name, got "project not found" and + * dropped the graph. Every installed guidance variant must say how to pick the + * project: the list_projects entry whose root_path contains the working + * directory, or that absolute directory itself, never a repo or folder name. */ +static int i1690_names_project_by_root(const char *text) { + return text && strstr(text, "root_path") != NULL && + strstr(text, "contains your working directory") != NULL && + strstr(text, "absolute directory") != NULL && + strstr(text, "never a repo or folder name") != NULL; +} + +TEST(cli_guidance_resolves_project_by_root_path_issue1690) { + const cbm_skill_t *skills = cbm_get_skills(); + ASSERT_NOT_NULL(skills); + for (int i = 0; i < CBM_SKILL_COUNT; i++) { + if (!i1690_names_project_by_root(skills[i].content)) { + fprintf(stderr, " [1690] skill %s does not say how to pick the project\n", + skills[i].name); + } + ASSERT(i1690_names_project_by_root(skills[i].content)); + } + ASSERT(i1690_names_project_by_root(cbm_get_agent_instructions())); + ASSERT(i1690_names_project_by_root(cbm_get_aider_instructions())); + PASS(); +} + TEST(cli_qwen_windows_hook_command_uses_powershell_schema) { char command[1024]; char shell[32]; @@ -11802,6 +12468,7 @@ TEST(cli_windows_claude_hook_scripts_migrate_and_uninstall_all_owned_shapes) { snprintf(appdata, sizeof(appdata), "%s/AppData/Roaming", tmpdir); snprintf(binary_path, sizeof(binary_path), "%s/.local/bin/codebase-memory-cli.exe", tmpdir); test_mkdirp(hooks_dir); + test_mark_claude_code_installed(config_dir); const char *const env_names[] = {"HOME", "PATH", "CLAUDE_CONFIG_DIR", "APPDATA", "CODEX_HOME", "OPENCODE_CONFIG", @@ -12182,6 +12849,260 @@ TEST(cli_hook_augment_deadline_breadcrumb_issue858) { #endif } +/* ── hook-augment breadcrumb log location ────────────────────────── + * Arming the deadline opens the breadcrumb log with O_CREAT. Its path was + * "$HOME/.cache/codebase-memory-mcp/logs/hook-augment-timeouts.log" built in + * 1 KiB buffers with no length check, so a long HOME created a file at a + * cut-off name, and CBM_CACHE_DIR, which every other cbm log honours, was + * ignored. These run the real entry point in a child with a no-op Bash + * payload: the deadline is armed (the log opened), then the hook passes + * through with exit 0. POSIX only: the in-process deadline and its log are + * compiled out on Windows. */ +#ifndef _WIN32 +enum { HL_PATH_CAP = 1024, HL_SEGMENT = 200 }; + +/* Exit code of `hook-augment` run in a child with HOME = home and + * CBM_CACHE_DIR = cache_dir (unset when NULL), or -1 when it did not exit. */ +static int hl_run_hook(const char *home, const char *cache_dir) { + static const char payload[] = "{\"hook_event_name\":\"PreToolUse\",\"tool_name\":\"Bash\"," + "\"tool_input\":{\"command\":\"ls\"}}"; + int fds[2]; + if (pipe(fds) != 0) { + return -1; + } + /* Queued before the fork, so no write can meet a reader that has gone. */ + ssize_t queued = write(fds[1], payload, sizeof(payload) - 1); + close(fds[1]); + if (queued != (ssize_t)(sizeof(payload) - 1)) { + close(fds[0]); + return -1; + } + fflush(NULL); + pid_t pid = fork(); + if (pid == 0) { + dup2(fds[0], STDIN_FILENO); + close(fds[0]); + cbm_setenv("HOME", home, 1); + cbm_unsetenv("USERPROFILE"); + cbm_unsetenv("CBM_HOOK_TIMEOUT_LOG"); + if (cache_dir) { + cbm_setenv("CBM_CACHE_DIR", cache_dir, 1); + } else { + cbm_unsetenv("CBM_CACHE_DIR"); + } + _exit(cbm_cmd_hook_augment(0, NULL)); + } + close(fds[0]); + int status = 0; + if (pid < 0 || waitpid(pid, &status, 0) != pid || !WIFEXITED(status)) { + return -1; + } + return WEXITSTATUS(status); +} + +/* A fresh synthetic root with its links resolved. The kernel bounds the + * EXPANDED path, and macOS /tmp is a link to /private/tmp: under the link a + * near-1 KiB path can be neither created nor probed, which hides the very + * cut-off file these tests look for. */ +static bool hl_make_root(char *root, size_t root_sz) { + char tmpl[256]; + char real[4096]; + snprintf(tmpl, sizeof(tmpl), "/tmp/cbm-hooklog-XXXXXX"); + if (!cbm_mkdtemp(tmpl)) { + return false; + } + if (!cbm_canonical_path(tmpl, real, sizeof(real))) { + th_rmtree(tmpl); + return false; + } + int n = snprintf(root, root_sz, "%s", real); + return n > 0 && (size_t)n < root_sz; +} + +/* "/hhh…/hhh…" of exactly `len` bytes, every component short of + * NAME_MAX. False when it cannot be built. */ +static bool hl_long_dir(char *out, size_t out_sz, const char *root, size_t len) { + int n = snprintf(out, out_sz, "%s", root); + if (n <= 0 || (size_t)n >= len || len >= out_sz) { + return false; + } + size_t at = (size_t)n; + while (at < len) { + out[at++] = '/'; + for (int seg = 0; seg < HL_SEGMENT && at < len; seg++) { + out[at++] = 'h'; + } + } + out[at] = '\0'; + return out[len - 1] != '/'; +} + +/* The first HL_PATH_CAP - 1 bytes of "/": the name the old + * unchecked 1 KiB buffer cut that path down to. */ +static void hl_cut_name(char *out, const char *base, const char *suffix) { + char full[2 * HL_PATH_CAP]; + snprintf(full, sizeof(full), "%s/%s", base, suffix); + snprintf(out, HL_PATH_CAP, "%.*s", HL_PATH_CAP - 1, full); +} + +static bool hl_is_ancestor_or_self(const char *path, const char *of) { + size_t n = strlen(path); + return of && strncmp(of, path, n) == 0 && (of[n] == '/' || of[n] == '\0'); +} + +/* Entries under dir that are neither keep_a, keep_b nor one of their + * ancestors: whatever the hook created. Each is printed by its tail. */ +static int hl_count_strays(const char *dir, const char *keep_a, const char *keep_b) { + cbm_dir_t *d = cbm_opendir(dir); + if (!d) { + return 0; + } + int strays = 0; + cbm_dirent_t *entry; + while ((entry = cbm_readdir(d)) != NULL) { + if (strcmp(entry->name, ".") == 0 || strcmp(entry->name, "..") == 0) { + continue; + } + char child[HL_PATH_CAP]; + int n = snprintf(child, sizeof(child), "%s/%s", dir, entry->name); + bool named = n > 0 && (size_t)n < sizeof(child); + if (named && + (hl_is_ancestor_or_self(child, keep_a) || hl_is_ancestor_or_self(child, keep_b))) { + if (entry->is_dir) { + strays += hl_count_strays(child, keep_a, keep_b); + } + continue; + } + size_t len = named ? (size_t)n : strlen(child); + printf(" stray entry: ...%s\n", child + (len > 60 ? len - 60 : 0)); + strays++; + } + cbm_closedir(d); + return strays; +} +#endif + +/* A HOME of ~1 KiB: the log path "/.cache/codebase-memory-cli/logs/ + * hook-augment-timeouts.log" does not fit the buffer. The old code opened its + * cut-off prefix ".../logs/hook-augme" with O_CREAT. Nothing may be created at + * a cut path (nothing at all under the synthetic root), and the hook must + * still exit 0: it never fails the host agent's hook call. */ +TEST(cli_hook_augment_log_long_home_creates_no_cut_path) { +#ifdef _WIN32 + SKIP_PLATFORM("in-process deadline breadcrumb log is POSIX-only"); +#else + enum { HL_HOME_LEN = 980 }; /* "/.cache/.../logs" fits 1 KiB, the file name does not */ + char root[256]; + if (!hl_make_root(root, sizeof(root))) { + FAIL("synthetic root failed"); + } + char home[HL_PATH_CAP]; + bool built = hl_long_dir(home, sizeof(home), root, HL_HOME_LEN) && th_mkdir_p(home) == 0; + char cut[HL_PATH_CAP]; + hl_cut_name(cut, home, ".cache/codebase-memory-cli/logs/hook-augment-timeouts.log"); + + int rc = built ? hl_run_hook(home, NULL) : -1; + bool cut_exists = cbm_file_exists(cut); + int strays = hl_count_strays(root, home, NULL); + test_rmdir_r(root); + + ASSERT_TRUE(built); + ASSERT_EQ(rc, 0); + if (cut_exists) { + printf(" log opened at the cut-off name ...%s\n", cut + strlen(cut) - 40); + } + ASSERT_FALSE(cut_exists); + ASSERT_EQ(strays, 0); + PASS(); +#endif +} + +/* CBM_CACHE_DIR is where every cbm log goes (/logs), so the hook's + * breadcrumb goes there too, and not under HOME. A CBM_CACHE_DIR of ~1 KiB + * whose log path does not fit is no log at all: no cut-off file, no stray + * directory, and the hook still exits 0. */ +TEST(cli_hook_augment_log_honours_cache_dir) { +#ifdef _WIN32 + SKIP_PLATFORM("in-process deadline breadcrumb log is POSIX-only"); +#else + enum { HL_CACHE_LEN = 1000 }; /* "/logs" fits 1 KiB, the file name does not */ + char root[256]; + if (!hl_make_root(root, sizeof(root))) { + FAIL("synthetic root failed"); + } + char home[512]; + char cache[512]; + char expected[640]; + snprintf(home, sizeof(home), "%s/home", root); + snprintf(cache, sizeof(cache), "%s/cache", root); + snprintf(expected, sizeof(expected), "%s/logs/hook-augment-timeouts.log", cache); + bool made = th_mkdir_p(home) == 0 && th_mkdir_p(cache) == 0; + + /* A configured cache dir holds the log; HOME is left alone. */ + int rc = made ? hl_run_hook(home, cache) : -1; + bool in_cache = cbm_file_exists(expected); + int strays = hl_count_strays(root, home, expected); + + /* A configured cache dir too long for the log path: no log anywhere. */ + char long_cache[HL_PATH_CAP]; + char cut[HL_PATH_CAP]; + bool long_built = hl_long_dir(long_cache, sizeof(long_cache), root, HL_CACHE_LEN) && + th_mkdir_p(long_cache) == 0; + hl_cut_name(cut, long_cache, "logs/hook-augment-timeouts.log"); + bool reset = th_rmtree(home) == 0 && th_rmtree(cache) == 0 && th_mkdir_p(home) == 0; + int long_rc = long_built && reset ? hl_run_hook(home, long_cache) : -1; + bool cut_exists = cbm_file_exists(cut); + int long_strays = hl_count_strays(root, home, long_cache); + test_rmdir_r(root); + + ASSERT_TRUE(made); + ASSERT_EQ(rc, 0); + if (!in_cache) { + printf(" no breadcrumb log under CBM_CACHE_DIR/logs\n"); + } + ASSERT_TRUE(in_cache); + ASSERT_EQ(strays, 0); + ASSERT_TRUE(long_built); + ASSERT_TRUE(reset); + ASSERT_EQ(long_rc, 0); + ASSERT_FALSE(cut_exists); + ASSERT_EQ(long_strays, 0); + PASS(); +#endif +} + +/* Positive control: with a normal HOME and no CBM_CACHE_DIR the breadcrumb log + * stays exactly where it always was, + * "$HOME/.cache/codebase-memory-cli/logs/hook-augment-timeouts.log", and the + * hook creates nothing else. */ +TEST(cli_hook_augment_log_default_location_unchanged) { +#ifdef _WIN32 + SKIP_PLATFORM("in-process deadline breadcrumb log is POSIX-only"); +#else + char root[256]; + if (!hl_make_root(root, sizeof(root))) { + FAIL("synthetic root failed"); + } + char home[512]; + char expected[640]; + snprintf(home, sizeof(home), "%s/home", root); + snprintf(expected, sizeof(expected), + "%s/.cache/codebase-memory-cli/logs/hook-augment-timeouts.log", home); + bool made = th_mkdir_p(home) == 0; + + int rc = made ? hl_run_hook(home, NULL) : -1; + bool at_default = cbm_file_exists(expected); + int strays = hl_count_strays(root, expected, NULL); + test_rmdir_r(root); + + ASSERT_TRUE(made); + ASSERT_EQ(rc, 0); + ASSERT_TRUE(at_default); + ASSERT_EQ(strays, 0); + PASS(); +#endif +} + TEST(cli_upsert_claude_hook_existing) { char tmpdir[256]; snprintf(tmpdir, sizeof(tmpdir), "/tmp/cli-hook-XXXXXX"); @@ -12837,6 +13758,40 @@ TEST(cli_config_watcher_enabled_default_and_persist) { PASS(); } +/* #1948: non-git roots are polled only when `watch_non_git` opts in. The key + * defaults OFF (behaviour unchanged, also for a NULL config) and is listed in + * the config-key table so `config list` / `config set` can find it. */ +TEST(cli_config_watch_non_git_default_off_and_listed_issue1948) { + char tmpdir[256]; + snprintf(tmpdir, sizeof(tmpdir), "/tmp/cli-cfg-ng-XXXXXX"); + if (!cbm_mkdtemp(tmpdir)) + FAIL("cbm_mkdtemp failed"); + + ASSERT_FALSE(cbm_config_watch_non_git(NULL)); + cbm_config_t *cfg = cbm_config_open(tmpdir); + ASSERT_NOT_NULL(cfg); + ASSERT_FALSE(cbm_config_watch_non_git(cfg)); + cbm_config_set(cfg, CBM_CONFIG_WATCH_NON_GIT, "true"); + ASSERT_TRUE(cbm_config_watch_non_git(cfg)); + cbm_config_close(cfg); + cfg = cbm_config_open(tmpdir); + ASSERT_NOT_NULL(cfg); + ASSERT_TRUE(cbm_config_watch_non_git(cfg)); + cbm_config_set(cfg, CBM_CONFIG_WATCH_NON_GIT, "false"); + ASSERT_FALSE(cbm_config_watch_non_git(cfg)); + cbm_config_close(cfg); + + bool listed = false; + for (size_t i = 0; i < cbm_cli_config_key_count_for_testing(); i++) { + const char *key = cbm_cli_config_key_at_for_testing(i); + listed = listed || (key && strcmp(key, CBM_CONFIG_WATCH_NON_GIT) == 0); + } + ASSERT_TRUE(listed); + + test_rmdir_r(tmpdir); + PASS(); +} + /* ═══════════════════════════════════════════════════════════════════ * Group H: cbm_replace_binary (update command helper) * ═══════════════════════════════════════════════════════════════════ */ @@ -13831,6 +14786,12 @@ TEST(cli_cp78_install_and_hook_plans_are_disjoint) { char claude_dir[512]; snprintf(claude_dir, sizeof(claude_dir), "%s/.claude", tmpdir); test_mkdirp(claude_dir); + /* #1180: a bare ~/.claude no longer counts as a Claude Code install, so + * give the fixture the settings.json Claude Code leaves. Without it the + * result depends on whether a `claude` CLI is on the host PATH. */ + char claude_settings[640]; + snprintf(claude_settings, sizeof(claude_settings), "%s/settings.json", claude_dir); + write_test_file(claude_settings, "{}\n"); char *assets = cbm_build_install_plan_json(tmpdir, "/opt/codebase-memory-cli"); char *hooks = cbm_build_hook_install_plan_json(tmpdir, "/opt/codebase-memory-cli"); @@ -14030,6 +14991,13 @@ SUITE(cli) { RUN_TEST(cli_update_agent_configs_finish_before_guard_release); RUN_TEST(cli_uninstall_quiesces_active_cohort_before_removing_binary_and_index); RUN_TEST(cli_uninstall_preserves_binary_and_index_when_cohort_does_not_drain); + RUN_TEST(cli_uninstall_auto_answers_keep_indexes_without_delete_indexes); + RUN_TEST(cli_uninstall_delete_indexes_is_explicit_and_overrides_no); + RUN_TEST(cli_uninstall_absence_check_preserves_dangling_symlink_entries); + RUN_TEST(cli_uninstall_delete_indexes_dry_run_keeps_files); + RUN_TEST(cli_uninstall_dry_run_without_delete_indexes_keeps_files); + RUN_TEST(cli_uninstall_non_tty_affirmative_input_keeps_indexes); + RUN_TEST(cli_uninstall_delete_indexes_help_and_invalid_options_preserve_files); #ifndef _WIN32 RUN_TEST(cli_uninstall_removes_binary_and_index_when_agent_config_cleanup_fails); RUN_TEST(cli_uninstall_cleans_user_owned_symlinked_config); @@ -14132,6 +15100,13 @@ SUITE(cli) { /* Agent detection (6 tests — group A) */ RUN_TEST(cli_detect_agents_finds_claude); RUN_TEST(cli_detect_agents_finds_claude_via_env); + RUN_TEST(cli_detect_claude_empty_dir_not_detected_issue1180); + RUN_TEST(cli_detect_claude_empty_config_dir_env_not_detected_issue1180); + RUN_TEST(cli_detect_claude_real_install_detected_issue1180); + RUN_TEST(cli_detect_opencode_desktop_storage_issue1180); + RUN_TEST(cli_hermes_home_windows_localappdata_issue1180); + RUN_TEST(cli_detect_hermes_desktop_home_issue1180); + RUN_TEST(cli_detect_client_non_install_control_issue1167_1180); RUN_TEST(cli_detect_agents_finds_codex); RUN_TEST(cli_detect_agents_finds_grok); RUN_TEST(cli_detect_agents_finds_cursor_issue222); @@ -14271,6 +15246,7 @@ SUITE(cli) { RUN_TEST(cli_upsert_instructions_no_duplicate); RUN_TEST(cli_remove_instructions); RUN_TEST(cli_agent_instructions_content); + RUN_TEST(cli_guidance_resolves_project_by_root_path_issue1690); RUN_TEST(cli_qwen_windows_hook_command_uses_powershell_schema); RUN_TEST(cli_windows_optional_hooks_require_a_documented_shell); RUN_TEST(cli_installed_skill_limits_match_server_contract); @@ -14287,6 +15263,9 @@ SUITE(cli) { RUN_TEST(cli_claude_exec_hooks_custom_dir_uninstall_issue1733); RUN_TEST(cli_hook_augment_path_is_abs); RUN_TEST(cli_hook_augment_deadline_breadcrumb_issue858); + RUN_TEST(cli_hook_augment_log_long_home_creates_no_cut_path); + RUN_TEST(cli_hook_augment_log_honours_cache_dir); + RUN_TEST(cli_hook_augment_log_default_location_unchanged); RUN_TEST(cli_upsert_claude_hook_fresh); RUN_TEST(cli_upsert_claude_hook_existing); RUN_TEST(cli_tool_hooks_preserve_foreign_same_matcher); @@ -14313,6 +15292,7 @@ SUITE(cli) { RUN_TEST(cli_config_delete); RUN_TEST(cli_config_persists); RUN_TEST(cli_config_watcher_enabled_default_and_persist); + RUN_TEST(cli_config_watch_non_git_default_off_and_listed_issue1948); /* Replace binary (update command helper — group H) */ #ifndef _WIN32 diff --git a/tests/test_config_toml_edit.c b/tests/test_config_toml_edit.c index 465f3d922b..8f928aa6e7 100644 --- a/tests/test_config_toml_edit.c +++ b/tests/test_config_toml_edit.c @@ -871,6 +871,181 @@ TEST(config_toml_managed_unbalanced_duplicate_fail_closed) { PASS(); } +/* #2228: the region between the managed markers is not only ours. Codex + * Desktop appended its own tables below a block that ended config.toml, the + * closing marker stayed the last line, and [mcp_servers.node_repl], [desktop] + * and the marketplaces ended up inside the markers. The next install rewrote + * the whole region and deleted them, together with a key the user had added + * to our own table. */ +static const char *CTE_MCP_BEGIN = "# >>> codebase-memory-mcp MCP >>>"; +static const char *CTE_MCP_END = "# <<< codebase-memory-mcp MCP <<<"; +static const char *CTE_MCP_DECLARATION = "[mcp_servers.codebase-memory-mcp]\n"; +#define CTE_MCP_BEGIN_LINE "# >>> codebase-memory-mcp MCP >>>\n" +#define CTE_MCP_END_LINE "# <<< codebase-memory-mcp MCP <<<\n" +#define CTE_MCP_PREFIX \ + "model = \"gpt-5\"\n\n[mcp_servers.codegraph]\ncommand = \"codegraph\"\n" \ + "args = [\"serve\", \"--mcp\"]\n" +/* The 0.10.4 shape of our block: no env_vars yet. */ +#define CTE_MCP_OLD_TABLE \ + "[mcp_servers.codebase-memory-mcp]\ncommand = \"/old/codebase-memory-mcp\"\nargs = []\n" +#define CTE_MCP_NEW_TABLE \ + "[mcp_servers.codebase-memory-mcp]\ncommand = \"/new/codebase-memory-mcp\"\nargs = []\n" \ + "env_vars = [\"CBM_CACHE_DIR\", \"CBM_RUNTIME_DIR\"]\n" +#define CTE_MCP_USER_LINES "# raised for a 1.5 GB index\nstartup_timeout_sec = 90\n" +#define CTE_MCP_FOREIGN \ + "[mcp_servers.node_repl]\nargs = []\n" \ + "command = \"/Applications/ChatGPT.app/Contents/Resources/cua_node/bin/node_repl\"\n" \ + "startup_timeout_sec = 120\n\n" \ + "[mcp_servers.node_repl.env]\nNODE_REPL_NATIVE_PIPE_CONNECT_TIMEOUT_MS = \"5000\"\n\n" \ + "[desktop]\nfollowUpQueueMode = \"steer\"\n\n" \ + "[marketplaces.openai-codex]\nsource_type = \"git\"\n" \ + "source = \"https://github.com/openai/codex-plugin-cc.git\"\n" +#define CTE_MCP_ISSUE2228 \ + CTE_MCP_PREFIX CTE_MCP_BEGIN_LINE CTE_MCP_OLD_TABLE CTE_MCP_USER_LINES \ + "\n" CTE_MCP_FOREIGN CTE_MCP_END_LINE +#define CTE_MCP_LEAD_AND_CHILD \ + CTE_MCP_PREFIX CTE_MCP_BEGIN_LINE \ + "# pinned by hand\n" CTE_MCP_OLD_TABLE \ + "\n[mcp_servers.codebase-memory-mcp.env]\nRUST_LOG = \"warn\"\n\n" \ + "[notice]\nhide_full_access_warning = true\n" CTE_MCP_END_LINE \ + "\n[profiles.fast]\nmodel = \"o4-mini\"\n" + +TEST(config_toml_managed_keeps_foreign_tables_and_user_keys_issue2228) { + char dir[CTE_PATH_CAP]; + char path[CTE_PATH_CAP]; + char actual[CTE_FILE_CAP]; + /* Our table is rewritten in the current shape and keeps the user's comment + * and key; the foreign tables move, byte for byte and in order, to just + * after the block, so from now on the markers bracket only our table. */ + const char *expected = CTE_MCP_PREFIX CTE_MCP_BEGIN_LINE CTE_MCP_NEW_TABLE CTE_MCP_USER_LINES + CTE_MCP_END_LINE CTE_MCP_FOREIGN; + ASSERT_EQ(cte_fixture(dir, sizeof(dir), path, sizeof(path)), 0); + ASSERT_EQ(th_write_file(path, CTE_MCP_ISSUE2228), 0); + ASSERT_EQ(cbm_toml_upsert_managed_block(path, CTE_MCP_BEGIN, CTE_MCP_END, CTE_MCP_NEW_TABLE), + 0); + ASSERT_EQ(cte_read(path, actual, sizeof(actual)), 0); + ASSERT_STR_EQ(actual, expected); + ASSERT_EQ(cbm_toml_upsert_managed_block(path, CTE_MCP_BEGIN, CTE_MCP_END, CTE_MCP_NEW_TABLE), + 0); + ASSERT_EQ(cte_read(path, actual, sizeof(actual)), 0); + ASSERT_STR_EQ(actual, expected); + th_cleanup(dir); + PASS(); +} + +/* Control: without foreign content the output is exactly what the whole-region + * rewrite always produced — unchanged for the current shape, the new shape for + * an old one. */ +TEST(config_toml_managed_control_without_foreign_content_issue2228) { + char dir[CTE_PATH_CAP]; + char path[CTE_PATH_CAP]; + char actual[CTE_FILE_CAP]; + const char *current = CTE_MCP_PREFIX CTE_MCP_BEGIN_LINE CTE_MCP_NEW_TABLE CTE_MCP_END_LINE + "\n[profiles.fast]\nmodel = \"o4-mini\"\n"; + const char *old = CTE_MCP_PREFIX CTE_MCP_BEGIN_LINE CTE_MCP_OLD_TABLE CTE_MCP_END_LINE + "\n[profiles.fast]\nmodel = \"o4-mini\"\n"; + ASSERT_EQ(cte_fixture(dir, sizeof(dir), path, sizeof(path)), 0); + ASSERT_EQ(th_write_file(path, current), 0); + ASSERT_EQ(cbm_toml_upsert_managed_block(path, CTE_MCP_BEGIN, CTE_MCP_END, CTE_MCP_NEW_TABLE), + 0); + ASSERT_EQ(cte_read(path, actual, sizeof(actual)), 0); + ASSERT_STR_EQ(actual, current); + ASSERT_EQ(th_write_file(path, old), 0); + ASSERT_EQ(cbm_toml_upsert_managed_block(path, CTE_MCP_BEGIN, CTE_MCP_END, CTE_MCP_NEW_TABLE), + 0); + ASSERT_EQ(cte_read(path, actual, sizeof(actual)), 0); + ASSERT_STR_EQ(actual, current); + th_cleanup(dir); + PASS(); +} + +/* A comment above our header belongs to no table of ours and moves above the + * block; a sub-table of our table is part of our server entry and stays with + * it (outside the markers it would collide with the next install). */ +TEST(config_toml_managed_keeps_lead_and_owned_subtable_issue2228) { + char dir[CTE_PATH_CAP]; + char path[CTE_PATH_CAP]; + char actual[CTE_FILE_CAP]; + const char *expected = CTE_MCP_PREFIX + "# pinned by hand\n" CTE_MCP_BEGIN_LINE CTE_MCP_NEW_TABLE + "[mcp_servers.codebase-memory-mcp.env]\nRUST_LOG = \"warn\"\n\n" CTE_MCP_END_LINE + "[notice]\nhide_full_access_warning = true\n" + "\n[profiles.fast]\nmodel = \"o4-mini\"\n"; + ASSERT_EQ(cte_fixture(dir, sizeof(dir), path, sizeof(path)), 0); + ASSERT_EQ(th_write_file(path, CTE_MCP_LEAD_AND_CHILD), 0); + ASSERT_EQ(cbm_toml_upsert_managed_block(path, CTE_MCP_BEGIN, CTE_MCP_END, CTE_MCP_NEW_TABLE), + 0); + ASSERT_EQ(cte_read(path, actual, sizeof(actual)), 0); + ASSERT_STR_EQ(actual, expected); + ASSERT_EQ(cbm_toml_upsert_managed_block(path, CTE_MCP_BEGIN, CTE_MCP_END, CTE_MCP_NEW_TABLE), + 0); + ASSERT_EQ(cte_read(path, actual, sizeof(actual)), 0); + ASSERT_STR_EQ(actual, expected); + th_cleanup(dir); + PASS(); +} + +/* Where ownership cannot be decided, nothing is written: a second [[hooks]] + * element we did not declare, or a move that would hand the keys below the + * block to a different table. */ +TEST(config_toml_managed_ambiguous_foreign_content_fails_closed_issue2228) { + char dir[CTE_PATH_CAP]; + char path[CTE_PATH_CAP]; + const char *hook = "[[hooks]]\nevent = \"UserPromptSubmit\"\n" + "command = \"cbm hook-augment --dialect kimi\"\ntimeout = 5\n"; + const char *extra_element = "# BEGIN codebase-memory-mcp\n" + "[[hooks]]\nevent = \"UserPromptSubmit\"\n" + "command = \"cbm hook-augment --dialect kimi\"\ntimeout = 5\n\n" + "[[hooks]]\nevent = \"Stop\"\ncommand = \"notify-send done\"\n" + "# END codebase-memory-mcp\n"; + const char *rescoped_tail = "# BEGIN codebase-memory-mcp\n" + "[notice]\nhide_full_access_warning = true\n\n" + "[[hooks]]\nevent = \"UserPromptSubmit\"\n" + "command = \"cbm hook-augment --dialect kimi\"\ntimeout = 5\n" + "# END codebase-memory-mcp\n" + "matcher = \"user\"\n"; + ASSERT_EQ(cte_fixture(dir, sizeof(dir), path, sizeof(path)), 0); + ASSERT(cte_assert_unchanged_after_managed_upsert(path, extra_element, hook)); + ASSERT(cte_assert_unchanged_after_managed_upsert(path, rescoped_tail, hook)); + th_cleanup(dir); + PASS(); +} + +/* Uninstall takes only what we declare -- our table with its keys and + * sub-tables -- and leaves everything else between the markers in place. */ +TEST(config_toml_managed_remove_owned_keeps_foreign_content_issue2228) { + char dir[CTE_PATH_CAP]; + char path[CTE_PATH_CAP]; + char actual[CTE_FILE_CAP]; + const char *extra_element = "keep = true\n# BEGIN codebase-memory-mcp\n" + "[[hooks]]\nevent = \"UserPromptSubmit\"\n\n" + "[[hooks]]\nevent = \"Stop\"\n" + "# END codebase-memory-mcp\n"; + ASSERT_EQ(cte_fixture(dir, sizeof(dir), path, sizeof(path)), 0); + ASSERT_EQ(th_write_file(path, CTE_MCP_ISSUE2228), 0); + ASSERT_EQ( + cbm_toml_remove_managed_block_owned(path, CTE_MCP_BEGIN, CTE_MCP_END, CTE_MCP_DECLARATION), + 0); + ASSERT_EQ(cte_read(path, actual, sizeof(actual)), 0); + ASSERT_STR_EQ(actual, CTE_MCP_PREFIX CTE_MCP_FOREIGN); + + ASSERT_EQ(th_write_file(path, CTE_MCP_LEAD_AND_CHILD), 0); + ASSERT_EQ( + cbm_toml_remove_managed_block_owned(path, CTE_MCP_BEGIN, CTE_MCP_END, CTE_MCP_DECLARATION), + 0); + ASSERT_EQ(cte_read(path, actual, sizeof(actual)), 0); + ASSERT_STR_EQ(actual, CTE_MCP_PREFIX "# pinned by hand\n" + "[notice]\nhide_full_access_warning = true\n" + "\n[profiles.fast]\nmodel = \"o4-mini\"\n"); + + ASSERT_EQ(th_write_file(path, extra_element), 0); + ASSERT_EQ(cbm_toml_remove_managed_block_owned(path, CTE_BEGIN, CTE_END, "[[hooks]]\n"), -1); + ASSERT_EQ(cte_read(path, actual, sizeof(actual)), 0); + ASSERT_STR_EQ(actual, extra_element); + th_cleanup(dir); + PASS(); +} + TEST(config_toml_vibe_insert_among_other_tables) { char dir[CTE_PATH_CAP]; char path[CTE_PATH_CAP]; @@ -1574,6 +1749,11 @@ SUITE(config_toml_edit) { RUN_TEST(config_toml_managed_remove); RUN_TEST(config_toml_managed_idempotent); RUN_TEST(config_toml_managed_unbalanced_duplicate_fail_closed); + RUN_TEST(config_toml_managed_keeps_foreign_tables_and_user_keys_issue2228); + RUN_TEST(config_toml_managed_control_without_foreign_content_issue2228); + RUN_TEST(config_toml_managed_keeps_lead_and_owned_subtable_issue2228); + RUN_TEST(config_toml_managed_ambiguous_foreign_content_fails_closed_issue2228); + RUN_TEST(config_toml_managed_remove_owned_keeps_foreign_content_issue2228); RUN_TEST(config_toml_vibe_insert_among_other_tables); RUN_TEST(config_toml_vibe_replace_target_preserves_comments_tables); RUN_TEST(config_toml_vibe_owned_table_installs_idempotently_and_removes_exact_state); diff --git a/tests/test_cypher.c b/tests/test_cypher.c index b683b6331b..eb16e145d9 100644 --- a/tests/test_cypher.c +++ b/tests/test_cypher.c @@ -9,6 +9,7 @@ #include "../src/foundation/compat_thread.h" #include #include +#include #include #include #include @@ -1269,6 +1270,114 @@ TEST(cypher_exec_aggregate_sees_all_edges_beyond_expansion_cap_issue1196) { PASS(); } +/* #1364: RETURN DISTINCT de-duplicated only AFTER the engine row cap, so a + * value that first appears past the cap was never seen: on a 150k-edge index + * `RETURN DISTINCT type(r)` returned 21 of 22 types (HAS_BRANCH, one edge, + * missing). The cap must count DISTINCT rows. Fixture: 20 CALLS edges, then + * one HAS_BRANCH edge from the last-inserted node; max_rows=5 puts the rare + * type far past the raw-row cap. */ +TEST(cypher_return_distinct_counts_distinct_rows_against_cap_issue1364) { + cbm_store_t *s = cbm_store_open_memory(); + ASSERT_NOT_NULL(s); + ASSERT_EQ(cbm_store_upsert_project(s, "test", "/tmp/test"), CBM_STORE_OK); + + int64_t ids[22]; + for (int i = 0; i < 22; i++) { + char name[32]; + char qn[64]; + snprintf(name, sizeof(name), "fn_%02d", i); + snprintf(qn, sizeof(qn), "test.%s", name); + cbm_node_t n = {.project = "test", + .label = "Function", + .name = name, + .qualified_name = qn, + .file_path = "a.py"}; + ids[i] = cbm_store_upsert_node(s, &n); + ASSERT_GT(ids[i], 0); + } + for (int i = 0; i < 20; i++) { + cbm_edge_t e = { + .project = "test", .source_id = ids[i], .target_id = ids[i + 1], .type = "CALLS"}; + cbm_store_insert_edge(s, &e); + } + cbm_edge_t rare = { + .project = "test", .source_id = ids[21], .target_id = ids[0], .type = "HAS_BRANCH"}; + cbm_store_insert_edge(s, &rare); + + /* Plain DISTINCT: both types, complete (2 distinct rows << cap of 5). */ + cbm_cypher_result_t r = {0}; + int rc = cbm_cypher_execute(s, "MATCH ()-[r]->() RETURN DISTINCT type(r) AS t", "test", 5, &r); + ASSERT_EQ(rc, 0); + ASSERT_EQ(r.row_count, 2); + bool saw_calls = false; + bool saw_branch = false; + for (int i = 0; i < r.row_count; i++) { + saw_calls = saw_calls || strcmp(r.rows[i][0], "CALLS") == 0; + saw_branch = saw_branch || strcmp(r.rows[i][0], "HAS_BRANCH") == 0; + } + ASSERT_TRUE(saw_calls); + ASSERT_TRUE(saw_branch); + ASSERT_FALSE(r.truncated); + cbm_cypher_result_free(&r); + + /* DISTINCT + ORDER BY shares the cap. */ + memset(&r, 0, sizeof(r)); + rc = cbm_cypher_execute(s, "MATCH ()-[r]->() RETURN DISTINCT type(r) AS t ORDER BY t DESC", + "test", 5, &r); + ASSERT_EQ(rc, 0); + ASSERT_EQ(r.row_count, 2); + ASSERT_STR_EQ(r.rows[0][0], "HAS_BRANCH"); + ASSERT_STR_EQ(r.rows[1][0], "CALLS"); + ASSERT_FALSE(r.truncated); + cbm_cypher_result_free(&r); + + /* DISTINCT + ORDER BY + SKIP/LIMIT: SKIP counts distinct rows. */ + memset(&r, 0, sizeof(r)); + rc = cbm_cypher_execute( + s, "MATCH ()-[r]->() RETURN DISTINCT type(r) AS t ORDER BY t SKIP 1 LIMIT 1", "test", 5, + &r); + ASSERT_EQ(rc, 0); + ASSERT_EQ(r.row_count, 1); + ASSERT_STR_EQ(r.rows[0][0], "HAS_BRANCH"); + ASSERT_FALSE(r.truncated); + cbm_cypher_result_free(&r); + + /* An explicit LIMIT on DISTINCT counts distinct rows too. */ + memset(&r, 0, sizeof(r)); + rc = cbm_cypher_execute(s, "MATCH ()-[r]->() RETURN DISTINCT type(r) AS t LIMIT 2", "test", 5, + &r); + ASSERT_EQ(rc, 0); + ASSERT_EQ(r.row_count, 2); + ASSERT_FALSE(r.truncated); + cbm_cypher_result_free(&r); + + /* More distinct rows than the cap: still bounded and reported truncated. */ + memset(&r, 0, sizeof(r)); + rc = cbm_cypher_execute(s, "MATCH (a)-[r]->(b) RETURN DISTINCT b.name", "test", 5, &r); + ASSERT_EQ(rc, 0); + ASSERT_EQ(r.row_count, 5); + ASSERT_TRUE(r.truncated); + cbm_cypher_result_free(&r); + + /* An oversized user LIMIT cannot bypass the engine projection budget. */ + const char *bounded_queries[] = { + "MATCH (n) RETURN n.name LIMIT 1000000", + "MATCH (n) RETURN DISTINCT n.name LIMIT 1000000", + "MATCH (n) RETURN * LIMIT 1000000", + }; + for (size_t i = 0; i < sizeof(bounded_queries) / sizeof(bounded_queries[0]); i++) { + memset(&r, 0, sizeof(r)); + rc = cbm_cypher_execute(s, bounded_queries[i], "test", 5, &r); + ASSERT_EQ(rc, 0); + ASSERT_EQ(r.row_count, 5); + ASSERT_TRUE(r.truncated); + cbm_cypher_result_free(&r); + } + + cbm_store_close(s); + PASS(); +} + /* #874: coalesce(var.prop, literal) in WHERE — null-safe numeric filters * for audit queries over OPTIONAL graph properties. The parser rejected the * call outright ("unexpected operator"); RETURN-side coalesce already @@ -1442,6 +1551,43 @@ TEST(cypher_exec_where_regex) { PASS(); } +/* An `=~` pattern over the regex wrapper's compile-size budget is refused + * before the platform compiler expands it; the comparison then matches nothing, + * as for any pattern that cannot be compiled, the result carries a warning + * naming the reason, and the engine keeps answering. The pattern is a 509-way + * alternation whose first branch is `.`: once compiled it matches every name, + * and it costs 509 + 509*509/8 = 32,894 units, just over the budget. */ +TEST(cypher_exec_where_regex_oversized_pattern_matches_nothing) { + cbm_store_t *s = setup_cypher_store(); + cbm_cypher_result_t r = {0}; + + char query[1200]; + char *w = query; + w += snprintf(w, sizeof(query), "MATCH (f:Function) WHERE f.name =~ \"(."); + for (int i = 0; i < 508; i++) { + *w++ = '|'; + *w++ = (char)('a' + i % 26); + } + snprintf(w, (size_t)(query + sizeof(query) - w), ")\""); + + int rc = cbm_cypher_execute(s, query, "test", 0, &r); + ASSERT_EQ(rc, 0); + ASSERT_EQ(r.row_count, 0); + ASSERT_NOT_NULL(r.warning); + ASSERT_NOT_NULL(strstr(r.warning, "too large to compile")); + cbm_cypher_result_free(&r); + + cbm_cypher_result_t r2 = {0}; + rc = cbm_cypher_execute(s, "MATCH (f:Function) WHERE f.name =~ \".*Order.*\"", "test", 0, &r2); + ASSERT_EQ(rc, 0); + ASSERT_EQ(r2.row_count, 3); + ASSERT_NULL(r2.warning); + cbm_cypher_result_free(&r2); + + cbm_store_close(s); + PASS(); +} + TEST(cypher_exec_where_contains) { cbm_store_t *s = setup_cypher_store(); cbm_cypher_result_t r = {0}; @@ -4980,6 +5126,28 @@ TEST(cypher_exec_deadline_allows_normal_query_issue601) { PASS(); } +/* The caller-supplied output-row limit sizes the initial binding array; a + * value above the engine ceiling is clamped to it before that happens, and a + * value inside the ceiling still bounds the row count. */ +TEST(cypher_exec_max_rows_above_ceiling_is_clamped) { + cbm_store_t *s = setup_cypher_store(); + cbm_cypher_result_t r = {0}; + + int rc = cbm_cypher_execute(s, "MATCH (f:Function)", "test", INT_MAX, &r); + ASSERT_EQ(rc, 0); + ASSERT_EQ(r.row_count, 4); + cbm_cypher_result_free(&r); + + memset(&r, 0, sizeof(r)); + rc = cbm_cypher_execute(s, "MATCH (f:Function)", "test", 2, &r); + ASSERT_EQ(rc, 0); + ASSERT_EQ(r.row_count, 2); + cbm_cypher_result_free(&r); + + cbm_store_close(s); + PASS(); +} + /* ══════════════════════════════════════════════════════════════════ */ SUITE(cypher) { @@ -5043,9 +5211,11 @@ SUITE(cypher) { RUN_TEST(cypher_exec_where_eq); RUN_TEST(cypher_exec_unlabeled_where_beyond_result_limit_issue1196); RUN_TEST(cypher_exec_aggregate_sees_all_edges_beyond_expansion_cap_issue1196); + RUN_TEST(cypher_return_distinct_counts_distinct_rows_against_cap_issue1364); RUN_TEST(cypher_exec_varlength_path_semantics_issue797); RUN_TEST(cypher_exec_where_coalesce_issue874); RUN_TEST(cypher_exec_where_regex); + RUN_TEST(cypher_exec_where_regex_oversized_pattern_matches_nothing); RUN_TEST(cypher_exec_where_contains); RUN_TEST(cypher_exec_where_starts_with); RUN_TEST(cypher_exec_return_properties); @@ -5213,4 +5383,5 @@ SUITE(cypher) { RUN_TEST(cypher_exec_prop_array_with_internal_commas); RUN_TEST(cypher_exec_prop_string_with_escaped_quote); RUN_TEST(cypher_single_hop_seeds_from_selective_far_node); + RUN_TEST(cypher_exec_max_rows_above_ceiling_is_clamped); } diff --git a/tests/test_daemon_application.c b/tests/test_daemon_application.c index 4d89c81643..c25a6f5c4a 100644 --- a/tests/test_daemon_application.c +++ b/tests/test_daemon_application.c @@ -1185,6 +1185,8 @@ TEST(daemon_application_prune_clears_logical_watch_for_reregistration) { enum { APP_FAKE_MAX_ATTEMPTS = 16 }; enum { APP_FAKE_ARGS_CAP = 2048 }; +/* Scripted response slot: a CLEAN exit that wrote no response (#1300). */ +static const char APP_FAKE_NO_RESPONSE[] = ""; typedef struct { atomic_int starts; @@ -1370,7 +1372,12 @@ static cbm_index_worker_poll_t app_fake_worker_poll(void *opaque, worker->attempt < APP_FAKE_MAX_ATTEMPTS && worker->context->responses[worker->attempt] ? worker->context->responses[worker->attempt] : "{\"status\":\"indexed\"}"; - if (outcome == CBM_PROC_CLEAN) { + bool silent = response == APP_FAKE_NO_RESPONSE; + if (silent) { + worker->result.response_missing = true; + snprintf(worker->result.last_phase, sizeof(worker->result.last_phase), "%s", + "incremental.purge.progress"); + } else if (outcome == CBM_PROC_CLEAN) { cbm_operation_result_t operation_result = cbm_operation_result_copy( response, worker->attempt < APP_FAKE_MAX_ATTEMPTS && worker->context->response_errors[worker->attempt]); @@ -3038,6 +3045,130 @@ TEST(daemon_application_disconnect_before_request_callback_is_sticky) { PASS(); } +/* #1300: a worker that exits cleanly without a response must reach the + * client as a named failure: said to be a clean exit with no response, with + * the last phase it reached and the worker log to inspect — not the generic + * "ended with clean" line. No recovery re-run for a worker that did not + * crash. */ +TEST(daemon_application_clean_exit_without_response_is_named_failure_issue1300) { + app_fake_worker_context_t fake; + app_fake_worker_context_init(&fake); + atomic_store(&fake.scripted, true); + fake.outcomes[0] = CBM_PROC_CLEAN; + fake.responses[0] = APP_FAKE_NO_RESPONSE; + cbm_daemon_application_worker_ops_t worker_ops = { + .context = &fake, + .start = app_fake_worker_start, + .poll = app_fake_worker_poll, + .cancel = app_fake_worker_cancel, + .log_path = app_fake_worker_log_path, + .destroy = app_fake_worker_destroy, + }; + cbm_daemon_application_config_t config = {.worker_ops = &worker_ops}; + cbm_daemon_application_t *application = cbm_daemon_application_new(&config); + cbm_daemon_runtime_application_callbacks_t callbacks = + cbm_daemon_application_runtime_callbacks(application); + cbm_daemon_runtime_application_session_t *session = app_test_open(&callbacks, 1300); + char root[APP_TEST_PATH_CAP]; + snprintf(root, sizeof(root), "%s/cbm-app-no-response-XXXXXX", cbm_tmpdir()); + bool root_ok = cbm_mkdtemp(root) != NULL; + uint8_t *context = NULL; + uint32_t context_length = 0; + char args[APP_TEST_PATH_CAP + 32]; + snprintf(args, sizeof(args), "{\"repo_path\":\"%s\"}", root); + uint8_t *tool = NULL; + uint32_t tool_length = 0; + bool setup = application && session && root_ok && + app_test_context_request(root, root, &context, &context_length) && + app_test_tool_request("index_repository", args, &tool, &tool_length); + uint8_t *response = NULL; + uint32_t response_length = 0; + setup = setup && app_test_request(&callbacks, session, context, context_length, &response, + &response_length) == CBM_DAEMON_RUNTIME_APPLICATION_OK; + free(response); + response = NULL; + response_length = 0; + cbm_daemon_runtime_application_status_t status = + setup + ? app_test_request(&callbacks, session, tool, tool_length, &response, &response_length) + : CBM_DAEMON_RUNTIME_APPLICATION_HANDLER_ERROR; + if (session) { + callbacks.session_close(callbacks.context, session); + } + bool stopped = application && cbm_daemon_application_shutdown(application, APP_TEST_TIMEOUT_MS); + cbm_daemon_application_free(application); + + ASSERT_TRUE(setup); + ASSERT_EQ(status, CBM_DAEMON_RUNTIME_APPLICATION_OK); + ASSERT_TRUE( + app_test_response_contains(response, response_length, "without writing a response")); + ASSERT_TRUE( + app_test_response_contains(response, response_length, "incremental.purge.progress")); + ASSERT_TRUE(app_test_response_contains(response, response_length, "/tmp/cbm-fake-worker.log")); + ASSERT_TRUE(!app_test_response_contains(response, response_length, "indexed")); + ASSERT_EQ(atomic_load(&fake.starts), 1); + ASSERT_TRUE(stopped); + + free(response); + free(context); + free(tool); + (void)cbm_rmdir(root); + PASS(); +} + +TEST(daemon_application_real_worker_no_response_preserves_named_failure) { + cbm_daemon_application_t *application = cbm_daemon_application_new(NULL); + cbm_daemon_runtime_application_callbacks_t callbacks = + cbm_daemon_application_runtime_callbacks(application); + cbm_daemon_runtime_application_session_t *session = app_test_open(&callbacks, 1300); + char root[APP_TEST_PATH_CAP]; + snprintf(root, sizeof(root), "%s/cbm-app-no-response-XXXXXX", cbm_tmpdir()); + bool root_ok = cbm_mkdtemp(root) != NULL; + uint8_t *context = NULL; + uint32_t context_length = 0; + char args[APP_TEST_PATH_CAP + 32]; + snprintf(args, sizeof(args), "{\"repo_path\":\"%s\",\"__cbm_test_worker\":\"silent-exit\"}", + root); + uint8_t *tool = NULL; + uint32_t tool_length = 0; + bool setup = application && session && root_ok && + app_test_context_request(root, root, &context, &context_length) && + app_test_tool_request("index_repository", args, &tool, &tool_length); + uint8_t *response = NULL; + uint32_t response_length = 0; + setup = setup && app_test_request(&callbacks, session, context, context_length, &response, + &response_length) == CBM_DAEMON_RUNTIME_APPLICATION_OK; + free(response); + response = NULL; + response_length = 0; + cbm_daemon_runtime_application_status_t status = + setup + ? app_test_request(&callbacks, session, tool, tool_length, &response, &response_length) + : CBM_DAEMON_RUNTIME_APPLICATION_HANDLER_ERROR; + if (session) { + callbacks.session_close(callbacks.context, session); + } + bool stopped = application && cbm_daemon_application_shutdown(application, APP_TEST_TIMEOUT_MS); + cbm_daemon_application_free(application); + + ASSERT_TRUE(setup); + ASSERT_EQ(status, CBM_DAEMON_RUNTIME_APPLICATION_OK); + ASSERT_TRUE( + app_test_response_contains(response, response_length, "without writing a response")); + ASSERT_TRUE(app_test_response_contains(response, response_length, "incremental.edge_snapshot")); + ASSERT_TRUE(app_test_response_contains(response, response_length, "worker_log")); + ASSERT_TRUE(!app_test_response_contains(response, response_length, "indexed")); + ASSERT_TRUE(response && response_length > 1 && response[0] == 1); + ASSERT_TRUE(app_test_response_contains(response, response_length, "no_response")); + ASSERT_TRUE(stopped); + + free(response); + free(context); + free(tool); + (void)cbm_rmdir(root); + PASS(); +} + TEST(daemon_application_request_cancel_preserves_persistent_watch_and_session) { app_watch_race_fixture_t fixture; bool fixture_ready = app_watch_race_fixture_init(&fixture, 3021); @@ -5875,4 +6006,6 @@ SUITE(daemon_application) { RUN_TEST(daemon_application_over_budget_response_passes_through_without_recovery); RUN_TEST(daemon_application_free_reports_retained_live_ownership); RUN_TEST(daemon_application_rejects_clean_exit_when_process_tree_is_not_contained); + RUN_TEST(daemon_application_clean_exit_without_response_is_named_failure_issue1300); + RUN_TEST(daemon_application_real_worker_no_response_preserves_named_failure); } diff --git a/tests/test_daemon_runtime.c b/tests/test_daemon_runtime.c index caae09a48d..11f9540390 100644 --- a/tests/test_daemon_runtime.c +++ b/tests/test_daemon_runtime.c @@ -3094,65 +3094,138 @@ TEST(daemon_runtime_authenticated_idle_connection_outlives_lease_interval) { PASS(); } -TEST(daemon_runtime_connection_cap_covers_slow_hello_and_stopping_is_terminal) { +/* Flaky-ledger item 12 (7 sightings, ASSERT(capacity_rejected)): the overflow + * client's HELLO raced the daemon's inline capacity rejection, which the + * accept loop writes WITHOUT reading that HELLO and follows with an immediate + * close. The hook pins one order per run so no window is left to race; the + * hold seam keeps the slow hello's slot until the test closes it, where the + * request deadline used to free it after 2 s. The waits below end on events + * that are guaranteed to arrive (see each comment), never on a bound. */ +typedef enum { + RUNTIME_CAP_HELLO_BEFORE_REJECTION = 0, + RUNTIME_CAP_REJECTION_BEFORE_HELLO = 1, +} runtime_cap_order_t; + +static atomic_int runtime_cap_order; +static atomic_bool runtime_cap_hello_sent; +static atomic_bool runtime_cap_rejection_closed; +static atomic_bool runtime_cap_order_pinned; +static cbm_daemon_runtime_service_t *runtime_cap_service; + +static void runtime_cap_hello_hook(cbm_daemon_runtime_hello_point_t point) { + int order = atomic_load(&runtime_cap_order); + if (point == CBM_DAEMON_RUNTIME_HELLO_POINT_CLIENT_SENT) { + atomic_store(&runtime_cap_hello_sent, true); + } else if (point == CBM_DAEMON_RUNTIME_HELLO_POINT_DAEMON_REJECTED) { + atomic_store(&runtime_cap_rejection_closed, true); + } else if (point == CBM_DAEMON_RUNTIME_HELLO_POINT_DAEMON_REJECTING && + order == RUNTIME_CAP_HELLO_BEFORE_REJECTION) { + /* Accept thread. The client always gets there: its connect already + * completed and a HELLO fits the transport buffer, so its send never + * waits on this thread. */ + while (!atomic_load(&runtime_cap_hello_sent)) { + cbm_usleep(200); + } + atomic_store(&runtime_cap_order_pinned, true); + } else if (point == CBM_DAEMON_RUNTIME_HELLO_POINT_CLIENT_CONNECTED && + order == RUNTIME_CAP_REJECTION_BEFORE_HELLO) { + /* Client (test) thread. Ends on the rejection, or on an admission (a + * broken cap): that worker is held by the seam, so a third slot stays + * counted until this HELLO goes out and the test fails on ACCEPTED. */ + while (!atomic_load(&runtime_cap_rejection_closed) && + cbm_daemon_runtime_service_active_connections(runtime_cap_service) <= 2) { + cbm_usleep(200); + } + atomic_store(&runtime_cap_order_pinned, atomic_load(&runtime_cap_rejection_closed)); + } +} + +typedef struct { + bool started; + bool slow_slot_counted; + bool order_pinned; + bool capacity_rejected; + bool slow_slot_released; + bool exited; + bool no_resurrection; +} runtime_cap_outcome_t; + +static cbm_daemon_runtime_client_t *runtime_cap_overflow_connect( + runtime_test_fixture_t *fixture, const cbm_daemon_build_identity_t *identity, + runtime_cap_order_t order, runtime_cap_outcome_t *outcome) { + cbm_daemon_runtime_connect_result_t result = {0}; + runtime_cap_service = fixture->service; + atomic_store(&runtime_cap_order, (int)order); + atomic_store(&runtime_cap_hello_sent, false); + atomic_store(&runtime_cap_rejection_closed, false); + atomic_store(&runtime_cap_order_pinned, false); + cbm_daemon_runtime_set_hello_hook_for_testing(runtime_cap_hello_hook); + cbm_daemon_runtime_client_t *overflow = cbm_daemon_runtime_client_connect( + fixture->endpoint, identity, RUNTIME_TEST_TIMEOUT_MS, &result); + cbm_daemon_runtime_set_hello_hook_for_testing(NULL); + /* Both counts are HELD here: the slow hello's slot by the seam, the + * accepted client by its open session. */ + size_t connections = cbm_daemon_runtime_service_active_connections(fixture->service); + size_t clients = cbm_daemon_runtime_service_active_clients(fixture->service); + outcome->order_pinned = atomic_load(&runtime_cap_order_pinned); + outcome->capacity_rejected = + overflow == NULL && result.status == CBM_DAEMON_RUNTIME_CONNECT_REJECTED && + strstr(result.message, "capacity") != NULL && connections == 2 && clients == 1; + if (!outcome->capacity_rejected) { + printf(" overflow: client=%s status=%d message=\"%s\" active_connections=%zu " + "active_clients=%zu order=%d pinned=%d\n", + overflow ? "admitted" : "null", (int)result.status, result.message, connections, + clients, (int)order, outcome->order_pinned); + } + return overflow; +} + +static void runtime_cap_scenario(runtime_cap_order_t order, runtime_cap_outcome_t *outcome) { + memset(outcome, 0, sizeof(*outcome)); cbm_daemon_build_identity_t identity = runtime_test_identity("2.4.0", runtime_test_self_build()); runtime_test_fixture_t fixture; - /* slow_hello must keep its slot until the test closes it. With the default - * 2 s hello deadline, two full client handshakes on a slow sanitized runner - * can outlast it, freeing the slot before the overflow connect. */ - runtime_test_fixture_request_timeout_ms = 30000; - bool started = runtime_test_fixture_start_limited(&fixture, "connection-cap", &identity, 2); - runtime_test_fixture_request_timeout_ms = 0; + cbm_daemon_runtime_hold_peer_waits_for_testing(true); + outcome->started = runtime_test_fixture_start_limited(&fixture, "connection-cap", &identity, 2); cbm_daemon_ipc_connection_t *slow_hello = NULL; cbm_daemon_runtime_client_t *accepted = NULL; cbm_daemon_runtime_client_t *overflow = NULL; cbm_daemon_runtime_client_t *resurrection = NULL; cbm_daemon_runtime_connect_result_t accepted_result = {0}; - cbm_daemon_runtime_connect_result_t overflow_result = {0}; cbm_daemon_runtime_connect_result_t resurrection_result = {0}; - bool slow_slot_counted = false; - bool capacity_rejected = false; - bool slow_slot_released = false; - bool exited = false; - bool no_resurrection = false; - if (started) { + if (outcome->started) { slow_hello = cbm_daemon_ipc_connect(fixture.endpoint, RUNTIME_TEST_TIMEOUT_MS); } if (slow_hello) { - slow_slot_counted = cbm_daemon_runtime_service_wait_for_connections( + outcome->slow_slot_counted = cbm_daemon_runtime_service_wait_for_connections( fixture.service, 1, RUNTIME_TEST_TIMEOUT_MS); accepted = cbm_daemon_runtime_client_connect(fixture.endpoint, &identity, RUNTIME_TEST_TIMEOUT_MS, &accepted_result); } if (accepted) { - overflow = cbm_daemon_runtime_client_connect(fixture.endpoint, &identity, - RUNTIME_TEST_TIMEOUT_MS, &overflow_result); - capacity_rejected = overflow == NULL && - overflow_result.status == CBM_DAEMON_RUNTIME_CONNECT_REJECTED && - strstr(overflow_result.message, "capacity") != NULL && - cbm_daemon_runtime_service_active_connections(fixture.service) == 2 && - cbm_daemon_runtime_service_active_clients(fixture.service) == 1; + overflow = runtime_cap_overflow_connect(&fixture, &identity, order, outcome); cbm_daemon_ipc_connection_close(slow_hello); slow_hello = NULL; - slow_slot_released = cbm_daemon_runtime_service_wait_for_connections( + outcome->slow_slot_released = cbm_daemon_runtime_service_wait_for_connections( fixture.service, 1, RUNTIME_TEST_TIMEOUT_MS); (void)cbm_daemon_runtime_client_close(accepted, RUNTIME_TEST_TIMEOUT_MS); accepted = NULL; - exited = cbm_daemon_runtime_service_wait_exited(fixture.service, RUNTIME_TEST_TIMEOUT_MS); + outcome->exited = + cbm_daemon_runtime_service_wait_exited(fixture.service, RUNTIME_TEST_TIMEOUT_MS); resurrection = cbm_daemon_runtime_client_connect(fixture.endpoint, &identity, 100, &resurrection_result); cbm_daemon_runtime_service_state_t terminal_state = cbm_daemon_runtime_service_state(fixture.service); - no_resurrection = resurrection == NULL && - (resurrection_result.status == CBM_DAEMON_RUNTIME_CONNECT_ERROR || - resurrection_result.status == CBM_DAEMON_RUNTIME_CONNECT_REJECTED) && - cbm_daemon_runtime_service_active_clients(fixture.service) == 0 && - (terminal_state == CBM_DAEMON_RUNTIME_SERVICE_STOPPING || - terminal_state == CBM_DAEMON_RUNTIME_SERVICE_EXITED); + outcome->no_resurrection = + resurrection == NULL && + (resurrection_result.status == CBM_DAEMON_RUNTIME_CONNECT_ERROR || + resurrection_result.status == CBM_DAEMON_RUNTIME_CONNECT_REJECTED) && + cbm_daemon_runtime_service_active_clients(fixture.service) == 0 && + (terminal_state == CBM_DAEMON_RUNTIME_SERVICE_STOPPING || + terminal_state == CBM_DAEMON_RUNTIME_SERVICE_EXITED); } if (resurrection) { @@ -3166,16 +3239,49 @@ TEST(daemon_runtime_connection_cap_covers_slow_hello_and_stopping_is_terminal) { } cbm_daemon_ipc_connection_close(slow_hello); runtime_test_fixture_finish(&fixture); + cbm_daemon_runtime_hold_peer_waits_for_testing(false); +} - ASSERT_TRUE(started); - ASSERT_TRUE(slow_slot_counted); - ASSERT_TRUE(capacity_rejected); - ASSERT_TRUE(slow_slot_released); - ASSERT_TRUE(exited); - ASSERT_TRUE(no_resurrection); +/* The daemon answers the overflow only after its HELLO is on the wire: the + * order every platform delivers deterministically. */ +TEST(daemon_runtime_connection_cap_covers_slow_hello_and_stopping_is_terminal) { + runtime_cap_outcome_t outcome; + runtime_cap_scenario(RUNTIME_CAP_HELLO_BEFORE_REJECTION, &outcome); + + ASSERT_TRUE(outcome.started); + ASSERT_TRUE(outcome.slow_slot_counted); + ASSERT_TRUE(outcome.order_pinned); + ASSERT_TRUE(outcome.capacity_rejected); + ASSERT_TRUE(outcome.slow_slot_released); + ASSERT_TRUE(outcome.exited); + ASSERT_TRUE(outcome.no_resurrection); PASS(); } +#ifndef _WIN32 +/* Item 12's production race, pinned: the daemon has written its capacity + * rejection AND closed before the client sends its HELLO. That send fails + * (EPIPE) while the complete rejection sits in the client's receive buffer; + * the client must still report it, not a bare transport error that bootstrap + * would classify by lock state instead of "capacity reached". + * POSIX only: a Windows server close discards unread pipe data, so there the + * rejection is delivered by the drain-before-close instead (which the other + * order covers) and this order cannot carry one by design. */ +TEST(daemon_runtime_capacity_rejection_survives_daemon_close_before_hello) { + runtime_cap_outcome_t outcome; + runtime_cap_scenario(RUNTIME_CAP_REJECTION_BEFORE_HELLO, &outcome); + + ASSERT_TRUE(outcome.started); + ASSERT_TRUE(outcome.slow_slot_counted); + ASSERT_TRUE(outcome.order_pinned); + ASSERT_TRUE(outcome.capacity_rejected); + ASSERT_TRUE(outcome.slow_slot_released); + ASSERT_TRUE(outcome.exited); + ASSERT_TRUE(outcome.no_resurrection); + PASS(); +} +#endif + TEST(daemon_runtime_rejects_forged_identity_extension) { cbm_daemon_build_identity_t identity = runtime_test_identity("2.4.0", runtime_test_self_build()); @@ -5521,6 +5627,9 @@ SUITE(daemon_runtime) { #endif RUN_TEST(daemon_runtime_authenticated_idle_connection_outlives_lease_interval); RUN_TEST(daemon_runtime_connection_cap_covers_slow_hello_and_stopping_is_terminal); +#ifndef _WIN32 + RUN_TEST(daemon_runtime_capacity_rejection_survives_daemon_close_before_hello); +#endif RUN_TEST(daemon_runtime_rejects_forged_identity_extension); RUN_TEST(daemon_runtime_application_response_roundtrip_is_byte_exact); RUN_TEST(daemon_runtime_final_disconnect_rejects_blocked_provisional_session); diff --git a/tests/test_edge_imports.c b/tests/test_edge_imports.c index 70a8b77530..b222734702 100644 --- a/tests/test_edge_imports.c +++ b/tests/test_edge_imports.c @@ -1243,6 +1243,240 @@ TEST(ei_php_interface_use) { PASS(); } +/* ═══════════════════════════════════════════════════════════════════════════ + * PHP PSR-4 — #1186 + * + * With composer.json `autoload.psr-4`, `use App\Models\Agency;` names exactly + * one file: /Models/Agency.php. The resolver instead fell through + * to the namespace bucket and bound every class import of App\Models to the + * FIRST file declaring that namespace (User.php), fabricating a hub. These + * tests pin the exact target file per local name; a missing class file must + * leave the import unresolved rather than land on a sibling. + * ═══════════════════════════════════════════════════════════════════════════ */ + +typedef struct { + const char *local_name; /* IMPORTS edge local_name */ + const char *want_path; /* expected target file_path; NULL = no edge */ +} EIPhpImportExpect; + +/* Find the IMPORTS edge of `importer` whose local_name is `local` and write + * its target's file_path into `out` ("" when there is no such edge). */ +static void ei_import_target_path(cbm_store_t *store, const char *project, const char *importer, + const char *local, char *out, size_t outsz) { + out[0] = '\0'; + int64_t src_id = ei_node_id_for_file_label(store, project, importer, "File"); + cbm_edge_t *edges = NULL; + int n = 0; + if (src_id <= 0 || + cbm_store_find_edges_by_source_type(store, src_id, "IMPORTS", &edges, &n) != CBM_STORE_OK) { + return; + } + char needle[256]; + snprintf(needle, sizeof(needle), "\"local_name\":\"%s\"", local); + for (int i = 0; i < n; i++) { + if (!edges[i].properties_json || !strstr(edges[i].properties_json, needle)) { + continue; + } + cbm_node_t target; + memset(&target, 0, sizeof(target)); + if (cbm_store_find_node_by_id(store, edges[i].target_id, &target) == CBM_STORE_OK) { + snprintf(out, outsz, "%s", + target.label && strcmp(target.label, "File") == 0 && target.file_path + ? target.file_path + : "?"); + cbm_node_free_fields(&target); + } + break; + } + cbm_store_free_edges(edges, n); +} + +/* Index `files` and check every expectation for `importer`. Returns 1 when + * all hold; prints each mismatch so a RED names the fabricated target. */ +static int ei_php_imports_match(const EILangFile *files, int nfiles, const char *importer, + const EIPhpImportExpect *want, int nwant) { + EILangProj lp; + cbm_store_t *store = ei_index_files(&lp, files, nfiles); + if (!store) { + ei_cleanup(&lp, store); + return 0; + } + int ok = 1; + for (int i = 0; i < nwant; i++) { + char got[512]; + ei_import_target_path(store, lp.project, importer, want[i].local_name, got, sizeof(got)); + const char *expect = want[i].want_path ? want[i].want_path : ""; + if (strcmp(got, expect) != 0) { + fprintf(stderr, " [IMPORTS %s] %s -> got \"%s\", want \"%s\"\n", importer, + want[i].local_name, got, expect); + ok = 0; + } + } + ei_cleanup(&lp, store); + return ok; +} + +#define EI_PHP_CLASS(ns, cls) " src/Domain/ over + * App\ -> app/), a package's own root maps its namespace, and autoload-dev + * maps Tests\. The decoy app/Domain/Order.php declares the same namespace and + * sorts first. */ +TEST(ei_php_psr4_multiple_roots_longest_prefix_issue1186) { + static const EILangFile f[] = { + {"composer.json", "{\"autoload\":{\"psr-4\":{\"App\\\\\":\"app/\"," + "\"App\\\\Domain\\\\\":\"src/Domain/\"}}," + "\"autoload-dev\":{\"psr-4\":{\"Tests\\\\\":\"tests/\"}}}\n"}, + {"tests/Support/Assert.php", EI_PHP_CLASS("Tests\\Support", "Assert")}, + {"tests/Support/Factory.php", EI_PHP_CLASS("Tests\\Support", "Factory")}, + {"packages/billing/composer.json", + "{\"name\":\"acme/billing\",\"autoload\":{\"psr-4\":{\"Billing\\\\\":\"src/\"}}}\n"}, + {"app/Domain/Order.php", EI_PHP_CLASS("App\\Domain", "Order")}, + {"src/Domain/Customer.php", EI_PHP_CLASS("App\\Domain", "Customer")}, + {"src/Domain/Order.php", EI_PHP_CLASS("App\\Domain", "Order")}, + {"packages/billing/src/Account.php", EI_PHP_CLASS("Billing", "Account")}, + {"packages/billing/src/Ledger.php", EI_PHP_CLASS("Billing", "Ledger")}, + {"app/Http/Checkout.php", " #include #include @@ -157,12 +158,19 @@ static int et_edge_present(const EtFile *files, int nfiles, const char *edge, in enum { ET_ROUTE_ASSERT_MAX = 16 }; +static cbm_store_t *et_index_parallel(EtProj *lp, const EtFile *meaningful, int n_mean); + /* Assert the exact Route node set. Edge-count smoke tests cannot catch partial * Spring paths such as "/orders" when the real route is "/api/orders", and a - * presence-only assertion would still allow stale partial Route nodes to leak. */ -static int et_routes_exact(const EtFile *files, int nfiles, const char **routes) { + * presence-only assertion would still allow stale partial Route nodes to leak. + * parallel pads the fixture past MIN_FILES_FOR_PARALLEL so the parallel + * resolver mints the routes; must_qn (optional) is a Route qualified_name that + * must exist -- the identity cross-repo HTTP matching joins on. */ +static int et_routes_exact_mode(const EtFile *files, int nfiles, const char **routes, bool parallel, + const char *must_qn) { EtProj lp; - cbm_store_t *store = et_index_files(&lp, files, nfiles); + cbm_store_t *store = + parallel ? et_index_parallel(&lp, files, nfiles) : et_index_files(&lp, files, nfiles); cbm_node_t *nodes = NULL; int node_count = 0; int wanted = 0; @@ -210,11 +218,26 @@ static int et_routes_exact(const EtFile *files, int nfiles, const char **routes) fprintf(stderr, "\n"); } + if (store && must_qn) { + cbm_node_t qn_node; + memset(&qn_node, 0, sizeof(qn_node)); + if (cbm_store_find_node_by_qn(store, lp.project, must_qn, &qn_node) != CBM_STORE_OK) { + fprintf(stderr, " [ET-ROUTE] FAIL missing qualified_name %s\n", must_qn); + ok = 0; + } else { + cbm_node_free_fields(&qn_node); + } + } + cbm_store_free_nodes(nodes, node_count); et_cleanup(&lp, store); return ok; } +static int et_routes_exact(const EtFile *files, int nfiles, const char **routes) { + return et_routes_exact_mode(files, nfiles, routes, false, NULL); +} + /* Index meaningful[] plus PARALLEL_PAD_FILES trivial pad files to force the * parallel pipeline path (MIN_FILES_FOR_PARALLEL = 50). */ enum { ET_PARALLEL_PAD = 52, ET_PAD_MAX = 68 /* 52 pad + 16 meaningful */ }; @@ -236,6 +259,136 @@ static cbm_store_t *et_index_parallel(EtProj *lp, const EtFile *meaningful, int return et_index_files(lp, files, n); } +/* #598: a call that resolves to a GraphQL client used to name its Route after + * the WHOLE operation text (minus a leading "query "/"mutation "), so an + * anonymous `query { ... }` became one long Route and even a named operation + * carried its variables and selection set in the name. GraphQL Routes are now + * keyed by operation name only; anonymous operations share one bounded key per + * operation type. The Route set is asserted exactly (QN + name). Parallel path + * only: GRAPHQL_CALLS is emitted by pass_parallel.c. */ +typedef struct { + const char *qn; + const char *name; +} EtGqlRoute; + +static int et_graphql_routes_exact(const EtFile *files, int nfiles, const EtGqlRoute *want, + int nwant) { + EtProj lp; + cbm_store_t *store = et_index_parallel(&lp, files, nfiles); + cbm_node_t *nodes = NULL; + int node_count = 0; + int ok = store != NULL; + int got = 0; + if (!store || cbm_store_find_nodes_by_label(store, lp.project, "Route", &nodes, &node_count) != + CBM_STORE_OK) { + ok = 0; + } + for (int ni = 0; nodes && ni < node_count; ni++) { + const char *qn = nodes[ni].qualified_name ? nodes[ni].qualified_name : ""; + const char *name = nodes[ni].name ? nodes[ni].name : ""; + if (strncmp(qn, "__graphql__", 11) != 0) { + continue; + } + got++; + int matched = 0; + for (int wi = 0; wi < nwant; wi++) { + if (strcmp(qn, want[wi].qn) == 0 && strcmp(name, want[wi].name) == 0) { + matched = 1; + break; + } + } + /* Operation text (selection sets, variables) must never leak into a key. */ + if (!matched || strchr(qn, '{') || strchr(name, '{') || strlen(qn) > 160) { + fprintf(stderr, " [ET-GQL] unexpected Route qn='%s' name='%s'\n", qn, name); + ok = 0; + } + } + if (got != nwant) { + fprintf(stderr, " [ET-GQL] FAIL graphql routes expected=%d actual=%d\n", nwant, got); + ok = 0; + } + int edges = store ? cbm_store_count_edges_by_type(store, lp.project, "GRAPHQL_CALLS") : -1; + if (edges < nwant) { + fprintf(stderr, " [ET-GQL] FAIL GRAPHQL_CALLS=%d expected>=%d\n", edges, nwant); + ok = 0; + } + cbm_store_free_nodes(nodes, node_count); + et_cleanup(&lp, store); + return ok; +} + +TEST(graphql_route_keyed_by_operation_name_issue598) { + static const EtFile f[] = { + {"graphql/client.py", "def gql(query):\n return query\n"}, + {"api/ops.py", + "from graphql.client import gql\n\n\n" + "def get_user():\n" + " return gql('query GetUser($id: ID!) { user(id: $id) { name } }')\n\n\n" + "def update_user():\n" + " return gql('mutation UpdateUser($id: ID!) { updateUser(id: $id) { id } }')\n\n\n" + "def list_users():\n" + " return gql('fragment U on User { id } query ListUsers { users { ...U } }')\n\n\n" + "def on_event():\n" + " return gql('subscription OnEvent { event { id } }')\n\n\n" + "def products():\n" + " return gql('query { products(search: \"x\") { items { sku } } }')\n\n\n" + "def orders():\n" + " return gql('{ orders { id total } }')\n\n\n" + "def logout():\n" + " return gql('mutation { logout }')\n"}}; + static const EtGqlRoute want[] = { + {"__graphql__GetUser", "GetUser"}, + {"__graphql__UpdateUser", "UpdateUser"}, + {"__graphql__ListUsers", "ListUsers"}, + {"__graphql__OnEvent", "OnEvent"}, + /* products + orders: `query { }` and the `{ }` shorthand are both + * anonymous queries and share one key. */ + {"__graphql__query__anonymous", "(anonymous query)"}, + {"__graphql__mutation__anonymous", "(anonymous mutation)"}, + }; + ASSERT_TRUE(et_graphql_routes_exact(f, (int)(sizeof(f) / sizeof(f[0])), want, + (int)(sizeof(want) / sizeof(want[0])))); + PASS(); +} + +/* #598 parser edges the end-to-end fixture cannot reach through one language's + * string extraction: comments/strings/interpolations that contain braces, a + * bare word, non-document arguments and name truncation. */ +TEST(graphql_operation_identity_parser_issue598) { + const char *type = NULL; + char name[16]; + static const struct { + const char *doc; + int named; + const char *type; + const char *name; + } cases[] = { + {"# fetch { all }\nquery Q1 { a }", 1, "query", "Q1"}, + {"${UserFields} query WithFrag($id: ID) { u { ...UserFields } }", 1, "query", "WithFrag"}, + {"fragment F on T @dir(x: \"{\") { a } mutation M2 { b }", 1, "mutation", "M2"}, + {"\"\"\"doc { brace\"\"\" subscription S3 { c }", 1, "subscription", "S3"}, + {" query \n ( $x: Int ) { a }", 0, "query", ""}, + {" { viewer { id } }", 0, "query", ""}, + {"query GetUserByIdentifier { a }", 1, "query", "GetUserByIdenti"}, /* bounded */ + {"data", 0, "operation", ""}, /* a bare word is not evidence of an operation */ + {"https://api.example.com/graphql", 0, "operation", ""}, + {"/graphql", 0, "operation", ""}, + {"", 0, "operation", ""}, + }; + for (size_t i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) { + bool named = cbm_service_pattern_graphql_operation(cases[i].doc, &type, name, sizeof(name)); + if ((int)named != cases[i].named || strcmp(type, cases[i].type) != 0 || + strcmp(name, cases[i].name) != 0) { + fprintf(stderr, " [ET-GQL] case %zu '%s' -> named=%d type=%s name=%s\n", i, + cases[i].doc, (int)named, type, name); + ASSERT_TRUE(0); + } + } + ASSERT_FALSE(cbm_service_pattern_graphql_operation(NULL, &type, name, sizeof(name))); + ASSERT_STR_EQ(type, "operation"); + PASS(); +} + /* #1085: count CALLS edges whose target node has `name`, indexing via the * PARALLEL path (et_index_parallel pads to >50 files). The per-file TS resolver * used to treat a raw tsconfig alias as a confident target. That false row made @@ -651,6 +804,123 @@ TEST(handles_laravel_facade_no_junk_routes_issue952) { PASS(); } +/* #1146: Laravel 11+ mounts the files named in bootstrap/app.php's + * `->withRouting(api: ...)` under `apiPrefix` (default 'api'); the `web:` file + * gets no prefix. The prefix is declared in a DIFFERENT file than the routes, + * so per-file extraction alone minted `/users/me` where the runtime route is + * `/api/users/me`, and cross-repo HTTP matching (which joins on the Route + * qualified_name) never linked real callers. */ +#define ET_L11_BOOTSTRAP_HEAD \ + "withRouting(\n" +#define ET_L11_BOOTSTRAP_TAIL \ + " )\n" \ + " ->withMiddleware(function ($middleware) {\n //\n })\n" \ + " ->create();\n" + +static const char ET_L11_API_ROUTES[] = + "middleware('auth')->group(function (): void {\n" + " Route::get('/me', [UserController::class, 'me']);\n" + "});\n" + "Route::post('/orders', [OrderController::class, 'store']);\n"; + +static const char ET_L11_WEB_ROUTES[] = + " '/api'. */ +TEST(routes_laravel_withrouting_api_default_prefix_issue1146) { + static const char *routes[] = {"/api/users/me", "/api/orders", NULL}; + static const EtFile f[] = {{"bootstrap/app.php", ET_L11_BOOTSTRAP_HEAD + " web: __DIR__.'/../routes/web.php',\n" + " api: __DIR__.'/../routes/api.php',\n" + " commands: __DIR__.'/../routes/console.php',\n" + " health: '/up',\n" ET_L11_BOOTSTRAP_TAIL}, + {"routes/api.php", ET_L11_API_ROUTES}}; + ASSERT_TRUE(et_routes_exact_mode(f, 2, routes, false, "__route__GET__/api/users/me")); + PASS(); +} + +/* Explicit `apiPrefix:` (multi-segment, array form of `api:`) replaces the + * default for every mounted file. */ +TEST(routes_laravel_withrouting_api_custom_prefix_issue1146) { + static const char *routes[] = {"/v1/api/users/me", "/v1/api/orders", "/v1/api/partners", NULL}; + static const EtFile f[] = { + {"bootstrap/app.php", ET_L11_BOOTSTRAP_HEAD + " api: [__DIR__.'/../routes/api.php', __DIR__.'/../routes/partner.php'],\n" + " apiPrefix: 'v1/api',\n" ET_L11_BOOTSTRAP_TAIL}, + {"routes/api.php", ET_L11_API_ROUTES}, + {"routes/partner.php", "group(base_path(...))` call) is not composed today and + * must stay exactly as it is -- no filename-based `/api` inference. */ +TEST(routes_laravel_routeserviceprovider_control_issue1146) { + static const char *routes[] = {"/users/me", "/orders", NULL}; + static const EtFile f[] = { + {"bootstrap/app.php", + "routes(function () {\n" + " Route::middleware('api')->prefix('api')" + "->group(base_path('routes/api.php'));\n" + " Route::middleware('web')->group(base_path('routes/web.php'));\n" + " });\n }\n}\n"}, + {"routes/api.php", ET_L11_API_ROUTES}}; + ASSERT_TRUE(et_routes_exact(f, 3, routes)); + PASS(); +} + +/* A non-literal `apiPrefix:` is unknown at index time: leave the paths alone + * rather than guess the default. */ +TEST(routes_laravel_withrouting_nonliteral_prefix_issue1146) { + static const char *routes[] = {"/users/me", "/orders", NULL}; + static const EtFile f[] = { + {"bootstrap/app.php", ET_L11_BOOTSTRAP_HEAD + " api: __DIR__.'/../routes/api.php',\n" + " apiPrefix: config('app.api_prefix'),\n" ET_L11_BOOTSTRAP_TAIL}, + {"routes/api.php", ET_L11_API_ROUTES}}; + ASSERT_TRUE(et_routes_exact(f, 2, routes)); + PASS(); +} + +/* Same contract on the PARALLEL resolver (> 50 files), with the Laravel app + * in a monorepo subdirectory: the nearest ancestor bootstrap/app.php owns it. */ +TEST(routes_laravel_withrouting_parallel_subdir_issue1146) { + static const char *routes[] = {"/api/users/me", "/api/orders", NULL}; + static const EtFile f[] = { + {"gateway/bootstrap/app.php", ET_L11_BOOTSTRAP_HEAD + " web: __DIR__.'/../routes/web.php',\n" + " api: __DIR__.'/../routes/api.php',\n" ET_L11_BOOTSTRAP_TAIL}, + {"gateway/routes/api.php", ET_L11_API_ROUTES}}; + ASSERT_TRUE(et_routes_exact_mode(f, 2, routes, true, "__route__POST__/api/orders")); + PASS(); +} + /* Rails (Ruby) — ActionDispatch router. The handler MUST be passed as a bare * identifier (not the idiomatic `to: 'list_items'` string, which extract_handler_arg * cannot capture). mapper.get resolves by name to the Mapper#get method whose QN @@ -1651,6 +1921,154 @@ TEST(override_go_interface) { PASS(); } +/* ══════════════════════════════════════════════════════════════════ + * ROUTE REGISTRATION vs the #725 cross-language guard. + * + * The route classifier recognises a registration by its callee + * (`Route::get`, `$router->get`, `app.get`) plus a path-shaped first + * argument. The registry still resolves the callee's bare name first, and in + * a mixed-language repo with imports it binds `get` by suffix_match to a + * same-named definition in ANOTHER language (a JS `get`, a Python `get`). + * The #725 guard rightly refuses that binding — but it dropped the whole + * call, before either resolver reached the route classification, so every + * GET registration vanished while POST (no `post` definition to collide + * with) survived: krayin/laravel-crm minted 1 of its 127 GET routes. + * The guard must keep dropping the false CALLS edge; the registration must + * survive. Each fixture also carries calls the guard must still drop. + * ══════════════════════════════════════════════════════════════════ */ + +/* Index `files` (padded onto the parallel path when `parallel`), then require + * the exact Route-name set `routes` AND zero CALLS edges into any definition + * named `get`: every `get` in these fixtures lives in a different language + * than its would-be callers, so any such edge is the binding #725 refuses. */ +static int et_xlang_routes(const EtFile *files, int nfiles, bool parallel, const char **routes) { + EtProj lp; + cbm_store_t *store = + parallel ? et_index_parallel(&lp, files, nfiles) : et_index_files(&lp, files, nfiles); + cbm_node_t *nodes = NULL; + int node_count = 0; + int wanted = 0; + int ok = store != NULL; + while (routes[wanted]) { + wanted++; + } + if (!store || cbm_store_find_nodes_by_label(store, lp.project, "Route", &nodes, &node_count) != + CBM_STORE_OK) { + ok = 0; + } else if (node_count != wanted) { + ok = 0; + } + for (int wi = 0; wi < wanted; wi++) { + int hit = 0; + for (int ni = 0; ni < node_count; ni++) { + if (nodes[ni].name && strcmp(nodes[ni].name, routes[wi]) == 0) { + hit = 1; + } + } + if (!hit) { + fprintf(stderr, " [ET-XLANG] missing Route %s\n", routes[wi]); + ok = 0; + } + } + if (!ok) { + fprintf(stderr, " [ET-XLANG] expected=%d actual=%d available:", wanted, node_count); + for (int ni = 0; ni < node_count; ni++) { + fprintf(stderr, " %s", nodes[ni].name ? nodes[ni].name : ""); + } + fprintf(stderr, "\n"); + } + int false_calls = 0; + cbm_node_t *gets = NULL; + int get_count = 0; + if (store && + cbm_store_find_nodes_by_name(store, lp.project, "get", &gets, &get_count) == CBM_STORE_OK) { + for (int gi = 0; gi < get_count; gi++) { + cbm_edge_t *in = NULL; + int in_count = 0; + if (cbm_store_find_edges_by_target_type(store, gets[gi].id, "CALLS", &in, &in_count) == + CBM_STORE_OK) { + false_calls += in_count; + cbm_store_free_edges(in, in_count); + } + } + cbm_store_free_nodes(gets, get_count); + } + if (false_calls != 0) { + fprintf(stderr, " [ET-XLANG] %d cross-language CALLS into `get`\n", false_calls); + ok = 0; + } + cbm_store_free_nodes(nodes, node_count); + et_cleanup(&lp, store); + return ok; +} + +/* Laravel: facade `Route::get` and `$router->get` beside two JS `get` + * definitions. Controls: POST is unchanged, and a bare PHP `get('/x')` (no + * route-registration callee) stays dropped — no Route, no CALLS into JS. */ +static const EtFile et_xlang_laravel[] = { + {"app/Http/Controllers/LeadController.php", "get('/reports', [LeadController::class, 'index']);\n\n" + "function report_file() { return get('/files/report'); }\n"}, + {"resources/js/http.js", "export function get(url) {\n return url;\n}\n"}, + {"resources/js/store.js", "export class Store {\n get(key) {\n return key;\n }\n}\n"}, +}; +static const char *et_xlang_laravel_routes[] = {"/leads", "/leads/store", "/reports", NULL}; + +TEST(routes_laravel_get_survives_xlang_guard) { + ASSERT_TRUE(et_xlang_routes(et_xlang_laravel, + (int)(sizeof(et_xlang_laravel) / sizeof(et_xlang_laravel[0])), + false, et_xlang_laravel_routes)); + PASS(); +} + +TEST(routes_laravel_get_survives_xlang_guard_parallel) { + ASSERT_TRUE(et_xlang_routes(et_xlang_laravel, + (int)(sizeof(et_xlang_laravel) / sizeof(et_xlang_laravel[0])), true, + et_xlang_laravel_routes)); + PASS(); +} + +/* Express: `app.get` beside two Python `get` definitions. Controls: POST is + * unchanged; a member `memo.get(key)` (route suffix, no path) and a bare + * `get('/cache/key')` stay dropped — no Route, no CALLS into Python. */ +static const EtFile et_xlang_express[] = { + {"web/server.js", "const express = require('express');\n" + "const { health } = require('./handlers');\n\n" + "const app = express();\n\n" + "app.get('/health', health);\n" + "app.post('/items', health);\n\n" + "function readCache(memo, key) {\n" + " return memo.get(key) || get('/cache/key');\n" + "}\n" + "module.exports = { readCache };\n"}, + {"web/handlers.js", + "function health(req, res) {\n return res;\n}\nmodule.exports = { health };\n"}, + {"tools/cache.py", "def get(key):\n return key\n"}, + {"tools/store.py", "class Store:\n def get(self, key):\n return key\n"}, +}; +static const char *et_xlang_express_routes[] = {"/health", "/items", NULL}; + +TEST(routes_express_get_survives_xlang_guard) { + ASSERT_TRUE(et_xlang_routes(et_xlang_express, + (int)(sizeof(et_xlang_express) / sizeof(et_xlang_express[0])), + false, et_xlang_express_routes)); + PASS(); +} + +TEST(routes_express_get_survives_xlang_guard_parallel) { + ASSERT_TRUE(et_xlang_routes(et_xlang_express, + (int)(sizeof(et_xlang_express) / sizeof(et_xlang_express[0])), true, + et_xlang_express_routes)); + PASS(); +} + /* ══════════════════════════════════════════════════════════════════ * SUITE * ══════════════════════════════════════════════════════════════════ */ @@ -1658,6 +2076,8 @@ TEST(override_go_interface) { SUITE(edge_types_probe) { /* HANDLES — route→handler across web frameworks */ RUN_TEST(calls_jsx_component_via_tsconfig_alias_parallel_issue1085); + RUN_TEST(graphql_route_keyed_by_operation_name_issue598); + RUN_TEST(graphql_operation_identity_parser_issue598); RUN_TEST(handles_flask_python); RUN_TEST(handles_fastapi_python); RUN_TEST(handles_drf_action_python); @@ -1675,6 +2095,12 @@ SUITE(edge_types_probe) { RUN_TEST(handles_laravel_php); RUN_TEST(handles_laravel_facade_routes_issue952); RUN_TEST(handles_laravel_facade_no_junk_routes_issue952); + RUN_TEST(routes_laravel_withrouting_api_default_prefix_issue1146); + RUN_TEST(routes_laravel_withrouting_api_custom_prefix_issue1146); + RUN_TEST(routes_laravel_withrouting_web_no_prefix_issue1146); + RUN_TEST(routes_laravel_routeserviceprovider_control_issue1146); + RUN_TEST(routes_laravel_withrouting_nonliteral_prefix_issue1146); + RUN_TEST(routes_laravel_withrouting_parallel_subdir_issue1146); RUN_TEST(handles_rails_ruby); RUN_TEST(handles_actix_rust); @@ -1733,4 +2159,10 @@ SUITE(edge_types_probe) { /* OVERRIDE — Go interface method override (parallel path) */ RUN_TEST(override_go_interface); + + /* Route registration vs the #725 cross-language guard (both resolvers) */ + RUN_TEST(routes_laravel_get_survives_xlang_guard); + RUN_TEST(routes_laravel_get_survives_xlang_guard_parallel); + RUN_TEST(routes_express_get_survives_xlang_guard); + RUN_TEST(routes_express_get_survives_xlang_guard_parallel); } diff --git a/tests/test_extraction.c b/tests/test_extraction.c index 636d041a34..b7763e4679 100644 --- a/tests/test_extraction.c +++ b/tests/test_extraction.c @@ -1186,6 +1186,62 @@ TEST(cpp_method_return_type_preserves_pointer_and_qualifier) { * ═══════════════════════════════════════════════════════════════════ */ /* --- Python --- */ +/* #1277: annotated Python instance fields reach result->field_types (the + * cross-file LSP's only source for them); nothing unannotated is guessed and + * no graph def is minted for a field. */ +static const char *py_field_type_of(const CBMFileResult *r, const char *cls_suffix, + const char *field) { + for (int i = 0; i < r->field_types.count; i++) { + const CBMFieldType *ft = &r->field_types.items[i]; + size_t cl = strlen(ft->class_qn); + size_t sl = strlen(cls_suffix); + if (cl >= sl && strcmp(ft->class_qn + cl - sl, cls_suffix) == 0 && + strcmp(ft->field_name, field) == 0) { + return ft->type_text; + } + } + return NULL; +} + +TEST(python_annotated_instance_fields_exported_issue1277) { + const char *src = "class Trainer(Base):\n" + " engine: Contract\n" + " limit: int = 3\n" + " plain = 5\n\n" + " def __init__(self, strategies: Contract, n, opt: 'Other' = None):\n" + " self.strategies = strategies\n" + " self.typed: Contract = strategies\n" + " self.count = n\n" + " self.made = make()\n" + " self.opt = opt\n" + " if n:\n" + " self.nested: Nested = n\n" + " def inner(other: Inner):\n" + " self.hidden = other\n\n" + " def later(self, x: Later):\n" + " self.late = x\n"; + CBMFileResult *r = extract(src, CBM_LANG_PYTHON, "t", "trainer.py"); + ASSERT_NOT_NULL(r); + ASSERT_STR_EQ(py_field_type_of(r, ".Trainer", "engine"), "Contract"); + ASSERT_STR_EQ(py_field_type_of(r, ".Trainer", "limit"), "int"); + ASSERT_STR_EQ(py_field_type_of(r, ".Trainer", "strategies"), "Contract"); + ASSERT_STR_EQ(py_field_type_of(r, ".Trainer", "typed"), "Contract"); + ASSERT_STR_EQ(py_field_type_of(r, ".Trainer", "opt"), "'Other'"); + ASSERT_STR_EQ(py_field_type_of(r, ".Trainer", "nested"), "Nested"); + /* Controls: no declared type, a nested function's binding, a non-__init__ method. */ + ASSERT_NULL(py_field_type_of(r, ".Trainer", "plain")); + ASSERT_NULL(py_field_type_of(r, ".Trainer", "count")); + ASSERT_NULL(py_field_type_of(r, ".Trainer", "made")); + ASSERT_NULL(py_field_type_of(r, ".Trainer", "hidden")); + ASSERT_NULL(py_field_type_of(r, ".Trainer", "late")); + ASSERT_EQ(r->field_types.count, 6); + for (int i = 0; i < r->defs.count; i++) { + ASSERT_FALSE(r->defs.items[i].label && strcmp(r->defs.items[i].label, "Field") == 0); + } + cbm_free_result(r); + PASS(); +} + TEST(python_function) { CBMFileResult *r = extract( "def greet(name):\n return f\"Hello {name}\"\n\ndef main():\n greet(\"World\")\n", @@ -7137,12 +7193,12 @@ TEST(extract_python_method_test_dir_marks_is_test_issue1294) { PASS(); } -/* #1017: docstring truncation at MAX_COMMENT_LEN (500 bytes) can split a +/* #1017: docstring truncation at MAX_COMMENT_LEN (500 bytes) could split a * multi-byte UTF-8 character, leaving an incomplete byte sequence. * Craft a Go comment whose 498th-500th bytes are a 3-byte CJK character - * (U+6210 = 成 = e6 88 90). The raw byte truncation at offset 500 lands - * one byte past the character start, splitting it. After the fix the - * truncated string must end on a complete codepoint boundary. */ + * (U+6210 = 成 = e6 88 90), where a raw byte cut at offset 500 lands one + * byte past the character start. Docstrings are no longer cut at all, so + * the whole comment is kept and still ends on a complete codepoint. */ TEST(docstring_utf8_truncation_boundary_issue1017) { /* Build a comment: "// " (3 bytes) + 495 ASCII 'A' + "成成成" (9 bytes) * Total comment text = 3 + 495 + 9 = 507 bytes. @@ -7183,10 +7239,10 @@ TEST(docstring_utf8_truncation_boundary_issue1017) { } ASSERT_NOT_NULL(doc); - /* Verify every byte in the truncated docstring is valid UTF-8: - * no trailing incomplete multi-byte sequence. */ + /* The whole comment, and every byte of it valid UTF-8: no trailing + * incomplete multi-byte sequence. */ + ASSERT_STR_EQ(doc, comment); size_t len = strlen(doc); - ASSERT_TRUE(len <= 500); const unsigned char *u = (const unsigned char *)doc; size_t i = 0; while (i < len) { @@ -7212,6 +7268,317 @@ TEST(docstring_utf8_truncation_boundary_issue1017) { PASS(); } +/* ═══════════════════════════════════════════════════════════════════ + * Doc comments: the whole comment run above a definition, above its + * attributes / export / declaration wrapper, never a detached or + * banner-only run, and never cut at 500 bytes. + * ═══════════════════════════════════════════════════════════════════ */ + +/* The def with this label and name, or NULL. */ +static const CBMDefinition *doc_def(CBMFileResult *r, const char *label, const char *name) { + for (int i = 0; i < r->defs.count; i++) { + const CBMDefinition *d = &r->defs.items[i]; + if (d->label && d->name && strcmp(d->label, label) == 0 && strcmp(d->name, name) == 0) { + return d; + } + } + return NULL; +} + +TEST(doc_go_comment_group_without_directives) { + CBMFileResult *r = extract("package knob\n\n" + "// Turn rotates the knob by one step.\n" + "// It is safe for concurrent use.\n" + "//go:noinline\n" + "func Turn() {}\n", + CBM_LANG_GO, "t", "knob.go"); + ASSERT_NOT_NULL(r); + const CBMDefinition *d = doc_def(r, "Function", "Turn"); + ASSERT_NOT_NULL(d); + /* go/ast CommentGroup.Text: the whole group, the //go: directive dropped. */ + ASSERT_STR_EQ(d->docstring, "// Turn rotates the knob by one step.\n" + "// It is safe for concurrent use."); + cbm_free_result(r); + PASS(); +} + +TEST(doc_go_struct_field) { + CBMFileResult *r = extract("package knob\n\n" + "// Knob is a rotary control.\n" + "type Knob struct {\n" + "\t// Steps counts the turns so far.\n" + "\tSteps int\n" + "}\n", + CBM_LANG_GO, "t", "knob.go"); + ASSERT_NOT_NULL(r); + const CBMDefinition *knob = doc_def(r, "Struct", "Knob"); + ASSERT_NOT_NULL(knob); + ASSERT_STR_EQ(knob->docstring, "// Knob is a rotary control."); + const CBMDefinition *steps = doc_def(r, "Field", "Steps"); + ASSERT_NOT_NULL(steps); + ASSERT_STR_EQ(steps->docstring, "// Steps counts the turns so far."); + cbm_free_result(r); + PASS(); +} + +TEST(doc_go_detached_comment_is_not_doc) { + /* go/doc: a comment group separated by a blank line documents nothing. */ + CBMFileResult *r = extract("package knob\n\n" + "// Section: helpers.\n" + "\n" + "func Reset() {}\n", + CBM_LANG_GO, "t", "knob.go"); + ASSERT_NOT_NULL(r); + const CBMDefinition *d = doc_def(r, "Function", "Reset"); + ASSERT_NOT_NULL(d); + ASSERT_NULL(d->docstring); + cbm_free_result(r); + PASS(); +} + +TEST(doc_python_comment_run_and_detached) { + CBMFileResult *r = extract("# Turns the knob.\n" + "# One step per call.\n" + "def turn():\n" + " pass\n" + "\n" + "# Section: helpers.\n" + "\n" + "def reset():\n" + " pass\n", + CBM_LANG_PYTHON, "t", "knob.py"); + ASSERT_NOT_NULL(r); + const CBMDefinition *turn = doc_def(r, "Function", "turn"); + ASSERT_NOT_NULL(turn); + ASSERT_STR_EQ(turn->docstring, "# Turns the knob.\n# One step per call."); + const CBMDefinition *reset = doc_def(r, "Function", "reset"); + ASSERT_NOT_NULL(reset); + ASSERT_NULL(reset->docstring); + cbm_free_result(r); + PASS(); +} + +TEST(doc_python_docstring_beats_comment_run) { + /* A `#` comment run above a def must not replace the def's own docstring + * (see extract_docstring): the in-body string is the documented API. */ + CBMFileResult *r = extract("# TODO: handle the zero case.\n" + "# noqa: E501\n" + "def scale(x):\n" + " \"\"\"Multiply x by the knob factor.\"\"\"\n" + " return x\n", + CBM_LANG_PYTHON, "t", "knob.py"); + ASSERT_NOT_NULL(r); + const CBMDefinition *scale = doc_def(r, "Function", "scale"); + ASSERT_NOT_NULL(scale); + ASSERT_NOT_NULL(scale->docstring); + ASSERT(strstr(scale->docstring, "Multiply x by the knob factor") != NULL); + ASSERT(strstr(scale->docstring, "TODO") == NULL); + cbm_free_result(r); + PASS(); +} + +TEST(doc_rust_attribute_impl_method_and_inner_doc) { + CBMFileResult *r = extract("//! The knob crate.\n" + "\n" + "/// Adds two steps.\n" + "/// Wraps on overflow.\n" + "#[inline]\n" + "pub fn add(a: u8, b: u8) -> u8 { a.wrapping_add(b) }\n" + "\n" + "pub struct Knob;\n" + "\n" + "impl Knob {\n" + " /// Turns the knob once.\n" + " pub fn turn(&self) {}\n" + "}\n", + CBM_LANG_RUST, "t", "lib.rs"); + ASSERT_NOT_NULL(r); + /* The doc sits above the attribute; a run is joined line by line. */ + const CBMDefinition *add = doc_def(r, "Function", "add"); + ASSERT_NOT_NULL(add); + ASSERT_STR_EQ(add->docstring, "/// Adds two steps.\n/// Wraps on overflow."); + /* impl methods get their doc too; a single comment keeps its bytes. */ + const CBMDefinition *turn = doc_def(r, "Method", "turn"); + ASSERT_NOT_NULL(turn); + ASSERT_STR_EQ(turn->docstring, "/// Turns the knob once.\n"); + /* The //! inner doc documents the crate (the File node), not the struct. */ + const CBMDefinition *knob = doc_def(r, "Struct", "Knob"); + ASSERT_NOT_NULL(knob); + ASSERT_NULL(knob->docstring); + cbm_free_result(r); + PASS(); +} + +TEST(doc_ts_export_and_arrow_wrapper) { + CBMFileResult *r = extract("/** Exported helper. */\n" + "export function helper(): number {\n" + " return 1;\n" + "}\n" + "\n" + "/** Arrow helper. */\n" + "export const arrow = (): number => 2;\n", + CBM_LANG_TYPESCRIPT, "t", "knob.ts"); + ASSERT_NOT_NULL(r); + const CBMDefinition *helper = doc_def(r, "Function", "helper"); + ASSERT_NOT_NULL(helper); + ASSERT_STR_EQ(helper->docstring, "/** Exported helper. */"); + const CBMDefinition *arrow = doc_def(r, "Function", "arrow"); + ASSERT_NOT_NULL(arrow); + ASSERT_STR_EQ(arrow->docstring, "/** Arrow helper. */"); + cbm_free_result(r); + PASS(); +} + +TEST(doc_csharp_xml_doc_run_and_top_level_function) { + CBMFileResult *r = extract("/// Adds one.\n" + "static int AddOne(int x) => x + 1;\n" + "\n" + "namespace Knobs {\n" + " public class Knob {\n" + " /// \n" + " /// Turns the knob.\n" + " /// \n" + " public void Turn() {}\n" + " }\n" + "}\n", + CBM_LANG_CSHARP, "t", "Knob.cs"); + ASSERT_NOT_NULL(r); + const CBMDefinition *turn = doc_def(r, "Method", "Turn"); + ASSERT_NOT_NULL(turn); + ASSERT_STR_EQ(turn->docstring, "/// \n/// Turns the knob.\n/// "); + const CBMDefinition *add = doc_def(r, "Function", "AddOne"); + ASSERT_NOT_NULL(add); + ASSERT_STR_EQ(add->docstring, "/// Adds one."); + cbm_free_result(r); + PASS(); +} + +TEST(doc_java_javadoc_over_line_comment_and_field) { + CBMFileResult *r = extract("class Knob {\n" + " /** Turns taken so far. */\n" + " private int steps;\n" + "\n" + " /** Turns the knob. */\n" + " // NOSONAR\n" + " void turn() {}\n" + "}\n", + CBM_LANG_JAVA, "t", "Knob.java"); + ASSERT_NOT_NULL(r); + /* A plain comment between the Javadoc and the method does not replace it. */ + const CBMDefinition *turn = doc_def(r, "Method", "turn"); + ASSERT_NOT_NULL(turn); + ASSERT_STR_EQ(turn->docstring, "/** Turns the knob. */"); + const CBMDefinition *steps = doc_def(r, "Field", "steps"); + ASSERT_NOT_NULL(steps); + ASSERT_STR_EQ(steps->docstring, "/** Turns taken so far. */"); + cbm_free_result(r); + PASS(); +} + +TEST(doc_kotlin_kdoc_folded_after_imports) { + /* tree-sitter-kotlin folds this KDoc into the node before the class, so it + * is no sibling of it. */ + CBMFileResult *r = extract("package knobs\n" + "\n" + "import knobs.core.Step\n" + "import knobs.core.Turn\n" + "\n" + "/** Base class of every knob. */\n" + "abstract class BaseKnob(step: Step) : Turn(step)\n", + CBM_LANG_KOTLIN, "t", "BaseKnob.kt"); + ASSERT_NOT_NULL(r); + const CBMDefinition *d = doc_def(r, "Class", "BaseKnob"); + ASSERT_NOT_NULL(d); + ASSERT_STR_EQ(d->docstring, "/** Base class of every knob. */"); + cbm_free_result(r); + PASS(); +} + +TEST(doc_c_typedef_struct) { + CBMFileResult *r = extract("/* A file event. */\n" + "typedef struct aeFileEvent {\n" + " int mask;\n" + "} aeFileEvent;\n", + CBM_LANG_C, "t", "ae.h"); + ASSERT_NOT_NULL(r); + const CBMDefinition *d = doc_def(r, "Class", "aeFileEvent"); + ASSERT_NOT_NULL(d); + ASSERT_STR_EQ(d->docstring, "/* A file event. */"); + cbm_free_result(r); + PASS(); +} + +TEST(doc_perl_pod_section_by_sub_name) { + CBMFileResult *r = extract("package Knob;\n" + "\n" + "sub turn { return 1; }\n" + "\n" + "1;\n" + "__END__\n" + "\n" + "=head1 METHODS\n" + "\n" + "=head2 turn\n" + "\n" + "Turns the knob once.\n" + "\n" + "=cut\n", + CBM_LANG_PERL, "t", "Knob.pm"); + ASSERT_NOT_NULL(r); + const CBMDefinition *d = doc_def(r, "Function", "turn"); + ASSERT_NOT_NULL(d); + ASSERT_STR_EQ(d->docstring, "=head2 turn\n\nTurns the knob once."); + cbm_free_result(r); + PASS(); +} + +TEST(doc_long_comment_kept_whole) { + /* One 12-line block comment well past the old 500-byte cut, kept byte for + * byte. */ + char src[4096]; + char want[2048]; + int w = snprintf(want, sizeof(want), "/*"); + for (int i = 0; i < 12; i++) { + w += snprintf(want + w, sizeof(want) - (size_t)w, + "\n * Line %02d of a long doc that keeps going past the cut.", i); + } + snprintf(want + w, sizeof(want) - (size_t)w, "\n */"); + snprintf(src, sizeof(src), "package knob\n\n%s\nfunc Long() {}\n", want); + ASSERT_GT((int)strlen(want), 500); + + CBMFileResult *r = extract(src, CBM_LANG_GO, "t", "knob.go"); + ASSERT_NOT_NULL(r); + const CBMDefinition *d = doc_def(r, "Function", "Long"); + ASSERT_NOT_NULL(d); + ASSERT_STR_EQ(d->docstring, want); + cbm_free_result(r); + PASS(); +} + +TEST(doc_divider_only_run_is_not_doc) { + CBMFileResult *r = extract("/*****************************/\n" + "void knob(void) {}\n" + "\n" + "// \xe2\x94\x80\xe2\x94\x80\xe2\x94\x80\xe2\x94\x80\xe2\x94\x80\n" + "void dial(void) {}\n" + "\n" + "/* Resets the dial. */\n" + "void reset(void) {}\n", + CBM_LANG_C, "t", "knob.c"); + ASSERT_NOT_NULL(r); + const CBMDefinition *knob = doc_def(r, "Function", "knob"); + ASSERT_NOT_NULL(knob); + ASSERT_NULL(knob->docstring); + const CBMDefinition *dial = doc_def(r, "Function", "dial"); /* a box-drawing banner */ + ASSERT_NOT_NULL(dial); + ASSERT_NULL(dial->docstring); + const CBMDefinition *reset = doc_def(r, "Function", "reset"); + ASSERT_NOT_NULL(reset); + ASSERT_STR_EQ(reset->docstring, "/* Resets the dial. */"); + cbm_free_result(r); + PASS(); +} + /* Reproduce-first (ms-typescript reallyLargeFile.ts, 2026-07-07): a file * whose root node has hundreds of thousands of FLAT SIBLINGS (580k //// * comment lines in the 3.5 MB fourslash fixture) hung extraction for over @@ -9066,6 +9433,7 @@ SUITE(extraction) { /* Scripting */ RUN_TEST(python_function); RUN_TEST(python_class); + RUN_TEST(python_annotated_instance_fields_exported_issue1277); RUN_TEST(js_function); RUN_TEST(js_class); RUN_TEST(ts_function); @@ -9370,6 +9738,20 @@ SUITE(extraction) { RUN_TEST(extract_c_test_dir_marks_is_test_issue1294); RUN_TEST(extract_python_method_test_dir_marks_is_test_issue1294); RUN_TEST(docstring_utf8_truncation_boundary_issue1017); + RUN_TEST(doc_go_comment_group_without_directives); + RUN_TEST(doc_go_struct_field); + RUN_TEST(doc_go_detached_comment_is_not_doc); + RUN_TEST(doc_python_comment_run_and_detached); + RUN_TEST(doc_python_docstring_beats_comment_run); + RUN_TEST(doc_rust_attribute_impl_method_and_inner_doc); + RUN_TEST(doc_ts_export_and_arrow_wrapper); + RUN_TEST(doc_csharp_xml_doc_run_and_top_level_function); + RUN_TEST(doc_java_javadoc_over_line_comment_and_field); + RUN_TEST(doc_kotlin_kdoc_folded_after_imports); + RUN_TEST(doc_c_typedef_struct); + RUN_TEST(doc_perl_pod_section_by_sub_name); + RUN_TEST(doc_long_comment_kept_whole); + RUN_TEST(doc_divider_only_run_is_not_doc); RUN_TEST(extract_ts_decorators_survive_interleaved_comment); /* #518/#519 — prose carried into docstring so nodes_fts can index it */ diff --git a/tests/test_incremental.c b/tests/test_incremental.c index 97fc9cb1c4..77447016ab 100644 --- a/tests/test_incremental.c +++ b/tests/test_incremental.c @@ -16,10 +16,14 @@ #include "test_operation_host.h" #include #include +#include #include #include #include +#include +#include +#include #include #include #include @@ -546,6 +550,85 @@ TEST(incr_formatter_run) { PASS(); } +static char g_purge_log[8192]; +static int g_purge_progress_lines; + +static void purge_log_sink(const char *line) { + if (!line || !strstr(line, "msg=incremental.purge")) { + return; + } + if (strstr(line, "msg=incremental.purge.progress")) { + g_purge_progress_lines++; + } + size_t used = strlen(g_purge_log); + (void)snprintf(g_purge_log + used, sizeof(g_purge_log) - used, "%s\n", line); +} + +/* #1300: the per-file purge after incremental.edge_snapshot logged nothing + * until it finished, so a stall there left no trace of where the run was. It + * now announces its counts and reports bounded progress — counts only, at + * most 20 progress lines however many files, ending at files_done == + * files_total. */ +TEST(incr_purge_reports_bounded_progress_issue1300) { + char *resp = index_repo(); + ASSERT(resp != NULL); + free(resp); + reformat_files("fastapi", 30); + /* The loop instrumented for #1300 sits on the legacy partial route (the + * production closure route purges in SQL and logs delta.snapshot_purge). */ + cbm_pipeline_incremental_test_force_legacy_partial_once(); + + CBMLogLevel saved_level = cbm_log_get_level(); + CBMLogFormat saved_format = cbm_log_get_format(); + cbm_log_set_level(CBM_LOG_INFO); + cbm_log_set_format(CBM_LOG_FORMAT_TEXT); + g_purge_log[0] = '\0'; + g_purge_progress_lines = 0; + cbm_log_set_sink_ex(purge_log_sink, CBM_LOG_SINK_TEE); + resp = index_repo(); + cbm_log_set_sink(NULL); + cbm_log_set_level(saved_level); + cbm_log_set_format(saved_format); + ASSERT(resp != NULL); + ASSERT(strstr(resp, "indexed") != NULL); + free(resp); + ASSERT_EQ(cbm_pipeline_incremental_test_last_route(), CBM_INCREMENTAL_ROUTE_LEGACY_PARTIAL); + + const char *start = strstr(g_purge_log, "msg=incremental.purge.start"); + ASSERT(start != NULL); + int changed = -1; + int deleted = -1; + const char *changed_at = strstr(start, "changed="); + const char *deleted_at = strstr(start, "deleted="); + ASSERT(changed_at != NULL && deleted_at != NULL); + const char *changed_val = changed_at + strlen("changed="); + const char *deleted_val = deleted_at + strlen("deleted="); + char *changed_end = NULL; + char *deleted_end = NULL; + errno = 0; + long parsed_changed = strtol(changed_val, &changed_end, 10); + ASSERT(errno == 0 && changed_end != changed_val && parsed_changed >= 0 && + parsed_changed <= INT_MAX && + (*changed_end == '\0' || isspace((unsigned char)*changed_end))); + errno = 0; + long parsed_deleted = strtol(deleted_val, &deleted_end, 10); + ASSERT(errno == 0 && deleted_end != deleted_val && parsed_deleted >= 0 && + parsed_deleted <= INT_MAX - parsed_changed && + (*deleted_end == '\0' || isspace((unsigned char)*deleted_end))); + changed = (int)parsed_changed; + deleted = (int)parsed_deleted; + int total = changed + deleted; + ASSERT_GT(total, 20); /* enough files that one line per file would exceed the bound */ + ASSERT_GT(g_purge_progress_lines, 0); + ASSERT(g_purge_progress_lines <= 21); + ASSERT(g_purge_progress_lines < total); /* never one line per file */ + char last[64]; + (void)snprintf(last, sizeof(last), "files_done=%d files_total=%d", total, total); + ASSERT(strstr(g_purge_log, last) != NULL); + ASSERT(strstr(g_purge_log, "msg=incremental.purge elapsed_ms=") != NULL); + PASS(); +} + TEST(incr_add_file) { int nodes_before = get_node_count(); @@ -3406,6 +3489,9 @@ SUITE(incremental) { RUN_TEST(tool_err_ingest_bad_project); RUN_TEST(tool_err_ingest_no_traces); + /* #1300: legacy-partial purge progress (the full rebuild below resets the graph) */ + RUN_TEST(incr_purge_reports_bounded_progress_issue1300); + /* Phase 40: index_repository mode=full + arch empty aspects */ RUN_TEST(tool_index_mode_full); RUN_TEST(tool_arch_empty_aspects); diff --git a/tests/test_index_resilience.c b/tests/test_index_resilience.c index a0e4440558..59d7a70807 100644 --- a/tests/test_index_resilience.c +++ b/tests/test_index_resilience.c @@ -279,6 +279,123 @@ TEST(index_clean_run_no_logfile) { PASS(); } +/* INV(long-log-override): CBM_INDEX_LOG was copied into the 1 KiB logfile path + * with no length check and opened "wb", so an override longer than that + * created (or truncated) a file at its cut-off prefix and echoed the cut name + * as the run's logfile. A path that does not fit is no logfile: the index still + * completes and reports the skip, nothing is written at the cut name (its + * directory stays empty), and the response names no logfile. */ +TEST(index_long_log_override_writes_no_cut_path) { + enum { RI_PATH_CAP = 1024, RI_DIR_SEGMENTS = 4, RI_SEGMENT = 200, RI_NAME = 255 }; + RProj lp; + memset(&lp, 0, sizeof(lp)); + snprintf(lp.tmpdir, sizeof(lp.tmpdir), "/tmp/cbm_resil_XXXXXX"); + if (!cbm_mkdtemp(lp.tmpdir)) { + FAIL("mkdtemp failed"); + } + rh_to_fwd_slashes(lp.tmpdir); + ri_write_text(lp.tmpdir, "good.py", "def alpha():\n return 1\n"); + ri_write_big(lp.tmpdir, "big.py", (size_t)2 * 1024 * 1024); /* ~2 MiB > 1 MiB cap */ + + /* Outside the indexed repo: /ddd…(x4) is a real directory, and the + * override adds a 255-byte file name, so the whole path runs past 1 KiB and + * its first 1023 bytes end inside the file name, in that directory. The + * root has its links resolved: the kernel bounds the EXPANDED path, and + * under macOS's /tmp -> /private/tmp link a near-1 KiB path can be neither + * created nor probed, which would hide the cut-off file. */ + /* cbm_mkdtemp writes the %TEMP%-expanded path back on Windows, so the + * template buffer needs the 256 bytes that compat.h asks for. */ + char tmpl[256]; + char logroot[4096]; + snprintf(tmpl, sizeof(tmpl), "/tmp/cbm_resil_log_XXXXXX"); + if (!cbm_mkdtemp(tmpl)) { + rh_cleanup(&lp, NULL); + FAIL("mkdtemp failed"); + } + if (!cbm_canonical_path(tmpl, logroot, sizeof(logroot))) { + th_rmtree(tmpl); + rh_cleanup(&lp, NULL); + FAIL("canonical log root failed"); + } + rh_to_fwd_slashes(logroot); + char logdir[RI_PATH_CAP]; + int root_len = snprintf(logdir, sizeof(logdir), "%s", logroot); + if (root_len <= 0 || + (size_t)root_len + RI_DIR_SEGMENTS * (RI_SEGMENT + 1) + 2 >= sizeof(logdir)) { + th_rmtree(logroot); + rh_cleanup(&lp, NULL); + FAIL("log root too long for the fixture"); + } + size_t at = (size_t)root_len; + for (int s = 0; s < RI_DIR_SEGMENTS; s++) { + logdir[at++] = '/'; + memset(logdir + at, 'd', RI_SEGMENT); + at += RI_SEGMENT; + } + logdir[at] = '\0'; + char name[RI_NAME + 1]; + memset(name, 'x', RI_NAME); + memcpy(name, "skip-", 5); + name[RI_NAME] = '\0'; + char override[2 * RI_PATH_CAP]; + snprintf(override, sizeof(override), "%s/%s", logdir, name); + char cut[RI_PATH_CAP]; + snprintf(cut, sizeof(cut), "%.*s", RI_PATH_CAP - 1, override); + bool made = cbm_mkdir_p(logdir, 0755) && strlen(override) >= RI_PATH_CAP && + strlen(cut) > strlen(logdir) + 1; + + cbm_setenv("CBM_MAX_FILE_BYTES", "1048576", 1); /* 1 MiB cap */ + cbm_setenv("CBM_INDEX_LOG", override, 1); + char *resp = NULL; + cbm_store_t *store = made ? ri_index_capture(&lp, &resp) : NULL; + cbm_unsetenv("CBM_MAX_FILE_BYTES"); + cbm_unsetenv("CBM_INDEX_LOG"); + + bool cut_exists = cbm_file_exists(cut); + int entries = 0; + cbm_dir_t *d = cbm_opendir(logdir); + cbm_dirent_t *entry; + while (d && (entry = cbm_readdir(d)) != NULL) { + if (strcmp(entry->name, ".") != 0 && strcmp(entry->name, "..") != 0) { + entries++; + } + } + if (d) { + cbm_closedir(d); + } + th_rmtree(logroot); + + if (!made) { + /* ASSERT_TRUE would return before the fixture is removed and leak the + * temporary project directory (store/resp are NULL here). */ + rh_cleanup(&lp, store); + FAIL("log override fixture setup failed"); + } + if (!resp) { + rh_cleanup(&lp, store); + FAIL("no operation response"); + } + yyjson_doc *doc = yyjson_read(resp, strlen(resp), 0); + ASSERT_NOT_NULL(doc); + yyjson_val *sc = yyjson_doc_get_root(doc); + ASSERT_NOT_NULL(sc); + const char *status = yyjson_get_str(yyjson_obj_get(sc, "status")); + ASSERT_NOT_NULL(status); + ASSERT_STR_EQ("indexed", status); + ASSERT_GTE(yyjson_get_int(yyjson_obj_get(sc, "skipped_count")), 1); + if (cut_exists) { + printf(" skip log written at the cut-off name ...%s\n", cut + strlen(cut) - 40); + } + ASSERT_FALSE(cut_exists); + ASSERT_EQ(entries, 0); + ASSERT_NULL(yyjson_obj_get(sc, "logfile")); + + yyjson_doc_free(doc); + free(resp); + rh_cleanup(&lp, store); + PASS(); +} + /* INV(parse-partial-reported, #963): a file whose parse tree contains * ERROR/MISSING regions (here: the preprocessor-blind #ifdef-split-brace C * pattern) is INDEXED — not skipped — and the best-effort coverage signal @@ -521,7 +638,7 @@ TEST(index_parse_unusable_names_the_range_end) { char *resp = NULL; cbm_store_t *store = ri_index_capture(&lp, &resp); if (!resp) { - FAIL("no MCP response"); + FAIL("no operation response"); } if (!store) { free(resp); @@ -977,6 +1094,7 @@ TEST(index_metrics_out_written) { SUITE(index_resilience) { RUN_TEST(index_oversized_file_reported); RUN_TEST(index_clean_run_no_logfile); + RUN_TEST(index_long_log_override_writes_no_cut_path); RUN_TEST(index_parse_partial_reported); RUN_TEST(index_parse_partial_clears_on_fix); RUN_TEST(index_parse_unusable_names_the_range_end); diff --git a/tests/test_index_supervisor.c b/tests/test_index_supervisor.c index 61e5f67916..52734c4982 100644 --- a/tests/test_index_supervisor.c +++ b/tests/test_index_supervisor.c @@ -10,6 +10,8 @@ #include "foundation/platform.h" #include "foundation/profile.h" #include "operations/index_supervisor.h" +#include "foundation/constants.h" +#include "operations/index.h" #include #include @@ -838,6 +840,87 @@ TEST(index_supervisor_oversized_response_is_contained_and_log_is_retained) { PASS(); } +static char g_index_supervisor_no_response_log[CBM_SZ_2K]; + +static void index_supervisor_test_no_response_sink(const char *line) { + if (strstr(line, "index.supervisor.no_response")) { + (void)snprintf(g_index_supervisor_no_response_log, + sizeof(g_index_supervisor_no_response_log), "%s", line); + } +} + +/* #1300: a worker that exits 0 without writing its response. The supervisor + * pre-creates the response file, so the old read returned "" and the run was + * a SUCCESS with an empty response; the clean-exit branch then deleted the + * worker log, the only record of where the run stopped. It must instead be a + * named failure: no response, the log retained, and the last phase the log + * recorded (incremental.edge_snapshot, not the plain stderr line after it). */ +static int index_supervisor_no_response_case(bool unreadable) { + char cache[INDEX_SUPERVISOR_TEST_PATH_CAP]; + (void)snprintf(cache, sizeof(cache), "%s/cbm-index-silent-XXXXXX", cbm_tmpdir()); + ASSERT_NOT_NULL(cbm_mkdtemp(cache)); + const char *old_cache = getenv("CBM_CACHE_DIR"); + char *saved_cache = old_cache ? cbm_strdup(old_cache) : NULL; + (void)cbm_setenv("CBM_CACHE_DIR", cache, 1); + CBMLogLevel saved_level = cbm_log_get_level(); + CBMLogFormat saved_format = cbm_log_get_format(); + cbm_log_set_level(CBM_LOG_INFO); + cbm_log_set_format(CBM_LOG_FORMAT_TEXT); + g_index_supervisor_no_response_log[0] = '\0'; + cbm_log_set_sink_ex(index_supervisor_test_no_response_sink, CBM_LOG_SINK_TEE); + + cbm_index_worker_handle_t *handle = NULL; + const char *args = unreadable ? "{\"__cbm_test_worker\":\"unreadable-response\"}" + : "{\"__cbm_test_worker\":\"silent-exit\"}"; + int start_rc = cbm_index_worker_start(args, 0, false, NULL, NULL, &handle); + char log_path[INDEX_SUPERVISOR_TEST_PATH_CAP] = {0}; + if (handle) { + (void)snprintf(log_path, sizeof(log_path), "%s", cbm_index_worker_log_path(handle)); + } + const cbm_index_worker_result_t *result = NULL; + bool terminal = handle && index_supervisor_test_poll_terminal( + handle, INDEX_SUPERVISOR_TEST_TERMINAL_MS, &result); + bool clean_exit = terminal && result && result->outcome == CBM_PROC_CLEAN && + result->exit_code == 0 && result->tree_quiesced; + bool no_response = terminal && result && result->response == NULL; + bool named = terminal && result && result->response_missing && + result->response_unreadable == unreadable && + strcmp(result->last_phase, "incremental.edge_snapshot") == 0 && + strcmp(result->worker_log, log_path) == 0; + bool not_success = + terminal && result && + cbm_index_supervised_result_disposition(0, result) != CBM_INDEX_SUPERVISED_RESULT_SUCCESS; + bool log_retained = log_path[0] && cbm_file_size(log_path) > 0; + if (terminal) { + cbm_index_worker_destroy(handle); + } else { + index_supervisor_test_dump("silent-exit worker log", log_path); + index_supervisor_test_cleanup_handle(handle); + } + cbm_log_set_sink(NULL); + cbm_log_set_level(saved_level); + cbm_log_set_format(saved_format); + bool event_named = + strstr(g_index_supervisor_no_response_log, + unreadable ? "reason=response_read_error" : "reason=worker_did_not_write") != NULL && + strstr(g_index_supervisor_no_response_log, "last_phase=incremental.edge_snapshot") != + NULL && + log_path[0] && strstr(g_index_supervisor_no_response_log, log_path) != NULL; + (void)cbm_unlink(log_path); + index_supervisor_test_restore_env("CBM_CACHE_DIR", saved_cache); + (void)th_rmtree(cache); + + ASSERT_EQ(start_rc, 0); + ASSERT_TRUE(terminal); + ASSERT_TRUE(clean_exit); + ASSERT_TRUE(no_response); + ASSERT_TRUE(not_success); + ASSERT_TRUE(named); + ASSERT_TRUE(log_retained); + ASSERT_TRUE(event_named); + PASS(); +} + /* #1070, #1130, #1132, #1133, #1145, #1450: six reports of an indexing worker * that died leaving "the worker log file is completely empty (0 KB)". Nothing * was ever flushed, so not one of them is reproducible or attributable — the @@ -941,6 +1024,14 @@ TEST(index_supervisor_job_memory_limit_has_floor_headroom_and_no_overflow) { PASS(); } +TEST(index_supervisor_clean_exit_without_response_is_named_failure_issue1300) { + return index_supervisor_no_response_case(false); +} + +TEST(index_supervisor_unreadable_response_reports_supervisor_error) { + return index_supervisor_no_response_case(true); +} + SUITE(index_supervisor) { RUN_TEST(index_supervisor_job_memory_limit_has_floor_headroom_and_no_overflow); RUN_TEST(index_supervisor_worker_argv_requires_exact_build_bound_grammar); @@ -950,5 +1041,7 @@ SUITE(index_supervisor) { RUN_TEST(index_supervisor_worker_keeps_default_info_liveness_heartbeat); RUN_TEST(index_supervisor_drains_terminal_backlog_into_request_progress_callback); RUN_TEST(index_supervisor_oversized_response_is_contained_and_log_is_retained); + RUN_TEST(index_supervisor_clean_exit_without_response_is_named_failure_issue1300); + RUN_TEST(index_supervisor_unreadable_response_reports_supervisor_error); RUN_TEST(index_supervisor_killed_worker_log_is_never_empty_and_names_the_run); } diff --git a/tests/test_infrascan.c b/tests/test_infrascan.c index a9bf1aef94..449998f028 100644 --- a/tests/test_infrascan.c +++ b/tests/test_infrascan.c @@ -113,6 +113,8 @@ TEST(infrascan_service_pattern_match_rejects_ids_inside_words) { {"gqlalchemy.Memgraph.execute", CBM_SVC_NONE}, {"celeryconfig.broker_url", CBM_SVC_NONE}, {"kafkaesque.story.tell", CBM_SVC_NONE}, + /* "Axios" glued to a lowercase continuation */ + {"geo.Axiosphere.map", CBM_SVC_NONE}, {NULL, CBM_SVC_NONE}, }; ASSERT_EQ(svc_case_mismatches(cases), 0); @@ -136,6 +138,11 @@ TEST(infrascan_service_pattern_match_keeps_real_library_qns) { {"package:dio/dio.dart", CBM_SVC_HTTP}, {"dio.Dio.get", CBM_SVC_HTTP}, {"pkg.net.curl.get", CBM_SVC_HTTP}, + /* axios under a capitalized default-import binding + * (`import Axios from "axios"`) and axios's own instance type */ + {"Axios.get", CBM_SVC_HTTP}, + {"Axios.post", CBM_SVC_HTTP}, + {"AxiosInstance.get", CBM_SVC_HTTP}, {"curl_exec", CBM_SVC_HTTP}, {"surf.get", CBM_SVC_HTTP}, {"hyper.Client.request", CBM_SVC_HTTP}, @@ -225,6 +232,25 @@ TEST(infrascan_service_pattern_match_keeps_real_library_qns) { PASS(); } +/* Document/log extensions are never HTTP routes (distilled from PR #1245): + * the arg-URL detector runs for every resolved call, so without the + * extension guard `open("/new/file.txt")` minted a Route + HTTP_CALLS. + * First segments deliberately avoid the filesystem-root list so ONLY the + * extension guard can reject them; `/api/...` marker paths stay routes. */ +TEST(infrascan_http_route_literal_guard_rejects_document_extensions) { + ASSERT_FALSE(cbm_service_pattern_is_http_route_literal("/new/file.txt", "open")); + ASSERT_FALSE(cbm_service_pattern_is_http_route_literal("/docs/guide.md", "read")); + ASSERT_FALSE(cbm_service_pattern_is_http_route_literal("/data/app.log", "write")); + ASSERT_FALSE(cbm_service_pattern_is_http_route_literal("/fake/path.pdf", "open")); + ASSERT_FALSE(cbm_service_pattern_is_http_route_literal("/docs/index.rst", "open")); + /* Same extensions under a query string are still files. */ + ASSERT_FALSE(cbm_service_pattern_is_http_route_literal("/docs/guide.md?raw=1", "open")); + /* Positive controls: extension-less routes and the same callees. */ + ASSERT_TRUE(cbm_service_pattern_is_http_route_literal("/api/items", "requests.get")); + ASSERT_TRUE(cbm_service_pattern_is_http_route_literal("/docs/guide", "open")); + PASS(); +} + TEST(infrascan_route_nodes_skip_bad_http_url_paths) { cbm_gbuf_t *gb = cbm_gbuf_new("test", "/tmp/cbm_infrascan_route_guard"); ASSERT_NOT_NULL(gb); @@ -305,6 +331,7 @@ TEST(infrascan_http_calls_join_matching_handler_route) { SUITE(infrascan) { RUN_TEST(infrascan_http_route_literal_guard_rejects_filesystem_paths); + RUN_TEST(infrascan_http_route_literal_guard_rejects_document_extensions); RUN_TEST(infrascan_service_pattern_match_rejects_ids_inside_words); RUN_TEST(infrascan_service_pattern_match_keeps_real_library_qns); RUN_TEST(infrascan_route_nodes_skip_bad_http_url_paths); diff --git a/tests/test_integration.c b/tests/test_integration.c index 6d946534a7..ea82c0ee60 100644 --- a/tests/test_integration.c +++ b/tests/test_integration.c @@ -1,3 +1,6 @@ +#include "foundation/subprocess.h" +#include "foundation/mem_core.h" +#include "callable_sig.h" /* * test_integration.c — End-to-end integration tests for the pure C pipeline. * @@ -14,7 +17,6 @@ #include "operations/output_budget.h" #include "operations/result_wire.h" #include "cbm.h" -#include "foundation/mem_core.h" #include #include #include @@ -1579,6 +1581,319 @@ TEST(integ_coverage_freshness_uses_indexer_mtime_source_issue1714) { PASS(); } +/* Step-5 ports: direct operations over real stores and source files. */ +static char *step5_execute(const char *operation, const char *extra) { + char args[CBM_SZ_4K]; + snprintf(args, sizeof(args), "{\"project\":\"%s\"%s}", g_project, extra ? extra : ""); + return cbm_test_operation_execute(g_srv, operation, args); +} + +TEST(integ_step5_search_filters_and_integer_bounds) { + static const char *filters[] = { + ",\"file_pattern\":\"missing-step5/*\"", + ",\"qn_pattern\":\"missing-step5\"", + ",\"relationship\":\"IMPORTS\",\"name_pattern\":\"greet\"", + ",\"min_degree\":2147483647", + }; + for (size_t i = 0; i < sizeof(filters) / sizeof(filters[0]); i++) { + char *response = step5_execute("search_graph", filters[i]); + ASSERT_NOT_NULL(response); + ASSERT_NOT_NULL(strstr(response, "No results match the current filters")); + free(response); + } + static const char *integers[] = {"4294967297", "-4294967297", "18446744073709551615"}; + char *expected = step5_execute("search_graph", ",\"label\":\"Function\",\"format\":\"json\""); + ASSERT_NOT_NULL(expected); + for (size_t i = 0; i < sizeof(integers) / sizeof(integers[0]); i++) { + char extra[256]; + snprintf(extra, sizeof(extra), ",\"label\":\"Function\",\"format\":\"json\",\"limit\":%s", + integers[i]); + char *response = step5_execute("search_graph", extra); + ASSERT_NOT_NULL(response); + ASSERT_STR_EQ(response, expected); + free(response); + } + free(expected); + PASS(); +} + +TEST(integ_step5_search_regex_errors_and_cleanup) { + size_t before = cbm_mem_class_live_blocks(CBM_MEM_CLASS_OPERATION_ARG); + static const char *bad[] = { + ",\"path_filter\":\".*\"", /* missing pattern */ + ",\"pattern\":\"greet\",\"path_filter\":\"(\"", + ",\"pattern\":\"a{1000000000}\",\"regex\":true", + ",\"name_pattern\":\"(\"", + ",\"pattern\":\"greet\",\"path_filter\":\"a{1000000000}\"", + ",\"pattern\":\"greet\",\"file_pattern\":\"quote'path\"", + }; + for (size_t i = 0; i < sizeof(bad) / sizeof(bad[0]); i++) { + char *response = step5_execute(i == 3 ? "search_graph" : "search_code", bad[i]); + ASSERT_NOT_NULL(response); + ASSERT_TRUE(strstr(response, "required") || strstr(response, "regex") || + strstr(response, "invalid") || strstr(response, "budget")); + free(response); + ASSERT_EQ(cbm_mem_class_live_blocks(CBM_MEM_CLASS_OPERATION_ARG), before); + } + cbm_operation_runtime_t runtime = {.search_scratch_dir = "/step5-no-such-scratch-parent"}; + cbm_operation_context_t context = {.runtime = &runtime}; + char args[CBM_SZ_1K]; + snprintf(args, sizeof(args), + "{\"project\":\"%s\",\"pattern\":\"greet\",\"path_filter\":\".*\"}", g_project); + cbm_operation_result_t result = + cbm_operation_execute(&context, CBM_OPERATION_SOURCE_SEARCH, args); + ASSERT_TRUE(result.is_error); + ASSERT_NOT_NULL(strstr(result.payload, "cannot create temp file")); + cbm_operation_result_dispose(&result); + ASSERT_EQ(cbm_mem_class_live_blocks(CBM_MEM_CLASS_OPERATION_ARG), before); + PASS(); +} + +TEST(integ_step5_source_context_bounds) { + char path[CBM_SZ_1K]; + snprintf(path, sizeof(path), "%s/step5-long.md", g_tmpdir); + FILE *fp = cbm_fopen(path, "wb"); + ASSERT_NOT_NULL(fp); + for (int line = 1; line <= 1000; line++) + fprintf(fp, line == 500 ? "needle-context-window\n" : "line %04d\n", line); + ASSERT_EQ(fclose(fp), 0); + cbm_store_t *store = cbm_store_open_path(g_dbpath); + ASSERT_NOT_NULL(store); + cbm_node_t node = {.project = g_project, + .label = "File", + .name = "long", + .qualified_name = "step5.long", + .file_path = "step5-long.md", + .start_line = 1, + .end_line = 1000}; + ASSERT_GT(cbm_store_upsert_node(store, &node), 0); + cbm_store_close(store); + static const char *values[] = {"200", "999999", "2147483647", "-5", "4294967297"}; + for (size_t i = 0; i < sizeof(values) / sizeof(values[0]); i++) { + char extra[256]; + snprintf(extra, sizeof(extra), + ",\"pattern\":\"needle-context-window\",\"format\":\"json\",\"context\":%s", + values[i]); + char *response = step5_execute("search_code", extra); + ASSERT_NOT_NULL(response); + yyjson_doc *doc = yyjson_read(response, strlen(response), 0); + ASSERT_NOT_NULL(doc); + yyjson_val *root = yyjson_doc_get_root(doc); + yyjson_val *cols = yyjson_obj_get(root, "cols"); + yyjson_val *rows = yyjson_obj_get(root, "rows"); + ASSERT_GT(yyjson_arr_size(rows), 0); + bool found = false; + for (size_t col = 0; col < yyjson_arr_size(cols); col++) { + const char *name = yyjson_get_str(yyjson_arr_get(cols, col)); + if (name && strcmp(name, "context") == 0) { + found = true; + yyjson_val *context = yyjson_arr_get(yyjson_arr_get(rows, 0), col); + ASSERT_EQ(yyjson_get_int(yyjson_obj_get(context, "context_start")), 300); + const char *text = yyjson_get_str(yyjson_obj_get(context, "context")); + ASSERT_NOT_NULL(text); + int lines = 0; + for (const char *p = text; *p; p++) + lines += *p == '\n'; + ASSERT_EQ(lines, 401); + } + } + ASSERT_EQ(found, i < 3); + yyjson_doc_free(doc); + free(response); + } + cbm_unlink(path); + store = cbm_store_open_path(g_dbpath); + ASSERT_NOT_NULL(store); + ASSERT_EQ(cbm_store_delete_nodes_by_file(store, g_project, "step5-long.md"), CBM_STORE_OK); + cbm_store_close(store); + PASS(); +} + +TEST(integ_step5_callable_lookup_and_grouping) { + cbm_store_t *store = cbm_store_open_path(g_dbpath); + ASSERT_NOT_NULL(store); + const char *qn = "step5.module.overloaded(pkg.Type)"; + cbm_node_t node = {.project = g_project, + .label = "Function", + .name = "overloaded", + .qualified_name = qn, + .file_path = "main.py", + .start_line = 1, + .end_line = 2}; + ASSERT_GT(cbm_store_upsert_node(store, &node), 0); + cbm_store_close(store); + char *response = + step5_execute("get_code_snippet", ",\"qualified_name\":\"step5.module.overloaded\""); + ASSERT_NOT_NULL(response); + ASSERT_NOT_NULL(strstr(response, "def greet")); + free(response); + response = step5_execute("get_code_snippet", ",\"qualified_name\":\"module.overloaded\""); + ASSERT_NOT_NULL(response); + ASSERT_NOT_NULL(strstr(response, "def greet")); + free(response); + response = step5_execute("trace_path", ",\"function_name\":\"step5.module.overloaded\""); + ASSERT_NOT_NULL(response); + ASSERT_NULL(strstr(response, "function not found")); + free(response); + response = + step5_execute("search_graph", ",\"name_pattern\":\"overloaded\",\"format\":\"json\""); + ASSERT_NOT_NULL(response); + ASSERT_NOT_NULL(strstr(response, "\"qn_prefix\":\"step5.module\"")); + free(response); + response = step5_execute("search_graph", + ",\"name_pattern\":\"overloaded\",\"fields\":[\"docstring\"]"); + ASSERT_NOT_NULL(response); + ASSERT_NOT_NULL(strstr(response, "step5.module (main.py):")); + ASSERT_NOT_NULL(strstr(response, "overloaded(pkg.Type)")); + free(response); + PASS(); +} + +static bool step5_seed_project(const char *cache, const char *name, const char *root, + const char *function) { + char path[CBM_SZ_4K]; + snprintf(path, sizeof(path), "%s/%s.db", cache, name); + cbm_store_t *store = cbm_store_open_path(path); + if (!store) + return false; + cbm_node_t node = {.project = name, + .label = "Function", + .name = function, + .qualified_name = function, + .file_path = "app.py", + .start_line = 1, + .end_line = 1}; + bool ok = cbm_store_upsert_project(store, name, root) == CBM_STORE_OK && + cbm_store_upsert_node(store, &node) > 0; + cbm_store_close(store); + return ok; +} + +TEST(integ_step5_named_path_owners_and_underscore_discovery) { + char temp[CBM_SZ_4K]; + snprintf(temp, sizeof(temp), "%s/cbm-step5-owners-XXXXXX", cbm_tmpdir()); + ASSERT_NOT_NULL(cbm_mkdtemp(temp)); + char cache[CBM_SZ_4K], root[CBM_SZ_4K], sub[CBM_SZ_4K], sibling[CBM_SZ_4K]; + snprintf(cache, sizeof(cache), "%s/cache", temp); + snprintf(root, sizeof(root), "%s/repo", temp); + snprintf(sub, sizeof(sub), "%s/repo/pkg", temp); + snprintf(sibling, sizeof(sibling), "%s/repo-sibling", temp); + ASSERT_TRUE(cbm_mkdir_p(cache, 0700)); + ASSERT_TRUE(cbm_mkdir_p(sub, 0700)); + ASSERT_TRUE(cbm_mkdir_p(sibling, 0700)); + const char *saved = getenv("CBM_CACHE_DIR"); + char *saved_copy = saved ? strdup(saved) : NULL; + cbm_setenv("CBM_CACHE_DIR", cache, 1); + bool seeded = step5_seed_project(cache, "__named", root, "parent_fn") && + step5_seed_project(cache, "_nested", sub, "nested_fn"); + cbm_operation_context_t context = {0}; + char args[CBM_SZ_8K]; + snprintf(args, sizeof(args), "{\"project\":\"%s\",\"name_pattern\":\".*\"}", root); + cbm_operation_result_t parent = cbm_operation_execute(&context, CBM_OPERATION_SEARCH, args); + snprintf(args, sizeof(args), "{\"project_name\":\"%s\",\"name_pattern\":\".*\"}", sub); + cbm_operation_result_t nested = cbm_operation_execute(&context, CBM_OPERATION_SEARCH, args); + snprintf(args, sizeof(args), "{\"project\":\"%s\"}", sibling); + cbm_operation_result_t absent = cbm_operation_execute(&context, CBM_OPERATION_SEARCH, args); + cbm_operation_result_t listed = cbm_operation_execute(&context, CBM_OPERATION_PROJECTS, "{}"); + seeded = seeded && step5_seed_project(cache, "_twin", sub, "twin_fn"); + snprintf(args, sizeof(args), "{\"project\":\"%s\"}", sub); + cbm_operation_result_t ambiguous = cbm_operation_execute(&context, CBM_OPERATION_SEARCH, args); + bool ok = seeded && !parent.is_error && parent.payload && strstr(parent.payload, "parent_fn") && + !nested.is_error && nested.payload && strstr(nested.payload, "nested_fn") && + absent.is_error && ambiguous.is_error && listed.payload && + strstr(listed.payload, "__named") && strstr(listed.payload, "_nested"); + cbm_operation_result_dispose(&parent); + cbm_operation_result_dispose(&nested); + cbm_operation_result_dispose(&absent); + cbm_operation_result_dispose(&listed); + cbm_operation_result_dispose(&ambiguous); + if (saved_copy) + cbm_setenv("CBM_CACHE_DIR", saved_copy, 1); + else + cbm_unsetenv("CBM_CACHE_DIR"); + free(saved_copy); + ASSERT_EQ(th_rmtree(temp), 0); + ASSERT_TRUE(ok); + PASS(); +} + +static bool step5_git(const char *repo, const char *const *args) { + const char *argv[32] = {"git", + "-C", + repo, + "-c", + "core.hooksPath=/dev/null", + "-c", + "commit.gpgsign=false", + "-c", + "user.name=cbm-test", + "-c", + "user.email=cbm-test@example.invalid"}; + size_t n = 11; + while (*args && n < 31) + argv[n++] = *args++; + cbm_proc_opts_t options = { + .bin = "git", .argv = argv, .quiet_timeout_ms = 10000, .strip_git_repo_env = true}; + cbm_proc_result_t result = {0}; + return cbm_subprocess_run(&options, &result) == 0 && result.outcome == CBM_PROC_CLEAN; +} + +TEST(integ_step5_detect_default_branch_and_explicit_base) { + static const char *branches[] = {"trunk", "master", "main", "develop", "trunk", "trunk"}; + for (size_t scenario = 0; scenario < sizeof(branches) / sizeof(branches[0]); scenario++) { + char repo[CBM_SZ_4K]; + snprintf(repo, sizeof(repo), "%s/cbm-step5-git-XXXXXX", cbm_tmpdir()); + ASSERT_NOT_NULL(cbm_mkdtemp(repo)); + const char *init[] = {"init", "-q", NULL}; + char head[256]; + snprintf(head, sizeof(head), "refs/heads/%s", branches[scenario]); + const char *head_args[] = {"symbolic-ref", "HEAD", head, NULL}; + const char *add[] = {"add", "base.c", NULL}; + const char *commit[] = {"commit", "-q", "-m", "fixture", NULL}; + bool ok = step5_git(repo, init) && step5_git(repo, head_args) && + th_write_file(TH_PATH(repo, "base.c"), "int base = 1;\n") == 0 && + step5_git(repo, add) && step5_git(repo, commit); + const char *remote[] = {"update-ref", "refs/remotes/origin/trunk", "HEAD", NULL}; + const char *remote_head[] = {"symbolic-ref", "refs/remotes/origin/HEAD", + "refs/remotes/origin/trunk", NULL}; + if (scenario == 0 || scenario == 4) + ok = ok && step5_git(repo, remote) && step5_git(repo, remote_head); + const char *checkout[] = {"checkout", "-q", "-b", "feature", NULL}; + const char *add_feature[] = {"add", "feature.c", NULL}; + ok = ok && step5_git(repo, checkout) && + th_write_file(TH_PATH(repo, "feature.c"), "int feature = 2;\n") == 0 && + step5_git(repo, add_feature) && step5_git(repo, commit); + if (scenario == 5) { + const char *remote_add[] = {"remote", "add", "origin", repo, NULL}; + const char *track_ref[] = {"update-ref", "refs/remotes/origin/trunk", "HEAD~1", NULL}; + const char *track_remote[] = {"config", "branch.feature.remote", "origin", NULL}; + const char *track_merge[] = {"config", "branch.feature.merge", "refs/heads/trunk", + NULL}; + ok = ok && step5_git(repo, remote_add) && step5_git(repo, track_ref) && + step5_git(repo, track_remote) && step5_git(repo, track_merge); + } + cbm_store_t *store = cbm_store_open_path(g_dbpath); + ASSERT_NOT_NULL(store); + ASSERT_EQ(cbm_store_upsert_project(store, g_project, repo), CBM_STORE_OK); + cbm_store_close(store); + char *response = step5_execute( + "detect_changes", + scenario == 4 ? ",\"scope\":\"files\",\"format\":\"json\",\"base_branch\":\"trunk\"" + : ",\"scope\":\"files\",\"format\":\"json\""); + bool answered = + response && (scenario == 3 ? strstr(response, "origin/HEAD") && strstr(response, "main") + : strstr(response, "feature.c") != NULL); + free(response); + ASSERT_EQ(th_rmtree(repo), 0); + ASSERT_TRUE(ok && answered); + } + cbm_store_t *store = cbm_store_open_path(g_dbpath); + ASSERT_NOT_NULL(store); + ASSERT_EQ(cbm_store_upsert_project(store, g_project, g_tmpdir), CBM_STORE_OK); + cbm_store_close(store); + PASS(); +} + SUITE(integration) { RUN_TEST(index_reports_excluded_subtrees_issue411); /* Set up: create temp project and index it */ @@ -1592,6 +1907,14 @@ SUITE(integration) { return; } + RUN_TEST(integ_step5_search_filters_and_integer_bounds); + RUN_TEST(integ_step5_search_regex_errors_and_cleanup); + RUN_TEST(integ_step5_source_context_bounds); + RUN_TEST(integ_step5_callable_lookup_and_grouping); + + RUN_TEST(integ_step5_named_path_owners_and_underscore_discovery); + RUN_TEST(integ_step5_detect_default_branch_and_explicit_base); + /* Pipeline result validation */ RUN_TEST(integ_index_has_nodes); RUN_TEST(integ_index_has_edges); diff --git a/tests/test_main.c b/tests/test_main.c index 21fc6ccec1..58a6b8b697 100644 --- a/tests/test_main.c +++ b/tests/test_main.c @@ -377,6 +377,19 @@ static void tf_index_worker_probe(const char *args_json, const char *response_ou fflush(NULL); _Exit(response ? 0 : 1); } + if (strstr(args_json, "\"silent-exit\"") || strstr(args_json, "\"unreadable-response\"")) { + /* #1300: reach a phase, then exit 0 without writing the response. The + * trailing plain-text line must not be mistaken for a phase. */ + bool readable = true; + if (strstr(args_json, "\"unreadable-response\"")) { + /* A real missing file exercises fopen failure in the supervisor. */ + readable = response_out && cbm_unlink(response_out) == 0; + } + cbm_log_info("incremental.edge_snapshot", "captured", "3"); + (void)fprintf(stderr, "async worker silent-exit probe\n"); + fflush(NULL); + _Exit(readable ? 0 : 1); + } if (strstr(args_json, "\"crash\"")) { (void)fprintf(stderr, "async worker crash probe\n"); fflush(NULL); @@ -876,6 +889,7 @@ extern void suite_subprocess(void); extern void suite_private_file_lock(void); extern void suite_lock_registry(void); extern void suite_extraction(void); +extern void suite_callable_sig(void); extern void suite_extraction_inheritance(void); extern void suite_extraction_imports(void); extern void suite_parse_coverage(void); @@ -1187,6 +1201,7 @@ int main(int argc, char **argv) { /* Existing C code regression tests */ RUN_SELECTED_SUITE(ac); RUN_SELECTED_SUITE(extraction); + RUN_SELECTED_SUITE(callable_sig); RUN_SELECTED_SUITE(extraction_inheritance); RUN_SELECTED_SUITE(extraction_imports); RUN_SELECTED_SUITE(parse_coverage); diff --git a/tests/test_path_alias.c b/tests/test_path_alias.c index a0bdc89364..60a3900040 100644 --- a/tests/test_path_alias.c +++ b/tests/test_path_alias.c @@ -9,6 +9,7 @@ #include "test_framework.h" #include "../src/pipeline/path_alias.h" #include "../src/foundation/compat.h" +#include "../src/foundation/compat_fs.h" #include #include @@ -112,8 +113,7 @@ TEST(path_alias_at_nested) { TEST(path_alias_specificity_longest_first) { // @/lib/* must beat @/* even though @/* would also match. - cbm_path_alias_map_t *m = - make_map(NULL, 2, "@/*", "src/*", "@/lib/*", "src/shared/lib/*"); + cbm_path_alias_map_t *m = make_map(NULL, 2, "@/*", "src/*", "@/lib/*", "src/shared/lib/*"); char *r = cbm_path_alias_resolve(m, "@/lib/auth"); ASSERT_NOT_NULL(r); ASSERT_STR_EQ(r, "src/shared/lib/auth"); @@ -205,8 +205,7 @@ TEST(path_alias_find_for_file_nearest_ancestor) { ASSERT_EQ(m1->count, 1); ASSERT_STR_EQ(m1->entries[0].alias_prefix, "@/"); - const cbm_path_alias_map_t *m2 = - cbm_path_alias_find_for_file(coll, "packages/utils/index.ts"); + const cbm_path_alias_map_t *m2 = cbm_path_alias_find_for_file(coll, "packages/utils/index.ts"); ASSERT_NOT_NULL(m2); ASSERT_EQ(m2->count, 1); ASSERT_STR_EQ(m2->entries[0].alias_prefix, "@root/"); @@ -242,14 +241,12 @@ TEST(path_alias_loader_monorepo) { char path[512]; snprintf(path, sizeof(path), "%s/tsconfig.json", root); - ASSERT_EQ(write_file(path, - "{\n \"compilerOptions\": {\n \"paths\": {\n" - " \"@root/*\": [\"shared/*\"]\n }\n }\n}\n"), + ASSERT_EQ(write_file(path, "{\n \"compilerOptions\": {\n \"paths\": {\n" + " \"@root/*\": [\"shared/*\"]\n }\n }\n}\n"), 0); snprintf(path, sizeof(path), "%s/apps/manager/tsconfig.json", root); - ASSERT_EQ(write_file(path, - "{\n // monorepo subpackage\n \"compilerOptions\": {\n" - " \"paths\": {\n \"@/*\": [\"./src/*\"]\n }\n },\n}\n"), + ASSERT_EQ(write_file(path, "{\n // monorepo subpackage\n \"compilerOptions\": {\n" + " \"paths\": {\n \"@/*\": [\"./src/*\"]\n }\n },\n}\n"), 0); cbm_path_alias_collection_t *coll = cbm_load_path_aliases(root); @@ -305,17 +302,15 @@ TEST(path_alias_loader_monorepo_dotdot_climb) { char path[512]; snprintf(path, sizeof(path), "%s/apps/web/tsconfig.json", root); - ASSERT_EQ(write_file(path, - "{\n \"compilerOptions\": {\n \"paths\": {\n" - " \"@shared/*\": [\"../../packages/shared/src/*\"]\n" - " }\n }\n}\n"), + ASSERT_EQ(write_file(path, "{\n \"compilerOptions\": {\n \"paths\": {\n" + " \"@shared/*\": [\"../../packages/shared/src/*\"]\n" + " }\n }\n}\n"), 0); cbm_path_alias_collection_t *coll = cbm_load_path_aliases(root); ASSERT_NOT_NULL(coll); - const cbm_path_alias_map_t *m = - cbm_path_alias_find_for_file(coll, "apps/web/src/feature/x.ts"); + const cbm_path_alias_map_t *m = cbm_path_alias_find_for_file(coll, "apps/web/src/feature/x.ts"); ASSERT_NOT_NULL(m); char *r = cbm_path_alias_resolve(m, "@shared/utils"); ASSERT_NOT_NULL(r); @@ -353,14 +348,12 @@ TEST(path_alias_loader_honors_discovery_exclusions) { char path[512]; snprintf(path, sizeof(path), "%s/tsconfig.json", root); - ASSERT_EQ(write_file(path, - "{\n \"compilerOptions\": {\n \"paths\": {\n" - " \"@root/*\": [\"shared/*\"]\n }\n }\n}\n"), + ASSERT_EQ(write_file(path, "{\n \"compilerOptions\": {\n \"paths\": {\n" + " \"@root/*\": [\"shared/*\"]\n }\n }\n}\n"), 0); snprintf(path, sizeof(path), "%s/big_generated/tsconfig.json", root); - ASSERT_EQ(write_file(path, - "{\n \"compilerOptions\": {\n \"paths\": {\n" - " \"@gen/*\": [\"./src/*\"]\n }\n }\n}\n"), + ASSERT_EQ(write_file(path, "{\n \"compilerOptions\": {\n \"paths\": {\n" + " \"@gen/*\": [\"./src/*\"]\n }\n }\n}\n"), 0); /* Control: the unexcluded loader collects BOTH configs. */ @@ -407,6 +400,144 @@ TEST(path_alias_loader_no_configs) { PASS(); } +/* ── Loader does not follow a linked directory ─────────────────── */ + +/* The walk must not leave the repository through a link, as discovery does + * not. A config in a real subdirectory is collected; one reachable only + * through a link to a directory outside the tree is not. */ +TEST(path_alias_loader_skips_linked_directory) { +#ifdef _WIN32 + /* symlink() does not exist on Windows, and creating a symbolic link there + * needs a privilege an ordinary account (and the CI runner) does not hold. + * The Windows shape of this case is a directory junction (cmd.exe mklink + * /J), which this file has no cmd.exe fixture for; the junction case is + * not covered here. */ + SKIP_PLATFORM("Windows: symlink() unavailable; links need a privilege"); +#else + char tmpl[256]; + snprintf(tmpl, sizeof(tmpl), "/tmp/cbm_palias_link_XXXXXX"); + char *root = cbm_mkdtemp(tmpl); + ASSERT_NOT_NULL(root); + char outside_tmpl[256]; + snprintf(outside_tmpl, sizeof(outside_tmpl), "/tmp/cbm_palias_outside_XXXXXX"); + char *outside = cbm_mkdtemp(outside_tmpl); + ASSERT_NOT_NULL(outside); + + char sub[512]; + snprintf(sub, sizeof(sub), "%s/pkg", root); + cbm_mkdir(sub); + + char path[512]; + snprintf(path, sizeof(path), "%s/pkg/tsconfig.json", root); + ASSERT_EQ(write_file(path, "{\n \"compilerOptions\": {\n \"paths\": {\n" + " \"@pkg/*\": [\"./src/*\"]\n }\n }\n}\n"), + 0); + snprintf(path, sizeof(path), "%s/tsconfig.json", outside); + ASSERT_EQ(write_file(path, "{\n \"compilerOptions\": {\n \"paths\": {\n" + " \"@out/*\": [\"./src/*\"]\n }\n }\n}\n"), + 0); + + char link[512]; + snprintf(link, sizeof(link), "%s/linked", root); + ASSERT_EQ(symlink(outside, link), 0); + + cbm_path_alias_collection_t *coll = cbm_load_path_aliases(root); + ASSERT_NOT_NULL(coll); + /* Only the real subdirectory contributes a scope. */ + ASSERT_EQ(coll->count, 1); + ASSERT_STR_EQ(coll->scopes[0].dir_prefix, "pkg"); + ASSERT_NULL(cbm_path_alias_find_for_file(coll, "linked/src/x.ts")); + cbm_path_alias_collection_free(coll); + + unlink(link); + snprintf(path, sizeof(path), "%s/pkg/tsconfig.json", root); + unlink(path); + snprintf(path, sizeof(path), "%s/pkg", root); + rmdir(path); + rmdir(root); + snprintf(path, sizeof(path), "%s/tsconfig.json", outside); + unlink(path); + rmdir(outside); + PASS(); +#endif +} + +#ifdef _WIN32 +/* ── Loader does not follow a directory junction (Windows) ─────── */ + +/* The Windows shape of the linked-directory case. A junction needs no + * privilege (a symbolic link does), and the directory listing reports it + * with the directory attribute, so is_dir alone would descend through it. + * Compiled out elsewhere: cmd.exe and junctions exist only on Windows, so + * this case runs on the Windows leg and nowhere else. */ +TEST(path_alias_loader_skips_junction) { + char root_tmpl[512]; + char outside_tmpl[512]; + snprintf(root_tmpl, sizeof(root_tmpl), "%s/cbm_palias_junction_XXXXXX", cbm_tmpdir()); + snprintf(outside_tmpl, sizeof(outside_tmpl), "%s/cbm_palias_outside_XXXXXX", cbm_tmpdir()); + char *root = cbm_mkdtemp(root_tmpl); + ASSERT_NOT_NULL(root); + char *outside = cbm_mkdtemp(outside_tmpl); + ASSERT_NOT_NULL(outside); + + char sub[600]; + snprintf(sub, sizeof(sub), "%s/pkg", root); + cbm_mkdir(sub); + + char path[700]; + snprintf(path, sizeof(path), "%s/pkg/tsconfig.json", root); + ASSERT_EQ(write_file(path, "{\n \"compilerOptions\": {\n \"paths\": {\n" + " \"@pkg/*\": [\"./src/*\"]\n }\n }\n}\n"), + 0); + snprintf(path, sizeof(path), "%s/tsconfig.json", outside); + ASSERT_EQ(write_file(path, "{\n \"compilerOptions\": {\n \"paths\": {\n" + " \"@out/*\": [\"./src/*\"]\n }\n }\n}\n"), + 0); + + /* cbm_tmpdir() can expose the MSYS spelling C:/msys64/...; cmd's mklink + * builtin treats the slash before "msys64" as another option delimiter. + * Native backslashes are required only at this cmd.exe fixture boundary. */ + char junction[600]; + snprintf(junction, sizeof(junction), "%s/linked", root); + char junction_native[sizeof(junction)]; + char outside_native[sizeof(outside_tmpl)]; + snprintf(junction_native, sizeof(junction_native), "%s", junction); + snprintf(outside_native, sizeof(outside_native), "%s", outside); + for (char *cursor = junction_native; *cursor; cursor++) { + if (*cursor == '/') { + *cursor = '\\'; + } + } + for (char *cursor = outside_native; *cursor; cursor++) { + if (*cursor == '/') { + *cursor = '\\'; + } + } + const char *junction_argv[] = {"cmd.exe", "/d", "/c", "mklink", "/J", + junction_native, outside_native, NULL}; + ASSERT_EQ(cbm_exec_no_shell(junction_argv), 0); + + cbm_path_alias_collection_t *coll = cbm_load_path_aliases(root); + ASSERT_NOT_NULL(coll); + /* Only the real subdirectory contributes a scope. */ + ASSERT_EQ(coll->count, 1); + ASSERT_STR_EQ(coll->scopes[0].dir_prefix, "pkg"); + ASSERT_NULL(cbm_path_alias_find_for_file(coll, "linked/src/x.ts")); + cbm_path_alias_collection_free(coll); + + /* Removing the junction removes the link, not its target. */ + cbm_rmdir(junction); + snprintf(path, sizeof(path), "%s/pkg/tsconfig.json", root); + cbm_unlink(path); + cbm_rmdir(sub); + cbm_rmdir(root); + snprintf(path, sizeof(path), "%s/tsconfig.json", outside); + cbm_unlink(path); + cbm_rmdir(outside); + PASS(); +} +#endif + void suite_path_alias(void); void suite_path_alias(void) { RUN_TEST(path_alias_at_wildcard); @@ -421,4 +552,8 @@ void suite_path_alias(void) { RUN_TEST(path_alias_loader_monorepo_dotdot_climb); RUN_TEST(path_alias_loader_honors_discovery_exclusions); RUN_TEST(path_alias_loader_no_configs); + RUN_TEST(path_alias_loader_skips_linked_directory); +#ifdef _WIN32 + RUN_TEST(path_alias_loader_skips_junction); +#endif } diff --git a/tests/test_pipeline.c b/tests/test_pipeline.c index 9c9db25d6a..35a653665e 100644 --- a/tests/test_pipeline.c +++ b/tests/test_pipeline.c @@ -1593,6 +1593,35 @@ static int named_edge_count(cbm_store_t *s, const char *project, const char *edg return matches; } +/* HTTP_CALLS edges out of the function `source_name` whose Route target's + * qualified name contains `route_qn_part` (e.g. "__route__GET__"). */ +static int http_calls_to_route(cbm_store_t *s, const char *project, const char *source_name, + const char *route_qn_part) { + cbm_edge_t *edges = NULL; + int edge_count = 0; + if (cbm_store_find_edges_by_type(s, project, "HTTP_CALLS", &edges, &edge_count) != + CBM_STORE_OK) { + return -1; + } + int matches = 0; + for (int i = 0; i < edge_count; i++) { + cbm_node_t source = {0}; + cbm_node_t target = {0}; + int source_ok = cbm_store_find_node_by_id(s, edges[i].source_id, &source) == CBM_STORE_OK; + int target_ok = cbm_store_find_node_by_id(s, edges[i].target_id, &target) == CBM_STORE_OK; + if (source_ok && target_ok && source.name && target.qualified_name && + strcmp(source.name, source_name) == 0 && strstr(target.qualified_name, route_qn_part)) { + matches++; + } + cbm_node_free_fields(&source); + cbm_node_free_fields(&target); + } + if (edges) { + cbm_store_free_edges(edges, edge_count); + } + return matches; +} + /* Like named_edge_count, but distinguish same-named targets by their source * file. Semantic-control fixtures intentionally keep the exported short name * identical in two modules so a project-wide unique-name fallback cannot make @@ -2939,6 +2968,115 @@ TEST(pipeline_incremental_repoints_call_reference_without_stale_edge) { PASS(); } +/* #1419: every index publishes through a private staging DB in WAL mode that + * cbm_store_seal_for_atomic_publish() checkpoints at synchronous=FULL and + * switches to DELETE mode before the atomic rename. A crash before that rename + * discards the stage, so a WAL commit at FULL only adds an fsync per + * transaction without protecting anything; SQLite documents NORMAL as + * corruption-safe under WAL. The probe reads the EFFECTIVE synchronous level + * of the committing handle from a WAL hook on every connection the run opens. + * + * SQLite installs its default autocheckpoint WAL hook after auto-extensions + * run, so the auto-extension installs a statement trace and the trace installs + * the probe once the connection is fully open. The probe keeps SQLite's + * default 1000-frame autocheckpoint, so it does not change what the run + * writes. */ +enum { I1419_SYNC_FULL = 2, I1419_DEFAULT_AUTOCHECKPOINT = 1000 }; +static atomic_int g_i1419_wal_commits; +static atomic_int g_i1419_full_sync_commits; + +static int i1419_wal_probe(void *arg, sqlite3 *db, const char *db_name, int wal_frames) { + (void)arg; + atomic_fetch_add(&g_i1419_wal_commits, 1); + sqlite3_stmt *stmt = NULL; + if (sqlite3_prepare_v2(db, "PRAGMA synchronous;", -1, &stmt, NULL) == SQLITE_OK && + sqlite3_step(stmt) == SQLITE_ROW && sqlite3_column_int(stmt, 0) >= I1419_SYNC_FULL) { + atomic_fetch_add(&g_i1419_full_sync_commits, 1); + } + sqlite3_finalize(stmt); + if (wal_frames >= I1419_DEFAULT_AUTOCHECKPOINT) { + (void)sqlite3_wal_checkpoint(db, db_name); + } + return SQLITE_OK; +} + +static int i1419_trace(unsigned type, void *ctx, void *stmt, void *sql) { + (void)type; + (void)ctx; + (void)sql; + (void)sqlite3_wal_hook(sqlite3_db_handle((sqlite3_stmt *)stmt), i1419_wal_probe, NULL); + return 0; +} + +static int i1419_install_probe(sqlite3 *db, char **errmsg, const sqlite3_api_routines *api) { + (void)errmsg; + (void)api; + return sqlite3_trace_v2(db, SQLITE_TRACE_STMT, i1419_trace, NULL); +} + +static int i1419_index(const char *repo, const char *db_path) { + cbm_pipeline_t *p = cbm_pipeline_new(repo, db_path, CBM_MODE_FULL); + if (!p) { + return -1; + } + int rc = cbm_pipeline_run(p); + cbm_pipeline_free(p); + return rc; +} + +TEST(pipeline_staged_publish_commits_below_full_sync_issue1419) { + char tmp[256]; + snprintf(tmp, sizeof(tmp), "/tmp/cbm_i1419_XXXXXX"); + if (!cbm_mkdtemp(tmp)) { + FAIL("tmpdir"); + } + /* Equal lengths + equal mtimes: the second run must route on content + * bytes to the closure-repair (delta) publish, not a full rebuild. */ + static const char initial_source[] = + "package syncprobe\n" + "func alphaSyncTarget() {}\n" + "func bravoSyncTarget() {}\n" + "func syncProbeAccept(callback func()) {}\n" + "func syncProbeSite() { syncProbeAccept(alphaSyncTarget) }\n"; + static const char replacement_source[] = + "package syncprobe\n" + "func alphaSyncTarget() {}\n" + "func bravoSyncTarget() {}\n" + "func syncProbeAccept(callback func()) {}\n" + "func syncProbeSite() { syncProbeAccept(bravoSyncTarget) }\n"; + ASSERT_EQ(sizeof(initial_source), sizeof(replacement_source)); + write_temp_file(tmp, "probe.go", initial_source); + char source_path[512]; + snprintf(source_path, sizeof(source_path), "%s/probe.go", tmp); + const time_t fixed_mtime = 1700000000; + ASSERT_EQ(pipeline_test_set_mtime(source_path, fixed_mtime, 123456789L), 0); + char db_path[512]; + snprintf(db_path, sizeof(db_path), "%s/i1419.db", tmp); + + atomic_store(&g_i1419_wal_commits, 0); + atomic_store(&g_i1419_full_sync_commits, 0); + ASSERT_EQ(sqlite3_auto_extension((void (*)(void))i1419_install_probe), SQLITE_OK); + /* Both publish routes run before any assertion, so the probe is always + * unregistered again even when a run fails. */ + int full_rc = i1419_index(tmp, db_path); + write_temp_file(tmp, "probe.go", replacement_source); + int mtime_rc = pipeline_test_set_mtime(source_path, fixed_mtime, 123456789L); + cbm_pipeline_incremental_test_reset_faults(); + int delta_rc = i1419_index(tmp, db_path); + cbm_incremental_route_t delta_route = cbm_pipeline_incremental_test_last_route(); + (void)sqlite3_cancel_auto_extension((void (*)(void))i1419_install_probe); + th_rmtree(tmp); + + ASSERT_EQ(full_rc, 0); + ASSERT_EQ(mtime_rc, 0); + ASSERT_EQ(delta_rc, 0); + ASSERT_EQ(delta_route, CBM_INCREMENTAL_ROUTE_CLOSURE_REPAIR); + /* Non-vacuous: both publishes commit through the WAL. */ + ASSERT_GT(atomic_load(&g_i1419_wal_commits), 0); + ASSERT_EQ(atomic_load(&g_i1419_full_sync_commits), 0); + PASS(); +} + /* SQL DDL becomes first-class Table/View nodes wired into FROM/JOIN lineage, * while the shared name registry must NOT leak those relations into other * languages' textual resolution: a Python call or identifier sharing the @@ -6228,6 +6366,254 @@ TEST(pipeline_ts_crossfile_new_instance_method_call_issue1354) { PASS(); } +/* A query failure is distinct from a valid zero-edge result. */ +static int ts514_pipeline_count(cbm_store_t *store, const char *sql, const char **values, + int value_count) { + sqlite3_stmt *stmt = NULL; + sqlite3 *db = cbm_store_get_db(store); + if (!db || sqlite3_prepare_v2(db, sql, -1, &stmt, NULL) != SQLITE_OK) { + sqlite3_finalize(stmt); + return -1; + } + bool bound = sqlite3_bind_parameter_count(stmt) == value_count; + for (int i = 0; bound && i < value_count; i++) + bound = values[i] && + sqlite3_bind_text(stmt, i + 1, values[i], -1, SQLITE_TRANSIENT) == SQLITE_OK; + int count = -1; + if (bound && sqlite3_step(stmt) == SQLITE_ROW) { + count = sqlite3_column_int(stmt, 0); + if (sqlite3_step(stmt) != SQLITE_DONE) + count = -1; + } + if (sqlite3_finalize(stmt) != SQLITE_OK) + count = -1; + return count; +} + +/* #514: constructor injection and new-initialized fields resolve through both + * drivers. Same-named methods make wrong-owner and self-loop claims meaningful. + * Every negative has a real extracted call, source and target, plus successful + * queries and typed positive controls in the same run. */ +TEST(pipeline_ts_param_property_injection_issue514) { + static const char *paths[] = {"cats/service.ts", "cats/controller.ts", "cats/reporter.ts"}; + static const char *sources[] = { + "export class BaseService { findAll(): string[] { return []; } }\n" + "export class CatService extends BaseService { findAll(): string[] { return ['cat']; } }\n", + "import { CatService } from './service';\n" + "export class CatController {\n" + " constructor(private readonly catService: CatService, plain: CatService) {}\n" + " findAll() { return this.catService.findAll(); }\n" + " direct(control: CatService) { return control.findAll(); }\n" + " viaPlain() { return this.plain.findAll(); }\n" + "}\n", + "import { CatService, BaseService } from './service';\n" + "function makeService(): CatService { return new CatService(); }\n" + "export class CatReporter {\n" + " private fresh = new CatService();\n" + " private annotated: BaseService = new CatService();\n" + " private factory = makeService();\n" + " constructor(protected cats: CatService) {}\n" + " reportInjected() { return this.cats.findAll(); }\n" + " reportLocal() { return this.fresh.findAll(); }\n" + " reportAnnotated() { return this.annotated.findAll(); }\n" + " reportFactory() { return this.factory.findAll(); }\n" + "}\n", + }; + static const struct { + int file; + const char *symbol; + const char *expression; + const char *callee; + const char *target; + int want; + } cases[] = { + {1, "CatController.direct", "control.findAll()", "control.findAll", "CatService.findAll", + 1}, + {1, "CatController.findAll", "this.catService.findAll()", "this.catService.findAll", + "CatService.findAll", 1}, + {2, "CatReporter.reportInjected", "this.cats.findAll()", "this.cats.findAll", + "CatService.findAll", 1}, + {2, "CatReporter.reportLocal", "this.fresh.findAll()", "this.fresh.findAll", + "CatService.findAll", 1}, + {2, "CatReporter.reportAnnotated", "this.annotated.findAll()", "this.annotated.findAll", + "BaseService.findAll", 1}, + {1, "CatController.viaPlain", "this.plain.findAll()", "this.plain.findAll", + "CatService.findAll", 0}, + {2, "CatReporter.reportFactory", "this.factory.findAll()", "this.factory.findAll", + "CatService.findAll", 0}, + }; + enum { NCASES = (int)(sizeof(cases) / sizeof(cases[0])) }; + static const char node_sql[] = "SELECT count(*) FROM nodes WHERE project=?1 AND " + "qualified_name=?2 AND file_path=?3 AND label='Method'"; + static const char edge_sql[] = + "SELECT count(*) FROM edges e JOIN nodes s ON s.id=e.source_id JOIN nodes t ON " + "t.id=e.target_id " + "WHERE e.project=?1 AND e.type='CALLS' AND s.qualified_name=?2 AND s.file_path=?3 " + "AND t.qualified_name=?4 AND t.file_path=?5 " + "AND (?6='' OR json_extract(e.properties,'$.strategy')=?6)"; + + CBMFileResult *fr[3] = {0}; + bool extraction_ok = true; + for (int i = 0; i < 3; i++) { + fr[i] = cbm_extract_file(sources[i], (int)strlen(sources[i]), CBM_LANG_TYPESCRIPT, + "ts514_probe", paths[i], 0, NULL, NULL); + extraction_ok = extraction_ok && fr[i] && !fr[i]->has_error; + } + int occurrences[NCASES] = {0}; + if (extraction_ok) { + for (int i = 0; i < NCASES; i++) { + int file = cases[i].file; + const char *site = strstr(sources[file], cases[i].expression); + char *caller = cbm_pipeline_fqn_compute("ts514_probe", paths[file], cases[i].symbol); + for (int c = 0; site && caller && c < fr[file]->calls.count; c++) { + const CBMCall *call = &fr[file]->calls.items[c]; + uint32_t start = (uint32_t)(site - sources[file]); + if (call->enclosing_func_qn && strcmp(call->enclosing_func_qn, caller) == 0 && + call->callee_name && strcmp(call->callee_name, cases[i].callee) == 0 && + call->site_start_byte == start && + call->site_end_byte == start + strlen(cases[i].expression) && + call->start_line > 0) + occurrences[i]++; + } + free(caller); + } + } + for (int i = 0; i < 3; i++) + cbm_free_result(fr[i]); + + const char *old_workers = getenv("CBM_WORKERS"); + const char *old_single = getenv("CBM_INDEX_SINGLE_THREAD"); + char *saved_workers = old_workers ? strdup(old_workers) : NULL; + char *saved_single = old_single ? strdup(old_single) : NULL; + if ((old_workers && !saved_workers) || (old_single && !saved_single)) { + free(saved_workers); + free(saved_single); + FAIL("save environment"); + } + char tmp[256] = "/tmp/cbm_ts_514_XXXXXX"; + if (!cbm_mkdtemp(tmp)) { + free(saved_workers); + free(saved_single); + FAIL("tmpdir"); + } + for (int i = 0; i < 3; i++) + write_temp_file(tmp, paths[i], sources[i]); + for (int i = 0; i < 52; i++) { + char name[64], body[128]; + snprintf(name, sizeof(name), "cats/filler%d.ts", i); + snprintf(body, sizeof(body), "export function filler%d(): number { return %d; }\n", i, i); + write_temp_file(tmp, name, body); + } + int run_rc[2] = {-1, -1}, files[2] = {-1, -1}; + int edges[2][NCASES], source_nodes[2][NCASES], target_nodes[2][NCASES]; + int unresolved_calls[2][2] = {{-1, -1}, {-1, -1}}; + int self_loop[2] = {-1, -1}, annotation_wrong[2] = {-1, -1}; + bool configured[2] = {false, false}, opened[2] = {false, false}; + for (int mode = 0; mode < 2; mode++) { + for (int i = 0; i < NCASES; i++) + edges[mode][i] = source_nodes[mode][i] = target_nodes[mode][i] = -1; + cbm_setenv("CBM_WORKERS", "4", 1); + if (mode == 0) + cbm_setenv("CBM_INDEX_SINGLE_THREAD", "1", 1); + else + cbm_unsetenv("CBM_INDEX_SINGLE_THREAD"); + const char *workers = getenv("CBM_WORKERS"); + const char *single = getenv("CBM_INDEX_SINGLE_THREAD"); + configured[mode] = workers && strcmp(workers, "4") == 0 && + (mode == 0 ? single && strcmp(single, "1") == 0 : !single); + char db_path[512]; + snprintf(db_path, sizeof(db_path), "%s/ts514_%d.db", tmp, mode); + cbm_pipeline_t *pipeline = + configured[mode] ? cbm_pipeline_new(tmp, db_path, CBM_MODE_FULL) : NULL; + run_rc[mode] = pipeline ? cbm_pipeline_run(pipeline) : -1; + cbm_store_t *store = run_rc[mode] == 0 ? cbm_store_open_path(db_path) : NULL; + opened[mode] = store != NULL; + if (store) { + const char *project = cbm_pipeline_project_name(pipeline); + const char *file_values[] = {project}; + files[mode] = ts514_pipeline_count( + store, "SELECT count(*) FROM nodes WHERE project=?1 AND label='File'", file_values, + 1); + for (int i = 0; i < NCASES; i++) { + char *src = + cbm_pipeline_fqn_compute(project, paths[cases[i].file], cases[i].symbol); + char *tgt = cbm_pipeline_fqn_compute(project, paths[0], cases[i].target); + const char *src_values[] = {project, src, paths[cases[i].file]}; + const char *tgt_values[] = {project, tgt, paths[0]}; + source_nodes[mode][i] = ts514_pipeline_count(store, node_sql, src_values, 3); + target_nodes[mode][i] = ts514_pipeline_count(store, node_sql, tgt_values, 3); + const char *edge_values[] = { + project, src, paths[cases[i].file], + tgt, paths[0], cases[i].want ? "lsp_ts_method" : ""}; + edges[mode][i] = ts514_pipeline_count(store, edge_sql, edge_values, 6); + if (!cases[i].want) + unresolved_calls[mode][i - 5] = ts514_pipeline_count( + store, + "SELECT count(*) FROM edges e JOIN nodes s ON s.id=e.source_id " + "WHERE e.project=?1 AND e.type='CALLS' " + "AND s.qualified_name=?2 AND s.file_path=?3", + src_values, 3); + if (i == 1) { + edge_values[3] = src; + edge_values[4] = paths[cases[i].file]; + edge_values[5] = ""; + self_loop[mode] = ts514_pipeline_count(store, edge_sql, edge_values, 6); + } + if (i == 4) { + char *wrong = cbm_pipeline_fqn_compute(project, paths[0], "CatService.findAll"); + edge_values[3] = wrong; + edge_values[5] = ""; + annotation_wrong[mode] = ts514_pipeline_count(store, edge_sql, edge_values, 6); + free(wrong); + } + free(src); + free(tgt); + } + cbm_store_close(store); + } + cbm_pipeline_free(pipeline); + } + if (saved_workers) + cbm_setenv("CBM_WORKERS", saved_workers, 1); + else + cbm_unsetenv("CBM_WORKERS"); + if (saved_single) + cbm_setenv("CBM_INDEX_SINGLE_THREAD", saved_single, 1); + else + cbm_unsetenv("CBM_INDEX_SINGLE_THREAD"); + const char *restored_workers = getenv("CBM_WORKERS"); + const char *restored_single = getenv("CBM_INDEX_SINGLE_THREAD"); + bool restored = + (saved_workers ? restored_workers && strcmp(saved_workers, restored_workers) == 0 + : !restored_workers) && + (saved_single ? restored_single && strcmp(saved_single, restored_single) == 0 + : !restored_single); + free(saved_workers); + free(saved_single); + th_rmtree(tmp); + + ASSERT_TRUE(extraction_ok); + ASSERT_TRUE(restored); + for (int i = 0; i < NCASES; i++) + ASSERT_EQ(occurrences[i], 1); + for (int mode = 0; mode < 2; mode++) { + ASSERT_TRUE(configured[mode]); + ASSERT_EQ(run_rc[mode], 0); + ASSERT_TRUE(opened[mode]); + ASSERT_GTE(files[mode], 55); + for (int i = 0; i < NCASES; i++) { + ASSERT_EQ(source_nodes[mode][i], 1); + ASSERT_EQ(target_nodes[mode][i], 1); + ASSERT_EQ(edges[mode][i], cases[i].want); + } + ASSERT_EQ(unresolved_calls[mode][0], 0); + ASSERT_EQ(unresolved_calls[mode][1], 0); + ASSERT_EQ(self_loop[mode], 0); + ASSERT_EQ(annotation_wrong[mode], 0); + } + PASS(); +} + TEST(pipeline_tsjs_receiver_suppresses_weak_method_edge) { char tmp[256]; snprintf(tmp, sizeof(tmp), "/tmp/cbm_tsjs_recv_XXXXXX"); @@ -6968,16 +7354,19 @@ TEST(pipeline_tsjs_receiver_parallel_keeps_service_edges) { cbm_store_t *s = cbm_store_open_path(db_path); ASSERT_NOT_NULL(s); - /* (1) Genuine HTTP_CALLS survive under the guard (>= 3): - * - axios.get('/api/orders') -> 2 edges (recognized lib #523 callee bypass - * + detect_url_in_args), and + /* (1) Genuine HTTP_CALLS survive under the guard: + * - axios.get('/api/orders') -> 1 edge, typed GET (recognized lib #523 + * callee bypass; no method-less arg-URL twin), and * - dev.load('/api/data') -> 1 edge via detect_url_in_args, which runs - * unconditionally after emit_service_edge's branch even when the plain - * fall-through is suppressed. + * after emit_service_edge's branch even when the plain fall-through is + * suppressed. * dev.load is the class the predicate-duplicating guard lost: `.load` is not * a route suffix and `dev` is not an HTTP lib, so it was dropped before - * emit_service_edge ran (RED on that guard: only axios's 2). */ - ASSERT_GTE(cbm_store_count_edges_by_type(s, project, "HTTP_CALLS"), 3); + * emit_service_edge ran (RED on that guard: only axios's edge). */ + ASSERT_EQ(http_calls_to_route(s, project, "callApi", "__route__GET__"), 1); + ASSERT_EQ(http_calls_to_route(s, project, "callApi", "__route__"), 1); + ASSERT_EQ(http_calls_to_route(s, project, "callLoad", "__route__ANY__"), 1); + ASSERT_EQ(cbm_store_count_edges_by_type(s, project, "HTTP_CALLS"), 2); /* (2) The verb-suffix + route-path member calls keep their route * registrations (edge type CALLS -> a Route node named by the path). These * classify as route_registration on main, NOT HTTP_CALLS — Option A preserves @@ -7007,6 +7396,120 @@ TEST(pipeline_tsjs_receiver_parallel_keeps_service_edges) { PASS(); } +/* Route-registration CALLS edges (via=route_registration) out of the function + * `source_name`. */ +static int route_registrations_from(cbm_store_t *s, const char *project, const char *source_name) { + cbm_edge_t *edges = NULL; + int edge_count = 0; + if (cbm_store_find_edges_by_type(s, project, "CALLS", &edges, &edge_count) != CBM_STORE_OK) { + return -1; + } + int matches = 0; + for (int i = 0; i < edge_count; i++) { + cbm_node_t source = {0}; + int source_ok = cbm_store_find_node_by_id(s, edges[i].source_id, &source) == CBM_STORE_OK; + if (source_ok && source.name && strcmp(source.name, source_name) == 0 && + edges[i].properties_json && + strstr(edges[i].properties_json, "\"via\":\"route_registration\"")) { + matches++; + } + cbm_node_free_fields(&source); + } + if (edges) { + cbm_store_free_edges(edges, edge_count); + } + return matches; +} + +/* `import Axios from "axios"` -- the default-import binding capitalized, a + * common convention -- must classify `Axios.get/post/delete(url)` as HTTP + * client calls exactly like `axios.*`. The library table matched only the + * lowercase id, so the calls fell through to the `.get`/`.post` route-suffix + * fallback and became route REGISTRATIONS: on + * microsoft/Game-Control-Puzzle-Event-Administration-Tools all 69 client + * calls masqueraded as server Routes. A real Express registration in the same + * project stays a registration. `fillers` > 0 forces the parallel resolver, + * 0 keeps the sequential one; both classify through the same table. */ +static int capitalized_axios_case(int fillers) { + char tmp[256]; + snprintf(tmp, sizeof(tmp), "/tmp/cbm_axios_cap_XXXXXX"); + if (!cbm_mkdtemp(tmp)) { + FAIL("tmpdir"); + } + + write_temp_file(tmp, "src/service.ts", + "import Axios from \"axios\";\n" + "export function getSettings(eventId: string) {\n" + " return Axios.get(`/api/admin/events/${eventId}/settings`);\n" + "}\n" + "export function addUser(eventId: string, participant: unknown) {\n" + " return Axios.post(`/api/admin/users/${eventId}`, participant);\n" + "}\n" + "export function removeUser(id: string) {\n" + " return Axios.delete(`/api/admin/users/${id}`);\n" + "}\n"); + write_temp_file(tmp, "src/server.ts", + "function listUsers(req: unknown, res: unknown) {}\n" + "export function mount(router: any) {\n" + " router.get('/api/admin/users', listUsers);\n" + "}\n"); + for (int i = 0; i < fillers; i++) { + char name[64]; + char body[128]; + snprintf(name, sizeof(name), "src/filler%d.ts", i); + snprintf(body, sizeof(body), "export function filler%d(): number {\n return %d;\n}\n", i, + i); + write_temp_file(tmp, name, body); + } + + char *old_workers = getenv("CBM_WORKERS"); + char *saved = old_workers ? strdup(old_workers) : NULL; + cbm_setenv("CBM_WORKERS", "4", 1); + + char db_path[512]; + snprintf(db_path, sizeof(db_path), "%s/axios_cap.db", tmp); + cbm_pipeline_t *p = cbm_pipeline_new(tmp, db_path, CBM_MODE_FULL); + ASSERT_NOT_NULL(p); + ASSERT_EQ(cbm_pipeline_run(p), 0); + const char *project = cbm_pipeline_project_name(p); + + cbm_store_t *s = cbm_store_open_path(db_path); + ASSERT_NOT_NULL(s); + + static const struct { + const char *caller; + const char *typed_route; + } client[] = {{"getSettings", "__route__GET__"}, + {"addUser", "__route__POST__"}, + {"removeUser", "__route__DELETE__"}}; + for (size_t i = 0; i < sizeof(client) / sizeof(client[0]); i++) { + ASSERT_EQ(route_registrations_from(s, project, client[i].caller), 0); + ASSERT_EQ(http_calls_to_route(s, project, client[i].caller, client[i].typed_route), 1); + } + /* Control: the server-side registration is untouched. */ + ASSERT_EQ(route_registrations_from(s, project, "mount"), 1); + ASSERT_EQ(http_calls_to_route(s, project, "mount", "__route__"), 0); + + cbm_store_close(s); + cbm_pipeline_free(p); + if (saved) { + cbm_setenv("CBM_WORKERS", saved, 1); + free(saved); + } else { + cbm_unsetenv("CBM_WORKERS"); + } + th_rmtree(tmp); + PASS(); +} + +TEST(pipeline_capitalized_axios_import_is_http_client) { + return capitalized_axios_case(52); +} + +TEST(pipeline_capitalized_axios_import_is_http_client_sequential) { + return capitalized_axios_case(0); +} + /* Python bare-call local-binding suppression, sequential path. The bare-call * counterpart of the receiver guard above: `run` is a PARAMETER, so `run()` * cannot be the module-level `run` and must not bind SatoriLive.run. @@ -7540,6 +8043,100 @@ TEST(pipeline_ts_config_object_url_http_calls_issue2235) { PASS(); } +/* Count HTTP_CALLS-style edges whose url_path property contains `keyword`. */ +static int count_edges_with_url_path(cbm_store_t *s, const char *project, const char *keyword) { + cbm_edge_t *es = NULL; + int n = 0; + cbm_store_find_edges_by_url_path(s, project, keyword, &es, &n); + if (es) { + cbm_store_free_edges(es, n); + } + return n; +} + +/* Distilled from PR #1245: detect_url_in_args (parallel resolver, >= 50 files) + * runs for EVERY resolved call with no callee gating, so a filesystem path + * with a document/log extension passed to a project-defined open/read/write + * helper passed normalize_url_arg (second slash, no space) and the route + * literal guard, minting a Route node + an HTTP_CALLS edge for a file. The + * callees are project functions on purpose: a bare builtin `open(...)` never + * resolves to a target and so never reaches emit_service_edge at all. The + * first path segments avoid the filesystem-root list (etc/var/tmp/...) so + * only the extension guard can reject them. requests.get('/api/items') is + * the positive control that must keep its Route + HTTP_CALLS. */ +TEST(pipeline_arg_url_rejects_document_file_paths) { + char tmp[256]; + snprintf(tmp, sizeof(tmp), "/tmp/cbm_arg_url_docs_XXXXXX"); + if (!cbm_mkdtemp(tmp)) { + FAIL("tmpdir"); + } + + write_temp_file(tmp, "src/files.py", + "import requests\n" + "def open_file(path):\n" + " return path\n" + "def read_file(path):\n" + " return path\n" + "def write_file(path):\n" + " return path\n" + "def load_docs():\n" + " open_file('/new/file.txt')\n" + " read_file('/docs/guide.md')\n" + " write_file('/data/app.log')\n" + " read_file('/reports/summary.pdf')\n" + " open_file('/docs/index.rst')\n" + "def fetch_items():\n" + " return requests.get('/api/items')\n"); + for (int i = 0; i < 52; i++) { + char name[64]; + char body[128]; + snprintf(name, sizeof(name), "src/filler%d.ts", i); + snprintf(body, sizeof(body), "export function filler%d(): number { return %d; }\n", i, i); + write_temp_file(tmp, name, body); + } + + char *old_workers = getenv("CBM_WORKERS"); + char *saved = old_workers ? strdup(old_workers) : NULL; + cbm_setenv("CBM_WORKERS", "4", 1); + + char db_path[512]; + snprintf(db_path, sizeof(db_path), "%s/arg_url_docs.db", tmp); + cbm_pipeline_t *p = cbm_pipeline_new(tmp, db_path, CBM_MODE_FULL); + ASSERT_NOT_NULL(p); + ASSERT_EQ(cbm_pipeline_run(p), 0); + const char *project = cbm_pipeline_project_name(p); + + cbm_store_t *s = cbm_store_open_path(db_path); + ASSERT_NOT_NULL(s); + + /* Positive control first: the genuine HTTP call still mints its route. */ + ASSERT_GTE(count_nodes_named(s, project, "/api/items"), 1); + ASSERT_GTE(count_edges_with_url_path(s, project, "/api/items"), 1); + /* No Route node for any filesystem document path... */ + ASSERT_EQ(count_nodes_named(s, project, "/new/file.txt"), 0); + ASSERT_EQ(count_nodes_named(s, project, "/docs/guide.md"), 0); + ASSERT_EQ(count_nodes_named(s, project, "/data/app.log"), 0); + ASSERT_EQ(count_nodes_named(s, project, "/reports/summary.pdf"), 0); + ASSERT_EQ(count_nodes_named(s, project, "/docs/index.rst"), 0); + /* ...and no HTTP_CALLS edge carrying one as its url_path either. */ + ASSERT_EQ(count_edges_with_url_path(s, project, "/new/file.txt"), 0); + ASSERT_EQ(count_edges_with_url_path(s, project, "/docs/guide.md"), 0); + ASSERT_EQ(count_edges_with_url_path(s, project, "/data/app.log"), 0); + ASSERT_EQ(count_edges_with_url_path(s, project, "/reports/summary.pdf"), 0); + ASSERT_EQ(count_edges_with_url_path(s, project, "/docs/index.rst"), 0); + + cbm_store_close(s); + cbm_pipeline_free(p); + if (saved) { + cbm_setenv("CBM_WORKERS", saved, 1); + free(saved); + } else { + cbm_unsetenv("CBM_WORKERS"); + } + th_rmtree(tmp); + PASS(); +} + /* Native `fetch()` (#856), sequential path (< 50 files → pass_calls.c). A bare * unqualified call to the global fetch API has no import and no local * definition anywhere in this project, so registry resolution comes back @@ -7784,6 +8381,103 @@ TEST(pipeline_local_fetch_shadow_not_classified_as_http) { PASS(); } +/* A call the HTTP service patterns classify (axios, an axios.create instance) + * gets exactly ONE HTTP_CALLS edge, typed with the verb its callee names. The + * arg-URL heuristic (detect_url_in_args, via=arg_url) exists for calls the + * patterns do not know -- a local fetch wrapper such as callApi('/api/orders') + * -- and must not add a second, method-less __route__ANY__ edge for the same + * call site. On microsoft/healthcare-ai-model-evaluator 59 of 129 HTTP_CALLS + * were such twins, and the cross-repo matcher bound them to server handlers of + * any method. `fillers` > 0 forces the parallel resolver (the RED case); 0 + * keeps the sequential one, which must agree. */ +static int typed_http_no_url_twin_case(int fillers) { + char tmp[256]; + snprintf(tmp, sizeof(tmp), "/tmp/cbm_http_twin_XXXXXX"); + if (!cbm_mkdtemp(tmp)) { + FAIL("tmpdir"); + } + + write_temp_file(tmp, "src/api.ts", + "import axios from 'axios';\n" + "const axiosInstance = axios.create({ baseURL: 'http://localhost:5000' });\n" + "export function getModel(id: string) {\n" + " return axiosInstance.get(`/api/models/${id}`);\n" + "}\n" + "export function addModel(body: unknown) {\n" + " return axiosInstance.post('/api/models', body);\n" + "}\n" + "export function removeModel(id: string) {\n" + " return axios.delete(`/api/models/${id}`);\n" + "}\n"); + write_temp_file(tmp, "src/wrapper.ts", + "function callApi(path: string): Promise {\n" + " return fetch(path);\n" + "}\n" + "export function listOrders() {\n" + " return callApi('/api/orders');\n" + "}\n"); + for (int i = 0; i < fillers; i++) { + char name[64]; + char body[128]; + snprintf(name, sizeof(name), "src/filler%d.ts", i); + snprintf(body, sizeof(body), "export function filler%d(): number {\n return %d;\n}\n", i, + i); + write_temp_file(tmp, name, body); + } + + char *old_workers = getenv("CBM_WORKERS"); + char *saved = old_workers ? strdup(old_workers) : NULL; + cbm_setenv("CBM_WORKERS", "4", 1); + + char db_path[512]; + snprintf(db_path, sizeof(db_path), "%s/http_twin.db", tmp); + cbm_pipeline_t *p = cbm_pipeline_new(tmp, db_path, CBM_MODE_FULL); + ASSERT_NOT_NULL(p); + ASSERT_EQ(cbm_pipeline_run(p), 0); + const char *project = cbm_pipeline_project_name(p); + + cbm_store_t *s = cbm_store_open_path(db_path); + ASSERT_NOT_NULL(s); + + static const struct { + const char *caller; + const char *typed_route; + } typed[] = {{"getModel", "__route__GET__"}, + {"addModel", "__route__POST__"}, + {"removeModel", "__route__DELETE__"}}; + for (size_t i = 0; i < sizeof(typed) / sizeof(typed[0]); i++) { + /* The typed edge stays ... */ + ASSERT_EQ(http_calls_to_route(s, project, typed[i].caller, typed[i].typed_route), 1); + /* ... and it is the only HTTP_CALLS edge of that call site. */ + ASSERT_EQ(http_calls_to_route(s, project, typed[i].caller, "__route__ANY__"), 0); + ASSERT_EQ(http_calls_to_route(s, project, typed[i].caller, "__route__"), 1); + } + if (fillers > 0) { + /* Control: the heuristic still covers the call no pattern knows. The + * sequential resolver has no arg-URL heuristic yet (#2235). */ + ASSERT_EQ(http_calls_to_route(s, project, "listOrders", "__route__ANY__"), 1); + } + + cbm_store_close(s); + cbm_pipeline_free(p); + if (saved) { + cbm_setenv("CBM_WORKERS", saved, 1); + free(saved); + } else { + cbm_unsetenv("CBM_WORKERS"); + } + th_rmtree(tmp); + PASS(); +} + +TEST(pipeline_typed_http_call_has_no_arg_url_twin) { + return typed_http_no_url_twin_case(52); +} + +TEST(pipeline_typed_http_call_has_no_arg_url_twin_sequential) { + return typed_http_no_url_twin_case(0); +} + /* ── Git history pass tests ─────────────────────────────────────── */ TEST(githistory_is_trackable) { @@ -8189,6 +8883,145 @@ static void teardown_usages_repo(void) { g_usages_tmpdir[0] = '\0'; } +/* #1277: typed instance fields of a Python class imported from another file. + * Counts CALLS edges caller -> callee (QN suffixes) whose properties name an + * LSP strategy. */ +static int count_lsp_calls(cbm_store_t *s, const char *project, const char *caller_suffix, + const char *callee_suffix) { + cbm_edge_t *edges = NULL; + int edge_count = 0; + cbm_store_find_edges_by_type(s, project, "CALLS", &edges, &edge_count); + int hits = 0; + for (int i = 0; i < edge_count; i++) { + cbm_node_t src = {0}; + cbm_node_t tgt = {0}; + if (cbm_store_find_node_by_id(s, edges[i].source_id, &src) == CBM_STORE_OK && + cbm_store_find_node_by_id(s, edges[i].target_id, &tgt) == CBM_STORE_OK) { + size_t sl = strlen(src.qualified_name); + size_t tl = strlen(tgt.qualified_name); + size_t cs = strlen(caller_suffix); + size_t ts = strlen(callee_suffix); + if (sl >= cs && tl >= ts && strcmp(src.qualified_name + sl - cs, caller_suffix) == 0 && + strcmp(tgt.qualified_name + tl - ts, callee_suffix) == 0 && + edges[i].properties_json && strstr(edges[i].properties_json, "lsp")) { + hits++; + } + } + cbm_node_free_fields(&src); + cbm_node_free_fields(&tgt); + } + if (edges) + cbm_store_free_edges(edges, edge_count); + return hits; +} + +static int check_python_crossfile_typed_field_calls_issue1277(bool parallel) { + const char *contracts = "class Contract:\n" + " def process_batch(self) -> None:\n" + " ...\n"; + const char *trainer = "from contracts import Contract\n\n" + "def make():\n" + " return None\n\n" + "class Trainer:\n" + " engine: Contract\n\n" + " def __init__(self, strategies: Contract) -> None:\n" + " self.strategies = strategies\n" + " self.typed: Contract = strategies\n" + " self.plain = make()\n"; + const char *loop = "from trainer import Trainer\n\n" + "def run(trainer: Trainer) -> None:\n" + " strategies = trainer.strategies\n" + " strategies.process_batch()\n\n" + "def run_typed(trainer: Trainer) -> None:\n" + " trainer.typed.process_batch()\n\n" + "def run_classlevel(trainer: Trainer) -> None:\n" + " trainer.engine.process_batch()\n\n" + "def run_untyped(trainer: Trainer) -> None:\n" + " trainer.plain.process_batch()\n"; + if (setup_usages_repo("contracts.py", contracts, "trainer.py", trainer) != 0) { + FAIL("failed to create temp dir"); + } + char path[512]; + snprintf(path, sizeof(path), "%s/loop.py", g_usages_tmpdir); + if (th_write_file(path, loop) != 0) { + teardown_usages_repo(); + FAIL("failed to write loop.py"); + } + if (parallel) { + /* Three semantic files plus 50 inert files exceed the production + * threshold for the fused parallel pipeline. */ + for (int i = 0; i < 50; i++) { + snprintf(path, sizeof(path), "%s/field_pad_%02d.py", g_usages_tmpdir, i); + if (th_write_file(path, "field_padding = 0\n") != 0) { + teardown_usages_repo(); + FAIL("failed to write parallel-selection fixture"); + } + } + } + + const char *old_workers = getenv("CBM_WORKERS"); + char *saved_workers = old_workers ? strdup(old_workers) : NULL; + const char *old_single = getenv("CBM_INDEX_SINGLE_THREAD"); + char *saved_single = old_single ? strdup(old_single) : NULL; + if ((old_workers && !saved_workers) || (old_single && !saved_single)) { + free(saved_workers); + free(saved_single); + teardown_usages_repo(); + FAIL("failed to save pipeline environment"); + } + int pin_workers_rc = parallel ? cbm_setenv("CBM_WORKERS", "4", 1) : 0; + int pin_single_rc = parallel ? cbm_unsetenv("CBM_INDEX_SINGLE_THREAD") : 0; + + char db_path[512]; + snprintf(db_path, sizeof(db_path), "%s/test_1277.db", g_usages_tmpdir); + cbm_pipeline_t *p = pin_workers_rc == 0 && pin_single_rc == 0 + ? cbm_pipeline_new(g_usages_tmpdir, db_path, CBM_MODE_FULL) + : NULL; + int run_rc = p ? cbm_pipeline_run(p) : -1; + cbm_store_t *s = run_rc == 0 ? cbm_store_open_path(db_path) : NULL; + bool store_opened = s != NULL; + int via_alias = -1, via_typed = -1, via_class = -1, via_untyped = -1; + if (s) { + const char *project = cbm_pipeline_project_name(p); + via_alias = count_lsp_calls(s, project, "loop.run", "contracts.Contract.process_batch"); + via_typed = + count_lsp_calls(s, project, "loop.run_typed", "contracts.Contract.process_batch"); + via_class = + count_lsp_calls(s, project, "loop.run_classlevel", "contracts.Contract.process_batch"); + via_untyped = count_lsp_calls(s, project, "loop.run_untyped", "process_batch"); + cbm_store_close(s); + } + + cbm_pipeline_free(p); + int restore_workers_rc = + saved_workers ? cbm_setenv("CBM_WORKERS", saved_workers, 1) : cbm_unsetenv("CBM_WORKERS"); + int restore_single_rc = saved_single ? cbm_setenv("CBM_INDEX_SINGLE_THREAD", saved_single, 1) + : cbm_unsetenv("CBM_INDEX_SINGLE_THREAD"); + free(saved_workers); + free(saved_single); + teardown_usages_repo(); + + ASSERT_EQ(pin_workers_rc, 0); + ASSERT_EQ(pin_single_rc, 0); + ASSERT_EQ(restore_workers_rc, 0); + ASSERT_EQ(restore_single_rc, 0); + ASSERT_EQ(run_rc, 0); + ASSERT_TRUE(store_opened); + ASSERT_EQ(via_alias, 1); /* self.x = */ + ASSERT_EQ(via_typed, 1); /* self.x: T = ... */ + ASSERT_EQ(via_class, 1); /* class-level x: T */ + ASSERT_EQ(via_untyped, 0); /* control: no annotation, no guess */ + PASS(); +} + +TEST(python_crossfile_typed_field_calls_issue1277) { + return check_python_crossfile_typed_field_calls_issue1277(false); +} + +TEST(python_crossfile_typed_field_calls_issue1277_parallel) { + return check_python_crossfile_typed_field_calls_issue1277(true); +} + TEST(usages_creates_edges) { /* Port of TestPassUsagesCreatesEdges. * Go source with callback reference → USAGE edge. */ @@ -9601,6 +10434,83 @@ TEST(pipeline_docstring_go_class) { PASS(); } +/* Index one file; true when the node with this label and name stores `want` in + * its properties JSON. */ +static bool doc_props_contain(const char *file, const char *content, const char *label, + const char *name, const char *want) { + const char *files[] = {file}; + const char *contents[] = {content}; + if (setup_lang_repo(files, contents, 1) != 0) { + return false; + } + char db[512]; + snprintf(db, sizeof(db), "%s/test.db", g_lang_tmpdir); + cbm_pipeline_t *p = cbm_pipeline_new(g_lang_tmpdir, db, CBM_MODE_FULL); + bool found = false; + if (p && cbm_pipeline_run(p) == 0) { + cbm_store_t *s = cbm_store_open_path(db); + cbm_node_t *nodes = NULL; + int nc = 0; + if (s && cbm_store_find_nodes_by_label(s, cbm_pipeline_project_name(p), label, &nodes, + &nc) == 0) { + for (int i = 0; i < nc; i++) { + if (nodes[i].name && strcmp(nodes[i].name, name) == 0 && nodes[i].properties_json && + strstr(nodes[i].properties_json, want)) { + found = true; + } + } + cbm_store_free_nodes(nodes, nc); + } + if (s) { + cbm_store_close(s); + } + } + if (p) { + cbm_pipeline_free(p); + } + teardown_lang_repo(); + return found; +} + +TEST(pipeline_doc_go_package_comment_on_file_node) { + /* go/doc: the comment group touching `package` documents the package; it is + * stored on the File node. */ + ASSERT_TRUE(doc_props_contain("knob.go", + "// Package knob turns knobs.\n" + "package knob\n\n" + "func Turn() {}\n", + "File", "knob.go", + "\"docstring\":\"// Package knob turns knobs.\"")); + PASS(); +} + +TEST(pipeline_doc_rust_inner_doc_on_file_node) { + ASSERT_TRUE(doc_props_contain("lib.rs", + "//! Knob crate.\n" + "//! Turns knobs.\n" + "\n" + "pub fn turn() {}\n", + "File", "lib.rs", + "\"docstring\":\"//! Knob crate.\\n//! Turns knobs.\"")); + PASS(); +} + +TEST(pipeline_doc_long_docstring_stored_whole) { + /* A 30-line doc (longer than the old fixed 2 KB properties buffer on its + * own) is stored whole, and the fields after it are not dropped. */ + char src[4096]; + int s = snprintf(src, sizeof(src), "package knob\n\n"); + for (int i = 0; i < 30; i++) { + s += snprintf(src + s, sizeof(src) - (size_t)s, + "// Line %02d of a long doc comment that keeps on going.\n", i); + } + snprintf(src + s, sizeof(src) - (size_t)s, "func Long(steps int) int { return steps }\n"); + ASSERT_TRUE(doc_props_contain("knob.go", src, "Function", "Long", + "that keeps on going.\\n// Line 29 of a long doc")); + ASSERT_TRUE(doc_props_contain("knob.go", src, "Function", "Long", "\"signature\":")); + PASS(); +} + TEST(project_name_from_path) { /* Port of TestProjectNameFromPath — more cases than integ_pipeline_project_name */ struct { @@ -16041,7 +16951,7 @@ TEST(pipeline_delta_patch_indexes_docstring_into_fts_body) { /* End-to-end for #518/#519: source → docstring → properties JSON → nodes_fts * `body` → findable. Each layer has its own test; this one proves they connect. - * It is also the guard on the size budget: build_def_props drops an oversized + * It is also the guard on the size budget: cbm_def_props_build drops an oversized * field ATOMICALLY, so a 500-byte section body that did not fit the 2 KB * properties buffer would vanish silently and every narrower test would still * pass. */ @@ -16377,6 +17287,402 @@ TEST(pipeline_objectscript_export_range_join_keeps_one_trailing_marker) { } #endif +/* #1153: a C++ method called on the object a static factory returns + * (`SecdManager::getInstance()->m()`, `Registry::instance().m()`, + * `auto w = Widget::create(); w->m()`) got no CALLS edge when the class lives + * in another file. The class is declared in a header, its methods are defined + * out of line in a .cpp, and the call is in a third file -- the reporter's + * layout. The cross-file C++ LSP resolved `Class::member` only under the + * caller's own module QN or as a bare QN, so the static call missed, its return + * type stayed unknown, and every receiver built from it dropped the method + * call. In one file the same code already resolved. + * + * `Other` declares the same method names in the same header: a correct + * resolution must never bind them. `Twin` is declared by two libraries, so + * `Twin::get()` names no single class and must stay unresolved. `Mystery` is + * never defined, so `m->paint()` has no receiver type to dispatch on. 52 + * fillers + CBM_WORKERS=4 force the parallel path; each test runs both. */ +/* write_temp_file creates one directory level; this layout nests two. */ +static void cpp_recv_write(const char *tmp, const char *name, const char *content) { + char dir[512]; + snprintf(dir, sizeof(dir), "%s/%s", tmp, name); + char *slash = strrchr(dir, '/'); + if (slash) { + *slash = '\0'; + cbm_mkdir_p(dir, 0755); + } + write_temp_file(tmp, name, content); +} + +static void cpp_recv_write_fixture(const char *tmp, bool parallel) { + cpp_recv_write(tmp, "app/src/SecdManager.h", + "#pragma once\n" + "#include \n\n" + "class SecdManager {\n" + "public:\n" + " static SecdManager *getInstance();\n" + " bool secd_retrieve_pic_fpc_i2c_id(int ifdh, int &fpc_id, int &pic_id);\n" + "};\n\n" + "class Registry {\n" + "public:\n" + " static Registry &instance();\n" + " void registerItem(int id);\n" + "};\n\n" + "class Widget {\n" + "public:\n" + " static std::unique_ptr create();\n" + " void paint();\n" + "};\n\n" + "class Session {\n" + "public:\n" + " static std::shared_ptr open();\n" + " void close();\n" + "};\n\n" + "class Other {\n" + "public:\n" + " bool secd_retrieve_pic_fpc_i2c_id(int ifdh, int &fpc_id, int &pic_id);\n" + " void registerItem(int id);\n" + " void paint();\n" + " void close();\n" + "};\n"); + cpp_recv_write( + tmp, "app/src/SecdManager.cpp", + "#include \"SecdManager.h\"\n\n" + "SecdManager *SecdManager::getInstance() {\n" + " static SecdManager inst;\n" + " return &inst;\n" + "}\n\n" + "bool SecdManager::secd_retrieve_pic_fpc_i2c_id(int ifdh, int &fpc_id, " + "int &pic_id) {\n" + " fpc_id = ifdh;\n" + " pic_id = ifdh;\n" + " return true;\n" + "}\n\n" + "Registry &Registry::instance() {\n" + " static Registry r;\n" + " return r;\n" + "}\n\n" + "void Registry::registerItem(int id) { (void)id; }\n\n" + "std::unique_ptr Widget::create() { return std::make_unique(); }\n\n" + "void Widget::paint() {}\n\n" + "std::shared_ptr Session::open() { return std::make_shared(); }\n\n" + "void Session::close() {}\n\n" + "bool Other::secd_retrieve_pic_fpc_i2c_id(int ifdh, int &fpc_id, int &pic_id) {\n" + " fpc_id = pic_id = ifdh;\n" + " return false;\n" + "}\n" + "void Other::registerItem(int id) { (void)id; }\n" + "void Other::paint() {}\n" + "void Other::close() {}\n"); + for (int lib = 0; lib < 2; lib++) { + char path[64]; + snprintf(path, sizeof(path), "lib_%c/Twin.h", 'a' + lib); + cpp_recv_write(tmp, path, + "#pragma once\n" + "class Twin {\n" + "public:\n" + " static Twin *get();\n" + " void run();\n" + "};\n"); + snprintf(path, sizeof(path), "lib_%c/Twin.cpp", 'a' + lib); + cpp_recv_write(tmp, path, + "#include \"Twin.h\"\n\n" + "Twin *Twin::get() {\n" + " static Twin t;\n" + " return &t;\n" + "}\n\n" + "void Twin::run() {}\n"); + } + cpp_recv_write( + tmp, "pil/src/SecdCfgHandlerThread.cpp", + "#include \"SecdManager.h\"\n" + "#include \"Twin.h\"\n\n" + "int GetExternalPhyPresent(int ifdh) {\n" + " int fpc_id = 0, pic_id = 0;\n" + " if (!SecdManager::getInstance()->secd_retrieve_pic_fpc_i2c_id(ifdh, fpc_id, " + "pic_id)) {\n" + " return -1;\n" + " }\n" + " return fpc_id + pic_id;\n" + "}\n\n" + "void UseRegistry() {\n" + " Registry::instance().registerItem(7);\n" + "}\n\n" + "void UseWidget() {\n" + " auto w = Widget::create();\n" + " w->paint();\n" + "}\n\n" + "void UseSession() {\n" + " auto s = Session::open();\n" + " s->close();\n" + "}\n\n" + "void UseUnknown(struct Mystery *m) {\n" + " m->paint();\n" + "}\n\n" + "void UseTwin() {\n" + " Twin::get()->run();\n" + "}\n"); + for (int i = 0; parallel && i < 52; i++) { + char name[64]; + char pad[128]; + snprintf(name, sizeof(name), "pad/cpp_pad_%02d.cpp", i); + snprintf(pad, sizeof(pad), "int cpp_pad_%02d() { return %d; }\n", i, i); + write_temp_file(tmp, name, pad); + } +} + +typedef struct { + char tmp[256]; + char db[512]; + char *saved_workers; + cbm_pipeline_t *p; + cbm_store_t *s; + const char *project; +} CppRecvRun; + +static void cpp_recv_close(CppRecvRun *r) { + if (r->s) { + cbm_store_close(r->s); + } + if (r->p) { + cbm_pipeline_free(r->p); + } + if (r->saved_workers) { + cbm_setenv("CBM_WORKERS", r->saved_workers, 1); + free(r->saved_workers); + } else { + cbm_unsetenv("CBM_WORKERS"); + } + if (r->tmp[0]) { + th_rmtree(r->tmp); + } + memset(r, 0, sizeof(*r)); +} + +/* Every node an assertion names must exist, or "no CALLS edge" would also + * pass for a fixture that never got indexed. */ +static bool cpp_recv_fixture_indexed(const CppRecvRun *r) { + static const char *const required[] = { + "pil.src.SecdCfgHandlerThread.GetExternalPhyPresent", + "pil.src.SecdCfgHandlerThread.UseRegistry", + "pil.src.SecdCfgHandlerThread.UseWidget", + "pil.src.SecdCfgHandlerThread.UseSession", + "pil.src.SecdCfgHandlerThread.UseUnknown", + "pil.src.SecdCfgHandlerThread.UseTwin", + "app.src.SecdManager.SecdManager.getInstance", + "app.src.SecdManager.SecdManager.secd_retrieve_pic_fpc_i2c_id", + "app.src.SecdManager.Registry.registerItem", + "app.src.SecdManager.Widget.paint", + "app.src.SecdManager.Session.close", + "app.src.SecdManager.Other.secd_retrieve_pic_fpc_i2c_id", + "app.src.SecdManager.Other.registerItem", + "app.src.SecdManager.Other.paint", + "app.src.SecdManager.Other.close", + "lib_a.Twin.Twin.get", + "lib_a.Twin.Twin.run", + "lib_b.Twin.Twin.get", + "lib_b.Twin.Twin.run", + }; + bool all = true; + for (size_t i = 0; i < sizeof(required) / sizeof(required[0]); i++) { + char qn[512]; + snprintf(qn, sizeof(qn), "%s.%s", r->project, required[i]); + cbm_node_t n; + memset(&n, 0, sizeof(n)); + if (cbm_store_find_node_by_qn(r->s, r->project, qn, &n) != CBM_STORE_OK) { + printf(" fixture node missing: %s\n", qn); + all = false; + } + cbm_node_free_fields(&n); + } + return all; +} + +/* Index the fixture; false (after cleanup) when the run itself fails. */ +static bool cpp_recv_open(CppRecvRun *r, bool parallel) { + memset(r, 0, sizeof(*r)); + snprintf(r->tmp, sizeof(r->tmp), "/tmp/cbm_cpp_recv_XXXXXX"); + if (!cbm_mkdtemp(r->tmp)) { + r->tmp[0] = '\0'; + return false; + } + cpp_recv_write_fixture(r->tmp, parallel); + const char *old_workers = getenv("CBM_WORKERS"); + r->saved_workers = old_workers ? strdup(old_workers) : NULL; + cbm_setenv("CBM_WORKERS", parallel ? "4" : "1", 1); + snprintf(r->db, sizeof(r->db), "%s/cpp_recv.db", r->tmp); + r->p = cbm_pipeline_new(r->tmp, r->db, CBM_MODE_FULL); + if (!r->p || cbm_pipeline_run(r->p) != 0) { + cpp_recv_close(r); + return false; + } + r->s = cbm_store_open_path(r->db); + r->project = cbm_pipeline_project_name(r->p); + if (!r->s || !r->project || !cpp_recv_fixture_indexed(r)) { + cpp_recv_close(r); + return false; + } + return true; +} + +/* Properties of the CALLS edge caller -> target (QNs below the project), or + * NULL when there is no such edge. The caller frees the result. */ +static char *cpp_recv_calls(const CppRecvRun *r, const char *caller, const char *target) { + char src_qn[512]; + char tgt_qn[512]; + snprintf(src_qn, sizeof(src_qn), "%s.%s", r->project, caller); + snprintf(tgt_qn, sizeof(tgt_qn), "%s.%s", r->project, target); + cbm_node_t src; + cbm_node_t tgt; + memset(&src, 0, sizeof(src)); + memset(&tgt, 0, sizeof(tgt)); + char *props = NULL; + if (cbm_store_find_node_by_qn(r->s, r->project, src_qn, &src) == CBM_STORE_OK && + cbm_store_find_node_by_qn(r->s, r->project, tgt_qn, &tgt) == CBM_STORE_OK) { + cbm_edge_t *edges = NULL; + int ec = 0; + cbm_store_find_edges_by_source_type(r->s, src.id, "CALLS", &edges, &ec); + for (int i = 0; i < ec && !props; i++) { + if (edges[i].target_id == tgt.id) { + props = strdup(edges[i].properties_json ? edges[i].properties_json : "{}"); + } + } + if (edges) { + cbm_store_free_edges(edges, ec); + } + } + cbm_node_free_fields(&src); + cbm_node_free_fields(&tgt); + return props; +} + +#define CPP_RECV_CALLER "pil.src.SecdCfgHandlerThread." +#define CPP_RECV_CLASS "app.src.SecdManager." + +/* 1 = an lsp_* CALLS edge, 0 = no edge, -1 = an edge from a name heuristic. */ +static int cpp_recv_edge_kind(const CppRecvRun *r, bool parallel, const char *caller, + const char *target) { + char *props = cpp_recv_calls(r, caller, target); + int kind = props ? (strstr(props, "\"strategy\":\"lsp_") ? 1 : -1) : 0; + if (kind != 1) { + printf(" [%s] %s -> %s: %s\n", parallel ? "parallel" : "sequential", caller, target, + props ? props : "(no CALLS edge)"); + } + free(props); + return kind; +} + +/* Reported shape: a raw pointer from a singleton accessor. */ +TEST(pipeline_cpp_static_factory_pointer_receiver_issue1153) { + for (int mode = 0; mode < 2; mode++) { + bool parallel = mode == 1; + CppRecvRun r; + ASSERT_TRUE(cpp_recv_open(&r, parallel)); + int factory = cpp_recv_edge_kind(&r, parallel, CPP_RECV_CALLER "GetExternalPhyPresent", + CPP_RECV_CLASS "SecdManager.getInstance"); + int method = cpp_recv_edge_kind(&r, parallel, CPP_RECV_CALLER "GetExternalPhyPresent", + CPP_RECV_CLASS "SecdManager.secd_retrieve_pic_fpc_i2c_id"); + char *wrong = cpp_recv_calls(&r, CPP_RECV_CALLER "GetExternalPhyPresent", + CPP_RECV_CLASS "Other.secd_retrieve_pic_fpc_i2c_id"); + cpp_recv_close(&r); + bool no_wrong = wrong == NULL; + free(wrong); + ASSERT_EQ(factory, 1); /* RED: only a unique_name guess */ + ASSERT_EQ(method, 1); /* RED: the call through the returned pointer dropped */ + ASSERT_TRUE(no_wrong); /* control: never the same-named Other method */ + } + PASS(); +} + +/* `Foo::instance().bar()`: a reference-returning accessor. */ +TEST(pipeline_cpp_static_factory_reference_receiver_issue1153) { + for (int mode = 0; mode < 2; mode++) { + bool parallel = mode == 1; + CppRecvRun r; + ASSERT_TRUE(cpp_recv_open(&r, parallel)); + int method = cpp_recv_edge_kind(&r, parallel, CPP_RECV_CALLER "UseRegistry", + CPP_RECV_CLASS "Registry.registerItem"); + char *wrong = + cpp_recv_calls(&r, CPP_RECV_CALLER "UseRegistry", CPP_RECV_CLASS "Other.registerItem"); + cpp_recv_close(&r); + bool no_wrong = wrong == NULL; + free(wrong); + ASSERT_EQ(method, 1); /* RED: dropped */ + ASSERT_TRUE(no_wrong); /* control: never the same-named Other method */ + } + PASS(); +} + +/* `auto w = Widget::create(); w->paint();` with std::unique_ptr. */ +TEST(pipeline_cpp_static_factory_unique_ptr_receiver_issue1153) { + for (int mode = 0; mode < 2; mode++) { + bool parallel = mode == 1; + CppRecvRun r; + ASSERT_TRUE(cpp_recv_open(&r, parallel)); + int method = cpp_recv_edge_kind(&r, parallel, CPP_RECV_CALLER "UseWidget", + CPP_RECV_CLASS "Widget.paint"); + char *wrong = cpp_recv_calls(&r, CPP_RECV_CALLER "UseWidget", CPP_RECV_CLASS "Other.paint"); + cpp_recv_close(&r); + bool no_wrong = wrong == NULL; + free(wrong); + ASSERT_EQ(method, 1); + ASSERT_TRUE(no_wrong); + } + PASS(); +} + +/* `auto s = Session::open(); s->close();` with std::shared_ptr. */ +TEST(pipeline_cpp_static_factory_shared_ptr_receiver_issue1153) { + for (int mode = 0; mode < 2; mode++) { + bool parallel = mode == 1; + CppRecvRun r; + ASSERT_TRUE(cpp_recv_open(&r, parallel)); + int method = cpp_recv_edge_kind(&r, parallel, CPP_RECV_CALLER "UseSession", + CPP_RECV_CLASS "Session.close"); + char *wrong = + cpp_recv_calls(&r, CPP_RECV_CALLER "UseSession", CPP_RECV_CLASS "Other.close"); + cpp_recv_close(&r); + bool no_wrong = wrong == NULL; + free(wrong); + ASSERT_EQ(method, 1); + ASSERT_TRUE(no_wrong); + } + PASS(); +} + +/* Controls: no receiver type, or a class name two libraries declare, gives + * the LSP nothing to dispatch on -- neither may gain an edge. */ +TEST(pipeline_cpp_unresolvable_receiver_stays_unbound_issue1153) { + static const char *const twin_targets[] = {"lib_a.Twin.Twin.get", "lib_a.Twin.Twin.run", + "lib_b.Twin.Twin.get", "lib_b.Twin.Twin.run"}; + for (int mode = 0; mode < 2; mode++) { + bool parallel = mode == 1; + CppRecvRun r; + ASSERT_TRUE(cpp_recv_open(&r, parallel)); + char *widget = + cpp_recv_calls(&r, CPP_RECV_CALLER "UseUnknown", CPP_RECV_CLASS "Widget.paint"); + char *other = + cpp_recv_calls(&r, CPP_RECV_CALLER "UseUnknown", CPP_RECV_CLASS "Other.paint"); + int twin_lsp = 0; + for (int t = 0; t < 4; t++) { + char *props = cpp_recv_calls(&r, CPP_RECV_CALLER "UseTwin", twin_targets[t]); + if (props && strstr(props, "\"strategy\":\"lsp_")) { + printf(" [%s] UseTwin -> %s: %s\n", parallel ? "parallel" : "sequential", + twin_targets[t], props); + twin_lsp++; + } + free(props); + } + cpp_recv_close(&r); + bool unknown_unbound = widget == NULL && other == NULL; + free(widget); + free(other); + ASSERT_TRUE(unknown_unbound); + ASSERT_EQ(twin_lsp, 0); + } + PASS(); +} + SUITE(pipeline) { RUN_TEST(pipeline_lsp_surface_decode_preserves_array_order_and_nulls); RUN_TEST(pipeline_nested_fixture_files_are_written); @@ -16451,6 +17757,7 @@ SUITE(pipeline) { RUN_TEST(pipeline_tsjs_receiver_suppresses_weak_method_edge); RUN_TEST(pipeline_axios_wrapper_baseurl_composes_http_calls_issue1916); RUN_TEST(pipeline_ts_crossfile_new_instance_method_call_issue1354); + RUN_TEST(pipeline_ts_param_property_injection_issue514); RUN_TEST(pipeline_python_receiver_suppresses_weak_method_edge); RUN_TEST(pipeline_python_receiver_keeps_specific_unique_name_member_call); RUN_TEST(pipeline_html_embedded_member_call_stays_unbound); @@ -16461,6 +17768,8 @@ SUITE(pipeline) { RUN_TEST(pipeline_go_bare_ref_never_binds_field); RUN_TEST(pipeline_go_bare_ref_never_binds_field_parallel); RUN_TEST(pipeline_tsjs_receiver_parallel_keeps_service_edges); + RUN_TEST(pipeline_capitalized_axios_import_is_http_client); + RUN_TEST(pipeline_capitalized_axios_import_is_http_client_sequential); RUN_TEST(pipeline_python_receiver_parallel_suppresses_weak_method_edges); RUN_TEST(pipeline_python_bare_local_binding_suppresses_weak_edge); RUN_TEST(pipeline_python_bare_local_binding_parallel_suppresses_weak_edge); @@ -16468,11 +17777,14 @@ SUITE(pipeline) { RUN_TEST(pipeline_parallel_rust_cross_only_macro_hidden_gets_synthetic_carrier); RUN_TEST(pipeline_arg_url_rejects_non_http_slash_arguments); RUN_TEST(pipeline_ts_config_object_url_http_calls_issue2235); + RUN_TEST(pipeline_arg_url_rejects_document_file_paths); RUN_TEST(pipeline_native_fetch_classified_as_http_calls); RUN_TEST(pipeline_swift_nested_url_makes_route_issue1892); RUN_TEST(pipeline_swift_http_call_makes_route_issue1892); RUN_TEST(pipeline_native_fetch_parallel_classified_as_http_calls); RUN_TEST(pipeline_local_fetch_shadow_not_classified_as_http); + RUN_TEST(pipeline_typed_http_call_has_no_arg_url_twin); + RUN_TEST(pipeline_typed_http_call_has_no_arg_url_twin_sequential); /* Git history pass */ RUN_TEST(githistory_is_trackable); RUN_TEST(githistory_compute_coupling); @@ -16486,6 +17798,8 @@ SUITE(pipeline) { RUN_TEST(implements_creates_override); RUN_TEST(implements_no_match); /* Usages pass (full pipeline integration) */ + RUN_TEST(python_crossfile_typed_field_calls_issue1277); + RUN_TEST(python_crossfile_typed_field_calls_issue1277_parallel); RUN_TEST(usages_creates_edges); RUN_TEST(usages_no_duplicate_calls); RUN_TEST(calls_edge_carries_call_site_line); @@ -16513,6 +17827,9 @@ SUITE(pipeline) { RUN_TEST(pipeline_docstring_java_method); RUN_TEST(pipeline_docstring_kotlin_function); RUN_TEST(pipeline_docstring_go_class); + RUN_TEST(pipeline_doc_go_package_comment_on_file_node); + RUN_TEST(pipeline_doc_rust_inner_doc_on_file_node); + RUN_TEST(pipeline_doc_long_docstring_stored_whole); /* Project name */ RUN_TEST(project_name_from_path); RUN_TEST(project_name_drive_letter_case_insensitive_issue394); @@ -16745,6 +18062,11 @@ SUITE(pipeline) { RUN_TEST(pipeline_markdown_and_config_prose_reaches_fts_body); RUN_TEST(pipeline_semantic_edges_no_functions); RUN_TEST(pipeline_semantic_batched_matches_unbatched); + RUN_TEST(pipeline_cpp_static_factory_pointer_receiver_issue1153); + RUN_TEST(pipeline_cpp_static_factory_reference_receiver_issue1153); + RUN_TEST(pipeline_cpp_static_factory_unique_ptr_receiver_issue1153); + RUN_TEST(pipeline_cpp_static_factory_shared_ptr_receiver_issue1153); + RUN_TEST(pipeline_cpp_unresolvable_receiver_stays_unbound_issue1153); } /* Focused semantic-manifest and publication contracts. Kept separate from the @@ -16753,6 +18075,7 @@ SUITE(pipeline) { SUITE(pipeline_semantic_manifest_repro) { RUN_TEST(incremental_downgrade_preserves_scope_and_artifact_across_change_noop_delete); RUN_TEST(pipeline_incremental_repoints_call_reference_without_stale_edge); + RUN_TEST(pipeline_staged_publish_commits_below_full_sync_issue1419); RUN_TEST(pipeline_sql_lineage_and_relation_isolation); RUN_TEST(pipeline_incremental_sql_table_rename_drops_stale_lineage); RUN_TEST(pipeline_dbt_jinja_lineage); diff --git a/tests/test_platform.c b/tests/test_platform.c index d2706b1f73..987fc47e4d 100644 --- a/tests/test_platform.c +++ b/tests/test_platform.c @@ -4,6 +4,7 @@ #include "test_framework.h" #include "../src/foundation/compat.h" /* cbm_setenv / cbm_unsetenv (Windows-portable) */ #include "../src/foundation/compat_fs.h" +#include "../src/foundation/compat_regex.h" #include "../src/foundation/constants.h" #include "../src/foundation/compat_thread.h" #include "../src/foundation/platform.h" @@ -1186,6 +1187,245 @@ TEST(cgroup_no_mem_files) { #endif /* __linux__ */ +/* ── compat_regex: compile-size guard ──────────────────────────── */ + +static uint64_t regex_units_ere(const char *pattern) { + return cbm_regcomp_estimate_units(pattern, CBM_REG_EXTENDED); +} + +static uint64_t regex_units_bre(const char *pattern) { + return cbm_regcomp_estimate_units(pattern, 0); +} + +/* Writes "(a|b|c|...)" with `branches` one-letter alternatives into buf, which + * must hold 2 * branches + 2 bytes. */ +static void regex_one_letter_alternation(char *buf, int branches) { + char *w = buf; + *w++ = '('; + for (int i = 0; i < branches; i++) { + if (i) { + *w++ = '|'; + } + *w++ = (char)('a' + i % 26); + } + *w++ = ')'; + *w = '\0'; +} + +/* Both regex backends expand `X{m,n}` into n copies of X at compile time, so + * nested intervals multiply and a 25-byte pattern can ask for millions of + * atoms. The wrapper sizes the pattern first and refuses one over the budget + * with CBM_REG_ETOOBIG before the platform compiler allocates anything, so + * there is nothing to free on refusal. The over-budget shapes here are small + * enough that an unguarded compile would still only cost tens of MB. */ +TEST(regex_compile_refuses_oversized_expansion) { + cbm_regex_t re; + /* 14^4 = 38,416 expanded atoms from 25 bytes: over the 32,768-unit budget. */ + const char *nested = "((((a){14}){14}){14}){14}"; + ASSERT_EQ(cbm_regcomp(&re, nested, CBM_REG_EXTENDED | CBM_REG_NOSUB), CBM_REG_ETOOBIG); + /* Every flag set the call sites use is refused the same way. */ + ASSERT_EQ(cbm_regcomp(&re, nested, CBM_REG_EXTENDED), CBM_REG_ETOOBIG); + ASSERT_EQ(cbm_regcomp(&re, nested, CBM_REG_EXTENDED | CBM_REG_NOSUB | CBM_REG_ICASE), + CBM_REG_ETOOBIG); + /* A repeated alternation multiplies too: (16 + 16*16/8) x 255 x 3 = 36,720. */ + ASSERT_EQ(cbm_regcomp(&re, "((a|b|c|d|e|f|g|h|i|j|k|l|m|n|o|p){255}){3}", + CBM_REG_EXTENDED | CBM_REG_NOSUB), + CBM_REG_ETOOBIG); + /* A plain alternation just over the budget: 509 one-letter branches cost + * 509 + 509*509/8 = 32,894 units; 508 cost 32,766 and stay accepted. */ + char alts[2 * 509 + 2]; + regex_one_letter_alternation(alts, 509); + ASSERT_EQ(regex_units_ere(alts), 32894); + ASSERT_EQ(cbm_regcomp(&re, alts, CBM_REG_EXTENDED | CBM_REG_NOSUB), CBM_REG_ETOOBIG); + regex_one_letter_alternation(alts, 508); + ASSERT_EQ(regex_units_ere(alts), 32766); + ASSERT_LTE(regex_units_ere(alts), (uint64_t)CBM_REGEX_COMPILE_BUDGET_UNITS); + /* TRE expands bracket members into union nodes, including case folds. */ + ASSERT_EQ(regex_units_ere("([abcdefghijklmnop]{255}){3}"), 36720); + ASSERT_EQ(cbm_regcomp(&re, "([abcdefghijklmnop]{255}){3}", CBM_REG_EXTENDED | CBM_REG_NOSUB), + CBM_REG_ETOOBIG); + ASSERT_EQ(cbm_regcomp_estimate_units("([abcdefgh]{255}){3}", CBM_REG_EXTENDED | CBM_REG_ICASE), + 36720); + ASSERT_EQ( + cbm_regcomp(&re, "([abcdefgh]{255}){3}", CBM_REG_EXTENDED | CBM_REG_NOSUB | CBM_REG_ICASE), + CBM_REG_ETOOBIG); + ASSERT_EQ(cbm_regcomp_estimate_units("[A-z]", CBM_REG_EXTENDED | CBM_REG_ICASE), 4); + ASSERT_GTE(cbm_regcomp_estimate_units("[[:lower:]]", CBM_REG_EXTENDED | CBM_REG_ICASE), 2); + /* Bounds too large to represent saturate instead of wrapping around. */ + ASSERT_EQ(cbm_regcomp(&re, "(a{99999999999999999999}){99999999999999999999}", + CBM_REG_EXTENDED | CBM_REG_NOSUB), + CBM_REG_ETOOBIG); + /* Nesting deeper than the sizing pass tracks is refused, not recursed into. */ + char deep[2 * 70 + 2]; + memset(deep, '(', 70); + deep[70] = 'a'; + memset(deep + 71, ')', 70); + deep[141] = '\0'; + ASSERT_EQ(cbm_regcomp(&re, deep, CBM_REG_EXTENDED | CBM_REG_NOSUB), CBM_REG_ETOOBIG); + PASS(); +} + +/* The same nested shape one step under the budget (13^4 = 28,561 units) + * compiles and matches, so the refusal starts only past the budget. */ +TEST(regex_compile_accepts_expansion_under_budget) { + cbm_regex_t re; + ASSERT_EQ(regex_units_ere("((((a){13}){13}){13}){13}"), 28561); + ASSERT_EQ(cbm_regcomp(&re, "((((a){13}){13}){13}){13}", CBM_REG_EXTENDED | CBM_REG_NOSUB), + CBM_REG_OK); + char *subject = malloc(28561 + 1); + ASSERT_NOT_NULL(subject); + memset(subject, 'a', 28561); + subject[28561] = '\0'; + ASSERT_EQ(cbm_regexec(&re, subject, 0, NULL, 0), CBM_REG_OK); + subject[28560] = '\0'; /* 28,560 a's: one short of 13^4 */ + ASSERT_EQ(cbm_regexec(&re, subject, 0, NULL, 0), CBM_REG_NOMATCH); + free(subject); + cbm_regfree(&re); + PASS(); +} + +/* A pattern longer than CBM_REGEX_PATTERN_MAX_BYTES is refused outright; one + * exactly at the cap still compiles (a literal at the cap is exactly the unit + * budget, which is accepted). */ +TEST(regex_compile_refuses_overlong_pattern) { + cbm_regex_t re; + char *pattern = malloc(CBM_REGEX_PATTERN_MAX_BYTES + 2); + ASSERT_NOT_NULL(pattern); + memset(pattern, 'a', CBM_REGEX_PATTERN_MAX_BYTES + 1); + pattern[CBM_REGEX_PATTERN_MAX_BYTES + 1] = '\0'; + ASSERT_EQ(cbm_regcomp(&re, pattern, CBM_REG_EXTENDED | CBM_REG_NOSUB), CBM_REG_ETOOBIG); + pattern[CBM_REGEX_PATTERN_MAX_BYTES] = '\0'; + ASSERT_EQ(cbm_regcomp(&re, pattern, CBM_REG_EXTENDED | CBM_REG_NOSUB), CBM_REG_OK); + cbm_regfree(&re); + free(pattern); + PASS(); +} + +/* Ordinary intervals, the pipeline's fixed patterns and patterns from the + * tool-level tests stay well inside the budget. */ +TEST(regex_compile_accepts_ordinary_intervals) { + static const char *const patterns[] = { + "[a-z]{1,64}", + ".{0,255}", + "(\\w{1,100}){1,20}", + "(foo|bar|baz){2,5}", + "^[A-Za-z_][A-Za-z0-9_]{0,255}$", + "[0-9]{4}-[0-9]{2}-[0-9]{2}", + /* pass_calls.c and pass_envscan.c */ + "Depends\\(([A-Za-z_][A-Za-z0-9_.]*)", + "^(ENV|ARG)[[:space:]]+([A-Za-z0-9_]+)[= ](.*)", + "(default|value)[[:space:]]*=[[:space:]]*\"(https?://[^\"]+)\"", + "(export[[:space:]]+)?([A-Za-z0-9_]+)=[\"']?(https?://[^ \t\"']+)", + /* test_mcp.c and test_cypher.c */ + ".*Order.*", + "^src/", + "^no_such_dir/", + "/api/.*", + "^design/", + }; + cbm_regex_t re; + for (size_t i = 0; i < sizeof(patterns) / sizeof(patterns[0]); i++) { + int rc = cbm_regcomp(&re, patterns[i], CBM_REG_EXTENDED | CBM_REG_NOSUB); + if (rc != CBM_REG_OK) { + printf(" pattern %zu refused with rc=%d: %s\n", i, rc, patterns[i]); + } + ASSERT_EQ(rc, CBM_REG_OK); + cbm_regfree(&re); + } + + /* A long alternation of identifiers, as a search for "any of these names" + * sends it: 400 names of up to nine characters is 3,761 bytes and about + * 23,400 units (20,000 of them the alternation share). It stays under both + * caps and compiles under the search_graph flag set. */ + static const char *const prefixes[] = {"parse", "emit", "visit", "bind", "scan"}; + char names[CBM_SZ_4K]; + char *w = names; + *w++ = '('; + for (int i = 0; i < 400; i++) { + w += snprintf(w, (size_t)(names + sizeof(names) - w), "%s%s_%03d", i ? "|" : "", + prefixes[i % 5], i); + } + *w++ = ')'; + *w = '\0'; + ASSERT_LTE(strlen(names), (size_t)CBM_REGEX_PATTERN_MAX_BYTES); + ASSERT_GTE(regex_units_ere(names), (uint64_t)20000); + ASSERT_LTE(regex_units_ere(names), (uint64_t)CBM_REGEX_COMPILE_BUDGET_UNITS); + ASSERT_EQ(cbm_regcomp(&re, names, CBM_REG_EXTENDED | CBM_REG_NOSUB | CBM_REG_ICASE), + CBM_REG_OK); + ASSERT_EQ(cbm_regexec(&re, "Bind_123", 0, NULL, 0), CBM_REG_OK); + ASSERT_EQ(cbm_regexec(&re, "bind_400", 0, NULL, 0), CBM_REG_NOMATCH); + cbm_regfree(&re); + + /* The same at realistic identifier length: 400 names of 20 characters is + * 8,401 bytes and 28,000 units, inside the budget — the length cap never + * decides a name list of this size, the unit budget does. */ + static const char *const long_prefixes[] = {"compute_checksum", "validate_request", + "serialize_record", "normalize_header", + "dispatch_message"}; + const size_t long_cap = 400 * 21 + 3; + char *long_names = malloc(long_cap); + ASSERT_NOT_NULL(long_names); + w = long_names; + *w++ = '('; + for (int i = 0; i < 400; i++) { + w += snprintf(w, long_cap - (size_t)(w - long_names), "%s%s_%03d", i ? "|" : "", + long_prefixes[i % 5], i); + } + *w++ = ')'; + *w = '\0'; + ASSERT_EQ(strlen(long_names), 8401); + ASSERT_EQ(regex_units_ere(long_names), 28000); + ASSERT_EQ(cbm_regcomp(&re, long_names, CBM_REG_EXTENDED | CBM_REG_NOSUB), CBM_REG_OK); + ASSERT_EQ(cbm_regexec(&re, "normalize_header_123", 0, NULL, 0), CBM_REG_OK); + ASSERT_EQ(cbm_regexec(&re, "normalize_header_400", 0, NULL, 0), CBM_REG_NOMATCH); + cbm_regfree(&re); + free(long_names); + + /* `.{0,500}` is 500 units, inside the budget. Whether it compiles is the + * platform's RE_DUP_MAX (255 on the TRE-derived libcs, 32767 on glibc), + * which the guard does not change. */ + ASSERT_LTE(regex_units_ere(".{0,500}"), (uint64_t)CBM_REGEX_COMPILE_BUDGET_UNITS); +#ifdef __linux__ + ASSERT_EQ(cbm_regcomp(&re, ".{0,500}", CBM_REG_EXTENDED | CBM_REG_NOSUB), CBM_REG_OK); + cbm_regfree(&re); +#endif + PASS(); +} + +/* Sizing must read the syntax the way the compilers do: nothing inside a + * bracket expression operates, a backslash escapes one character, braces that + * do not form an interval are literals, and only an interval multiplies. */ +TEST(regex_estimate_counts_brackets_and_escapes_literally) { + ASSERT_EQ(regex_units_ere("[{]{2}"), 2); + ASSERT_EQ(regex_units_ere("\\{"), 1); + ASSERT_EQ(regex_units_ere("a\\{2\\}"), 4); + ASSERT_EQ(regex_units_ere("[]a]{2}"), 4); + ASSERT_EQ(regex_units_ere("[^]a]{3}"), 12); + ASSERT_GTE(regex_units_ere("[[:alpha:]]{4}"), 4); + ASSERT_EQ(regex_units_ere("[a-z{3}]"), 6); + ASSERT_EQ(regex_units_ere("a{"), 2); + ASSERT_EQ(regex_units_ere("a{2"), 3); + ASSERT_EQ(regex_units_ere("a{x}"), 4); + ASSERT_EQ(regex_units_ere("a{2}"), 2); + ASSERT_EQ(regex_units_ere("a{2,}"), 2); + ASSERT_EQ(regex_units_ere("a{,3}"), 3); + ASSERT_EQ(regex_units_ere("a{2,5}"), 5); + ASSERT_EQ(regex_units_ere("a*b+c?"), 3); + ASSERT_EQ(regex_units_ere("a{3}{3}"), 9); + ASSERT_EQ(regex_units_ere("(ab){3}"), 6); + ASSERT_EQ(regex_units_ere("((ab){3}){4}"), 24); + /* A K-way alternation is charged K*K/8 on top of its branches. */ + ASSERT_EQ(regex_units_ere("a|b|c|d"), 4 + 2); + ASSERT_EQ(regex_units_ere("(a|b|c|d){10}"), 60); + /* Basic syntax: bare braces and parens are literals; `\{ \}` and `\( \)` + * operate. No call site uses it today, but the guard is shared. */ + ASSERT_EQ(regex_units_bre("a{2}"), 4); + ASSERT_EQ(regex_units_bre("a\\{2\\}"), 2); + ASSERT_EQ(regex_units_bre("\\(ab\\)\\{3\\}"), 6); + ASSERT_EQ(regex_units_bre("(ab){3}"), 7); + PASS(); +} + SUITE(platform) { RUN_TEST(platform_file_apis_survive_max_path_overflow); RUN_TEST(platform_mkdir_p_follows_own_symlink_only_when_opted_in); @@ -1221,6 +1461,11 @@ SUITE(platform) { RUN_TEST(platform_env_long_reads_a_clean_number); RUN_TEST(platform_env_long_refuses_what_it_cannot_read); RUN_TEST(platform_system_info); + RUN_TEST(regex_compile_refuses_oversized_expansion); + RUN_TEST(regex_compile_accepts_expansion_under_budget); + RUN_TEST(regex_compile_refuses_overlong_pattern); + RUN_TEST(regex_compile_accepts_ordinary_intervals); + RUN_TEST(regex_estimate_counts_brackets_and_escapes_literally); #ifdef __linux__ RUN_TEST(cgroup_v2_cpu_quota); RUN_TEST(cgroup_v2_cpu_quota_rounds_up); diff --git a/tests/test_py_lsp.c b/tests/test_py_lsp.c index 3aaadee519..2c80434342 100644 --- a/tests/test_py_lsp.c +++ b/tests/test_py_lsp.c @@ -16,8 +16,8 @@ /* ── Helpers — same shape as test_go_lsp.c ──────────────────────── */ static CBMFileResult *extract_py(const char *source) { - return cbm_extract_file(source, (int)strlen(source), CBM_LANG_PYTHON, - "test", "main.py", 0, NULL, NULL); + return cbm_extract_file(source, (int)strlen(source), CBM_LANG_PYTHON, "test", "main.py", 0, + NULL, NULL); } static int find_resolved(const CBMFileResult *r, const char *callerSub, const char *calleeSub) { @@ -32,18 +32,16 @@ static int find_resolved(const CBMFileResult *r, const char *callerSub, const ch /* Avoid unused-static-function warnings: helpers compiled but not yet used * outside the smoke tests will be referenced in Phase 3+ tests. */ -__attribute__((unused)) -static int require_resolved(const CBMFileResult *r, const char *callerSub, const char *calleeSub) { +__attribute__((unused)) static int require_resolved(const CBMFileResult *r, const char *callerSub, + const char *calleeSub) { int idx = find_resolved(r, callerSub, calleeSub); if (idx < 0) { - printf(" MISSING resolved call: caller~%s -> callee~%s (have %d)\n", - callerSub, calleeSub, r->resolved_calls.count); + printf(" MISSING resolved call: caller~%s -> callee~%s (have %d)\n", callerSub, calleeSub, + r->resolved_calls.count); for (int i = 0; i < r->resolved_calls.count; i++) { const CBMResolvedCall *rc = &r->resolved_calls.items[i]; - printf(" %s -> %s [%s %.2f]\n", - rc->caller_qn ? rc->caller_qn : "(null)", - rc->callee_qn ? rc->callee_qn : "(null)", - rc->strategy ? rc->strategy : "(null)", + printf(" %s -> %s [%s %.2f]\n", rc->caller_qn ? rc->caller_qn : "(null)", + rc->callee_qn ? rc->callee_qn : "(null)", rc->strategy ? rc->strategy : "(null)", rc->confidence); } } @@ -61,9 +59,8 @@ TEST(pylsp_smoke_empty) { } TEST(pylsp_smoke_one_function) { - CBMFileResult *r = extract_py( - "def greet(name):\n" - " return name\n"); + CBMFileResult *r = extract_py("def greet(name):\n" + " return name\n"); ASSERT_NOT_NULL(r); /* Phase 2 stub: no resolutions yet, but extraction must succeed and * the result must be addressable without crashes. */ @@ -73,12 +70,11 @@ TEST(pylsp_smoke_one_function) { } TEST(pylsp_smoke_one_class) { - CBMFileResult *r = extract_py( - "class Greeter:\n" - " def __init__(self, name):\n" - " self.name = name\n" - " def greet(self):\n" - " return self.name\n"); + CBMFileResult *r = extract_py("class Greeter:\n" + " def __init__(self, name):\n" + " self.name = name\n" + " def greet(self):\n" + " return self.name\n"); ASSERT_NOT_NULL(r); /* Class + 2 methods at minimum */ ASSERT_GTE(r->defs.count, 1); @@ -89,10 +85,9 @@ TEST(pylsp_smoke_one_class) { TEST(pylsp_no_crash_on_syntax_error) { /* Tree-sitter recovers from errors but we must not crash on the * recovered tree. */ - CBMFileResult *r = extract_py( - "def broken(\n" - " x = 1\n" - "class\n"); + CBMFileResult *r = extract_py("def broken(\n" + " x = 1\n" + "class\n"); ASSERT_NOT_NULL(r); cbm_free_result(r); PASS(); @@ -102,13 +97,12 @@ TEST(pylsp_smoke_imports_passed_through) { /* Imports populate ctx->import_local_names — Phase 2 just verifies * the unified extractor still produces them; resolution happens in * Phase 3. */ - CBMFileResult *r = extract_py( - "import os\n" - "import json as j\n" - "from pathlib import Path\n" - "from . import sibling\n" - "def use():\n" - " return os.getcwd()\n"); + CBMFileResult *r = extract_py("import os\n" + "import json as j\n" + "from pathlib import Path\n" + "from . import sibling\n" + "def use():\n" + " return os.getcwd()\n"); ASSERT_NOT_NULL(r); ASSERT_GTE(r->imports.count, 3); cbm_free_result(r); @@ -120,8 +114,7 @@ TEST(pylsp_smoke_imports_passed_through) { /* Build a context, register one or more imports, run the binding pass, * and let the caller verify scope state. */ static void bind_imports_into_ctx(PyLSPContext *ctx, CBMArena *a, CBMTypeRegistry *reg, - const char *const *locals, const char *const *qns, - int count) { + const char *const *locals, const char *const *qns, int count) { py_lsp_init(ctx, a, "", 0, reg, "test.main", NULL); for (int i = 0; i < count; i++) { py_lsp_add_import(ctx, locals[i], qns[i]); @@ -279,12 +272,11 @@ TEST(pylsp_import_multi_pass_through_extract_file) { /* End-to-end: extract_file + run_py_lsp populate scope via imports. * We can't peek into the embedded ctx, but we verify imports survive * to the result and bind correctly when re-traversed. */ - CBMFileResult *r = extract_py( - "import os\n" - "import json as j\n" - "from pathlib import Path\n" - "def use():\n" - " return Path('.')\n"); + CBMFileResult *r = extract_py("import os\n" + "import json as j\n" + "from pathlib import Path\n" + "def use():\n" + " return Path('.')\n"); ASSERT_NOT_NULL(r); ASSERT_GTE(r->imports.count, 3); cbm_free_result(r); @@ -296,11 +288,10 @@ TEST(pylsp_import_multi_pass_through_extract_file) { TEST(pylsp_direct_function_call) { /* def helper(): return 1 * def main(): return helper() */ - CBMFileResult *r = extract_py( - "def helper():\n" - " return 1\n" - "def main():\n" - " return helper()\n"); + CBMFileResult *r = extract_py("def helper():\n" + " return 1\n" + "def main():\n" + " return helper()\n"); ASSERT_NOT_NULL(r); ASSERT_GTE(require_resolved(r, "main", "helper"), 0); cbm_free_result(r); @@ -312,12 +303,11 @@ TEST(pylsp_method_call_simple) { * def m(self): return 1 * def use(c): * c.m() -- with annotation */ - CBMFileResult *r = extract_py( - "class C:\n" - " def m(self):\n" - " return 1\n" - "def use(c: C):\n" - " return c.m()\n"); + CBMFileResult *r = extract_py("class C:\n" + " def m(self):\n" + " return 1\n" + "def use(c: C):\n" + " return c.m()\n"); ASSERT_NOT_NULL(r); ASSERT_GTE(require_resolved(r, "use", "m"), 0); cbm_free_result(r); @@ -325,12 +315,11 @@ TEST(pylsp_method_call_simple) { } TEST(pylsp_method_via_self) { - CBMFileResult *r = extract_py( - "class C:\n" - " def helper(self):\n" - " return 1\n" - " def caller(self):\n" - " return self.helper()\n"); + CBMFileResult *r = extract_py("class C:\n" + " def helper(self):\n" + " return 1\n" + " def caller(self):\n" + " return self.helper()\n"); ASSERT_NOT_NULL(r); ASSERT_GTE(require_resolved(r, "caller", "helper"), 0); cbm_free_result(r); @@ -342,13 +331,12 @@ TEST(pylsp_constructor_call_returns_instance) { * def use(): * f = Foo() * f.method() -- requires inferring f as Foo */ - CBMFileResult *r = extract_py( - "class Foo:\n" - " def method(self):\n" - " return 1\n" - "def use():\n" - " f = Foo()\n" - " return f.method()\n"); + CBMFileResult *r = extract_py("class Foo:\n" + " def method(self):\n" + " return 1\n" + "def use():\n" + " f = Foo()\n" + " return f.method()\n"); ASSERT_NOT_NULL(r); ASSERT_GTE(require_resolved(r, "use", "method"), 0); cbm_free_result(r); @@ -356,13 +344,12 @@ TEST(pylsp_constructor_call_returns_instance) { } TEST(pylsp_method_via_inheritance) { - CBMFileResult *r = extract_py( - "class Base:\n" - " def shared(self):\n" - " return 1\n" - "class Child(Base):\n" - " def go(self):\n" - " return self.shared()\n"); + CBMFileResult *r = extract_py("class Base:\n" + " def shared(self):\n" + " return 1\n" + "class Child(Base):\n" + " def go(self):\n" + " return self.shared()\n"); ASSERT_NOT_NULL(r); ASSERT_GTE(require_resolved(r, "go", "shared"), 0); cbm_free_result(r); @@ -372,9 +359,8 @@ TEST(pylsp_method_via_inheritance) { TEST(pylsp_no_false_positive_on_unknown_method) { /* Calling a method on an UNKNOWN type should NOT emit a high-confidence * resolution. */ - CBMFileResult *r = extract_py( - "def f(x):\n" - " return x.something_unknown_42()\n"); + CBMFileResult *r = extract_py("def f(x):\n" + " return x.something_unknown_42()\n"); ASSERT_NOT_NULL(r); /* Should produce no high-confidence match for "something_unknown_42" */ int idx = find_resolved(r, "f", "something_unknown_42"); @@ -390,13 +376,12 @@ TEST(pylsp_no_false_positive_on_unknown_method) { TEST(pylsp_decorated_function_resolves) { /* Decorated functions still resolve as their bare-name target. */ - CBMFileResult *r = extract_py( - "import functools\n" - "@functools.cache\n" - "def helper():\n" - " return 1\n" - "def main():\n" - " return helper()\n"); + CBMFileResult *r = extract_py("import functools\n" + "@functools.cache\n" + "def helper():\n" + " return 1\n" + "def main():\n" + " return helper()\n"); ASSERT_NOT_NULL(r); ASSERT_GTE(require_resolved(r, "main", "helper"), 0); cbm_free_result(r); @@ -404,13 +389,12 @@ TEST(pylsp_decorated_function_resolves) { } TEST(pylsp_classmethod_resolves) { - CBMFileResult *r = extract_py( - "class C:\n" - " @classmethod\n" - " def make(cls):\n" - " return cls()\n" - "def use():\n" - " return C.make()\n"); + CBMFileResult *r = extract_py("class C:\n" + " @classmethod\n" + " def make(cls):\n" + " return cls()\n" + "def use():\n" + " return C.make()\n"); ASSERT_NOT_NULL(r); ASSERT_GTE(require_resolved(r, "use", "make"), 0); cbm_free_result(r); @@ -418,13 +402,12 @@ TEST(pylsp_classmethod_resolves) { } TEST(pylsp_staticmethod_resolves) { - CBMFileResult *r = extract_py( - "class C:\n" - " @staticmethod\n" - " def add(a, b):\n" - " return a + b\n" - "def use():\n" - " return C.add(1, 2)\n"); + CBMFileResult *r = extract_py("class C:\n" + " @staticmethod\n" + " def add(a, b):\n" + " return a + b\n" + "def use():\n" + " return C.add(1, 2)\n"); ASSERT_NOT_NULL(r); ASSERT_GTE(require_resolved(r, "use", "add"), 0); cbm_free_result(r); @@ -434,17 +417,16 @@ TEST(pylsp_staticmethod_resolves) { TEST(pylsp_dataclass_constructor) { /* @dataclass synthesizes __init__. We don't emit __init__ explicitly, * but the constructor call should still link to the class qn. */ - CBMFileResult *r = extract_py( - "from dataclasses import dataclass\n" - "@dataclass\n" - "class Point:\n" - " x: int\n" - " y: int\n" - " def magnitude(self):\n" - " return self.x + self.y\n" - "def use():\n" - " p = Point(1, 2)\n" - " return p.magnitude()\n"); + CBMFileResult *r = extract_py("from dataclasses import dataclass\n" + "@dataclass\n" + "class Point:\n" + " x: int\n" + " y: int\n" + " def magnitude(self):\n" + " return self.x + self.y\n" + "def use():\n" + " p = Point(1, 2)\n" + " return p.magnitude()\n"); ASSERT_NOT_NULL(r); ASSERT_GTE(require_resolved(r, "use", "magnitude"), 0); cbm_free_result(r); @@ -452,13 +434,12 @@ TEST(pylsp_dataclass_constructor) { } TEST(pylsp_super_call) { - CBMFileResult *r = extract_py( - "class Base:\n" - " def greet(self):\n" - " return 'hi'\n" - "class Child(Base):\n" - " def greet(self):\n" - " return super().greet()\n"); + CBMFileResult *r = extract_py("class Base:\n" + " def greet(self):\n" + " return 'hi'\n" + "class Child(Base):\n" + " def greet(self):\n" + " return super().greet()\n"); ASSERT_NOT_NULL(r); /* super().greet() should resolve to Base.greet, not Child.greet. */ int idx = find_resolved(r, "greet", "greet"); @@ -472,17 +453,16 @@ TEST(pylsp_super_call) { } TEST(pylsp_multi_inheritance_first_base) { - CBMFileResult *r = extract_py( - "class A:\n" - " def a_method(self):\n" - " return 1\n" - "class B:\n" - " def b_method(self):\n" - " return 2\n" - "class C(A, B):\n" - " def use(self):\n" - " self.a_method()\n" - " self.b_method()\n"); + CBMFileResult *r = extract_py("class A:\n" + " def a_method(self):\n" + " return 1\n" + "class B:\n" + " def b_method(self):\n" + " return 2\n" + "class C(A, B):\n" + " def use(self):\n" + " self.a_method()\n" + " self.b_method()\n"); ASSERT_NOT_NULL(r); ASSERT_GTE(require_resolved(r, "use", "a_method"), 0); ASSERT_GTE(require_resolved(r, "use", "b_method"), 0); @@ -492,12 +472,11 @@ TEST(pylsp_multi_inheritance_first_base) { TEST(pylsp_pep695_generic_class) { /* PEP 695: class Box[T]: -- our implementation ignores the [T] part */ - CBMFileResult *r = extract_py( - "class Box:\n" - " def get(self):\n" - " return 1\n" - "def use(b: Box):\n" - " return b.get()\n"); + CBMFileResult *r = extract_py("class Box:\n" + " def get(self):\n" + " return 1\n" + "def use(b: Box):\n" + " return b.get()\n"); ASSERT_NOT_NULL(r); ASSERT_GTE(require_resolved(r, "use", "get"), 0); cbm_free_result(r); @@ -510,8 +489,8 @@ static int find_resolved_arr(const CBMResolvedCallArray *arr, const char *caller const char *calleeSub) { for (int i = 0; i < arr->count; i++) { const CBMResolvedCall *rc = &arr->items[i]; - if (rc->caller_qn && strstr(rc->caller_qn, callerSub) && - rc->callee_qn && strstr(rc->callee_qn, calleeSub)) + if (rc->caller_qn && strstr(rc->caller_qn, callerSub) && rc->callee_qn && + strstr(rc->callee_qn, calleeSub)) return i; } return -1; @@ -547,10 +526,9 @@ TEST(pylsp_crossfile_method_dispatch) { /* file svc.py defines class RedisStore with Get(); file main.py calls * the method on a typed parameter. Reuses CBMLSPDef to feed the * cross-file definition into the resolver. */ - const char *source = - "from svc import RedisStore\n" - "def process(s: RedisStore):\n" - " return s.Get('k')\n"; + const char *source = "from svc import RedisStore\n" + "def process(s: RedisStore):\n" + " return s.Get('k')\n"; CBMLSPDef defs[2]; memset(defs, 0, sizeof(defs)); @@ -580,6 +558,119 @@ TEST(pylsp_crossfile_method_dispatch) { PASS(); } +/* Issue #1277 (reproducer contributed by @Enferlain): a method call through a + * typed instance field of a class imported from ANOTHER file. The field type + * reaches the resolver only through CBMLSPDef.field_defs on the class def. */ +TEST(pylsp_crossfile_receiver_through_typed_field) { + const char *source = "from trainer import Trainer\n" + "def run(trainer: Trainer):\n" + " strategies = trainer.strategies\n" + " return strategies.process_batch()\n"; + + CBMLSPDef defs[3]; + memset(defs, 0, sizeof(defs)); + + defs[0].qualified_name = "contracts.Contract"; + defs[0].short_name = "Contract"; + defs[0].label = "Class"; + defs[0].def_module_qn = "contracts"; + + defs[1].qualified_name = "contracts.Contract.process_batch"; + defs[1].short_name = "process_batch"; + defs[1].label = "Method"; + defs[1].receiver_type = "contracts.Contract"; + defs[1].def_module_qn = "contracts"; + + defs[2].qualified_name = "trainer.Trainer"; + defs[2].short_name = "Trainer"; + defs[2].label = "Class"; + defs[2].def_module_qn = "trainer"; + defs[2].field_defs = "strategies:contracts.Contract"; + + const char *imp_names[] = {"Trainer"}; + const char *imp_qns[] = {"trainer.Trainer"}; + + CBMArena arena; + cbm_arena_init(&arena); + CBMResolvedCallArray out = {0}; + + cbm_run_py_lsp_cross(&arena, source, (int)strlen(source), "test.loop", defs, 3, imp_names, + imp_qns, 1, NULL, &out, NULL); + + ASSERT_GTE(find_resolved_arr(&out, "run", "process_batch"), 0); + cbm_arena_destroy(&arena); + PASS(); +} + +/* #1277 on the production path: the sealed shared Tier-2 registry. Covers the + * direct chain, the local alias, a field inherited from a base class, and two + * controls: an undeclared field and a field whose type has no such method + * must stay unresolved (never guessed). */ +TEST(pylsp_crossfile_typed_field_shared_registry) { + CBMArena arena; + cbm_arena_init(&arena); + + CBMLSPDef defs[5]; + memset(defs, 0, sizeof(defs)); + defs[0].qualified_name = "contracts.Contract"; + defs[0].short_name = "Contract"; + defs[0].label = "Class"; + defs[0].def_module_qn = "contracts"; + defs[1].qualified_name = "contracts.Contract.process_batch"; + defs[1].short_name = "process_batch"; + defs[1].label = "Method"; + defs[1].receiver_type = "contracts.Contract"; + defs[1].def_module_qn = "contracts"; + defs[2].qualified_name = "trainer.Trainer"; + defs[2].short_name = "Trainer"; + defs[2].label = "Class"; + defs[2].def_module_qn = "trainer"; + defs[2].field_defs = "strategies:contracts.Contract|name:builtins.str"; + defs[3].qualified_name = "trainer.SubTrainer"; + defs[3].short_name = "SubTrainer"; + defs[3].label = "Class"; + defs[3].def_module_qn = "trainer"; + defs[3].embedded_types = "trainer.Trainer"; + defs[4].qualified_name = "trainer.Other"; + defs[4].short_name = "Other"; + defs[4].label = "Class"; + defs[4].def_module_qn = "trainer"; + for (int i = 0; i < 5; i++) { + defs[i].lang = CBM_LANG_PYTHON; + } + + CBMTypeRegistry *reg = cbm_py_build_cross_registry(&arena, defs, 5); + ASSERT_NOT_NULL(reg); + ASSERT_TRUE(reg->read_only); + + const char *src = "from trainer import Trainer, SubTrainer, Other\n" + "def direct(t: Trainer):\n" + " return t.strategies.process_batch()\n" + "def alias(t: Trainer):\n" + " s = t.strategies\n" + " return s.process_batch()\n" + "def inherited(t: SubTrainer):\n" + " return t.strategies.process_batch()\n" + "def undeclared(t: Other):\n" + " return t.strategies.process_batch()\n" + "def wrong_type(t: Trainer):\n" + " return t.name.process_batch()\n"; + const char *imp_names[] = {"Trainer", "SubTrainer", "Other"}; + const char *imp_qns[] = {"trainer.Trainer", "trainer.SubTrainer", "trainer.Other"}; + CBMResolvedCallArray out = {0}; + cbm_run_py_lsp_cross_with_registry(&arena, src, (int)strlen(src), "test.loop", reg, imp_names, + imp_qns, 3, NULL, &out, NULL); + + ASSERT_GTE(find_resolved_arr(&out, "direct", "Contract.process_batch"), 0); + ASSERT_GTE(find_resolved_arr(&out, "alias", "Contract.process_batch"), 0); + ASSERT_GTE(find_resolved_arr(&out, "inherited", "Contract.process_batch"), 0); + ASSERT_EQ(find_resolved_arr(&out, "undeclared", "process_batch"), -1); + ASSERT_EQ(find_resolved_arr(&out, "wrong_type", "process_batch"), -1); + + cbm_arena_destroy(&arena); + PASS(); +} + /* Graph-quality guard for the Python seal fix (per-file field overlay). In the * FUSED path the shared Tier-2 registry is read_only, so `self.b = Bar()` can no * longer be recorded by mutating the shared registry; it is recorded in the per-file @@ -642,10 +733,9 @@ TEST(pylsp_fused_self_attr_chain_via_overlay) { * the method showed in/out degree 0 and was flagged as dead code. Distinct from * pylsp_crossfile_method_dispatch (which dispatches on a typed *instance*). */ TEST(pylsp_crossfile_classmethod_on_class_issue228) { - const char *source = - "from core.schemas import ActionRecordX\n" - "def run_plain_flow():\n" - " return ActionRecordX.build_from_text('hello')\n"; + const char *source = "from core.schemas import ActionRecordX\n" + "def run_plain_flow():\n" + " return ActionRecordX.build_from_text('hello')\n"; CBMLSPDef defs[2]; memset(defs, 0, sizeof(defs)); @@ -679,11 +769,10 @@ TEST(pylsp_crossfile_inheritance) { /* svc.py defines class Base with shared(); main.py defines class Child(Base) * and calls self.shared(). Caller passes ALL relevant defs (cross-file * Base/shared + local Child/go) — same convention as test_go_lsp.c. */ - const char *source = - "from svc import Base\n" - "class Child(Base):\n" - " def go(self):\n" - " return self.shared()\n"; + const char *source = "from svc import Base\n" + "class Child(Base):\n" + " def go(self):\n" + " return self.shared()\n"; CBMLSPDef defs[4]; memset(defs, 0, sizeof(defs)); @@ -726,13 +815,11 @@ TEST(pylsp_crossfile_inheritance) { } TEST(pylsp_batch_two_files) { - const char *src_a = - "def helper():\n" - " return 1\n"; - const char *src_b = - "from a import helper\n" - "def main():\n" - " return helper()\n"; + const char *src_a = "def helper():\n" + " return 1\n"; + const char *src_b = "from a import helper\n" + "def main():\n" + " return helper()\n"; CBMLSPDef a_defs[1]; memset(a_defs, 0, sizeof(a_defs)); @@ -788,8 +875,8 @@ TEST(pylsp_batch_two_files) { } static int pylsp_exact_reference_count(const CBMResolvedCallArray *out, const char *caller, - const char *callee_qn, const char *reason, - uint32_t site_start, uint32_t site_end) { + const char *callee_qn, const char *reason, + uint32_t site_start, uint32_t site_end) { int count = 0; for (int i = 0; out && i < out->count; i++) { const CBMResolvedCall *resolved = &out->items[i]; @@ -810,9 +897,11 @@ typedef struct { int second_target; } PyImportReferenceCounts; -static PyImportReferenceCounts pylsp_import_reference_counts( - const char *source, const char *local_name, const char *import_qn, const char *first_target_qn, - const char *second_target_qn) { +static PyImportReferenceCounts pylsp_import_reference_counts(const char *source, + const char *local_name, + const char *import_qn, + const char *first_target_qn, + const char *second_target_qn) { PyImportReferenceCounts counts = {0}; const char *call = strstr(source, " accept("); if (!call) @@ -861,11 +950,11 @@ static PyImportReferenceCounts pylsp_import_reference_counts( cbm_arena_init(&arena); CBMResolvedCallArray out = {0}; cbm_batch_py_lsp_cross(&arena, &file, 1, &out); - counts.first_target = pylsp_exact_reference_count( - &out, "crossArgument", first_target_qn, local_name, site_start, site_end); + counts.first_target = pylsp_exact_reference_count(&out, "crossArgument", first_target_qn, + local_name, site_start, site_end); if (second_target_qn) { - counts.second_target = pylsp_exact_reference_count( - &out, "crossArgument", second_target_qn, local_name, site_start, site_end); + counts.second_target = pylsp_exact_reference_count(&out, "crossArgument", second_target_qn, + local_name, site_start, site_end); } cbm_arena_destroy(&arena); return counts; @@ -1012,7 +1101,7 @@ TEST(pylsp_batch_cross_file_callable_value_preserves_exact_reference_site) { cbm_batch_py_lsp_cross(&arena, &file, 1, &out); int exact = pylsp_exact_reference_count(&out, "crossArgument", "target.handler", NULL, - site_start, site_end); + site_start, site_end); cbm_arena_destroy(&arena); ASSERT_EQ(exact, 1); @@ -1042,9 +1131,8 @@ TEST(pylsp_batch_cross_file_callable_value_preserves_exact_reference_site) { cbm_arena_init(&arena); cbm_batch_py_lsp_cross(&arena, &file, 1, &out); - int alias_exact = pylsp_exact_reference_count(&out, "crossArgument", - "project.target.handler", "callback", - site_start, site_end); + int alias_exact = pylsp_exact_reference_count(&out, "crossArgument", "project.target.handler", + "callback", site_start, site_end); cbm_arena_destroy(&arena); ASSERT_EQ(alias_exact, 1); @@ -1278,10 +1366,9 @@ TEST(pylsp_scratch_cross_dunder_carrier_survives_copy) { TEST(pylsp_stdlib_os_getcwd) { /* Top-level module attribute resolution against the stdlib registry. */ - CBMFileResult *r = extract_py( - "import os\n" - "def use():\n" - " return os.getcwd()\n"); + CBMFileResult *r = extract_py("import os\n" + "def use():\n" + " return os.getcwd()\n"); ASSERT_NOT_NULL(r); int idx = find_resolved(r, "use", "getcwd"); ASSERT_GTE(idx, 0); @@ -1290,10 +1377,9 @@ TEST(pylsp_stdlib_os_getcwd) { } TEST(pylsp_stdlib_collections_defaultdict) { - CBMFileResult *r = extract_py( - "from collections import defaultdict\n" - "def use():\n" - " return defaultdict(list)\n"); + CBMFileResult *r = extract_py("from collections import defaultdict\n" + "def use():\n" + " return defaultdict(list)\n"); ASSERT_NOT_NULL(r); /* defaultdict(list) is a constructor — emits lsp_constructor edge */ int idx = find_resolved(r, "use", "defaultdict"); @@ -1303,10 +1389,9 @@ TEST(pylsp_stdlib_collections_defaultdict) { } TEST(pylsp_stdlib_pathlib_path_method) { - CBMFileResult *r = extract_py( - "from pathlib import Path\n" - "def use(p: Path):\n" - " return p.exists()\n"); + CBMFileResult *r = extract_py("from pathlib import Path\n" + "def use(p: Path):\n" + " return p.exists()\n"); ASSERT_NOT_NULL(r); int idx = find_resolved(r, "use", "exists"); ASSERT_GTE(idx, 0); @@ -1315,10 +1400,9 @@ TEST(pylsp_stdlib_pathlib_path_method) { } TEST(pylsp_stdlib_logging_getlogger) { - CBMFileResult *r = extract_py( - "import logging\n" - "def use():\n" - " return logging.getLogger('app')\n"); + CBMFileResult *r = extract_py("import logging\n" + "def use():\n" + " return logging.getLogger('app')\n"); ASSERT_NOT_NULL(r); int idx = find_resolved(r, "use", "getLogger"); ASSERT_GTE(idx, 0); @@ -1332,10 +1416,9 @@ TEST(pylsp_round1_dotted_import_walk) { /* `import os.path` — `os` and `os.path` should both be navigable * through attribute access so `os.path.join('a', 'b')` resolves to * the registered os.path.join function. */ - CBMFileResult *r = extract_py( - "import os.path\n" - "def use():\n" - " return os.path.join('a', 'b')\n"); + CBMFileResult *r = extract_py("import os.path\n" + "def use():\n" + " return os.path.join('a', 'b')\n"); ASSERT_NOT_NULL(r); ASSERT_EQ(r->imports.count, 1); ASSERT_STR_EQ(r->imports.items[0].local_name, "os"); @@ -1366,14 +1449,13 @@ TEST(pylsp_round1_dotted_import_alias_matching_root) { TEST(pylsp_round1_typing_cast) { /* cast(Foo, x) returns NAMED("Foo"), enabling subsequent method * dispatch to resolve. */ - CBMFileResult *r = extract_py( - "from typing import cast\n" - "class Foo:\n" - " def m(self):\n" - " return 1\n" - "def use(x):\n" - " f = cast(Foo, x)\n" - " return f.m()\n"); + CBMFileResult *r = extract_py("from typing import cast\n" + "class Foo:\n" + " def m(self):\n" + " return 1\n" + "def use(x):\n" + " f = cast(Foo, x)\n" + " return f.m()\n"); ASSERT_NOT_NULL(r); ASSERT_GTE(require_resolved(r, "use", "m"), 0); cbm_free_result(r); @@ -1383,14 +1465,13 @@ TEST(pylsp_round1_typing_cast) { TEST(pylsp_round1_assert_type_passthrough) { /* assert_type(x, T) is a no-op at runtime; the returned value's type * is unchanged. We type the result as type-of(x). */ - CBMFileResult *r = extract_py( - "from typing import assert_type\n" - "class Foo:\n" - " def m(self):\n" - " return 1\n" - "def use(x: Foo):\n" - " f = assert_type(x, Foo)\n" - " return f.m()\n"); + CBMFileResult *r = extract_py("from typing import assert_type\n" + "class Foo:\n" + " def m(self):\n" + " return 1\n" + "def use(x: Foo):\n" + " f = assert_type(x, Foo)\n" + " return f.m()\n"); ASSERT_NOT_NULL(r); ASSERT_GTE(require_resolved(r, "use", "m"), 0); cbm_free_result(r); @@ -1399,12 +1480,11 @@ TEST(pylsp_round1_assert_type_passthrough) { TEST(pylsp_round1_forward_reference) { /* def f(x: "Foo") — quoted annotation must resolve as if unquoted. */ - CBMFileResult *r = extract_py( - "class Foo:\n" - " def m(self):\n" - " return 1\n" - "def use(x: \"Foo\"):\n" - " return x.m()\n"); + CBMFileResult *r = extract_py("class Foo:\n" + " def m(self):\n" + " return 1\n" + "def use(x: \"Foo\"):\n" + " return x.m()\n"); ASSERT_NOT_NULL(r); ASSERT_GTE(require_resolved(r, "use", "m"), 0); cbm_free_result(r); @@ -1417,17 +1497,16 @@ TEST(pylsp_round1_self_return_chains) { * def step2(self) -> Self: return self * def build(self): return ... * Builder().step1().step2().build() — must chain through Self. */ - CBMFileResult *r = extract_py( - "from typing import Self\n" - "class Builder:\n" - " def step1(self) -> Self:\n" - " return self\n" - " def step2(self) -> Self:\n" - " return self\n" - " def build(self):\n" - " return 1\n" - "def use():\n" - " return Builder().step1().step2().build()\n"); + CBMFileResult *r = extract_py("from typing import Self\n" + "class Builder:\n" + " def step1(self) -> Self:\n" + " return self\n" + " def step2(self) -> Self:\n" + " return self\n" + " def build(self):\n" + " return 1\n" + "def use():\n" + " return Builder().step1().step2().build()\n"); ASSERT_NOT_NULL(r); /* Each chain link should resolve. We assert the final .build() does. */ ASSERT_GTE(require_resolved(r, "use", "build"), 0); @@ -1438,13 +1517,12 @@ TEST(pylsp_round1_self_return_chains) { TEST(pylsp_round1_generic_subscript_annotation) { /* `def f(items: list[Foo])` — the generic subscript should not * confuse annotation resolution; we drop the [Foo] part for v1. */ - CBMFileResult *r = extract_py( - "from typing import Optional\n" - "class Foo:\n" - " def m(self):\n" - " return 1\n" - "def use(x: Optional[Foo]):\n" - " return x.m()\n"); + CBMFileResult *r = extract_py("from typing import Optional\n" + "class Foo:\n" + " def m(self):\n" + " return 1\n" + "def use(x: Optional[Foo]):\n" + " return x.m()\n"); ASSERT_NOT_NULL(r); /* x has type Optional which strips to Optional, then we look up * .m on it. This SHOULD NOT resolve in v1 since Optional is just @@ -1464,13 +1542,12 @@ TEST(pylsp_round1_generic_subscript_annotation) { TEST(pylsp_round2_isinstance_narrow) { /* if isinstance(x, Foo): x.method() — x narrowed to Foo */ - CBMFileResult *r = extract_py( - "class Foo:\n" - " def method(self):\n" - " return 1\n" - "def use(x):\n" - " if isinstance(x, Foo):\n" - " return x.method()\n"); + CBMFileResult *r = extract_py("class Foo:\n" + " def method(self):\n" + " return 1\n" + "def use(x):\n" + " if isinstance(x, Foo):\n" + " return x.method()\n"); ASSERT_NOT_NULL(r); ASSERT_GTE(require_resolved(r, "use", "method"), 0); cbm_free_result(r); @@ -1481,14 +1558,13 @@ TEST(pylsp_round2_is_not_none_narrow) { /* def f(x: Optional[Foo]): * if x is not None: * x.method() */ - CBMFileResult *r = extract_py( - "from typing import Optional\n" - "class Foo:\n" - " def method(self):\n" - " return 1\n" - "def use(x: Optional[Foo]):\n" - " if x is not None:\n" - " return x.method()\n"); + CBMFileResult *r = extract_py("from typing import Optional\n" + "class Foo:\n" + " def method(self):\n" + " return 1\n" + "def use(x: Optional[Foo]):\n" + " if x is not None:\n" + " return x.method()\n"); ASSERT_NOT_NULL(r); /* Optional strips to Foo for v1 (we drop generic args). x: Optional[Foo] * binds x as NAMED("Optional"). After narrowing, ideally NAMED("Foo"). @@ -1505,15 +1581,14 @@ TEST(pylsp_round2_is_not_none_narrow) { TEST(pylsp_round2_isinstance_no_false_positive_in_else) { /* In the else branch, narrowing must NOT apply. */ - CBMFileResult *r = extract_py( - "class Foo:\n" - " def method(self):\n" - " return 1\n" - "def use(x):\n" - " if isinstance(x, Foo):\n" - " return 1\n" - " else:\n" - " return x.method()\n"); + CBMFileResult *r = extract_py("class Foo:\n" + " def method(self):\n" + " return 1\n" + "def use(x):\n" + " if isinstance(x, Foo):\n" + " return 1\n" + " else:\n" + " return x.method()\n"); ASSERT_NOT_NULL(r); /* No high-confidence resolution should exist for x.method() in the * else branch, since x is UNKNOWN there. */ @@ -1531,17 +1606,16 @@ TEST(pylsp_round2_isinstance_no_false_positive_in_else) { TEST(pylsp_round2_narrow_after_call) { /* Without walrus: x = compute(); if x is not None: x.method(). * Tests narrow on UNION return-type-of-call. */ - CBMFileResult *r = extract_py( - "from typing import Optional\n" - "class Foo:\n" - " def method(self):\n" - " return 1\n" - "def compute() -> Optional[Foo]:\n" - " return None\n" - "def use():\n" - " x = compute()\n" - " if x is not None:\n" - " return x.method()\n"); + CBMFileResult *r = extract_py("from typing import Optional\n" + "class Foo:\n" + " def method(self):\n" + " return 1\n" + "def compute() -> Optional[Foo]:\n" + " return None\n" + "def use():\n" + " x = compute()\n" + " if x is not None:\n" + " return x.method()\n"); ASSERT_NOT_NULL(r); ASSERT_GTE(require_resolved(r, "use", "method"), 0); cbm_free_result(r); @@ -1550,16 +1624,15 @@ TEST(pylsp_round2_narrow_after_call) { TEST(pylsp_round2_walrus_binds) { /* if (x := compute()) is not None: x.method() */ - CBMFileResult *r = extract_py( - "from typing import Optional\n" - "class Foo:\n" - " def method(self):\n" - " return 1\n" - "def compute() -> Optional[Foo]:\n" - " return None\n" - "def use():\n" - " if (x := compute()) is not None:\n" - " return x.method()\n"); + CBMFileResult *r = extract_py("from typing import Optional\n" + "class Foo:\n" + " def method(self):\n" + " return 1\n" + "def compute() -> Optional[Foo]:\n" + " return None\n" + "def use():\n" + " if (x := compute()) is not None:\n" + " return x.method()\n"); ASSERT_NOT_NULL(r); ASSERT_GTE(require_resolved(r, "use", "method"), 0); cbm_free_result(r); @@ -1568,12 +1641,11 @@ TEST(pylsp_round2_walrus_binds) { TEST(pylsp_round3_listcomp_element_method) { /* [x.method() for x in items] where items: list[Foo] */ - CBMFileResult *r = extract_py( - "class Foo:\n" - " def method(self):\n" - " return 1\n" - "def use(items: list[Foo]):\n" - " return [x.method() for x in items]\n"); + CBMFileResult *r = extract_py("class Foo:\n" + " def method(self):\n" + " return 1\n" + "def use(items: list[Foo]):\n" + " return [x.method() for x in items]\n"); ASSERT_NOT_NULL(r); ASSERT_GTE(require_resolved(r, "use", "method"), 0); cbm_free_result(r); @@ -1582,13 +1654,12 @@ TEST(pylsp_round3_listcomp_element_method) { TEST(pylsp_round3_for_loop_element_method) { /* for x in items: x.method() — items: list[Foo] */ - CBMFileResult *r = extract_py( - "class Foo:\n" - " def method(self):\n" - " return 1\n" - "def use(items: list[Foo]):\n" - " for x in items:\n" - " x.method()\n"); + CBMFileResult *r = extract_py("class Foo:\n" + " def method(self):\n" + " return 1\n" + "def use(items: list[Foo]):\n" + " for x in items:\n" + " x.method()\n"); ASSERT_NOT_NULL(r); ASSERT_GTE(require_resolved(r, "use", "method"), 0); cbm_free_result(r); @@ -1598,14 +1669,13 @@ TEST(pylsp_round3_for_loop_element_method) { TEST(pylsp_round3_optional_narrow_with_union) { /* def f(x: Optional[Foo]): * if x is not None: x.method() */ - CBMFileResult *r = extract_py( - "from typing import Optional\n" - "class Foo:\n" - " def method(self):\n" - " return 1\n" - "def use(x: Optional[Foo]):\n" - " if x is not None:\n" - " return x.method()\n"); + CBMFileResult *r = extract_py("from typing import Optional\n" + "class Foo:\n" + " def method(self):\n" + " return 1\n" + "def use(x: Optional[Foo]):\n" + " if x is not None:\n" + " return x.method()\n"); ASSERT_NOT_NULL(r); ASSERT_GTE(require_resolved(r, "use", "method"), 0); cbm_free_result(r); @@ -1616,19 +1686,18 @@ TEST(pylsp_round3_optional_narrow_with_union) { TEST(pylsp_round3_match_case_class_pattern) { /* match x: case Foo(): subject narrows to Foo */ - CBMFileResult *r = extract_py( - "class Foo:\n" - " def method(self):\n" - " return 1\n" - "class Bar:\n" - " def method(self):\n" - " return 2\n" - "def use(x):\n" - " match x:\n" - " case Foo():\n" - " return x.method()\n" - " case _:\n" - " return None\n"); + CBMFileResult *r = extract_py("class Foo:\n" + " def method(self):\n" + " return 1\n" + "class Bar:\n" + " def method(self):\n" + " return 2\n" + "def use(x):\n" + " match x:\n" + " case Foo():\n" + " return x.method()\n" + " case _:\n" + " return None\n"); ASSERT_NOT_NULL(r); int idx = find_resolved(r, "use", "method"); ASSERT_GTE(idx, 0); @@ -1644,15 +1713,14 @@ TEST(pylsp_round3_match_case_class_pattern) { TEST(pylsp_round3_async_await_pass_through) { /* await expr returns expr's type. async def f() -> int registers * with return int. await f() should resolve as int. */ - CBMFileResult *r = extract_py( - "class Foo:\n" - " def method(self):\n" - " return 1\n" - "async def make() -> Foo:\n" - " return Foo()\n" - "async def use():\n" - " f = await make()\n" - " return f.method()\n"); + CBMFileResult *r = extract_py("class Foo:\n" + " def method(self):\n" + " return 1\n" + "async def make() -> Foo:\n" + " return Foo()\n" + "async def use():\n" + " f = await make()\n" + " return f.method()\n"); ASSERT_NOT_NULL(r); ASSERT_GTE(require_resolved(r, "use", "method"), 0); cbm_free_result(r); @@ -1667,29 +1735,27 @@ TEST(pylsp_round4_instance_attribute_init) { * self.cfg = cfg # field cfg : Config * def use(self): * return self.cfg.display() # resolves through field type */ - CBMFileResult *r = extract_py( - "class Config:\n" - " def display(self):\n" - " return 1\n" - "class App:\n" - " def __init__(self, cfg: Config):\n" - " self.cfg = cfg\n" - " def use(self):\n" - " self.cfg.display()\n"); + CBMFileResult *r = extract_py("class Config:\n" + " def display(self):\n" + " return 1\n" + "class App:\n" + " def __init__(self, cfg: Config):\n" + " self.cfg = cfg\n" + " def use(self):\n" + " self.cfg.display()\n"); /* Note: extra indentation simulates a body block; real test * mirrors realistic Python code. */ ASSERT_NOT_NULL(r); /* The above source has bad indent — replace with proper test source. */ cbm_free_result(r); - r = extract_py( - "class Config:\n" - " def display(self):\n" - " return 1\n" - "class App:\n" - " def __init__(self, cfg: Config):\n" - " self.cfg = cfg\n" - " def use(self):\n" - " return self.cfg.display()\n"); + r = extract_py("class Config:\n" + " def display(self):\n" + " return 1\n" + "class App:\n" + " def __init__(self, cfg: Config):\n" + " self.cfg = cfg\n" + " def use(self):\n" + " return self.cfg.display()\n"); ASSERT_NOT_NULL(r); ASSERT_GTE(require_resolved(r, "use", "display"), 0); cbm_free_result(r); @@ -1700,14 +1766,13 @@ TEST(pylsp_round4_instance_attribute_class_annotation) { /* class C: * x: Foo * def use(self): return self.x.method() */ - CBMFileResult *r = extract_py( - "class Foo:\n" - " def method(self):\n" - " return 1\n" - "class C:\n" - " x: Foo\n" - " def use(self):\n" - " return self.x.method()\n"); + CBMFileResult *r = extract_py("class Foo:\n" + " def method(self):\n" + " return 1\n" + "class C:\n" + " x: Foo\n" + " def use(self):\n" + " return self.x.method()\n"); ASSERT_NOT_NULL(r); ASSERT_GTE(require_resolved(r, "use", "method"), 0); cbm_free_result(r); @@ -1718,15 +1783,14 @@ TEST(pylsp_round4_instance_attribute_class_annotation) { TEST(pylsp_round5_dict_subscript_value_type) { /* self.cache: dict[str, Foo]; self.cache[k].method() resolves. */ - CBMFileResult *r = extract_py( - "class Foo:\n" - " def method(self):\n" - " return 1\n" - "class C:\n" - " def __init__(self):\n" - " self.cache: dict[str, Foo] = {}\n" - " def use(self, k):\n" - " return self.cache[k].method()\n"); + CBMFileResult *r = extract_py("class Foo:\n" + " def method(self):\n" + " return 1\n" + "class C:\n" + " def __init__(self):\n" + " self.cache: dict[str, Foo] = {}\n" + " def use(self, k):\n" + " return self.cache[k].method()\n"); ASSERT_NOT_NULL(r); ASSERT_GTE(require_resolved(r, "use", "method"), 0); cbm_free_result(r); @@ -1734,12 +1798,11 @@ TEST(pylsp_round5_dict_subscript_value_type) { } TEST(pylsp_round5_list_subscript_value_type) { - CBMFileResult *r = extract_py( - "class Foo:\n" - " def method(self):\n" - " return 1\n" - "def use(items: list[Foo]):\n" - " return items[0].method()\n"); + CBMFileResult *r = extract_py("class Foo:\n" + " def method(self):\n" + " return 1\n" + "def use(items: list[Foo]):\n" + " return items[0].method()\n"); ASSERT_NOT_NULL(r); ASSERT_GTE(require_resolved(r, "use", "method"), 0); cbm_free_result(r); @@ -1747,14 +1810,13 @@ TEST(pylsp_round5_list_subscript_value_type) { } TEST(pylsp_round5_super_init) { - CBMFileResult *r = extract_py( - "class Base:\n" - " def __init__(self, root):\n" - " self.root = root\n" - "class Child(Base):\n" - " def __init__(self, root, extra):\n" - " super().__init__(root)\n" - " self.extra = extra\n"); + CBMFileResult *r = extract_py("class Base:\n" + " def __init__(self, root):\n" + " self.root = root\n" + "class Child(Base):\n" + " def __init__(self, root, extra):\n" + " super().__init__(root)\n" + " self.extra = extra\n"); ASSERT_NOT_NULL(r); int idx = find_resolved(r, "__init__", "__init__"); ASSERT_GTE(idx, 0); @@ -1986,16 +2048,15 @@ TEST(pylsp_ordinary_same_leaf_calls_join_by_exact_site) { TEST(pylsp_round6_generator_yields_iterable) { /* def gen() -> Generator[Foo, None, None]: yield Foo() * for x in gen(): x.method() — x : Foo via element-of(generator) */ - CBMFileResult *r = extract_py( - "from typing import Generator\n" - "class Foo:\n" - " def method(self):\n" - " return 1\n" - "def gen() -> Generator[Foo, None, None]:\n" - " yield Foo()\n" - "def use():\n" - " for x in gen():\n" - " x.method()\n"); + CBMFileResult *r = extract_py("from typing import Generator\n" + "class Foo:\n" + " def method(self):\n" + " return 1\n" + "def gen() -> Generator[Foo, None, None]:\n" + " yield Foo()\n" + "def use():\n" + " for x in gen():\n" + " x.method()\n"); ASSERT_NOT_NULL(r); ASSERT_GTE(require_resolved(r, "use", "method"), 0); cbm_free_result(r); @@ -2004,17 +2065,16 @@ TEST(pylsp_round6_generator_yields_iterable) { TEST(pylsp_round6_dataclass_field_access) { /* @dataclass class Point: x: Foo; def use(p: Point): p.x.method() */ - CBMFileResult *r = extract_py( - "from dataclasses import dataclass\n" - "class Foo:\n" - " def method(self):\n" - " return 1\n" - "@dataclass\n" - "class Point:\n" - " x: Foo\n" - " y: int\n" - "def use(p: Point):\n" - " return p.x.method()\n"); + CBMFileResult *r = extract_py("from dataclasses import dataclass\n" + "class Foo:\n" + " def method(self):\n" + " return 1\n" + "@dataclass\n" + "class Point:\n" + " x: Foo\n" + " y: int\n" + "def use(p: Point):\n" + " return p.x.method()\n"); ASSERT_NOT_NULL(r); ASSERT_GTE(require_resolved(r, "use", "method"), 0); cbm_free_result(r); @@ -2025,16 +2085,15 @@ TEST(pylsp_round6_property_access_chains) { /* class C: @property def thing(self) -> Foo: ... * def use(c: C): c.thing.method() -- thing is a property; access * returns its getter's return type. */ - CBMFileResult *r = extract_py( - "class Foo:\n" - " def method(self):\n" - " return 1\n" - "class C:\n" - " @property\n" - " def thing(self) -> Foo:\n" - " return Foo()\n" - "def use(c: C):\n" - " return c.thing.method()\n"); + CBMFileResult *r = extract_py("class Foo:\n" + " def method(self):\n" + " return 1\n" + "class C:\n" + " @property\n" + " def thing(self) -> Foo:\n" + " return Foo()\n" + "def use(c: C):\n" + " return c.thing.method()\n"); ASSERT_NOT_NULL(r); ASSERT_GTE(require_resolved(r, "use", "method"), 0); cbm_free_result(r); @@ -2054,24 +2113,24 @@ TEST(pylsp_issue710_deep_call_chain_resolves) { * finish and the suite times out, rather than passing silently. * We also require the FINAL link to actually resolve — the fix must * preserve resolution quality, not just terminate. */ - CBMFileResult *r = extract_py( - "from typing import Self\n" - "class G:\n" - " def add(self, x) -> Self:\n" - " return self\n" - " def compile(self):\n" - " return 1\n" - "def build():\n" - " return (\n" - " G()\n" - " .add(1).add(2).add(3).add(4).add(5).add(6).add(7).add(8)\n" - " .add(9).add(10).add(11).add(12).add(13).add(14).add(15)\n" - " .add(16).add(17).add(18).add(19).add(20).add(21).add(22)\n" - " .add(23).add(24).add(25).add(26).add(27).add(28).add(29)\n" - " .add(30).add(31).add(32).add(33).add(34).add(35).add(36)\n" - " .add(37).add(38).add(39).add(40)\n" - " .compile()\n" - " )\n"); + CBMFileResult *r = + extract_py("from typing import Self\n" + "class G:\n" + " def add(self, x) -> Self:\n" + " return self\n" + " def compile(self):\n" + " return 1\n" + "def build():\n" + " return (\n" + " G()\n" + " .add(1).add(2).add(3).add(4).add(5).add(6).add(7).add(8)\n" + " .add(9).add(10).add(11).add(12).add(13).add(14).add(15)\n" + " .add(16).add(17).add(18).add(19).add(20).add(21).add(22)\n" + " .add(23).add(24).add(25).add(26).add(27).add(28).add(29)\n" + " .add(30).add(31).add(32).add(33).add(34).add(35).add(36)\n" + " .add(37).add(38).add(39).add(40)\n" + " .compile()\n" + " )\n"); ASSERT_NOT_NULL(r); ASSERT_GTE(require_resolved(r, "build", "G.add"), 0); ASSERT_GTE(require_resolved(r, "build", "G.compile"), 0); @@ -2089,19 +2148,18 @@ TEST(pylsp_issue710_heterogeneous_receiver_chain) { * resolves wrongly or not at all. Each link here returns a DIFFERENT * class so any such aliasing changes an assertable QN. Keying by node * identity (TSNode.id) keeps every link distinct. */ - CBMFileResult *r = extract_py( - "class Result:\n" - " def fetch(self):\n" - " return 1\n" - "class Session:\n" - " def execute(self) -> \"Result\":\n" - " return Result()\n" - "class Client:\n" - " def session(self) -> \"Session\":\n" - " return Session()\n" - "def run():\n" - " c = Client()\n" - " return c.session().execute().fetch()\n"); + CBMFileResult *r = extract_py("class Result:\n" + " def fetch(self):\n" + " return 1\n" + "class Session:\n" + " def execute(self) -> \"Result\":\n" + " return Result()\n" + "class Client:\n" + " def session(self) -> \"Session\":\n" + " return Session()\n" + "def run():\n" + " c = Client()\n" + " return c.session().execute().fetch()\n"); ASSERT_NOT_NULL(r); ASSERT_GTE(require_resolved(r, "run", "Client.session"), 0); ASSERT_GTE(require_resolved(r, "run", "Session.execute"), 0); @@ -2193,6 +2251,8 @@ SUITE(py_lsp) { RUN_TEST(pylsp_pep695_generic_class); /* Phase 9 — cross-file + batch */ RUN_TEST(pylsp_crossfile_method_dispatch); + RUN_TEST(pylsp_crossfile_receiver_through_typed_field); + RUN_TEST(pylsp_crossfile_typed_field_shared_registry); RUN_TEST(pylsp_fused_self_attr_chain_via_overlay); RUN_TEST(pylsp_crossfile_classmethod_on_class_issue228); RUN_TEST(pylsp_crossfile_inheritance); diff --git a/tests/test_setup_scripts_contract.sh b/tests/test_setup_scripts_contract.sh new file mode 100644 index 0000000000..845a9ae98a --- /dev/null +++ b/tests/test_setup_scripts_contract.sh @@ -0,0 +1,330 @@ +#!/usr/bin/env bash +# Contract: the setup scripts install through the installers' verified path. +# +# scripts/setup.sh and scripts/setup-windows.ps1 are the README's "Automated +# download + install" one-liners. They used to resolve the latest release, +# download the archive and unpack it on their own -- with none of the checks +# install.sh and install.ps1 make before anything is installed or run: +# checksums.txt must download, must list the archive, a hash tool must exist, +# and the digest must match. Both setup scripts now hand the download to the +# installer, so there is exactly ONE implementation of download + verify. +# +# Functional leg (macOS/Linux host): setup.sh runs against a loopback fixture +# release built from a harmless stub "binary", in four variants. A correct +# checksums.txt installs (file mode and `curl | bash` pipe mode); a wrong +# digest, a missing checksums.txt and a PATH without any hash tool each stop +# the script with a non-zero exit, nothing installed and the stub never run. +# Everything lives in a scratch HOME under a private work directory; no real +# release is ever downloaded or executed, and the network is unreachable for +# the whole run (every proxy variable points at a closed loopback port; the +# fixture is reached with --noproxy). +# +# Static leg (every host): setup-windows.ps1 must delegate to install.ps1, +# must not resolve, download or unpack the release itself, must stage in a +# fresh directory of its own (not directly under %TEMP%), must honour CBM_DOWNLOAD_URL, and must stay +# pure ASCII (Windows PowerShell 5.1 reads a BOM-less file as ANSI). The +# functional equivalent is the Windows VM leg, which serves the same kind of +# fixture to install.ps1 (test-infrastructure/vm/vm-smoke.sh). + +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +SETUP_SH="$ROOT/scripts/setup.sh" +SETUP_PS1="$ROOT/scripts/setup-windows.ps1" + +fail() { + echo "FAIL: $*" >&2 + exit 1 +} + +# ── Static contract: scripts/setup.sh ────────────────────────────────────── +# +# These gate the functional leg on purpose: a setup.sh that ignores +# CBM_DOWNLOAD_URL cannot be pointed at the fixture and would reach for the +# real release. The contract refuses to run that script at all. +grep -q 'CBM_DOWNLOAD_URL' "$SETUP_SH" || + fail "scripts/setup.sh must honour CBM_DOWNLOAD_URL (the installers' download-base override)" +grep -q 'install\.sh' "$SETUP_SH" || + fail "scripts/setup.sh must install through install.sh" +if grep -Eq 'tar +-[a-zA-Z]*x' "$SETUP_SH"; then + fail "scripts/setup.sh must not unpack the release archive itself" +fi +if grep -Eq 'api\.github\.com|releases/download' "$SETUP_SH"; then + fail "scripts/setup.sh must not resolve or download the release itself (install.sh owns that)" +fi + +# ── Static contract: scripts/setup-windows.ps1 ───────────────────────────── +if LC_ALL=C grep -n $'[^ -~\t]' "$SETUP_PS1"; then + fail "scripts/setup-windows.ps1 must be pure ASCII (PS 5.1 reads BOM-less files as ANSI)" +fi +if grep -q $'\r' "$SETUP_PS1"; then + fail "scripts/setup-windows.ps1 must use LF line endings in the repository" +fi +grep -q 'install\.ps1' "$SETUP_PS1" || + fail "scripts/setup-windows.ps1 must install through install.ps1" +grep -q 'CBM_DOWNLOAD_URL' "$SETUP_PS1" || + fail "scripts/setup-windows.ps1 must honour CBM_DOWNLOAD_URL (the installers' download-base override)" +if grep -q 'Expand-Archive' "$SETUP_PS1"; then + fail "scripts/setup-windows.ps1 must not unpack the release archive itself" +fi +if grep -Eq 'api\.github\.com|releases/download' "$SETUP_PS1"; then + fail "scripts/setup-windows.ps1 must not resolve or download the release itself (install.ps1 owns that)" +fi +if grep -q 'env:TEMP' "$SETUP_PS1"; then + fail "scripts/setup-windows.ps1 must stage in a fresh directory of its own, not directly under %TEMP%" +fi +grep -q 'GetTempPath' "$SETUP_PS1" && grep -q 'Get-Random' "$SETUP_PS1" || + fail "scripts/setup-windows.ps1 must stage in a freshly created private directory (as install.ps1 does)" + +# No PowerShell parser is available on every host; a bracket balance over the +# file with comments and quoted strings removed still catches a truncated or +# mis-pasted script before the Windows leg does. +python3 - "$SETUP_PS1" <<'PY' || fail "scripts/setup-windows.ps1 has unbalanced brackets" +import re +import sys + +text = open(sys.argv[1], encoding="ascii").read() +text = re.sub(r"<#.*?#>", "", text, flags=re.S) +text = re.sub(r"#[^\n]*", "", text) +text = re.sub(r"'(?:[^']|'')*'", "''", text) +text = re.sub(r'"(?:[^"\\`]|`.|\\.)*"', '""', text) +for open_ch, close_ch in (("{", "}"), ("(", ")"), ("[", "]")): + if text.count(open_ch) != text.count(close_ch): + print(f"unbalanced {open_ch}{close_ch}: {text.count(open_ch)} vs {text.count(close_ch)}") + sys.exit(1) +PY + +echo "OK: static contract (setup.sh delegates to install.sh, setup-windows.ps1 to install.ps1)" + +# ── Functional leg: setup.sh against a loopback fixture release ──────────── +case "$(uname -s)" in + Darwin) OS=darwin ;; + Linux) OS=linux ;; + *) + # WHY: setup.sh is the macOS/Linux one-liner; on a Windows host it would + # select no archive at all. The Windows setup script's functional + # coverage is the VM leg that serves a fixture to install.ps1 (the + # script it delegates to). Tried: running the POSIX leg under MSYS -- + # install.sh then targets a windows-*.zip, which is not what setup.sh + # ever installs, so the run would prove nothing about this contract. + echo "SKIP: functional leg needs a macOS/Linux host ($(uname -s)); static contract passed" + exit 0 + ;; +esac +case "$(uname -m)" in + arm64 | aarch64) ARCH=arm64 ;; + x86_64 | amd64) + # Match install.sh: a Rosetta shell reports x86_64 on Apple Silicon. + if [ "$OS" = "darwin" ] && + sysctl -n machdep.cpu.brand_string 2>/dev/null | grep -qi apple; then + ARCH=arm64 + else + ARCH=amd64 + fi + ;; + *) fail "unsupported host architecture $(uname -m)" ;; +esac +PORTABLE="" +[ "$OS" = "linux" ] && PORTABLE="-portable" +ARCHIVE="codebase-memory-cli-${OS}-${ARCH}${PORTABLE}.tar.gz" + +for tool in python3 curl tar; do + command -v "$tool" >/dev/null 2>&1 || fail "functional leg needs $tool" +done +command -v sha256sum >/dev/null 2>&1 || command -v shasum >/dev/null 2>&1 || + fail "functional leg needs sha256sum or shasum" + +WORK=$(mktemp -d "${TMPDIR:-/tmp}/cbm-setup-contract-XXXXXX") +chmod 700 "$WORK" +SERVER_PID="" +cleanup() { + if [ -n "$SERVER_PID" ]; then + kill "$SERVER_PID" 2>/dev/null || true + wait "$SERVER_PID" 2>/dev/null || true + fi + rm -rf "$WORK" +} +trap cleanup EXIT + +FIXTURE="$WORK/fixture" +STAGE="$WORK/stage" +mkdir -p "$FIXTURE/good" "$FIXTURE/bad" "$FIXTURE/none" "$STAGE" "$WORK/tmp" \ + "$WORK/cache" "$WORK/runtime" "$WORK/trace" "$WORK/cwd" + +# The fixture "binary": a stub that answers --version and performs the minimal +# `install` the installer asks of the product (copy itself into --dir). It +# records every invocation so the failure cases can prove it never ran. +cat > "$STAGE/codebase-memory-cli" <<'STUB' +#!/bin/sh +set -eu +if [ -n "${CBM_SETUP_STUB_TRACE:-}" ]; then + printf '%s\n' "$*" >> "$CBM_SETUP_STUB_TRACE" +fi +case "${1:-}" in + --version) + echo "codebase-memory-cli setup-contract-stub 0.0.0" + ;; + install) + dir="" + for arg in "$@"; do + case "$arg" in --dir=*) dir="${arg#--dir=}" ;; esac + done + [ -n "$dir" ] || { echo "stub: install needs --dir=" >&2; exit 2; } + mkdir -p "$dir" + cp "$0" "$dir/codebase-memory-cli" + chmod 755 "$dir/codebase-memory-cli" + ;; + *) + echo "stub: unexpected invocation: $*" >&2 + exit 2 + ;; +esac +STUB +chmod 755 "$STAGE/codebase-memory-cli" +cp "$ROOT/LICENSE" "$ROOT/install.sh" "$STAGE/" +printf 'fixture notices\n' > "$STAGE/THIRD_PARTY_NOTICES.md" + +# Member set and order mirror scripts/package-release.sh; install.sh refuses +# any other inventory. +tar -czf "$STAGE/$ARCHIVE" -C "$STAGE" \ + codebase-memory-cli LICENSE install.sh THIRD_PARTY_NOTICES.md +if command -v sha256sum >/dev/null 2>&1; then + DIGEST=$(sha256sum "$STAGE/$ARCHIVE" | awk '{print $1}') +else + DIGEST=$(shasum -a 256 "$STAGE/$ARCHIVE" | awk '{print $1}') +fi + +for variant in good bad none; do + cp "$STAGE/$ARCHIVE" "$ROOT/install.sh" "$FIXTURE/$variant/" +done +printf '%s %s\n' "$DIGEST" "$ARCHIVE" > "$FIXTURE/good/checksums.txt" +WRONG_DIGEST=$(printf '%064d' 0) +printf '%s %s\n' "$WRONG_DIGEST" "$ARCHIVE" > "$FIXTURE/bad/checksums.txt" +# "none" ships no checksums.txt at all. + +# The server owns the ephemeral bind and prints its banner only after the +# port is published; reading that banner from a FIFO is the readiness signal +# (no polling, no timer). Request logs go to a file so nothing is written to +# the FIFO after the banner. +mkfifo "$WORK/server.fifo" +python3 "$ROOT/scripts/smoke-fixture-server.py" \ + --directory "$FIXTURE" --port-file "$WORK/port" \ + > "$WORK/server.fifo" 2> "$WORK/server.log" & +SERVER_PID=$! +IFS= read -r SERVER_BANNER < "$WORK/server.fifo" || + fail "fixture server exited before publishing its port" +PORT=$(tr -d '[:space:]' < "$WORK/port") +case "$PORT" in + '' | *[!0-9]*) fail "fixture server published no usable port ($SERVER_BANNER)" ;; +esac + +# Every tool setup.sh, install.sh and the stub may need, EXCEPT a hash tool. +# The no-hash-tool variant runs with PATH restricted to this directory. +mkdir -p "$WORK/tools" +for tool in bash sh env curl wget mktemp uname sysctl grep wc tr awk tar \ + xattr codesign chmod cp mv rm mkdir cat sed head dirname tput; do + if resolved=$(command -v "$tool" 2>/dev/null) && [ -n "$resolved" ]; then + ln -s "$resolved" "$WORK/tools/$tool" + fi +done + +# run_setup