diff --git a/Makefile.cbm b/Makefile.cbm index 96a90a1580..7ba4efc096 100644 --- a/Makefile.cbm +++ b/Makefile.cbm @@ -451,7 +451,7 @@ PREPROCESSOR_SRC = $(CBM_DIR)/preprocessor.cpp SQLITE_WRITER_SRC = $(CBM_DIR)/sqlite_writer.c # Store module (new) -STORE_SRCS = src/store/store.c src/store/architecture_projection.c src/store/store_impact.c src/store/store_graph_digest.c src/store/store_content_digest.c +STORE_SRCS = src/store/store.c src/store/graph_mem.c src/store/architecture_projection.c src/store/store_impact.c src/store/store_graph_digest.c src/store/store_content_digest.c # Cypher module (new) CYPHER_SRCS = src/cypher/cypher.c @@ -541,7 +541,9 @@ PIPELINE_SRCS = \ src/pipeline/pass_operator.c \ src/pipeline/pass_shell.c \ src/pipeline/pass_terraform.c \ + src/pipeline/pass_ansible.c \ src/pipeline/pass_docker.c \ + src/pipeline/pass_envjoin.c \ src/pipeline/pass_ci.c \ src/pipeline/pass_similarity.c \ src/pipeline/pass_semantic_edges.c \ @@ -745,6 +747,7 @@ TEST_STORE_SRCS = \ tests/test_store_impact.c \ tests/test_store_scope.c \ tests/test_store_graph_digest.c \ + tests/test_graph_mem.c \ tests/test_store_search.c \ tests/test_store_arch.c \ tests/test_architecture_projection.c \ @@ -760,6 +763,7 @@ TEST_OPERATION_SRCS = \ tests/test_index_supervisor.c \ tests/test_changes_operation.c \ tests/test_trace_architecture_operations.c \ + tests/test_source_search.c \ $(wildcard tests/test_test_impact*.c) TEST_DAEMON_SRCS = \ diff --git a/README.md b/README.md index 17253f87ed..e02f87d8d4 100644 --- a/README.md +++ b/README.md @@ -60,6 +60,8 @@ It includes: - definitions, calls, imports, usages, routes, channels, and cross-service links; - architecture summaries, change-impact analysis, dead-code queries, ADRs, and Cypher queries; - BM25, semantic, structural, and graph-augmented source search; +- infrastructure and configuration linking for Ansible, Kubernetes, Helm, Compose, Terraform, TOML and INI, including `CONNECTS_TO` topology; +- in-memory adjacency acceleration for trace traversal and degree-filtered search; - cross-repository graph intelligence and the CBM Atlas web UI: an explorer and read-only reader, a semantic twin that follows the caret, flow, bug-hunt and change-scope reading modes, the 3D galaxy and project management, served by the binary on loopback (`localhost:9749`) through native `/api/*` endpoints and `POST /api/tool`; - a persistent local index with incremental updates and optional shared graph artifacts; - a coordination daemon, supervised workers, and filesystem watcher for reliable background indexing. diff --git a/internal/cbm/cbm.h b/internal/cbm/cbm.h index 9fac8cf557..86586dd15b 100644 --- a/internal/cbm/cbm.h +++ b/internal/cbm/cbm.h @@ -701,23 +701,35 @@ typedef struct { /* A fact a Kubernetes manifest states about other objects (extract_k8s.c), * resolved by the K8s pass against the objects of the source's bundle. * CBM_K8S_FACT_REF: object def_index names a kind/ns/name object at a - * reference path (via), drawn as edge (USAGE, INFRA_MAPS or DEPENDS_ON); - * ns NULL = the source's own namespace. + * reference path (via), drawn as edge (USAGE, INFRA_MAPS, DEPENDS_ON or + * CONNECTS_TO); ns NULL = the source's own namespace. A CONNECTS_TO + * reference (via "env") is the Service the value of environment variable + * `key` of container `container` addresses -- the value is not kept. * CBM_K8S_FACT_CRD: the CRD def_index declares kind `kind` in group `name`. * CBM_K8S_FACT_DEPLOY: object def_index (a Flux Kustomization, an Argo * Application) deploys the repository directory `name` into namespace - * `ns` (NULL: the objects' own); via "flux" / "argo". + * `ns` (NULL: the objects' own); via "flux" / "argo". via "kustomize" + * (def_index -1: the kustomization itself): a `helmCharts` entry + * inflates chart `name` of repository `url` at `version` as release + * `release` into namespace `ns`, with values file `path`. * CBM_K8S_FACT_ROUTE: object def_index (an Ingress, an HTTPRoute) exposes * `route` ("") and sends it to the kind/ns/name backend * (name NULL: none); via "ingress" / "httproute". * CBM_K8S_FACT_SOURCE: object def_index (a Flux OCIRepository) fetches * from the URL `name`; a HelmRelease whose chartRef names it deploys the * chart there. - * CBM_K8S_FACT_IMAGE: object def_index runs a container of image `name` - * (a literal reference); via "container". + * CBM_K8S_FACT_IMAGE: object def_index runs its container `container` + * (NULL: unnamed) from image `name` (a literal reference); via "container". * CBM_K8S_FACT_ENV: object def_index sets environment variable `name` in * its container `container` (NULL: unnamed) at `line`; via = where the - * value comes from ("value", or the valueFrom kind) -- never the value. */ + * value comes from ("value", or the valueFrom kind) -- never the value. + * CBM_K8S_FACT_PATCH: a kustomization's `patches` entry at `line` patches + * the objects of its build named by kind / group / version / name / ns + * (any NULL: any) -- its `target`, else an inline patch document's own + * kind / group / name / ns; `path` is + * the patch file (as written, relative to the kustomization), NULL for an + * inline patch. kind and name both NULL: the patch file's documents name + * them. def_index -1: the kustomization itself is the source. */ typedef enum { CBM_K8S_FACT_REF = 0, CBM_K8S_FACT_CRD, @@ -725,7 +737,8 @@ typedef enum { CBM_K8S_FACT_ROUTE, CBM_K8S_FACT_SOURCE, CBM_K8S_FACT_IMAGE, - CBM_K8S_FACT_ENV + CBM_K8S_FACT_ENV, + CBM_K8S_FACT_PATCH } CBMK8sFactKind; typedef struct { CBMK8sFactKind fact; @@ -737,6 +750,13 @@ typedef struct { const char *edge; const char *route; const char *container; + const char *key; /* CONNECTS_TO: the environment variable addressing it */ + const char *group; /* PATCH: the API group named ("core"), NULL: any */ + const char *version; /* PATCH: the API version named, NULL: any */ + const char *path; /* PATCH: the patch file, NULL: inline */ + const char *selector; /* PATCH: the target's labelSelector, NULL: none */ + const char *url; /* DEPLOY (kustomize): the chart repository */ + const char *release; /* DEPLOY (kustomize): the release name */ int line; } CBMK8sFact; diff --git a/internal/cbm/extract_defs.c b/internal/cbm/extract_defs.c index 98df278a6f..1150a90381 100644 --- a/internal/cbm/extract_defs.c +++ b/internal/cbm/extract_defs.c @@ -7197,18 +7197,19 @@ static char *find_hcl_block_name(CBMArena *a, TSNode node, const char *source) { // Append the block's quoted labels so each block gets a distinct, // query-friendly name: resource "aws_instance" "web" -> resource.aws_instance.web // rather than every resource collapsing to the bare keyword "resource" (#337). - // HCL stores labels as string_lit -> template_literal children. + // HCL stores quoted labels as string_lit -> template_literal children and + // bare ones (`variable region {}`, valid HCL) as identifiers after the + // keyword's own -- both name the block. uint32_t cc = ts_node_named_child_count(node); for (uint32_t i = 0; i < cc; i++) { TSNode ch = ts_node_named_child(node, i); - if (strcmp(ts_node_type(ch), "string_lit") != 0) { - continue; - } - TSNode lit = cbm_find_child_by_kind(ch, "template_literal"); - if (ts_node_is_null(lit)) { - continue; + char *label = NULL; + if (strcmp(ts_node_type(ch), "string_lit") == 0) { + TSNode lit = cbm_find_child_by_kind(ch, "template_literal"); + label = ts_node_is_null(lit) ? NULL : cbm_node_text(a, lit, source); + } else if (strcmp(ts_node_type(ch), "identifier") == 0 && !ts_node_eq(ch, id)) { + label = cbm_node_text(a, ch, source); } - char *label = cbm_node_text(a, lit, source); if (!label || !label[0]) { continue; } diff --git a/internal/cbm/extract_k8s.c b/internal/cbm/extract_k8s.c index e6602c3f23..a3572763e8 100644 --- a/internal/cbm/extract_k8s.c +++ b/internal/cbm/extract_k8s.c @@ -15,6 +15,7 @@ #include "helpers.h" #include "tree_sitter/api.h" #include "foundation/constants.h" +#include #include #include #include @@ -22,6 +23,7 @@ /* Local constants. */ enum { K8S_BUF_SIZE = 256, + K8S_PROPS_KV = 5, /* extra_props: version, labels (key + value each), NULL */ }; // --------------------------------------------------------------------------- @@ -217,6 +219,302 @@ static TSNode unwrap_pair_value(TSNode pair) { return val_node; } +// --------------------------------------------------------------------------- +// Kustomize patches +// --------------------------------------------------------------------------- + +// One `patches` entry of a kustomization (the kustomize v5 mapping form). The +// patch file of `path` is an import (local name "patches"); what the entry +// patches is a PATCH fact the K8s pass resolves in the kustomization's build: +// its `target`, else each document of an inline `patch`; a path patch +// without a target names the objects of its own documents, which the pass +// reads from the patch file's Resources. A target the pass cannot evaluate +// (a label or annotation selector) names nothing. + +typedef struct { + const char *kind; + const char *group; + const char *version; + const char *name; + const char *ns; + const char *selector; /* labelSelector */ + bool unsupported; /* annotationSelector: not evaluated */ +} kust_target_t; + +static void kust_target(CBMExtractCtx *ctx, TSNode map, kust_target_t *t) { + CBMArena *a = ctx->arena; + k8s_kids_t pairs = k8s_kids(map); + TSNode pair; + while (k8s_kid(&pairs, false, &pair)) { + if (strcmp(ts_node_type(pair), "block_mapping_pair") != 0) { + continue; + } + const char *key = get_scalar_text(a, ts_node_named_child(pair, 0), ctx->source); + TSNode v = unwrap_pair_value(pair); + const char *val = key && !ts_node_is_null(v) ? get_scalar_text(a, v, ctx->source) : NULL; + if (!val || !val[0]) { + continue; + } + if (strcmp(key, "kind") == 0) { + t->kind = val; + } else if (strcmp(key, "group") == 0) { + t->group = val; + } else if (strcmp(key, "version") == 0) { + t->version = val; + } else if (strcmp(key, "name") == 0) { + t->name = val; + } else if (strcmp(key, "namespace") == 0) { + t->ns = val; + } else if (strcmp(key, "labelSelector") == 0) { + t->selector = val; + } else if (strcmp(key, "annotationSelector") == 0) { + t->unsupported = true; + } + } +} + +// The block mapping a sequence item holds, null when it holds none (a +// string entry, which emit_kustomize_sequence imports). +static TSNode kust_item_mapping(TSNode item) { + TSNode v = first_content_child(item); + if (!ts_node_is_null(v) && strcmp(ts_node_type(v), "block_node") == 0) { + v = first_content_child(v); + } + if (ts_node_is_null(v) || strcmp(ts_node_type(v), "block_mapping") != 0) { + TSNode null_node = {0}; + return null_node; + } + return v; +} + +// The text of an inline patch: a block scalar's lines after its header line +// (`|`, `|-`, `|2`...), or a one-line scalar. +static const char *kust_inline_text(CBMExtractCtx *ctx, TSNode v) { + if (strcmp(ts_node_type(v), "block_scalar") == 0) { + const char *raw = cbm_node_text(ctx->arena, v, ctx->source); + const char *nl = raw ? strchr(raw, '\n') : NULL; + return nl ? nl + SKIP_ONE : NULL; + } + return get_scalar_text(ctx->arena, v, ctx->source); +} + +// One line of an inline patch: its indent and `key: value` (the value +// unquoted, a trailing comment dropped); false for a blank or comment line. +static bool kust_line(const char *p, size_t len, int *indent, char *key, size_t ksz, char *val, + size_t vsz) { + size_t i = 0; + while (i < len && p[i] == ' ') { + i++; + } + if (i == len || p[i] == '#') { + return false; + } + *indent = (int)i; + size_t k = i; + while (k < len && p[k] != ':') { + k++; + } + snprintf(key, ksz, "%.*s", (int)(k - i), p + i); + val[0] = '\0'; + size_t s = k < len ? k + SKIP_ONE : len; + while (s < len && p[s] == ' ') { + s++; + } + size_t e = s; + while (e < len && !(p[e] == '#' && p[e - SKIP_ONE] == ' ')) { + e++; + } + while (e > s && (p[e - SKIP_ONE] == ' ' || p[e - SKIP_ONE] == '\r')) { + e--; + } + if (e - s >= PAIR_LEN && (p[s] == '"' || p[s] == '\'') && p[e - SKIP_ONE] == p[s]) { + s++; + e--; + } + snprintf(val, vsz, "%.*s", (int)(e - s), p + s); + return true; +} + +// Each document of an inline patch with a kind and a metadata.name: a PATCH +// fact naming that object -- its kind, the group of its apiVersion ("core" +// for the core group, as on a Resource), its name and namespace. A JSON6902 +// operation list names nothing. +static const char KUST_DOC_END[] = "---"; + +static void kust_inline_facts(CBMExtractCtx *ctx, const char *text, CBMK8sFact base) { + CBMArena *a = ctx->arena; + CBMK8sFact f = base; + int top = -1; /* indent of the document's keys */ + int meta = -1; /* indent of metadata's keys, -1: not inside metadata */ + bool in_meta = false; + for (const char *p = text; p;) { + const char *eol = strchr(p, '\n'); + size_t len = eol ? (size_t)(eol - p) : strlen(p); + size_t b = strspn(p, " "); + bool doc_end = b + strlen(KUST_DOC_END) <= len && + strncmp(p + b, KUST_DOC_END, strlen(KUST_DOC_END)) == 0; + int ind = 0; + char key[K8S_BUF_SIZE]; + char val[K8S_BUF_SIZE]; + if (!doc_end && kust_line(p, len, &ind, key, sizeof(key), val, sizeof(val))) { + if (top < 0) { + top = ind; + } + if (ind == top) { + in_meta = strcmp(key, "metadata") == 0 && !val[0]; + meta = -1; + if (strcmp(key, "kind") == 0 && val[0]) { + f.kind = cbm_arena_strdup(a, val); + } else if (strcmp(key, "apiVersion") == 0 && val[0]) { + const char *slash = strchr(val, '/'); + f.group = slash ? cbm_arena_strndup(a, val, (size_t)(slash - val)) : "core"; + } + } else if (in_meta && ind > top) { + meta = meta < 0 ? ind : meta; + if (ind == meta && strcmp(key, "name") == 0 && val[0]) { + f.name = cbm_arena_strdup(a, val); + } else if (ind == meta && strcmp(key, "namespace") == 0 && val[0]) { + f.ns = cbm_arena_strdup(a, val); + } + } + } + if (doc_end || !eol) { + if (f.kind && f.name) { + cbm_k8sfact_push(&ctx->result->k8s_facts, a, f); + } + f = base; + top = -1; + in_meta = false; + } + p = eol ? eol + SKIP_ONE : NULL; + } +} + +// The mapping entries of a kustomization's `patches` list. +static void emit_kustomize_patches(CBMExtractCtx *ctx, TSNode seq) { + CBMArena *a = ctx->arena; + k8s_kids_t items = k8s_kids(seq); + TSNode item; + while (k8s_kid(&items, false, &item)) { + TSNode map = strcmp(ts_node_type(item), "block_sequence_item") == 0 + ? kust_item_mapping(item) + : (TSNode){0}; + if (ts_node_is_null(map)) { + continue; + } + const char *path = NULL; + const char *inline_text = NULL; + kust_target_t t = {0}; + bool has_target = false; + k8s_kids_t pairs = k8s_kids(map); + TSNode pair; + while (k8s_kid(&pairs, false, &pair)) { + if (strcmp(ts_node_type(pair), "block_mapping_pair") != 0) { + continue; + } + const char *key = get_scalar_text(a, ts_node_named_child(pair, 0), ctx->source); + TSNode v = unwrap_pair_value(pair); + if (!key || ts_node_is_null(v)) { + continue; + } + if (strcmp(key, "path") == 0) { + path = get_scalar_text(a, v, ctx->source); + } else if (strcmp(key, "patch") == 0) { + inline_text = kust_inline_text(ctx, v); + } else if (strcmp(key, "target") == 0 && + strcmp(ts_node_type(v), "block_mapping") == 0) { + kust_target(ctx, v, &t); + has_target = true; + } + } + if (path && path[0]) { + CBMImport imp = {.local_name = cbm_arena_strdup(a, "patches"), + .module_path = cbm_arena_strdup(a, path)}; + cbm_imports_push(&ctx->result->imports, a, imp); + } + CBMK8sFact base = {.fact = CBM_K8S_FACT_PATCH, + .def_index = -1, + .via = "patch", + .edge = "OVERRIDE", + .path = path && path[0] ? path : NULL, + .line = (int)ts_node_start_point(item).row + TS_LINE_OFFSET}; + if (has_target) { + if (!t.unsupported && (t.kind || t.name || t.selector)) { + CBMK8sFact f = base; + f.kind = t.kind; + f.group = t.group; + f.version = t.version; + f.name = t.name; + f.ns = t.ns; + f.selector = t.selector; + cbm_k8sfact_push(&ctx->result->k8s_facts, a, f); + } + } else if (inline_text) { + kust_inline_facts(ctx, inline_text, base); + } else if (base.path) { + cbm_k8sfact_push(&ctx->result->k8s_facts, a, base); + } + } +} + +// The `helmCharts` list of a kustomization: each chart it inflates into its +// build is a DEPLOY fact (via "kustomize") the K8s pass draws to that chart; +// its values file an import (local name "valuesFile"). +static void emit_kustomize_charts(CBMExtractCtx *ctx, TSNode seq) { + CBMArena *a = ctx->arena; + k8s_kids_t items = k8s_kids(seq); + TSNode item; + while (k8s_kid(&items, false, &item)) { + TSNode map = strcmp(ts_node_type(item), "block_sequence_item") == 0 + ? kust_item_mapping(item) + : (TSNode){0}; + if (ts_node_is_null(map)) { + continue; + } + CBMK8sFact f = {.fact = CBM_K8S_FACT_DEPLOY, + .def_index = -1, + .via = "kustomize", + .edge = "DEPLOYS", + .line = (int)ts_node_start_point(item).row + TS_LINE_OFFSET}; + k8s_kids_t pairs = k8s_kids(map); + TSNode pair; + while (k8s_kid(&pairs, false, &pair)) { + if (strcmp(ts_node_type(pair), "block_mapping_pair") != 0) { + continue; + } + const char *key = get_scalar_text(a, ts_node_named_child(pair, 0), ctx->source); + TSNode v = unwrap_pair_value(pair); + const char *val = + key && !ts_node_is_null(v) ? get_scalar_text(a, v, ctx->source) : NULL; + if (!val || !val[0]) { + continue; + } + if (strcmp(key, "name") == 0) { + f.name = val; + } else if (strcmp(key, "repo") == 0) { + f.url = val; + } else if (strcmp(key, "version") == 0) { + f.version = val; + } else if (strcmp(key, "releaseName") == 0) { + f.release = val; + } else if (strcmp(key, "namespace") == 0) { + f.ns = val; + } else if (strcmp(key, "valuesFile") == 0) { + f.path = val; + } + } + if (!f.name) { + continue; + } + if (f.path) { + CBMImport imp = {.local_name = cbm_arena_strdup(a, "valuesFile"), + .module_path = cbm_arena_strdup(a, f.path)}; + cbm_imports_push(&ctx->result->imports, a, imp); + } + cbm_k8sfact_push(&ctx->result->k8s_facts, a, f); + } +} + // Process a single block_mapping_pair for kustomize list keys. static void process_kustomize_pair(CBMExtractCtx *ctx, TSNode pair) { if (strcmp(ts_node_type(pair), "block_mapping_pair") != 0) { @@ -227,15 +525,24 @@ static void process_kustomize_pair(CBMExtractCtx *ctx, TSNode pair) { return; } const char *key_text = get_scalar_text(ctx->arena, key_node, ctx->source); - if (!key_text || !is_kustomize_list_key(key_text)) { + if (!key_text) { return; } - TSNode val_node = unwrap_pair_value(pair); if (ts_node_is_null(val_node) || strcmp(ts_node_type(val_node), "block_sequence") != 0) { return; } + if (strcmp(key_text, "helmCharts") == 0) { + emit_kustomize_charts(ctx, val_node); + return; + } + if (!is_kustomize_list_key(key_text)) { + return; + } emit_kustomize_sequence(ctx, val_node, key_text); + if (strcmp(key_text, "patches") == 0) { + emit_kustomize_patches(ctx, val_node); + } } // Emit a "Class" def named after the document's `kind` scalar. A kustomization @@ -310,8 +617,72 @@ typedef struct { char kind[K8S_BUF_SIZE]; char name[K8S_BUF_SIZE]; char ns[K8S_BUF_SIZE]; + const char *labels; // metadata.labels as "k=v,k2=v2" (text order), NULL: none } k8s_doc_id_t; +// A label a selector term can name: no templated part (a skeleton's `<...>`, +// `{{`), no `,` or `=` (the separators of the list), nothing JSON escapes +// (`"`, `\`, control characters -- none in a valid label). +static bool k8s_label_plain(const char *s) { + if (s[0] == '<' || strstr(s, "{{") || strpbrk(s, ",=\"\\")) { + return false; + } + for (const char *p = s; *p; p++) { + if ((unsigned char)*p < ' ') { + return false; + } + } + return true; +} + +static int k8s_cmp_label(const void *x, const void *y) { + return strcmp(*(const char *const *)x, *(const char *const *)y); +} + +// metadata.labels as "k=v,k2=v2" in text order (the arena's), NULL when it +// has none a selector can name. +static const char *k8s_labels(CBMArena *a, TSNode map, const char *source) { + if (strcmp(ts_node_type(map), "block_mapping") != 0) { + return NULL; + } + int cap = (int)ts_node_named_child_count(map); + const char **items = cap > 0 ? cbm_arena_alloc(a, (size_t)cap * sizeof(*items)) : NULL; + int n = 0; + size_t len = 0; + k8s_kids_t pairs = k8s_kids(map); + TSNode p; + while (items && n < cap && k8s_kid(&pairs, false, &p)) { + if (strcmp(ts_node_type(p), "block_mapping_pair") != 0) { + continue; + } + const char *k = get_scalar_text(a, ts_node_named_child(p, 0), source); + TSNode vn = unwrap_pair_value(p); + const char *v = ts_node_is_null(vn) ? NULL : get_scalar_text(a, vn, source); + if (k && k[0] && v && k8s_label_plain(k) && k8s_label_plain(v)) { + items[n] = cbm_arena_sprintf(a, "%s=%s", k, v); + len += items[n] ? strlen(items[n]) + SKIP_ONE : 0; + n += items[n] ? SKIP_ONE : 0; + } + } + k8s_kids_done(&pairs); + if (n == 0) { + return NULL; + } + qsort((void *)items, (size_t)n, sizeof(*items), k8s_cmp_label); + char *out = cbm_arena_alloc(a, len); + if (!out) { + return NULL; + } + size_t w = 0; + for (int i = 0; i < n; i++) { + size_t l = strlen(items[i]); + memcpy(out + w, items[i], l); + w += l; + out[w++] = i + SKIP_ONE < n ? ',' : '\0'; + } + return out; +} + // Read metadata.name and metadata.namespace from a metadata block_mapping. static void extract_metadata(CBMArena *a, TSNode meta_mapping, const char *source, k8s_doc_id_t *id) { @@ -331,10 +702,14 @@ static void extract_metadata(CBMArena *a, TSNode meta_mapping, const char *sourc const char *mkey_text = get_scalar_text(a, mkey, source); bool is_name = mkey_text && strcmp(mkey_text, "name") == 0; bool is_ns = mkey_text && strcmp(mkey_text, "namespace") == 0; + TSNode mval = unwrap_pair_value(mpair); + if (mkey_text && strcmp(mkey_text, "labels") == 0 && !ts_node_is_null(mval)) { + id->labels = k8s_labels(a, mval, source); + continue; + } if (!is_name && !is_ns) { continue; } - TSNode mval = unwrap_pair_value(mpair); const char *text = ts_node_is_null(mval) ? NULL : get_scalar_text(a, mval, source); if (text) { snprintf(is_name ? id->name : id->ns, K8S_BUF_SIZE, "%s", text); @@ -773,35 +1148,13 @@ static void k8s_emit_ref(k8s_walk_t *w, const char *kind, const char *ns, const cbm_k8sfact_push(&w->ctx->result->k8s_facts, a, f); } -// A container's literal image: `path` ends in containers[].image, -// initContainers[].image or ephemeralContainers[].image at any depth (pod -// templates, CronJobs, the pod templates of custom resources). -static bool k8s_image_leaf(const char *path, const char *value) { - static const char *const LEAVES[] = {"containers[].image", "initContainers[].image", - "ephemeralContainers[].image"}; - if (!value[0] || strchr(value, '$') || strchr(value, ' ') || strstr(value, "{{")) { - return false; - } - size_t pl = strlen(path); - for (size_t i = 0; i < sizeof(LEAVES) / sizeof(LEAVES[0]); i++) { - size_t ll = strlen(LEAVES[i]); - if (pl >= ll && strcmp(path + pl - ll, LEAVES[i]) == 0 && - (pl == ll || path[pl - ll - SKIP_ONE] == '.')) { - return true; - } - } - return false; +// A literal image reference: no variable, no template, no blank. +static bool k8s_image_literal(const char *value) { + return value && value[0] && !strchr(value, '$') && !strchr(value, ' ') && !strstr(value, "{{"); } // A scalar at `path`: a leaf reference, or a CRD's group / kind. static void k8s_leaf(k8s_walk_t *w, const char *path, const char *value) { - if (k8s_image_leaf(path, value)) { - CBMK8sFact f = {.fact = CBM_K8S_FACT_IMAGE, - .def_index = w->def_index, - .name = cbm_arena_strdup(w->ctx->arena, value), - .via = "container"}; - cbm_k8sfact_push(&w->ctx->result->k8s_facts, w->ctx->arena, f); - } const char *pod = k8s_pod_prefix(w->kind); size_t pl = strlen(pod); if (pl && strncmp(path, pod, pl) == 0 && path[pl] == '.') { @@ -1086,10 +1439,39 @@ static const char *k8s_env_source(k8s_walk_t *w, TSNode item) { return "value"; } -// The environment variables a container (`path` ends in containers[], -// initContainers[] or ephemeralContainers[], any depth) sets: each env[] -// name, with where its value comes from -- never the value. -static void k8s_container_env(k8s_walk_t *w, const char *path, TSNode container) { +// The Service a container's literal env value addresses (`host:port`, a URL, +// `..svc...`, a bare name on an address key): a CONNECTS_TO +// reference, resolved in the bundle like every other. Only the service's +// name and namespace are kept -- never the value. +static void k8s_env_address(k8s_walk_t *w, TSNode item, const char *key, const char *cname) { + enum { K8S_DNS_LABEL = 64 }; + const char *value = k8s_text(w, item, "value"); + char svc[K8S_DNS_LABEL]; + char ns[K8S_DNS_LABEL]; + if (!value || cbm_env_address(key, value, svc, sizeof(svc), ns, sizeof(ns)) == 0) { + return; + } + CBMArena *a = w->ctx->arena; + CBMK8sFact f = {.fact = CBM_K8S_FACT_REF, + .def_index = w->def_index, + .kind = "Service", + .ns = ns[0] ? cbm_arena_strdup(a, ns) : NULL, + .name = cbm_arena_strdup(a, svc), + .via = "env", + .edge = "CONNECTS_TO", + .container = cname, + .key = key, + .line = (int)ts_node_start_point(item).row + SKIP_ONE}; + cbm_k8sfact_push(&w->ctx->result->k8s_facts, a, f); +} + +// A container (`path` ends in containers[], initContainers[] or +// ephemeralContainers[], any depth: pod templates, CronJobs, the pod +// templates of custom resources): its literal image, and the environment +// variables it sets -- each env[] name, with where its value comes from, +// never the value. Both carry the container's name, so a variable joins the +// program of its own container's image. +static void k8s_container(k8s_walk_t *w, const char *path, TSNode container) { static const char *const ITEMS[] = {"containers[]", "initContainers[]", "ephemeralContainers[]"}; size_t pl = strlen(path); @@ -1104,6 +1486,15 @@ static void k8s_container_env(k8s_walk_t *w, const char *path, TSNode container) } const char *cname = k8s_text(w, container, "name"); cname = cname && cname[0] && !strstr(cname, "{{") ? cname : NULL; + const char *image = k8s_text(w, container, "image"); + if (k8s_image_literal(image)) { + CBMK8sFact f = {.fact = CBM_K8S_FACT_IMAGE, + .def_index = w->def_index, + .name = image, + .container = cname, + .via = "container"}; + cbm_k8sfact_push(&w->ctx->result->k8s_facts, w->ctx->arena, f); + } k8s_iter_t env = k8s_iter(k8s_child(w, container, "env")); TSNode item; while (k8s_next(&env, &item)) { @@ -1118,12 +1509,13 @@ static void k8s_container_env(k8s_walk_t *w, const char *path, TSNode container) .via = k8s_env_source(w, item), .line = (int)ts_node_start_point(item).row + SKIP_ONE}; cbm_k8sfact_push(&w->ctx->result->k8s_facts, w->ctx->arena, f); + k8s_env_address(w, item, key, cname); } } // A mapping at `path` that holds a reference in its kind/name/namespace. static void k8s_mapping(k8s_walk_t *w, const char *path, TSNode mapping) { - k8s_container_env(w, path, mapping); + k8s_container(w, path, mapping); for (size_t i = 0; i < sizeof(k8s_mapping_refs) / sizeof(k8s_mapping_refs[0]); i++) { const k8s_mapping_ref_t *r = &k8s_mapping_refs[i]; if (k8s_kind_in(r->kinds, w->kind) && strcmp(path, r->path) == 0) { @@ -1332,6 +1724,26 @@ static void extract_k8s_manifest(CBMExtractCtx *ctx) { def.file_path = ctx->rel_path; def.start_line = ts_node_start_point(mapping).row + TS_LINE_OFFSET; def.end_line = ts_node_end_point(mapping).row + TS_LINE_OFFSET; + // The version of its apiVersion (`v1beta2` of cluster.x-k8s.io/v1beta2, + // `v1` of v1), which the QN leaves out (one object across versions); a + // bare group without one (`apiVersion: example.io`) has none. + const char *slash = strchr(id.api_version, '/'); + const char *version = slash ? slash + SKIP_ONE : id.api_version; + bool versioned = slash || (version[0] == 'v' && isdigit((unsigned char)version[1])); + const char **kv = (const char **)cbm_arena_alloc(a, K8S_PROPS_KV * sizeof(char *)); + int kn = 0; + if (kv && version[0] && versioned && id.api_version[0] != '<') { + kv[kn++] = "version"; + kv[kn++] = cbm_arena_strdup(a, version); + } + if (kv && id.labels) { + kv[kn++] = "labels"; + kv[kn++] = id.labels; + } + if (kv && kn > 0) { + kv[kn] = NULL; + def.extra_props = kv; + } cbm_defs_push(&ctx->result->defs, a, def); k8s_emit_facts(ctx, mapping, group, id.kind, ctx->result->defs.count - SKIP_ONE); } diff --git a/internal/cbm/extract_unified.c b/internal/cbm/extract_unified.c index f8e4a228fa..96e6ac865c 100644 --- a/internal/cbm/extract_unified.c +++ b/internal/cbm/extract_unified.c @@ -2634,6 +2634,12 @@ void cbm_extract_unified(CBMExtractCtx *ctx) { #else const uint32_t budget_nodes = ctx->walk_budget_nodes; #endif + /* A Terraform / OpenTofu module file's references resolve inside its + * module directory (pass_terraform.c); a call or usage by name here would + * meet same-named blocks of any module and a builtin function's name. */ + const char *ext = ctx->rel_path ? strrchr(ctx->rel_path, '.') : NULL; + bool tf_module = ctx->language == CBM_LANG_HCL && ext && + (strcmp(ext, ".tf") == 0 || strcmp(ext, ".tofu") == 0); for (;;) { TSNode node = ts_tree_cursor_current_node(&cursor); @@ -2659,8 +2665,11 @@ void cbm_extract_unified(CBMExtractCtx *ctx) { handle_string_constants(ctx, node, &state); handle_objectscript_type_map(ctx, node, &state); handle_url_builders(ctx, node, &state); - CBMInvocationDescriptor invocation = handle_calls(ctx, node, spec, &state); - handle_usages(ctx, node, spec, &state); + CBMInvocationDescriptor invocation = + tf_module ? (CBMInvocationDescriptor){0} : handle_calls(ctx, node, spec, &state); + if (!tf_module) { + handle_usages(ctx, node, spec, &state); + } handle_throws(ctx, node, spec, &state); handle_readwrites(ctx, node, spec, &state); handle_type_refs(ctx, node, spec, &state); diff --git a/internal/cbm/helpers.c b/internal/cbm/helpers.c index c02c025d27..2aa62191df 100644 --- a/internal/cbm/helpers.c +++ b/internal/cbm/helpers.c @@ -2151,6 +2151,170 @@ static bool is_env_var_pattern(const char *s, int len) { return has_upper && has_underscore; } +/* ── Service addresses in environment values ──────────────────────── */ + +static bool env_addr_key(const char *key) { + static const char *const SUFFIXES[] = {"_ADDR", "_ADDRESS", "_HOST", + "_URL", "_ENDPOINT", "_URI"}; + size_t kl = strlen(key); + for (size_t i = 0; i < sizeof(SUFFIXES) / sizeof(SUFFIXES[0]); i++) { + size_t sl = strlen(SUFFIXES[i]); + if (kl >= sl && strcmp(key + kl - sl, SUFFIXES[i]) == 0) { + return true; + } + } + return false; +} + +static bool env_label_char(char c) { + return islower((unsigned char)c) || isdigit((unsigned char)c) || c == '-'; +} + +/* A bare DNS name: `[a-z0-9][a-z0-9-]*(\.[a-z0-9-]+)*`, not only digits. */ +static bool env_bare_name(const char *v) { + bool digits_only = true; + if (!env_label_char(v[0]) || v[0] == '-') { + return false; + } + for (const char *p = v; *p; p++) { + if (*p == '.' ? (p[SKIP_ONE] == '\0' || p[SKIP_ONE] == '.') : !env_label_char(*p)) { + return false; + } + digits_only = digits_only && (isdigit((unsigned char)*p) || *p == '.'); + } + return !digits_only; +} + +static bool env_copy(char *out, size_t sz, const char *s, size_t n) { + if (n == 0 || n >= sz) { + return false; + } + memcpy(out, s, n); + out[n] = '\0'; + return true; +} + +/* The host of a URL value (after any user@, before :port). */ +static bool env_url_host(const char *v, char *host, size_t sz) { + const char *q = v; + if (!islower((unsigned char)*q)) { + return false; + } + while (islower((unsigned char)*q) || isdigit((unsigned char)*q) || *q == '+' || *q == '.' || + *q == '-') { + q++; + } + if (strncmp(q, "://", strlen("://")) != 0) { + return false; + } + const char *h = q + strlen("://"); + size_t hl = strcspn(h, "/ \t\r\n\"'"); + for (size_t i = hl; i > 0; i--) { + if (h[i - SKIP_ONE] == '@') { + hl -= i; + h += i; + break; + } + } + size_t colon = strcspn(h, ":"); + return env_copy(host, sz, h, colon < hl ? colon : hl); +} + +/* `host:port` with a 2-5 digit port. */ +static bool env_hostport(const char *v, char *host, size_t sz) { + enum { PORT_MIN = 2, PORT_MAX = 5 }; + const char *c = strrchr(v, ':'); + size_t pl = c ? strlen(c + SKIP_ONE) : 0; + if (!c || c == v || !isalnum((unsigned char)v[0]) || pl < PORT_MIN || pl > PORT_MAX || + strspn(c + SKIP_ONE, "0123456789") != pl) { + return false; + } + for (const char *p = v; p < c; p++) { + if (!isalnum((unsigned char)*p) && *p != '.' && *p != '-') { + return false; + } + } + return env_copy(host, sz, v, (size_t)(c - v)); +} + +/* A `..svc` name anywhere in the value. */ +static bool env_svc_dns(const char *v, char *host, size_t sz) { + for (const char *m = strstr(v, ".svc"); m; m = strstr(m + SKIP_ONE, ".svc")) { + char after = m[strlen(".svc")]; + if (isalnum((unsigned char)after) || after == '_') { + continue; + } + const char *ns = m; + while (ns > v && env_label_char(ns[-SKIP_ONE])) { + ns--; + } + if (ns == m || ns == v || ns[-SKIP_ONE] != '.' || ns[0] == '-') { + continue; + } + const char *name = ns - SKIP_ONE; + while (name > v && env_label_char(name[-SKIP_ONE])) { + name--; + } + bool word_start = + name == v || !(isalnum((unsigned char)name[-SKIP_ONE]) || name[-SKIP_ONE] == '_'); + if (name == ns - SKIP_ONE || name[0] == '-' || !word_start) { + continue; + } + return env_copy(host, sz, name, (size_t)(m + strlen(".svc") - name)); + } + return false; +} + +int cbm_env_address(const char *key, const char *value, char *svc, size_t svc_sz, char *ns, + size_t ns_sz) { + char host[CBM_SZ_256]; + if (!key || !value || !svc || !ns || svc_sz == 0 || ns_sz == 0) { + return 0; + } + bool found = env_addr_key(key) && env_bare_name(value) && + env_copy(host, sizeof(host), value, strlen(value)); + if (!found && (strstr(value, "{{") || strchr(value, '$'))) { + return 0; + } + found = found || env_url_host(value, host, sizeof(host)) || + env_hostport(value, host, sizeof(host)) || env_svc_dns(value, host, sizeof(host)); + if (!found) { + return 0; + } + int labels = 0; + const char *third = NULL; + for (const char *p = host; p;) { + const char *dot = strchr(p, '.'); + size_t len = dot ? (size_t)(dot - p) : strlen(p); + if (labels == 0 && !env_copy(svc, svc_sz, p, len)) { + return 0; + } + if (labels == SKIP_ONE && !env_copy(ns, ns_sz, p, len)) { + return 0; + } + if (labels == PAIR_LEN) { + third = p; + } + labels++; + p = dot ? dot + SKIP_ONE : NULL; + } + if (labels == SKIP_ONE) { + ns[0] = '\0'; + } + for (const char *p = svc; *p; p++) { + if (!env_label_char(*p)) { + return 0; + } + } + if (strcmp(svc, "localhost") == 0 || strspn(svc, "0123456789") == strlen(svc)) { + return 0; + } + bool svc_form = + labels <= PAIR_LEN || (third && strncmp(third, "svc", strlen("svc")) == 0 && + (third[strlen("svc")] == '\0' || third[strlen("svc")] == '.')); + return svc_form ? labels : 0; +} + int cbm_classify_string(const char *str, int len) { if (!str || len < PAIR_LEN) { return NOT_FOUND; diff --git a/internal/cbm/helpers.h b/internal/cbm/helpers.h index 1eb2726652..635ebceee7 100644 --- a/internal/cbm/helpers.h +++ b/internal/cbm/helpers.h @@ -25,6 +25,17 @@ bool cbm_is_keyword(const char *name, CBMLanguage lang); // stay in sync with kPyBuiltinNodes in internal/cbm/lsp/py_builtins.c. bool cbm_is_resolvable_builtin(const char *name, CBMLanguage lang); +// The service an environment variable's value addresses -- never the value +// itself: the host of a URL, of `host:port`, a `..svc[.]` +// name inside it, or the whole value when `key` names an address (*_ADDR, +// *_ADDRESS, *_HOST, *_URL, *_ENDPOINT, *_URI) and it is a bare DNS name. A +// templated value, localhost or an IP addresses nothing. Writes the host's +// first label to `svc` and its second (`svc.ns`, `svc.ns.svc...`) to `ns` +// ("" for one label), and returns the label count when the host is a +// service DNS name (`svc`, `svc.ns`, `svc.ns.svc[...]`), else 0. +int cbm_env_address(const char *key, const char *value, char *svc, size_t svc_sz, char *ns, + size_t ns_sz); + // Classify a string literal as URL, config, or neither. // Returns CBM_STRREF_URL (0), CBM_STRREF_CONFIG (1), or -1 for neither. int cbm_classify_string(const char *str, int len); diff --git a/scripts/check-lsp-originality.sh b/scripts/check-lsp-originality.sh index 56c0fdcde9..36633076d4 100755 --- a/scripts/check-lsp-originality.sh +++ b/scripts/check-lsp-originality.sh @@ -82,8 +82,8 @@ REFS=( # Infrastructure hybrid LSP sources outside internal/cbm/lsp/ (scanned too). INFRA_SRC=(internal/cbm/extract_k8s.c src/pipeline/pass_k8s.c src/pipeline/pass_operator.c - src/pipeline/pass_shell.c src/pipeline/pass_terraform.c - src/pipeline/pass_ci.c src/pipeline/pass_docker.c) + src/pipeline/pass_shell.c src/pipeline/pass_terraform.c src/pipeline/pass_ansible.c + src/pipeline/pass_ci.c src/pipeline/pass_docker.c src/pipeline/pass_envjoin.c) ONLY_LANG=""; REFRESH=0; LIST_ONLY=0 while [ $# -gt 0 ]; do diff --git a/scripts/memory-core-baseline.txt b/scripts/memory-core-baseline.txt index 88692b3121..f2ea7d7ac3 100644 --- a/scripts/memory-core-baseline.txt +++ b/scripts/memory-core-baseline.txt @@ -23,7 +23,7 @@ src/cli/config_toml_edit.c 73 src/cli/config_yaml_edit.c 117 src/cli/hook_augment.c 27 src/cli/legacy_agent_profiles.c 7 -src/cypher/cypher.c 128 +src/cypher/cypher.c 125 src/daemon/application.c 113 src/daemon/bootstrap.c 13 src/daemon/daemon.c 31 diff --git a/src/cli/cli.c b/src/cli/cli.c index 94a1d1b024..02862ec9d9 100644 --- a/src/cli/cli.c +++ b/src/cli/cli.c @@ -1561,7 +1561,9 @@ static const char skill_content[] = "HANDLES, IMPLEMENTS, OVERRIDE, USAGE, CALL_REFERENCE, CONFIGURES, REFERENCES_FILE,\n" "MENTIONS (doc comment -> referenced code), FILE_CHANGES_WITH, SIMILAR_TO,\n" "SEMANTICALLY_RELATED, CONTAINS_FILE, CONTAINS_FOLDER,\n" - "INFRA_MAPS (K8s Service -> selected workloads), DEPENDS_ON (chart / manifest -> dependency)\n" + "INFRA_MAPS (K8s Service -> selected workloads), DEPENDS_ON (chart / manifest -> dependency),\n" + "DEPLOYS (what puts manifests / charts on a cluster -> what it puts there),\n" + "CONNECTS_TO (workload / compose service -> the Service its environment addresses)\n" "\n" "## Sessions and delegation\n" "\n" diff --git a/src/cypher/cypher.c b/src/cypher/cypher.c index 6789b640a8..f22261e781 100644 --- a/src/cypher/cypher.c +++ b/src/cypher/cypher.c @@ -1830,7 +1830,7 @@ static int parse_return_or_with(parser_t *p, cbm_return_clause_t **out, bool is_ } while (check(p, TOK_COMMA)); /* Projection is materialized per row into fixed-width stack arrays sized at - * CBM_SZ_32 columns (execute_return_simple and its siblings). Bound the + * CBM_SZ_32 columns (ret_sink_feed_rows and its siblings). Bound the * parsed item count to that width so an over-wide RETURN is rejected here * instead of writing past those arrays downstream. * @@ -2619,6 +2619,28 @@ static void binding_set(binding_t *b, const char *var, const cbm_node_t *node) { b->var_count++; } +/* binding_set for a node the caller owns and is done with: the binding takes + * its strings instead of copying them, and *node is left empty, so the + * caller's node_fields_free of it is a no-op. A binding with no free slot + * takes nothing, as binding_set stores nothing. */ +static void binding_take_node(binding_t *b, const char *var, cbm_node_t *node) { + for (int i = 0; i < b->var_count; i++) { + if (strcmp(b->var_names[i], var) == 0) { + node_fields_free(&b->var_nodes[i]); + b->var_nodes[i] = *node; + memset(node, 0, sizeof(*node)); + return; + } + } + if (b->var_count >= CYP_MAX_VARS) { + return; + } + b->var_names[b->var_count] = var; /* not owned — points to AST string */ + b->var_nodes[b->var_count] = *node; + memset(node, 0, sizeof(*node)); + b->var_count++; +} + static const char *eval_multiarg_func(binding_t *b, const cbm_return_item_t *item, char *buf, size_t bufsz); @@ -3169,6 +3191,9 @@ static _Thread_local bool g_cypher_timed_out = false; /* Sticky for one cbm_cypher_execute call. Unlike row_count, this survives caps * that happen before DISTINCT, aggregation, ORDER BY, or the final projection. */ static _Thread_local bool g_cypher_truncated = false; +/* Set when the streamed start-node scan failed in this execution: the query + * fails with this store error rather than answer from the rows read so far. */ +static _Thread_local char g_cypher_scan_error[CBM_SZ_512]; static _Thread_local int64_t g_cypher_deadline_override_ms = -1; /* test hook; <0 = default */ #ifdef CBM_ENABLE_TEST_SEAMS @@ -3515,7 +3540,7 @@ static void process_edges(cbm_store_t *store, cbm_edge_t *edges, int edge_count, (*match_count)++; /* a real neighbour exists, OOM or not */ binding_t nb = {0}; binding_copy(&nb, b); - binding_set(&nb, to_var, &found); + binding_take_node(&nb, to_var, &found); if (rel_var) { binding_set_edge(&nb, rel_var, &edges[ei]); } @@ -3723,6 +3748,113 @@ static void expand_pattern_rels(cbm_store_t *store, cbm_pattern_t *pat, binding_ } } +/* ── Sorting and key lookup ──────────────────────────────────────── + * ORDER BY, GROUP BY and DISTINCT used to compare every row with every other + * row (bubble sorts, linear group and duplicate scans). These helpers keep the + * exact same comparisons and keys, in O(n log n) and O(n). */ + +/* Compare items a and b; positive means a sorts after b. */ +typedef int (*cypher_index_cmp_fn)(const void *ctx, int a, int b); + +/* Stable sort of idx[0..n). Both this and the bubble sorts it replaces are + * stable, so for any consistent ordering they give the same permutation. An + * item moves before an earlier one only when cmp(earlier, item) > 0, which is + * the bubble sorts' swap test. Without scratch memory it degrades to an + * insertion sort with the same test and result. */ +static void cypher_stable_sort(int *idx, int n, cypher_index_cmp_fn cmp, const void *ctx) { + if (n < PAIR_LEN) { + return; + } + int *tmp = cbm_alloc(CBM_MEM_CLASS_OTHER, (size_t)n * sizeof(int)); + if (!tmp) { + for (int i = SKIP_ONE; i < n; i++) { + for (int j = i; j > 0 && cmp(ctx, idx[j - SKIP_ONE], idx[j]) > 0; j--) { + int swap = idx[j]; + idx[j] = idx[j - SKIP_ONE]; + idx[j - SKIP_ONE] = swap; + } + } + return; + } + for (int64_t width = SKIP_ONE; width < n; width *= PAIR_LEN) { + for (int64_t lo = 0; lo < n; lo += PAIR_LEN * width) { + int64_t mid = lo + width < n ? lo + width : n; + int64_t hi = mid + width < n ? mid + width : n; + int64_t i = lo; + int64_t j = mid; + int64_t k = lo; + while (i < mid && j < hi) { + tmp[k++] = cmp(ctx, idx[i], idx[j]) > 0 ? idx[j++] : idx[i++]; + } + while (i < mid) { + tmp[k++] = idx[i++]; + } + while (j < hi) { + tmp[k++] = idx[j++]; + } + } + memcpy(idx, tmp, (size_t)n * sizeof(int)); + } + cbm_free(CBM_MEM_CLASS_OTHER, tmp); +} + +/* String key → the index it was first recorded with. Keys are copied: the + * arrays they mirror are reallocated as they grow. */ +typedef struct { + CBMHashTable *table; + char **keys; + int count; + int cap; + bool failed; /* an allocation failed: callers fall back to their scans */ +} cypher_key_index_t; + +/* The recorded index, CYP_FOUND_NONE, or KEY_INDEX_UNKNOWN after a failed + * allocation (the caller then scans, so a result never depends on memory). */ +enum { KEY_INDEX_UNKNOWN = -2 }; + +static int key_index_find(const cypher_key_index_t *ix, const char *key) { + if (ix->failed) { + return KEY_INDEX_UNKNOWN; + } + void *value = ix->table ? cbm_ht_get(ix->table, key) : NULL; + return value ? (int)((intptr_t)value - SKIP_ONE) : CYP_FOUND_NONE; +} + +static void key_index_add(cypher_key_index_t *ix, const char *key, int index) { + if (ix->failed) { + return; + } + if (!ix->table) { + ix->table = cbm_ht_create(CBM_SZ_256); + } + if (ix->table && ix->count >= ix->cap) { + int cap = ix->cap ? ix->cap * PAIR_LEN : CBM_SZ_256; + char **keys = cbm_realloc(CBM_MEM_CLASS_OTHER, ix->keys, (size_t)cap * sizeof(char *)); + if (keys) { + ix->keys = keys; + ix->cap = cap; + } + } + char *copy = ix->table && ix->count < ix->cap ? cbm_mem_strdup(CBM_MEM_CLASS_OTHER, key) : NULL; + if (!copy) { + ix->failed = true; + return; + } + ix->keys[ix->count++] = copy; + (void)cbm_ht_set(ix->table, copy, (void *)(intptr_t)(index + SKIP_ONE)); +} + +static void key_index_free(cypher_key_index_t *ix) { + if (ix->table) { + cbm_ht_free(ix->table); + } + for (int i = 0; i < ix->count; i++) { + cbm_free(CBM_MEM_CLASS_OTHER, ix->keys[i]); + } + cbm_free(CBM_MEM_CLASS_OTHER, ix->keys); + memset(ix, 0, sizeof(*ix)); +} + /* ── Result postprocessing helpers ─────────────────────────────── */ /* Find the column index for one ORDER BY key, checking both column names and @@ -3762,6 +3894,43 @@ static bool rb_is_numeric_column(const result_builder_t *rb, int col) { return false; } +typedef struct { + const int *cols; + const bool *numeric; + const bool *descs; + int keys; + const char ***rows; /* the rows in their pre-sort order */ + const int *values; /* numeric keys parsed once: values[row * keys + k], or NULL */ +} rb_order_ctx_t; + +/* One ORDER BY comparison: numeric columns by their (int)strtol value, + * others by strcmp, a later key breaking ties; positive = a after b. + * va/vb are the rows' parsed numeric keys, or NULL to parse here. */ +static int rb_order_compare_rows(const rb_order_ctx_t *ctx, const char **ra, const char **rb_row, + const int *va, const int *vb) { + int cmp = 0; + for (int k = 0; k < ctx->keys && cmp == 0; k++) { + if (ctx->numeric[k]) { + int a = va ? va[k] : (int)strtol(ra[ctx->cols[k]], NULL, CBM_DECIMAL_BASE); + int b = vb ? vb[k] : (int)strtol(rb_row[ctx->cols[k]], NULL, CBM_DECIMAL_BASE); + cmp = (a > b) - (a < b); + } else { + cmp = strcmp(ra[ctx->cols[k]], rb_row[ctx->cols[k]]); + } + if (ctx->descs[k]) { + cmp = -cmp; + } + } + return cmp; +} + +static int rb_order_compare(const void *context, int a, int b) { + const rb_order_ctx_t *ctx = context; + const int *va = ctx->values ? ctx->values + (size_t)a * (size_t)ctx->keys : NULL; + const int *vb = ctx->values ? ctx->values + (size_t)b * (size_t)ctx->keys : NULL; + return rb_order_compare_rows(ctx, ctx->rows[a], ctx->rows[b], va, vb); +} + static void rb_apply_order_by(result_builder_t *rb, const cbm_return_clause_t *ret) { if (ret->order_key_count == 0) { return; @@ -3783,30 +3952,46 @@ static void rb_apply_order_by(result_builder_t *rb, const cbm_return_clause_t *r descs[keys] = ret->order_descs[k]; keys++; } - if (keys == 0) { + if (keys == 0 || rb->row_count < PAIR_LEN) { return; } - for (int i = 0; i < rb->row_count - SKIP_ONE; i++) { - for (int j = 0; j < rb->row_count - i - SKIP_ONE; j++) { - int cmp = 0; - for (int k = 0; k < keys && cmp == 0; k++) { - if (numeric[k]) { - cmp = (int)strtol(rb->rows[j][cols[k]], NULL, CBM_DECIMAL_BASE) - - (int)strtol(rb->rows[j + SKIP_ONE][cols[k]], NULL, CBM_DECIMAL_BASE); - } else { - cmp = strcmp(rb->rows[j][cols[k]], rb->rows[j + SKIP_ONE][cols[k]]); - } - if (descs[k]) { - cmp = -cmp; - } - } - if (cmp > 0) { + int n = rb->row_count; + rb_order_ctx_t ctx = {.cols = cols, .numeric = numeric, .descs = descs, .keys = keys}; + int *idx = cbm_alloc(CBM_MEM_CLASS_OTHER, (size_t)n * sizeof(int)); + const char ***rows = cbm_alloc(CBM_MEM_CLASS_OTHER, (size_t)n * sizeof(const char **)); + if (!idx || !rows) { + /* No scratch memory: insertion sort in place, same comparisons. */ + cbm_free(CBM_MEM_CLASS_OTHER, idx); + cbm_free(CBM_MEM_CLASS_OTHER, rows); + for (int i = SKIP_ONE; i < n; i++) { + for (int j = i; j > 0 && rb_order_compare_rows(&ctx, rb->rows[j - SKIP_ONE], + rb->rows[j], NULL, NULL) > 0; + j--) { const char **tmp = rb->rows[j]; - rb->rows[j] = rb->rows[j + SKIP_ONE]; - rb->rows[j + SKIP_ONE] = tmp; + rb->rows[j] = rb->rows[j - SKIP_ONE]; + rb->rows[j - SKIP_ONE] = tmp; } } + return; + } + int *values = cbm_alloc(CBM_MEM_CLASS_OTHER, (size_t)n * (size_t)keys * sizeof(int)); + for (int r = 0; r < n; r++) { + idx[r] = r; + rows[r] = rb->rows[r]; + for (int k = 0; values && k < keys; k++) { + values[(size_t)r * (size_t)keys + (size_t)k] = + numeric[k] ? (int)strtol(rb->rows[r][cols[k]], NULL, CBM_DECIMAL_BASE) : 0; + } + } + ctx.rows = rows; + ctx.values = values; + cypher_stable_sort(idx, n, rb_order_compare, &ctx); + for (int r = 0; r < n; r++) { + rb->rows[r] = rows[idx[r]]; } + cbm_free(CBM_MEM_CLASS_OTHER, values); + cbm_free(CBM_MEM_CLASS_OTHER, rows); + cbm_free(CBM_MEM_CLASS_OTHER, idx); } static void rb_apply_skip_limit(result_builder_t *rb, int skip_n, int limit, @@ -3845,25 +4030,70 @@ static void rb_apply_skip_limit(result_builder_t *rb, int skip_n, int limit, } } +/* A row's DISTINCT identity: every column, length-prefixed so no value can + * run into the next — equal keys exactly when every column is equal. Built in + * *buf (grown as needed); NULL when out of memory. */ +static const char *rb_row_key(const result_builder_t *rb, const char **row, char **buf, + size_t *cap) { + size_t need = SKIP_ONE; + for (int c = 0; c < rb->col_count; c++) { + need += strlen(row[c]) + CBM_SZ_32; + } + if (need > *cap) { + char *grown = cbm_realloc(CBM_MEM_CLASS_OTHER, *buf, need); + if (!grown) { + return NULL; + } + *buf = grown; + *cap = need; + } + size_t used = 0; + for (int c = 0; c < rb->col_count; c++) { + size_t len = strlen(row[c]); + char prefix[CBM_SZ_32]; + int prefix_len = snprintf(prefix, sizeof(prefix), "%zu:", len); + if (prefix_len <= 0 || (size_t)prefix_len >= sizeof(prefix) || + (size_t)prefix_len + len >= *cap - used) { + return NULL; /* cannot happen: `need` reserved this */ + } + memcpy(*buf + used, prefix, (size_t)prefix_len); + used += (size_t)prefix_len; + memcpy(*buf + used, row[c], len); + used += len; + } + (*buf)[used] = '\0'; + return *buf; +} + +/* Whether row i equals one of the first `kept` rows, column by column. */ +static bool rb_row_seen_before(const result_builder_t *rb, int i, int kept) { + for (int j = 0; j < kept; j++) { + bool same = true; + for (int c = 0; c < rb->col_count && same; c++) { + same = strcmp(rb->rows[i][c], rb->rows[j][c]) == 0; + } + if (same) { + return true; + } + } + return false; +} + static void rb_apply_distinct(result_builder_t *rb) { if (rb->row_count <= SKIP_ONE) { return; } - int kept = SKIP_ONE; - for (int i = SKIP_ONE; i < rb->row_count; i++) { - bool dup = false; - for (int j = 0; j < kept && !dup; j++) { - bool same = true; - for (int c = 0; c < rb->col_count && same; c++) { - if (strcmp(rb->rows[i][c], rb->rows[j][c]) != 0) { - same = false; - } - } - if (same) { - dup = true; - } - } + cypher_key_index_t seen = {0}; + char *key_buf = NULL; + size_t key_cap = 0; + int kept = 0; + for (int i = 0; i < rb->row_count; i++) { + const char *key = rb_row_key(rb, rb->rows[i], &key_buf, &key_cap); + seen.failed = seen.failed || !key; + int found = key_index_find(&seen, key); + bool dup = found == KEY_INDEX_UNKNOWN ? rb_row_seen_before(rb, i, kept) : found >= 0; if (!dup) { + key_index_add(&seen, key, kept); if (kept != i) { rb->rows[kept] = rb->rows[i]; } @@ -3876,6 +4106,8 @@ static void rb_apply_distinct(result_builder_t *rb) { } } rb->row_count = kept; + key_index_free(&seen); + cbm_free(CBM_MEM_CLASS_OTHER, key_buf); } static void rb_free(result_builder_t *rb) { @@ -4120,21 +4352,57 @@ static bool is_aggregate_func(const char *func) { strcmp(func, "MIN") == 0 || strcmp(func, "MAX") == 0 || strcmp(func, "COLLECT") == 0); } +/* Grow a string list for one more entry at `idx`: the capacity doubles at + * every power of two, so n appends cost O(n) copying, not O(n^2). */ +static char **string_list_reserve(char **list, int idx) { + if (idx > 0 && (idx & (idx - SKIP_ONE)) != 0) { + return list; /* capacity is the next power of two above idx */ + } + return safe_realloc(list, (size_t)(idx ? idx * PAIR_LEN : SKIP_ONE) * sizeof(char *)); +} + +enum { DISTINCT_SET_MIN = 16 }; /* below this a set is scanned, not hashed */ + /* Append `val` to a string list only if not already present — i.e. maintain a - * set of distinct values. Used by COUNT(DISTINCT x) (#239). */ -static void distinct_list_add(char ***list, int *count, const char *val) { - for (int i = 0; i < *count; i++) { - if (strcmp((*list)[i], val) == 0) { + * set of distinct values. Used by COUNT(DISTINCT x) (#239). Past + * DISTINCT_SET_MIN values the membership test goes through *set, whose keys + * are the list's own strings; a set that cannot be built stays a scan. */ +static void distinct_list_add(char ***list, int *count, CBMHashTable **set, const char *val) { + if (*set) { + if (cbm_ht_has(*set, val)) { return; } + } else { + for (int i = 0; i < *count; i++) { + if (strcmp((*list)[i], val) == 0) { + return; + } + } } - int idx = (*count)++; - *list = safe_realloc(*list, (size_t)(idx + SKIP_ONE) * sizeof(char *)); - if (!*list) { - *count = 0; /* safe_realloc freed the old list */ + int idx = *count; + *list = string_list_reserve(*list, idx); + char *copy = *list ? heap_strdup(val) : NULL; + if (!copy) { + /* Out of memory: keep a consistent (possibly emptied) set. */ + if (!*list) { + *count = 0; + if (*set) { + cbm_ht_free(*set); + *set = NULL; + } + } return; } - (*list)[idx] = heap_strdup(val); + (*list)[idx] = copy; + (*count)++; + if (*set) { + (void)cbm_ht_set(*set, (*list)[idx], (void *)SKIP_ONE); + } else if (*count >= DISTINCT_SET_MIN) { + *set = cbm_ht_create(CBM_SZ_64); + for (int i = 0; *set && i < *count; i++) { + (void)cbm_ht_set(*set, (*list)[i], (void *)SKIP_ONE); + } + } } /* Resolve one WITH ORDER BY key on a projected binding. The key is either a @@ -4161,31 +4429,110 @@ static const char *order_key_value(binding_t *b, const char *key) { return binding_get_virtual(b, var, dot + SKIP_ONE); } -/* Sort bindings by the ORDER BY key list (virtual variables) using bubble - * sort; later keys break ties, direction is per key (#1334). */ +/* One WITH ORDER BY key comparison: numerically when both values start with a + * number, else by strcmp; positive = a after b. */ +static int with_order_compare_value(const char *va, const char *vb) { + char *ea = NULL; + char *eb = NULL; + double da = strtod(va, &ea); + double db = strtod(vb, &eb); + return (ea != va && eb != vb) ? ((da > db) - (da < db)) : strcmp(va, vb); +} + +/* The full comparison over the ORDER BY key list: later keys break ties, + * direction is per key (#1334). */ +static int with_order_compare_bindings(const cbm_return_clause_t *wc, binding_t *a, binding_t *b) { + int cmp = 0; + for (int k = 0; k < wc->order_key_count && cmp == 0; k++) { + cmp = with_order_compare_value(order_key_value(a, wc->order_keys[k]), + order_key_value(b, wc->order_keys[k])); + if (wc->order_descs[k]) { + cmp = -cmp; + } + } + return cmp; +} + +typedef struct { + const cbm_return_clause_t *wc; + char **values; /* values[i * keys + k]: owned copy of binding i's key k */ + bool *numbered; /* whether strtod consumed a number from that value */ + double *numbers; +} with_order_ctx_t; + +/* The same comparison on values resolved once per binding. order_key_value + * may return a rotating thread-local buffer, so the values are copies. */ +static int with_order_compare(const void *context, int a, int b) { + const with_order_ctx_t *ctx = context; + size_t keys = (size_t)ctx->wc->order_key_count; + int cmp = 0; + for (size_t k = 0; k < keys && cmp == 0; k++) { + size_t ia = (size_t)a * keys + k; + size_t ib = (size_t)b * keys + k; + if (ctx->numbered[ia] && ctx->numbered[ib]) { + cmp = (ctx->numbers[ia] > ctx->numbers[ib]) - (ctx->numbers[ia] < ctx->numbers[ib]); + } else { + cmp = strcmp(ctx->values[ia], ctx->values[ib]); + } + if (ctx->wc->order_descs[k]) { + cmp = -cmp; + } + } + return cmp; +} + +/* Sort bindings by the ORDER BY key list (virtual variables), stably. */ static void sort_bindings(binding_t *vbindings, int count, const cbm_return_clause_t *wc) { - for (int i = 0; i < count - SKIP_ONE; i++) { - for (int j = 0; j < count - i - SKIP_ONE; j++) { - int cmp = 0; - for (int k = 0; k < wc->order_key_count && cmp == 0; k++) { - const char *va = order_key_value(&vbindings[j], wc->order_keys[k]); - const char *vb2 = order_key_value(&vbindings[j + SKIP_ONE], wc->order_keys[k]); - char *ea = NULL; - char *eb = NULL; - double da = strtod(va, &ea); - double db = strtod(vb2, &eb); - cmp = (ea != va && eb != vb2) ? ((da > db) - (da < db)) : strcmp(va, vb2); - if (wc->order_descs[k]) { - cmp = -cmp; - } - } - if (cmp > 0) { + if (count < PAIR_LEN || wc->order_key_count <= 0) { + return; + } + size_t keys = (size_t)wc->order_key_count; + size_t cells = (size_t)count * keys; + with_order_ctx_t ctx = {.wc = wc, + .values = cbm_calloc(CBM_MEM_CLASS_OTHER, cells * sizeof(char *)), + .numbered = cbm_calloc(CBM_MEM_CLASS_OTHER, cells * sizeof(bool)), + .numbers = cbm_calloc(CBM_MEM_CLASS_OTHER, cells * sizeof(double))}; + int *idx = cbm_alloc(CBM_MEM_CLASS_OTHER, (size_t)count * sizeof(int)); + binding_t *sorted = cbm_alloc(CBM_MEM_CLASS_OTHER, (size_t)count * sizeof(binding_t)); + bool ready = ctx.values && ctx.numbered && ctx.numbers && idx && sorted; + for (int i = 0; ready && i < count; i++) { + idx[i] = i; + for (size_t k = 0; ready && k < keys; k++) { + size_t cell = (size_t)i * keys + k; + ctx.values[cell] = cbm_mem_strdup(CBM_MEM_CLASS_OTHER, + order_key_value(&vbindings[i], wc->order_keys[k])); + ready = ctx.values[cell] != NULL; + char *end = NULL; + ctx.numbers[cell] = ready ? strtod(ctx.values[cell], &end) : 0; + ctx.numbered[cell] = ready && end != ctx.values[cell]; + } + } + if (ready) { + cypher_stable_sort(idx, count, with_order_compare, &ctx); + for (int i = 0; i < count; i++) { + sorted[i] = vbindings[idx[i]]; + } + memcpy(vbindings, sorted, (size_t)count * sizeof(binding_t)); + } else { + /* No scratch memory: insertion sort in place, same comparisons. */ + for (int i = SKIP_ONE; i < count; i++) { + for (int j = i; j > 0 && with_order_compare_bindings(wc, &vbindings[j - SKIP_ONE], + &vbindings[j]) > 0; + j--) { binding_t tmp = vbindings[j]; - vbindings[j] = vbindings[j + SKIP_ONE]; - vbindings[j + SKIP_ONE] = tmp; + vbindings[j] = vbindings[j - SKIP_ONE]; + vbindings[j - SKIP_ONE] = tmp; } } } + for (size_t c = 0; ctx.values && c < cells; c++) { + cbm_free(CBM_MEM_CLASS_OTHER, ctx.values[c]); + } + cbm_free(CBM_MEM_CLASS_OTHER, ctx.values); + cbm_free(CBM_MEM_CLASS_OTHER, ctx.numbered); + cbm_free(CBM_MEM_CLASS_OTHER, ctx.numbers); + cbm_free(CBM_MEM_CLASS_OTHER, idx); + cbm_free(CBM_MEM_CLASS_OTHER, sorted); } /* Apply skip and limit to a binding array, freeing discarded entries */ @@ -4241,9 +4588,10 @@ typedef struct { double *sums; int *counts; double *mins, *maxs; - char ***distinct_lists; /* per-item set of seen values for COUNT(DISTINCT) */ - int *distinct_n; /* per-item distinct count (#239) */ - cbm_node_t *group_nodes; /* per-item carried node (deep copy; id 0 = not a node) */ + char ***distinct_lists; /* per-item set of seen values for COUNT(DISTINCT) */ + int *distinct_n; /* per-item distinct count (#239) */ + CBMHashTable **distinct_sets; /* per-item hash over distinct_lists (lazy) */ + cbm_node_t *group_nodes; /* per-item carried node (deep copy; id 0 = not a node) */ } with_agg_t; /* A WITH item that is a bare, bound node variable (`WITH f`, `WITH f AS g`) @@ -4289,18 +4637,24 @@ static int with_agg_build_key(cbm_return_clause_t *wc, binding_t *b, char *key, /* Find or create an aggregation group. Returns index. */ static int with_agg_find_or_create(with_agg_t **aggs, int *agg_cnt, int *agg_cap, - cbm_return_clause_t *wc, binding_t *b, const char *key) { - for (int a = 0; a < *agg_cnt; a++) { + cypher_key_index_t *groups, cbm_return_clause_t *wc, + binding_t *b, const char *key) { + int existing = key_index_find(groups, key); + for (int a = 0; existing == KEY_INDEX_UNKNOWN && a < *agg_cnt; a++) { if (strcmp((*aggs)[a].group_key, key) == 0) { - return a; + existing = a; } } + if (existing >= 0) { + return existing; + } if (*agg_cnt >= *agg_cap) { *agg_cap *= PAIR_LEN; *aggs = safe_realloc(*aggs, *agg_cap * sizeof(with_agg_t)); } int found = (*agg_cnt)++; snprintf((*aggs)[found].group_key, sizeof((*aggs)[found].group_key), "%s", key); + key_index_add(groups, (*aggs)[found].group_key, found); (*aggs)[found].group_vals = calloc(wc->count, sizeof(const char *)); (*aggs)[found].sums = calloc(wc->count, sizeof(double)); (*aggs)[found].counts = calloc(wc->count, sizeof(int)); @@ -4308,6 +4662,8 @@ static int with_agg_find_or_create(with_agg_t **aggs, int *agg_cnt, int *agg_cap (*aggs)[found].maxs = calloc(wc->count, sizeof(double)); (*aggs)[found].distinct_lists = calloc(wc->count, sizeof(char **)); (*aggs)[found].distinct_n = calloc(wc->count, sizeof(int)); + (*aggs)[found].distinct_sets = + cbm_calloc(CBM_MEM_CLASS_OTHER, (size_t)wc->count * sizeof(CBMHashTable *)); (*aggs)[found].group_nodes = calloc(wc->count, sizeof(cbm_node_t)); for (int ci = 0; ci < wc->count; ci++) { (*aggs)[found].mins[ci] = CYP_DBL_MAX; @@ -4361,7 +4717,8 @@ static void with_agg_accumulate(with_agg_t *agg, cbm_return_clause_t *wc, bindin agg->counts[ci]++; const char *raw = binding_get_virtual(b, wc->items[ci].variable, wc->items[ci].property); if (wc->items[ci].distinct && strcmp(wc->items[ci].func, "COUNT") == 0) { - distinct_list_add(&agg->distinct_lists[ci], &agg->distinct_n[ci], raw); + distinct_list_add(&agg->distinct_lists[ci], &agg->distinct_n[ci], + &agg->distinct_sets[ci], raw); } double dv = strtod(raw, NULL); agg->sums[ci] += dv; @@ -4407,6 +4764,9 @@ static void with_agg_free(with_agg_t *aggs, int agg_cnt, int item_count) { for (int a = 0; a < agg_cnt; a++) { for (int ci = 0; ci < item_count; ci++) { safe_str_free(&aggs[a].group_vals[ci]); + if (aggs[a].distinct_sets && aggs[a].distinct_sets[ci]) { + cbm_ht_free(aggs[a].distinct_sets[ci]); + } if (aggs[a].distinct_lists && aggs[a].distinct_lists[ci]) { for (int j = 0; j < aggs[a].distinct_n[ci]; j++) { free(aggs[a].distinct_lists[ci][j]); @@ -4421,6 +4781,7 @@ static void with_agg_free(with_agg_t *aggs, int agg_cnt, int item_count) { free(aggs[a].maxs); free(aggs[a].distinct_lists); free(aggs[a].distinct_n); + cbm_free(CBM_MEM_CLASS_OTHER, aggs[a].distinct_sets); if (aggs[a].group_nodes) { for (int ci = 0; ci < item_count; ci++) { node_fields_free(&aggs[a].group_nodes[ci]); @@ -4431,30 +4792,57 @@ static void with_agg_free(with_agg_t *aggs, int agg_cnt, int item_count) { free(aggs); } -/* Execute WITH aggregation path */ -static void execute_with_aggregate(cbm_return_clause_t *wc, binding_t *bindings, int bind_count, - binding_t **vbindings, int *vcount) { - int agg_cap = CBM_SZ_256; - with_agg_t *aggs = calloc(agg_cap, sizeof(with_agg_t)); - int agg_cnt = 0; - - for (int bi = 0; bi < bind_count; bi++) { - char key[CBM_SZ_1K] = ""; - with_agg_build_key(wc, &bindings[bi], key, sizeof(key)); - int found = with_agg_find_or_create(&aggs, &agg_cnt, &agg_cap, wc, &bindings[bi], key); - with_agg_accumulate(&aggs[found], wc, &bindings[bi]); +/* WITH as a consumer of the binding stream (execute_chunk): aggregation groups + * and projected rows accumulate across feeds in input order, so a chunked run + * keeps the first-seen group order and row order of a single pass. */ +typedef struct { + cbm_return_clause_t *wc; /* NULL: the query has no WITH */ + bool has_agg; + bool fed_any; + cbm_store_t *store; /* the first fed binding's store */ + with_agg_t *aggs; + int agg_cnt; + int agg_cap; + cypher_key_index_t groups; + binding_t *vbindings; /* simple projection: one row per binding (mem core, class OTHER) */ + int vcount; + int vcap; + bool oom; /* a row buffer could not grow: the query fails */ +} with_sink_t; + +static void with_sink_init(with_sink_t *s, cbm_return_clause_t *wc) { + memset(s, 0, sizeof(*s)); + s->wc = wc; + if (!wc) { + return; } + for (int i = 0; i < wc->count; i++) { + if (is_aggregate_func(wc->items[i].func)) { + s->has_agg = true; + break; + } + } + if (s->has_agg) { + s->agg_cap = CBM_SZ_256; + s->aggs = calloc(s->agg_cap, sizeof(with_agg_t)); + } +} - *vbindings = safe_realloc(*vbindings, (agg_cnt + SKIP_ONE) * sizeof(binding_t)); - if (!*vbindings) { - with_agg_free(aggs, agg_cnt, wc->count); - return; +/* One projected row per aggregation group, in first-seen group order. */ +static binding_t *with_agg_rows(with_sink_t *s, int *vcount) { + cbm_return_clause_t *wc = s->wc; + with_agg_t *aggs = s->aggs; + int agg_cnt = s->agg_cnt; + binding_t *vbindings = + cbm_alloc(CBM_MEM_CLASS_OTHER, ((size_t)agg_cnt + SKIP_ONE) * sizeof(binding_t)); + if (!vbindings) { + return NULL; } for (int a = 0; a < agg_cnt; a++) { binding_t vb = {0}; /* Carry the store so node_prop can re-fetch a carried node's properties * (and compute in_degree/out_degree) on the projected virtual binding. */ - vb.store = (bind_count > 0) ? bindings[0].store : NULL; + vb.store = s->store; for (int ci = 0; ci < wc->count; ci++) { char name_buf[CBM_SZ_256]; const char *alias = resolve_item_alias(&wc->items[ci], name_buf, sizeof(name_buf)); @@ -4472,9 +4860,9 @@ static void execute_with_aggregate(cbm_return_clause_t *wc, binding_t *bindings, with_add_vbinding_var(&vb, alias, aggs[a].group_vals[ci]); } } - (*vbindings)[(*vcount)++] = vb; + vbindings[(*vcount)++] = vb; } - with_agg_free(aggs, agg_cnt, wc->count); + return vbindings; } /* Execute WITH simple (non-aggregate) projection */ @@ -4500,6 +4888,49 @@ static void execute_with_simple(cbm_return_clause_t *wc, binding_t *bindings, in } } +static void with_sink_feed(with_sink_t *s, binding_t *bindings, int bind_count) { + if (bind_count > 0 && !s->fed_any) { + s->fed_any = true; + s->store = bindings[0].store; + } + if (!s->has_agg) { + if (s->vcount + bind_count + SKIP_ONE > s->vcap) { + int need = s->vcount + bind_count + SKIP_ONE; + int cap = s->vcap * CYP_GROWTH_2 > need ? s->vcap * CYP_GROWTH_2 : need; + binding_t *grown = + cbm_realloc(CBM_MEM_CLASS_OTHER, s->vbindings, (size_t)cap * sizeof(binding_t)); + if (!grown) { + s->oom = true; /* the query fails rather than lose rows */ + return; + } + s->vbindings = grown; + s->vcap = cap; + } + execute_with_simple(s->wc, bindings, bind_count, s->vbindings, &s->vcount); + return; + } + for (int bi = 0; bi < bind_count; bi++) { + char key[CBM_SZ_1K] = ""; + with_agg_build_key(s->wc, &bindings[bi], key, sizeof(key)); + int found = with_agg_find_or_create(&s->aggs, &s->agg_cnt, &s->agg_cap, &s->groups, s->wc, + &bindings[bi], key); + with_agg_accumulate(&s->aggs[found], s->wc, &bindings[bi]); + } +} + +/* Release a WITH sink that will not be finished (a failed query). */ +static void with_sink_free(with_sink_t *s) { + if (s->has_agg) { + key_index_free(&s->groups); + with_agg_free(s->aggs, s->agg_cnt, s->wc->count); + } + for (int i = 0; i < s->vcount; i++) { + binding_free(&s->vbindings[i]); + } + cbm_free(CBM_MEM_CLASS_OTHER, s->vbindings); + memset(s, 0, sizeof(*s)); +} + /* Apply post-WITH WHERE filter */ static void filter_bindings_where(const cbm_where_clause_t *where, binding_t *vbindings, int *vcount) { @@ -4539,22 +4970,23 @@ static void with_proj_key(cbm_return_clause_t *wc, binding_t *b, char *key, size /* Apply WITH DISTINCT: drop projected rows whose value tuple duplicates an * earlier one, keeping first occurrence (#238 — previously silently ignored). */ static void with_apply_distinct(cbm_return_clause_t *wc, binding_t *vbindings, int *vcount) { + cypher_key_index_t seen = {0}; int kept = 0; for (int i = 0; i < *vcount; i++) { char key[CBM_SZ_1K]; with_proj_key(wc, &vbindings[i], key, sizeof(key)); - bool dup = false; - for (int j = 0; j < kept; j++) { + int found = key_index_find(&seen, key); + for (int j = 0; found == KEY_INDEX_UNKNOWN && j < kept; j++) { char pkey[CBM_SZ_1K]; with_proj_key(wc, &vbindings[j], pkey, sizeof(pkey)); if (strcmp(key, pkey) == 0) { - dup = true; - break; + found = j; } } - if (dup) { + if (found >= 0) { binding_free(&vbindings[i]); } else { + key_index_add(&seen, key, kept); if (kept != i) { vbindings[kept] = vbindings[i]; } @@ -4562,32 +4994,23 @@ static void with_apply_distinct(cbm_return_clause_t *wc, binding_t *vbindings, i } } *vcount = kept; + key_index_free(&seen); } -static void execute_with_clause(cbm_query_t *q, binding_t **bindings_ptr, int *bind_count_ptr) { - cbm_return_clause_t *wc = q->with_clause; - if (!wc) { - return; - } - binding_t *bindings = *bindings_ptr; - int bind_count = *bind_count_ptr; - - binding_t *vbindings = malloc((bind_count + SKIP_ONE) * sizeof(binding_t)); - int vcount = 0; - - bool has_agg = false; - for (int i = 0; i < wc->count; i++) { - if (is_aggregate_func(wc->items[i].func)) { - has_agg = true; - break; - } - } - - if (has_agg) { - execute_with_aggregate(wc, bindings, bind_count, &vbindings, &vcount); - } else { - execute_with_simple(wc, bindings, bind_count, vbindings, &vcount); +/* The WITH output once every binding has been fed: projected rows (one per + * group or per binding), de-duplicated, sorted, skipped/limited and filtered + * by the post-WITH WHERE. The caller owns the returned rows. */ +static binding_t *with_sink_finish(with_sink_t *s, cbm_query_t *q, int *out_count) { + cbm_return_clause_t *wc = s->wc; + binding_t *vbindings = s->vbindings; + int vcount = s->vcount; + if (s->has_agg) { + key_index_free(&s->groups); + vcount = 0; + vbindings = with_agg_rows(s, &vcount); + with_agg_free(s->aggs, s->agg_cnt, wc->count); } + memset(s, 0, sizeof(*s)); /* WITH DISTINCT: dedup projected rows (no-op for aggregation, which already * collapses to one row per group). */ @@ -4597,17 +5020,12 @@ static void execute_with_clause(cbm_query_t *q, binding_t **bindings_ptr, int *b with_sort_skip_limit(wc, vbindings, &vcount); - for (int bi = 0; bi < bind_count; bi++) { - binding_free(&bindings[bi]); - } - free(bindings); - if (q->post_with_where) { filter_bindings_where(q->post_with_where, vbindings, &vcount); } - *bindings_ptr = vbindings; - *bind_count_ptr = vcount; + *out_count = vcount; + return vbindings; } /* ── Execute a single query (no UNION recursion) ──────────────── */ @@ -4693,71 +5111,80 @@ static void project_star_row(binding_t *b, const char **vars, int vc, const char } } +/* ── RETURN as a consumer of the binding stream ─────────────────── + * execute_chunk feeds the projection chunk by chunk. Each projection walks its + * input in order and keeps its state between feeds (row cap, DISTINCT set, + * aggregation groups, bindings seen), so a chunked run yields exactly the + * rows, row order, first-seen group order and truncation of a single pass + * over all bindings. A query with a WITH feeds the WITH rows once. */ +typedef enum { + RET_SINK_DEFAULT, /* no RETURN: name, qualified_name, label per pattern-0 var */ + RET_SINK_SIMPLE, + RET_SINK_AGG, + RET_SINK_STAR, + RET_SINK_STAR_AFTER_WITH, +} ret_sink_kind_t; + +typedef struct ret_agg_entry ret_agg_entry_t; + +typedef struct { + cbm_return_clause_t *ret; /* NULL for the default projection */ + result_builder_t *rb; + int max_rows; + ret_sink_kind_t kind; + bool has_agg; + bool done; /* the row cap is full, or the deadline tripped: later rows add nothing */ + int64_t fed; /* bindings fed so far */ + int cap; /* projection row cap */ + bool cap_is_engine_budget; + rb_distinct_t seen; + const char *vars[CBM_SZ_32]; /* projected variables, or the WITH names of RETURN * */ + int vc; + char names[CYP_MAX_VARS][CBM_SZ_128]; /* RETURN * after WITH: computed names */ + ret_agg_entry_t *aggs; + int agg_count; + int agg_cap; + cypher_key_index_t groups; +} ret_sink_t; + /* RETURN * after a WITH. * * The pattern's variables are out of scope by this point — the WITH replaced * them with the names it made. Each of those names holds one value, not a * node, so each is ONE column rather than the four a node variable gets. * - * Reading the pattern here instead is the fault this function exists to avoid: - * it named variables the bindings no longer hold, found nothing for every one - * of them, and answered a full result of empty strings with no error. */ -static void execute_return_star_after_with(cbm_query_t *q, binding_t *bindings, int bind_count, - int max_rows, result_builder_t *rb) { + * Reading the pattern here instead is the fault this projection exists to + * avoid: it named variables the bindings no longer hold, found nothing for + * every one of them, and answered a full result of empty strings with no + * error. */ +static void ret_sink_init_star_after_with(ret_sink_t *s, const cbm_query_t *q) { cbm_return_clause_t *wc = q->with_clause; - char name_bufs[CYP_MAX_VARS][CBM_SZ_128]; - const char *cols[CYP_MAX_VARS]; + s->kind = RET_SINK_STAR_AFTER_WITH; /* parse_return_or_with refuses a WITH wider than CYP_MAX_VARS, so this - * clamp cannot fire. It stays as the bound this function relies on. */ - int col_n = wc->count < CYP_MAX_VARS ? wc->count : CYP_MAX_VARS; - for (int i = 0; i < col_n; i++) { - cols[i] = resolve_item_alias(&wc->items[i], name_bufs[i], sizeof(name_bufs[i])); - } - rb_set_columns(rb, cols, col_n); - rb_distinct_t seen; - rb_distinct_init(&seen, q->ret && q->ret->distinct); - for (int bi = 0; bi < bind_count; bi++) { - const char *vals[CYP_MAX_VARS]; - for (int i = 0; i < col_n; i++) { - cbm_node_t *vn = binding_get(&bindings[bi], cols[i]); - vals[i] = vn && vn->name ? vn->name : ""; - } - if (!rb_add_row_capped(rb, &seen, vals, max_rows)) { - g_cypher_truncated = true; - break; - } - } - rb_distinct_free(&seen); -} - -static void execute_return_star(cbm_query_t *q, binding_t *bindings, int bind_count, int max_rows, - result_builder_t *rb) { - if (q->with_clause) { - execute_return_star_after_with(q, bindings, bind_count, max_rows, rb); - return; - } - const char *vars[CBM_SZ_32]; - int vc = collect_pattern_vars(q, vars, CBM_SZ_32); - build_star_columns(rb, vars, vc); - int projection_cap = max_rows; - bool cap_is_engine_budget = true; - cbm_return_clause_t *ret = q->ret; - if (ret && ret->limit >= 0 && ret->limit <= max_rows && ret->order_key_count == 0 && + * clamp cannot fire. It stays as the bound this projection relies on. */ + s->vc = wc->count < CYP_MAX_VARS ? wc->count : CYP_MAX_VARS; + for (int i = 0; i < s->vc; i++) { + s->vars[i] = resolve_item_alias(&wc->items[i], s->names[i], sizeof(s->names[i])); + } + rb_set_columns(s->rb, s->vars, s->vc); + s->cap = s->max_rows; + s->cap_is_engine_budget = true; + rb_distinct_init(&s->seen, s->ret->distinct); +} + +static void ret_sink_init_star(ret_sink_t *s, cbm_query_t *q) { + s->kind = RET_SINK_STAR; + s->vc = collect_pattern_vars(q, s->vars, CBM_SZ_32); + build_star_columns(s->rb, s->vars, s->vc); + cbm_return_clause_t *ret = s->ret; + s->cap = s->max_rows; + s->cap_is_engine_budget = true; + if (ret->limit >= 0 && ret->limit <= s->max_rows && ret->order_key_count == 0 && ret->skip <= 0) { - projection_cap = ret->limit; /* DISTINCT: the cap counts distinct rows (#1364) */ - cap_is_engine_budget = false; - } - rb_distinct_t seen; - rb_distinct_init(&seen, ret && ret->distinct); - for (int bi = 0; bi < bind_count; bi++) { - const char *vals[CBM_SZ_128]; - project_star_row(&bindings[bi], vars, vc, vals); - if (!rb_add_row_capped(rb, &seen, vals, projection_cap)) { - g_cypher_truncated = g_cypher_truncated || cap_is_engine_budget; - break; - } + s->cap = ret->limit; /* DISTINCT: the cap counts distinct rows (#1364) */ + s->cap_is_engine_budget = false; } - rb_distinct_free(&seen); + rb_distinct_init(&s->seen, ret->distinct); } /* Format an aggregate value into buf based on function name */ @@ -4799,8 +5226,8 @@ static void format_agg_value(const char *func, int count, double sum, double min } } -/* RETURN aggregation entry */ -typedef struct { +/* RETURN aggregation entry (ret_agg_entry_t, declared with ret_sink_t) */ +struct ret_agg_entry { char group_key[CBM_SZ_1K]; const char **group_vals; double *sums; @@ -4808,7 +5235,8 @@ typedef struct { double *mins, *maxs; char ***collect_lists; int *collect_counts; -} ret_agg_entry_t; + CBMHashTable **distinct_sets; /* per-item hash over a COUNT(DISTINCT) list (lazy) */ +}; /* Initialize a new RETURN aggregation group */ static void ret_agg_init_group(ret_agg_entry_t *entry, const char *key, int item_count, @@ -4824,6 +5252,8 @@ static void ret_agg_init_group(ret_agg_entry_t *entry, const char *key, int item entry->maxs = calloc(item_count, sizeof(double)); entry->collect_lists = calloc(item_count, sizeof(char **)); entry->collect_counts = calloc(item_count, sizeof(int)); + entry->distinct_sets = + cbm_calloc(CBM_MEM_CLASS_OTHER, (size_t)item_count * sizeof(CBMHashTable *)); for (int ci = 0; ci < item_count; ci++) { entry->mins[ci] = CYP_DBL_MAX; entry->maxs[ci] = -CYP_DBL_MAX; @@ -4852,13 +5282,18 @@ static void ret_agg_accumulate(ret_agg_entry_t *entry, cbm_return_clause_t *ret, entry->maxs[ci] = dv; } if (strcmp(ret->items[ci].func, "COLLECT") == 0) { - int idx = entry->collect_counts[ci]++; - entry->collect_lists[ci] = - safe_realloc(entry->collect_lists[ci], (idx + SKIP_ONE) * sizeof(char *)); + int idx = entry->collect_counts[ci]; + entry->collect_lists[ci] = string_list_reserve(entry->collect_lists[ci], idx); + if (!entry->collect_lists[ci]) { + entry->collect_counts[ci] = 0; + continue; + } entry->collect_lists[ci][idx] = heap_strdup(raw); + entry->collect_counts[ci]++; } else if (ret->items[ci].distinct && strcmp(ret->items[ci].func, "COUNT") == 0) { /* COUNT(DISTINCT x): track unique values; emit the set size (#239). */ - distinct_list_add(&entry->collect_lists[ci], &entry->collect_counts[ci], raw); + distinct_list_add(&entry->collect_lists[ci], &entry->collect_counts[ci], + &entry->distinct_sets[ci], raw); } } } @@ -4868,6 +5303,9 @@ static void ret_agg_free(ret_agg_entry_t *aggs, int agg_count, int item_count) { for (int a = 0; a < agg_count; a++) { for (int ci = 0; ci < item_count; ci++) { safe_str_free(&aggs[a].group_vals[ci]); + if (aggs[a].distinct_sets && aggs[a].distinct_sets[ci]) { + cbm_ht_free(aggs[a].distinct_sets[ci]); + } for (int j = 0; j < aggs[a].collect_counts[ci]; j++) { free(aggs[a].collect_lists[ci][j]); } @@ -4880,6 +5318,7 @@ static void ret_agg_free(ret_agg_entry_t *aggs, int agg_count, int item_count) { free(aggs[a].maxs); free(aggs[a].collect_lists); free(aggs[a].collect_counts); + cbm_free(CBM_MEM_CLASS_OTHER, aggs[a].distinct_sets); } free(aggs); } @@ -4932,16 +5371,13 @@ static void ret_agg_emit_row(cbm_return_clause_t *ret, ret_agg_entry_t *agg, res rb_add_row(rb, row); } -static void execute_return_agg(cbm_return_clause_t *ret, binding_t *bindings, int bind_count, - result_builder_t *rb) { - int agg_cap = CBM_SZ_256; - ret_agg_entry_t *aggs = calloc(agg_cap, sizeof(ret_agg_entry_t)); - int agg_count = 0; - - for (int bi = 0; bi < bind_count; bi++) { - /* #601: grouping is O(bindings x groups) — the dominant cost on a - * whole-graph GROUP BY. Abort if we blow the wall-clock budget. */ - if ((bi & CYPHER_DEADLINE_CHECK_MASK) == 0 && cypher_deadline_exceeded()) { +static void ret_sink_feed_agg(ret_sink_t *s, binding_t *bindings, int bind_count) { + cbm_return_clause_t *ret = s->ret; + for (int bi = 0; bi < bind_count && !s->done; bi++) { + /* #601: a whole-graph GROUP BY visits every binding. Abort if we blow + * the wall-clock budget. */ + if (((s->fed + bi) & CYPHER_DEADLINE_CHECK_MASK) == 0 && cypher_deadline_exceeded()) { + s->done = true; break; } char key[CBM_SZ_1K] = ""; @@ -4949,28 +5385,23 @@ static void execute_return_agg(cbm_return_clause_t *ret, binding_t *bindings, in char valbufs[CBM_SZ_32][CBM_SZ_512]; ret_agg_build_key(ret, &bindings[bi], key, sizeof(key), vals, valbufs); - int found = CYP_FOUND_NONE; - for (int a = 0; a < agg_count; a++) { - if (strcmp(aggs[a].group_key, key) == 0) { + int found = key_index_find(&s->groups, key); + for (int a = 0; found == KEY_INDEX_UNKNOWN && a < s->agg_count; a++) { + if (strcmp(s->aggs[a].group_key, key) == 0) { found = a; - break; } } if (found < 0) { - if (agg_count >= agg_cap) { - agg_cap *= PAIR_LEN; - aggs = safe_realloc(aggs, agg_cap * sizeof(ret_agg_entry_t)); + if (s->agg_count >= s->agg_cap) { + s->agg_cap *= PAIR_LEN; + s->aggs = safe_realloc(s->aggs, s->agg_cap * sizeof(ret_agg_entry_t)); } - found = agg_count++; - ret_agg_init_group(&aggs[found], key, ret->count, vals); + found = s->agg_count++; + ret_agg_init_group(&s->aggs[found], key, ret->count, vals); + key_index_add(&s->groups, s->aggs[found].group_key, found); } - ret_agg_accumulate(&aggs[found], ret, &bindings[bi]); - } - - for (int a = 0; a < agg_count; a++) { - ret_agg_emit_row(ret, &aggs[a], rb); + ret_agg_accumulate(&s->aggs[found], ret, &bindings[bi]); } - ret_agg_free(aggs, agg_count, ret->count); } /* Build RETURN column names from items */ @@ -5003,30 +5434,31 @@ static void build_return_columns(result_builder_t *rb, cbm_return_clause_t *ret) } } -/* Execute simple (non-aggregate) RETURN projection */ -static void execute_return_simple(cbm_return_clause_t *ret, binding_t *bindings, int bind_count, - int max_rows, result_builder_t *rb) { - int proj_cap = max_rows; - bool cap_is_engine_budget = true; - if (ret->limit >= 0 && ret->limit <= max_rows && ret->order_key_count == 0 && ret->skip <= 0) { - proj_cap = ret->limit; /* DISTINCT: the cap counts distinct rows (#1364) */ - cap_is_engine_budget = false; - } - rb_distinct_t seen; - rb_distinct_init(&seen, ret->distinct); - for (int bi = 0; bi < bind_count; bi++) { - const char *vals[CBM_SZ_32]; +/* Row projections (simple RETURN, RETURN *, RETURN * after WITH): one row per + * binding until the row cap is full. */ +static void ret_sink_feed_rows(ret_sink_t *s, binding_t *bindings, int bind_count) { + for (int bi = 0; bi < bind_count && !s->done; bi++) { + const char *vals[CBM_SZ_128]; + char func_bufs[CBM_SZ_32][CBM_SZ_512]; - for (int ci = 0; ci < ret->count; ci++) { - vals[ci] = - project_item(&bindings[bi], &ret->items[ci], func_bufs[ci], sizeof(func_bufs[ci])); + if (s->kind == RET_SINK_SIMPLE) { + for (int ci = 0; ci < s->ret->count; ci++) { + vals[ci] = project_item(&bindings[bi], &s->ret->items[ci], func_bufs[ci], + sizeof(func_bufs[ci])); + } + } else if (s->kind == RET_SINK_STAR) { + project_star_row(&bindings[bi], s->vars, s->vc, vals); + } else { + for (int i = 0; i < s->vc; i++) { + cbm_node_t *vn = binding_get(&bindings[bi], s->vars[i]); + vals[i] = vn && vn->name ? vn->name : ""; + } } - if (!rb_add_row_capped(rb, &seen, vals, proj_cap)) { - g_cypher_truncated = g_cypher_truncated || cap_is_engine_budget; - break; + if (!rb_add_row_capped(s->rb, &s->seen, vals, s->cap)) { + g_cypher_truncated = g_cypher_truncated || s->cap_is_engine_budget; + s->done = true; } } - rb_distinct_free(&seen); } /* Build default 3-column headers (name, qualified_name, label) per variable */ @@ -5049,24 +5481,26 @@ static void build_default_columns(result_builder_t *rb, const char **vars, int v } /* Default projection when no RETURN clause */ -static void execute_default_projection(cbm_pattern_t *pat0, binding_t *bindings, int bind_count, - int max_rows, result_builder_t *rb) { - const char *vars[CYP_MAX_VARS]; - int vc = 0; +static void ret_sink_init_default(ret_sink_t *s, const cbm_pattern_t *pat0) { + s->kind = RET_SINK_DEFAULT; for (int ni = 0; ni < pat0->node_count; ni++) { if (pat0->nodes[ni].variable) { - if (vc < CYP_MAX_VARS) { - vars[vc++] = pat0->nodes[ni].variable; + if (s->vc < CYP_MAX_VARS) { + s->vars[s->vc++] = pat0->nodes[ni].variable; } else { g_cypher_truncated = true; } } } - build_default_columns(rb, vars, vc); - for (int bi = 0; bi < bind_count && rb->row_count < max_rows; bi++) { + build_default_columns(s->rb, s->vars, s->vc); +} + +static void ret_sink_feed_default(ret_sink_t *s, binding_t *bindings, int bind_count) { + result_builder_t *rb = s->rb; + for (int bi = 0; bi < bind_count && rb->row_count < s->max_rows; bi++) { const char *vals[CYP_COL_BUF]; - for (int v = 0; v < vc; v++) { - cbm_node_t *n = binding_get(&bindings[bi], vars[v]); + for (int v = 0; v < s->vc; v++) { + cbm_node_t *n = binding_get(&bindings[bi], s->vars[v]); vals[(size_t)v * CYP_EDGE_COLS] = n && n->name ? n->name : ""; vals[((size_t)v * CYP_EDGE_COLS) + SKIP_ONE] = n && n->qualified_name ? n->qualified_name : ""; @@ -5074,9 +5508,98 @@ static void execute_default_projection(cbm_pattern_t *pat0, binding_t *bindings, } rb_add_row(rb, vals); } - if (bind_count > max_rows) { - g_cypher_truncated = true; +} + +static void ret_sink_init(ret_sink_t *s, cbm_query_t *q, int max_rows, result_builder_t *rb) { + memset(s, 0, sizeof(*s)); + s->ret = q->ret; + s->rb = rb; + s->max_rows = max_rows; + rb_init(rb); + cbm_return_clause_t *ret = q->ret; + if (!ret) { + ret_sink_init_default(s, &q->patterns[0]); + return; + } + for (int i = 0; i < ret->count; i++) { + if (is_aggregate_func(ret->items[i].func)) { + s->has_agg = true; + break; + } } + if (ret->star) { + if (q->with_clause) { + ret_sink_init_star_after_with(s, q); + } else { + ret_sink_init_star(s, q); + } + return; + } + build_return_columns(rb, ret); + if (s->has_agg) { + s->kind = RET_SINK_AGG; + s->agg_cap = CBM_SZ_256; + s->aggs = calloc(s->agg_cap, sizeof(ret_agg_entry_t)); + return; + } + s->kind = RET_SINK_SIMPLE; + s->cap = max_rows; + s->cap_is_engine_budget = true; + if (ret->limit >= 0 && ret->limit <= max_rows && ret->order_key_count == 0 && ret->skip <= 0) { + s->cap = ret->limit; /* DISTINCT: the cap counts distinct rows (#1364) */ + s->cap_is_engine_budget = false; + } + rb_distinct_init(&s->seen, ret->distinct); +} + +static void ret_sink_feed(ret_sink_t *s, binding_t *bindings, int bind_count) { + if (s->kind == RET_SINK_DEFAULT) { + ret_sink_feed_default(s, bindings, bind_count); + } else if (s->kind == RET_SINK_AGG) { + ret_sink_feed_agg(s, bindings, bind_count); + } else { + ret_sink_feed_rows(s, bindings, bind_count); + } + s->fed += bind_count; +} + +/* Release the sink's own state; the projected rows stay in the builder. */ +static void ret_sink_free(ret_sink_t *s) { + if (s->kind == RET_SINK_AGG) { + key_index_free(&s->groups); + ret_agg_free(s->aggs, s->agg_count, s->ret->count); + s->aggs = NULL; + s->agg_count = 0; + } + rb_distinct_free(&s->seen); +} + +/* Emit the aggregation groups, then DISTINCT / ORDER BY / SKIP / LIMIT. */ +static void ret_sink_finish(ret_sink_t *s) { + result_builder_t *rb = s->rb; + cbm_return_clause_t *ret = s->ret; + if (s->kind == RET_SINK_DEFAULT) { + if (s->fed > s->max_rows) { + g_cypher_truncated = true; + } + return; + } + if (s->kind == RET_SINK_AGG) { + for (int a = 0; a < s->agg_count; a++) { + ret_agg_emit_row(ret, &s->aggs[a], rb); + } + } + ret_sink_free(s); + + /* The simple and star projections de-duplicate while projecting (#1364); + * only aggregate rows still need the post-pass. */ + if (ret->distinct && s->has_agg && !ret->star) { + rb_apply_distinct(rb); + } + rb_apply_order_by(rb, ret); + bool limit_is_engine_budget = ret->limit < 0; + rb_apply_skip_limit(rb, ret->skip, limit_is_engine_budget ? s->max_rows : ret->limit, + limit_is_engine_budget); } /* Worst-case binding_t slot count for a node cross-join: one row per @@ -5269,7 +5792,7 @@ static void expand_from_bound_terminal(cbm_store_t *store, cbm_pattern_t *patn, match_count++; binding_t nb = {0}; binding_copy(&nb, b); - binding_set(&nb, start_var, &found); + binding_take_node(&nb, start_var, &found); if (rel->variable) { binding_set_edge(&nb, rel->variable, &edges[ei]); } @@ -5346,39 +5869,6 @@ static int expand_additional_patterns(cbm_store_t *store, cbm_query_t *q, const return 0; } -/* Project RETURN clause results */ -static void execute_return_clause(cbm_query_t *q, cbm_return_clause_t *ret, binding_t *bindings, - int bind_count, int max_rows, result_builder_t *rb) { - bool has_agg = false; - for (int i = 0; i < ret->count; i++) { - if (is_aggregate_func(ret->items[i].func)) { - has_agg = true; - break; - } - } - - if (ret->star) { - execute_return_star(q, bindings, bind_count, max_rows, rb); - } else { - build_return_columns(rb, ret); - if (has_agg) { - execute_return_agg(ret, bindings, bind_count, rb); - } else { - execute_return_simple(ret, bindings, bind_count, max_rows, rb); - } - } - - /* The simple and star projections de-duplicate while projecting (#1364); - * only aggregate rows still need the post-pass. */ - if (ret->distinct && has_agg && !ret->star) { - rb_apply_distinct(rb); - } - rb_apply_order_by(rb, ret); - bool limit_is_engine_budget = ret->limit < 0; - rb_apply_skip_limit(rb, ret->skip, limit_is_engine_budget ? max_rows : ret->limit, - limit_is_engine_budget); -} - /* ── Planner: seed a single-hop pattern from its selective end ────────────── * * The executor scans pattern node 0 and expands from there. Written as @@ -5486,33 +5976,247 @@ static void cypher_plan_seed_from_selective_end(cbm_query_t *q) { } } -static int execute_single(cbm_store_t *store, cbm_query_t *q, const char *project, int max_rows, - result_builder_t *rb) { - cypher_plan_seed_from_selective_end(q); - cbm_pattern_t *pat0 = &q->patterns[0]; +/* ── Streamed execution ────────────────────────────────────────────── + * + * The start nodes run through the pipeline (pattern-0 hops, late WHERE, WITH, + * RETURN) in chunks of CYPHER_STREAM_CHUNK instead of materialising every + * binding of the whole set first. On the kernel graph (7.5 M nodes, 1.9 M + * CALLS edges) `MATCH (a)-[:CALLS]->(b) RETURN count(*)` held a deep-copied + * binding per start node and per expanded row at once: 23 GB RSS, killed in a + * 12 GB container. Each hop emits a binding's expansions in the order of its + * input bindings, and every consumer keeps its state between chunks, so the + * concatenated chunk outputs are the single-pass output, order included. + * + * Additional MATCH patterns keep one pass: they pick their join strategy from + * the first binding of the whole set and refuse a cross join by its size. */ +#define CYPHER_STREAM_CHUNK 1024 + +#ifdef CBM_ENABLE_TEST_SEAMS +static _Thread_local int g_cypher_stream_chunk = 0; /* test override; 0 = default */ + +void cbm_cypher_test_set_stream_chunk(int start_nodes) { + g_cypher_stream_chunk = start_nodes; +} +#endif + +static int cypher_stream_chunk(void) { +#ifdef CBM_ENABLE_TEST_SEAMS + if (g_cypher_stream_chunk > 0) { + return g_cypher_stream_chunk; + } +#endif + return CYPHER_STREAM_CHUNK; +} + +typedef struct { + with_sink_t with; + ret_sink_t ret; +} cypher_sink_t; + +static void sink_feed(cypher_sink_t *s, binding_t *bindings, int bind_count) { + if (s->with.wc) { + with_sink_feed(&s->with, bindings, bind_count); + } else { + ret_sink_feed(&s->ret, bindings, bind_count); + } +} + +static void sink_finish(cypher_sink_t *s, cbm_query_t *q) { + if (s->with.wc) { + int vcount = 0; + binding_t *vbindings = with_sink_finish(&s->with, q, &vcount); + ret_sink_feed(&s->ret, vbindings, vcount); + for (int bi = 0; bi < vcount; bi++) { + binding_free(&vbindings[bi]); + } + cbm_free(CBM_MEM_CLASS_OTHER, vbindings); + } + ret_sink_finish(&s->ret); +} + +static void sink_free(cypher_sink_t *s) { + if (s->with.wc) { + with_sink_free(&s->with); + } + ret_sink_free(&s->ret); +} + +/* Pattern 0's start nodes in scan_pattern_nodes' order, pulled a chunk at a + * time from the store's streamed scan instead of materialised up front: a + * label alternation reads each label's scan in turn, and the inline property + * filter drops nodes as they arrive. */ +typedef struct { + cbm_store_t *store; + const char *project; + const cbm_node_pattern_t *node; + char *labels; /* label alternation "A|B": strtok_r copy, else NULL */ + char *labels_save; + bool labels_started; + cbm_store_node_scan_t *scan; + bool done; + bool failed; /* the scan could not be read to its end: the query fails */ +} start_source_t; + +/* A start-node scan failed part-way. The rows read so far must not answer the + * query as if they were all of them: record the store's error for + * cbm_cypher_execute, which fails the query with it. */ +static void start_source_fail(start_source_t *src, const char *why) { + src->failed = true; + src->done = true; + snprintf(g_cypher_scan_error, sizeof(g_cypher_scan_error), "%s", + why && why[0] ? why : "unknown store error"); + cbm_log_error("cypher.scan_failed", "error", g_cypher_scan_error); +} + +static void start_source_open(start_source_t *src, cbm_store_t *store, const char *project, + const cbm_node_pattern_t *node) { + memset(src, 0, sizeof(*src)); + src->store = store; + src->project = project; + src->node = node; + if (node->label && strchr(node->label, '|')) { + src->labels = cbm_mem_strdup(CBM_MEM_CLASS_OTHER, node->label); + if (!src->labels) { + start_source_fail(src, "out of memory"); + } + return; /* one scan per label, opened as the previous one ends */ + } + src->scan = cbm_store_node_scan_open(store, project, node->label); + if (!src->scan) { + start_source_fail(src, cbm_store_error(store)); + } +} + +/* Open the next label's scan of an alternation; false when none is left. */ +static bool start_source_next_label(start_source_t *src) { + char *tok = strtok_r(src->labels_started ? NULL : src->labels, "|", &src->labels_save); + src->labels_started = true; + if (!tok) { + return false; + } + src->scan = cbm_store_node_scan_open(src->store, src->project, tok); + if (!src->scan) { + start_source_fail(src, cbm_store_error(src->store)); + } + return true; +} + +/* Fill out[0..max) with the next start nodes. Fewer than max only once the + * scan is exhausted or has failed. */ +static int start_source_next(start_source_t *src, cbm_node_t *out, int max) { + const cbm_node_pattern_t *node = src->node; + int got = 0; + while (got < max && !src->done) { + if (!src->scan) { + if (!src->labels || !start_source_next_label(src)) { + src->done = true; + } + continue; + } + int n = 0; + if (cbm_store_node_scan_next(src->scan, out + got, max - got, &n) != CBM_STORE_OK) { + start_source_fail(src, cbm_store_error(src->store)); + break; + } + if (n == 0) { + cbm_store_node_scan_close(src->scan); + src->scan = NULL; + src->done = !src->labels; + continue; + } + /* Apply inline property filters — free rejected nodes' strings */ + int kept = got; + for (int i = got; i < got + n; i++) { + if (node->prop_count == 0 || + check_inline_props(&out[i], node->props, node->prop_count, src->store)) { + if (kept != i) { + out[kept] = out[i]; + } + kept++; + } else { + node_fields_free(&out[i]); + } + } + got = kept; + } + return got; +} + +static void start_source_close(start_source_t *src) { + cbm_store_node_scan_close(src->scan); + cbm_free(CBM_MEM_CLASS_OTHER, src->labels); + memset(src, 0, sizeof(*src)); +} + +/* Release a start-node batch: the strings the store gave each node (left + * empty once moved into a binding) and the array itself. */ +static void start_nodes_free(cbm_node_t *nodes, int count) { + for (int i = 0; nodes && i < count; i++) { + node_fields_free(&nodes[i]); + } + cbm_free(CBM_MEM_CLASS_OTHER, nodes); +} - /* Step 1: Scan initial nodes */ - cbm_node_t *scanned = NULL; - int scan_count = 0; - scan_pattern_nodes(store, project, &pat0->nodes[0], &scanned, &scan_count); +/* Pull up to `chunk` start nodes (INT_MAX: all of them) into *out. */ +static int pull_start_nodes(start_source_t *src, int chunk, cbm_node_t **out, int *count) { + int cap = chunk < CYPHER_STREAM_CHUNK ? chunk : CYPHER_STREAM_CHUNK; + cbm_node_t *nodes = cbm_alloc(CBM_MEM_CLASS_OTHER, (size_t)cap * sizeof(cbm_node_t)); + int n = 0; + while (nodes) { + n += start_source_next(src, nodes + n, cap - n); + if (n < cap || cap == chunk) { + break; /* scan exhausted, or the chunk is full */ + } + int grown_cap = cap > chunk / PAIR_LEN ? chunk : cap * PAIR_LEN; + cbm_node_t *grown = + cbm_realloc(CBM_MEM_CLASS_OTHER, nodes, (size_t)grown_cap * sizeof(cbm_node_t)); + if (!grown) { + start_nodes_free(nodes, n); + nodes = NULL; + break; + } + nodes = grown; + cap = grown_cap; + } + *out = nodes; + *count = nodes ? n : 0; + return nodes ? 0 : CBM_NOT_FOUND; +} + +/* Run the next chunk of start nodes through the pipeline into the sink. The + * start nodes' strings move into their bindings and are released with them. + * *start_index is the chunk's position in the whole start set. */ +static int execute_chunk(cbm_store_t *store, cbm_query_t *q, const char *project, int max_rows, + start_source_t *src, int chunk, int *start_index, cypher_sink_t *sink) { + cbm_pattern_t *pat0 = &q->patterns[0]; + cbm_node_t *nodes = NULL; + int node_count = 0; + if (pull_start_nodes(src, chunk, &nodes, &node_count) != 0) { + return CBM_NOT_FOUND; /* start node array refused */ + } + if (src->failed) { + start_nodes_free(nodes, node_count); + return CBM_NOT_FOUND; /* the scan failed: no answer from part of it */ + } /* Build initial bindings with early WHERE */ - int bind_cap = scan_count > max_rows ? scan_count : (max_rows > 0 ? max_rows : SKIP_ONE); + int bind_cap = node_count; binding_t *bindings = malloc(((size_t)bind_cap + SKIP_ONE) * sizeof(binding_t)); if (!bindings) { - cbm_store_free_nodes(scanned, scan_count); + start_nodes_free(nodes, node_count); return CBM_NOT_FOUND; /* initial binding array refused */ } int bind_count = 0; const char *var_name = pat0->nodes[0].variable ? pat0->nodes[0].variable : CYP_ANON_HEAD_VAR; - for (int i = 0; i < scan_count && bind_count < bind_cap; i++) { - if ((i & CYPHER_DEADLINE_CHECK_MASK) == 0 && cypher_deadline_exceeded()) { + int first = *start_index; + for (int i = 0; i < node_count; i++) { + if (((first + i) & CYPHER_DEADLINE_CHECK_MASK) == 0 && cypher_deadline_exceeded()) { break; } binding_t b = {0}; b.store = store; - binding_set(&b, var_name, &scanned[i]); + binding_take_node(&b, var_name, &nodes[i]); bool pass = eval_where_partial(q->where, &b) != CYP_PARTIAL_FALSE; if (pass) { bindings[bind_count++] = b; @@ -5520,44 +6224,64 @@ static int execute_single(cbm_store_t *store, cbm_query_t *q, const char *projec binding_free(&b); } } + *start_index = first + node_count; + start_nodes_free(nodes, node_count); /* + strings of nodes a deadline left untaken */ - /* Step 2: Expand first pattern's relationships */ + /* Expand first pattern's relationships */ expand_pattern_rels(store, pat0, &bindings, &bind_count, &bind_cap, &var_name, q->pattern_optional[0]); - /* Step 2b: Additional patterns */ - if (expand_additional_patterns(store, q, project, max_rows, &bindings, &bind_count, - &bind_cap) != 0) { - for (int bi = 0; bi < bind_count; bi++) { - binding_free(&bindings[bi]); + /* Additional patterns */ + int rc = + expand_additional_patterns(store, q, project, max_rows, &bindings, &bind_count, &bind_cap); + if (rc == 0) { + /* Late WHERE */ + if (q->where && (pat0->rel_count > 0 || q->pattern_count > SKIP_ONE)) { + filter_bindings_where(q->where, bindings, &bind_count); + } + sink_feed(sink, bindings, bind_count); + if (sink->with.oom) { + rc = CBM_NOT_FOUND; /* a WITH row buffer could not grow */ } - free(bindings); - cbm_store_free_nodes(scanned, scan_count); - return CBM_NOT_FOUND; /* cross-join allocation refused/failed */ } - /* Step 3: Late WHERE */ - if (q->where && (pat0->rel_count > 0 || q->pattern_count > SKIP_ONE)) { - filter_bindings_where(q->where, bindings, &bind_count); + for (int bi = 0; bi < bind_count; bi++) { + binding_free(&bindings[bi]); } + free(bindings); + return rc != 0 ? CBM_NOT_FOUND : 0; /* an allocation was refused or failed */ +} - /* Step 3b: WITH clause */ - execute_with_clause(q, &bindings, &bind_count); +static int execute_single(cbm_store_t *store, cbm_query_t *q, const char *project, int max_rows, + result_builder_t *rb) { + cypher_plan_seed_from_selective_end(q); + cbm_pattern_t *pat0 = &q->patterns[0]; - /* Step 4: Project results */ - rb_init(rb); - if (q->ret) { - execute_return_clause(q, q->ret, bindings, bind_count, max_rows, rb); - } else { - execute_default_projection(pat0, bindings, bind_count, max_rows, rb); - } + /* Step 1: Scan initial nodes, streamed */ + start_source_t src; + start_source_open(&src, store, project, &pat0->nodes[0]); - for (int bi = 0; bi < bind_count; bi++) { - binding_free(&bindings[bi]); + cypher_sink_t sink; + with_sink_init(&sink.with, q->with_clause); + ret_sink_init(&sink.ret, q, max_rows, rb); + + /* Steps 2-4 per chunk: expand, late WHERE, WITH / RETURN. The pipeline + * runs at least once, so an empty scan still reaches the additional + * patterns and the projection exactly as a single pass does. */ + int chunk = q->pattern_count > SKIP_ONE ? INT_MAX : cypher_stream_chunk(); + int rc = 0; + int start_index = 0; + do { + rc = execute_chunk(store, q, project, max_rows, &src, chunk, &start_index, &sink); + } while (rc == 0 && !src.done && !g_cypher_timed_out); + start_source_close(&src); + + if (rc == 0) { + sink_finish(&sink, q); + } else { + sink_free(&sink); } - free(bindings); - cbm_store_free_nodes(scanned, scan_count); - return 0; + return rc; } /* ── Main entry point ─────────────────────────────────────────── */ @@ -5805,6 +6529,18 @@ static char *check_projection_scope(const cbm_query_t *q) { return NULL; } +/* The error of an execution execute_single aborted: a failed start-node scan + * names the store's error, anything else was an allocation. */ +static char *cypher_abort_error(void) { + if (g_cypher_scan_error[0]) { + char buf[CBM_SZ_1K]; + snprintf(buf, sizeof(buf), "query aborted: reading the graph failed (%s)", + g_cypher_scan_error); + return heap_strdup(buf); + } + return heap_strdup("query aborted: out of memory or an allocation limit was reached"); +} + int cbm_cypher_execute(cbm_store_t *store, const char *query, const char *project, int max_rows, cbm_cypher_result_t *out) { memset(out, 0, sizeof(*out)); @@ -5812,6 +6548,7 @@ int cbm_cypher_execute(cbm_store_t *store, const char *query, const char *projec g_cypher_trail_truncated = 0; g_cypher_truncated = false; g_cypher_regex_refused = false; + g_cypher_scan_error[0] = '\0'; cypher_deadline_arm(); /* #601: start the wall-clock budget for this query */ /* max_rows sizes the initial binding array: non-positive means the * ceiling, and nothing above the ceiling is ever materialized anyway. */ @@ -5846,7 +6583,7 @@ int cbm_cypher_execute(cbm_store_t *store, const char *query, const char *projec rb_free(&rb); cbm_query_free(q); out->truncated = g_cypher_truncated; - out->error = heap_strdup("query aborted: out of memory or an allocation limit was reached"); + out->error = cypher_abort_error(); return CBM_NOT_FOUND; } @@ -5859,8 +6596,7 @@ int cbm_cypher_execute(cbm_store_t *store, const char *query, const char *projec rb_free(&rb2); cbm_query_free(q); out->truncated = g_cypher_truncated; - out->error = - heap_strdup("query aborted: out of memory or an allocation limit was reached"); + out->error = cypher_abort_error(); return CBM_NOT_FOUND; } /* Concatenate rows from rb2 into rb */ diff --git a/src/cypher/cypher.h b/src/cypher/cypher.h index dda0ecbe3d..10ef0aeec6 100644 --- a/src/cypher/cypher.h +++ b/src/cypher/cypher.h @@ -359,6 +359,9 @@ void cbm_cypher_test_set_deadline_ms(int64_t budget_ms); /* Override only the deadline clock on this thread; NULL restores real time. */ typedef uint64_t (*cbm_cypher_test_clock_fn)(void); void cbm_cypher_test_set_deadline_clock(cbm_cypher_test_clock_fn clock_fn); +/* Start nodes per streamed execution chunk on this thread; 0 restores the + * default. INT_MAX runs the whole start set as one chunk (a single pass). */ +void cbm_cypher_test_set_stream_chunk(int start_nodes); #endif /* Worst-case binding slot count for a node cross-join. Computes the count in diff --git a/src/daemon/application.c b/src/daemon/application.c index f20088503f..76f1222126 100644 --- a/src/daemon/application.c +++ b/src/daemon/application.c @@ -27,6 +27,7 @@ #include "operations/store_host.h" #include "operations/reliability_events.h" #include "pipeline/pipeline.h" +#include "store/graph_mem.h" #include "ui/config.h" #include "watcher/watcher.h" @@ -2126,7 +2127,13 @@ static size_t application_active_job_count_locked(cbm_daemon_application_t *appl /* Decision 2 (#1997 #832): a worker's memory slice is the aggregate budget * divided by the jobs active at spawn time. The job being spawned is already * in the table, so the divisor never drops below one; a zero aggregate means - * "no cap" and the worker falls back to its own RAM-fraction budget. */ + * "no cap" and the worker falls back to its own RAM-fraction budget. + * + * The daemon's resident adjacency layers (store/graph_mem.h) are part of the + * same aggregate, so workers split what is left after them. They are capped at + * a quarter of the daemon budget, which the aggregate defaults to; only a + * smaller configured aggregate can be exceeded, and then every job still gets + * at least one byte, as above — a zero slice would read as "no cap". */ static size_t application_worker_memory_slice_locked(cbm_daemon_application_t *application, size_t *active_jobs_out) { size_t active = application_active_job_count_locked(application); @@ -2136,7 +2143,16 @@ static size_t application_worker_memory_slice_locked(cbm_daemon_application_t *a if (active_jobs_out) { *active_jobs_out = active; } - return application->aggregate_memory_budget_bytes / active; + size_t aggregate = application->aggregate_memory_budget_bytes; + if (aggregate == 0) { + return 0; + } + size_t held = cbm_graph_mem_cached_bytes(); + size_t available = held < aggregate ? aggregate - held : 0; + if (available < active) { + available = active; + } + return available / active; } /* Compare the effective index request, not its JSON spelling. yyjson's deep diff --git a/src/foundation/mem_core.c b/src/foundation/mem_core.c index 783a8d4fab..bb52386ac3 100644 --- a/src/foundation/mem_core.c +++ b/src/foundation/mem_core.c @@ -66,8 +66,9 @@ typedef struct { static mem_class_stats_t g_classes[CBM_MEM_CLASS_COUNT]; static const char *const g_class_names[CBM_MEM_CLASS_COUNT] = { - "other", "gbuf_node", "gbuf_edge", "gbuf_string", "gbuf_index", "extract", "arena", - "ts_tree", "semantic", "dump", "store", "hash_table", "dyn_array", "operation_arg", + "other", "gbuf_node", "gbuf_edge", "gbuf_string", "gbuf_index", + "extract", "arena", "ts_tree", "semantic", "dump", + "store", "hash_table", "dyn_array", "operation_arg", "graph_cache", }; const char *cbm_mem_class_name(cbm_mem_class_t cls) { diff --git a/src/foundation/mem_core.h b/src/foundation/mem_core.h index fb609a0172..cfbf303426 100644 --- a/src/foundation/mem_core.h +++ b/src/foundation/mem_core.h @@ -67,6 +67,8 @@ typedef enum { CBM_MEM_CLASS_HASH_TABLE, /* CBMHashTable buckets/entries not claimed by an owner class */ CBM_MEM_CLASS_DYN_ARRAY, /* CBM_DYN_ARRAY item storage (every cbm_da_* user) */ CBM_MEM_CLASS_OPERATION_ARG, /* transient strings parsed from operation arguments */ + /* Daemon adjacency layers are resident across requests and separate from SQLite memory. */ + CBM_MEM_CLASS_GRAPH_CACHE, CBM_MEM_CLASS_COUNT } cbm_mem_class_t; diff --git a/src/foundation/workspace.c b/src/foundation/workspace.c index c338c0d15b..018c6c0efb 100644 --- a/src/foundation/workspace.c +++ b/src/foundation/workspace.c @@ -73,14 +73,19 @@ bool cbm_path_within_root(const char *root_path, const char *abs_path) { return false; } char real_root[4096]; - char real_file[4096]; - if (resolve_canonical_path(root_path, real_root, sizeof(real_root)) && - resolve_canonical_path(abs_path, real_file, sizeof(real_file))) { - if (canonical_path_has_root(real_root, real_file)) { - return true; - } + if (!resolve_canonical_path(root_path, real_root, sizeof(real_root))) { + return false; } - return false; + return cbm_path_within_canonical_root(real_root, abs_path); +} + +bool cbm_path_within_canonical_root(const char *canonical_root, const char *abs_path) { + if (!canonical_root || !abs_path) { + return false; + } + char real_file[4096]; + return resolve_canonical_path(abs_path, real_file, sizeof(real_file)) && + canonical_path_has_root(canonical_root, real_file); } enum { diff --git a/src/foundation/workspace.h b/src/foundation/workspace.h index 9b114888c1..2efcdd7a6f 100644 --- a/src/foundation/workspace.h +++ b/src/foundation/workspace.h @@ -55,6 +55,8 @@ int cbm_workspace_path_depth(const char *canonical_path); * and junction resolution? This is shared foundation policy used by every * source-reading surface. */ bool cbm_path_within_root(const char *root_path, const char *abs_path); +/* Check a path against a root already canonicalized by cbm_canonical_path. */ +bool cbm_path_within_canonical_root(const char *canonical_root, const char *abs_path); /* * ── Grant store ────────────────────────────────────────────────────────────── diff --git a/src/git/git_context.c b/src/git/git_context.c index f0be0c4b53..66431333b4 100644 --- a/src/git/git_context.c +++ b/src/git/git_context.c @@ -44,11 +44,14 @@ static bool git_validate_repo_path(const char *repo_path) { return cbm_validate_shell_path_arg(repo_path); } -static int git_capture(const char *repo_path, const char *git_args, char **out) { - if (!out) { +/* Run `git -C repo_path git_args` and copy the first output line (newlines + * trimmed) into buf. Returns 0 only when git exited 0 with a non-empty line. */ +static int git_capture_into(const char *repo_path, const char *git_args, char *buf, + size_t buf_size) { + if (!buf || buf_size == 0) { return CBM_NOT_FOUND; } - *out = NULL; + buf[0] = '\0'; if (!repo_path || !git_args || !git_validate_repo_path(repo_path)) { return CBM_NOT_FOUND; } @@ -76,8 +79,8 @@ static int git_capture(const char *repo_path, const char *git_args, char **out) } cbm_log_debug("git.capture", "stage", "read", "args", git_args); - char buf[GIT_OUTPUT_MAX]; - if (!fgets(buf, sizeof(buf), fp)) { + if (!fgets(buf, (int)buf_size, fp)) { + buf[0] = '\0'; cbm_log_debug("git.capture", "stage", "close", "args", git_args, "output", "none"); cbm_pclose(fp); cbm_log_debug("git.capture", "stage", "done", "args", git_args); @@ -89,9 +92,21 @@ static int git_capture(const char *repo_path, const char *git_args, char **out) int rc = cbm_pclose(fp); cbm_log_debug("git.capture", "stage", "done", "args", git_args); if (rc != 0 || buf[0] == '\0') { + buf[0] = '\0'; return CBM_NOT_FOUND; } + return 0; +} +static int git_capture(const char *repo_path, const char *git_args, char **out) { + if (!out) { + return CBM_NOT_FOUND; + } + *out = NULL; + char buf[GIT_OUTPUT_MAX]; + if (git_capture_into(repo_path, git_args, buf, sizeof(buf)) != 0) { + return CBM_NOT_FOUND; + } *out = git_strdup(buf); return *out ? 0 : CBM_NOT_FOUND; } @@ -313,6 +328,30 @@ int cbm_git_context_resolve(const char *path, cbm_git_context_t *out) { return 0; } +int cbm_git_branch_resolve(const char *path, bool *is_git, char *branch, size_t branch_size) { + if (!is_git || !branch || branch_size == 0) { + return CBM_NOT_FOUND; + } + *is_git = false; + branch[0] = '\0'; + if (!path || !path[0]) { + return CBM_NOT_FOUND; + } + if (!cbm_is_dir(path)) { + return 0; + } + char worktree_root[GIT_OUTPUT_MAX]; + if (git_capture_into(path, "rev-parse --show-toplevel", worktree_root, sizeof(worktree_root)) != + 0) { + return 0; + } + *is_git = true; + if (git_capture_into(path, "symbolic-ref --quiet --short HEAD", branch, branch_size) != 0) { + snprintf(branch, branch_size, "%s", "DETACHED"); + } + return 0; +} + char *cbm_git_context_branch_qn(const char *project_name, const cbm_git_context_t *ctx) { const char *project = project_name && project_name[0] ? project_name : "project"; const char *slug = "working-tree"; diff --git a/src/git/git_context.h b/src/git/git_context.h index 876309eb6a..2ac4195228 100644 --- a/src/git/git_context.h +++ b/src/git/git_context.h @@ -2,6 +2,7 @@ #define CBM_GIT_CONTEXT_H #include +#include typedef struct { bool is_git; @@ -20,6 +21,12 @@ typedef struct { } cbm_git_context_t; int cbm_git_context_resolve(const char *path, cbm_git_context_t *out); +/* Only the is_git and branch fields of cbm_git_context_resolve, from the same + * two git commands it uses for them (2 git processes instead of 7). branch + * gets "" when path is not in a git work tree; pass CBM_GIT_BRANCH_MAX bytes + * to read exactly what cbm_git_context_resolve reads. */ +#define CBM_GIT_BRANCH_MAX 4096 +int cbm_git_branch_resolve(const char *path, bool *is_git, char *branch, size_t branch_size); void cbm_git_context_free(cbm_git_context_t *ctx); char *cbm_git_context_branch_qn(const char *project_name, const cbm_git_context_t *ctx); int cbm_git_context_props_json(const cbm_git_context_t *ctx, char *buf, int buf_size); diff --git a/src/graph_buffer/graph_buffer.c b/src/graph_buffer/graph_buffer.c index 7065814b5a..670717ac18 100644 --- a/src/graph_buffer/graph_buffer.c +++ b/src/graph_buffer/graph_buffer.c @@ -1550,10 +1550,10 @@ static bool gbuf_node_touched(const cbm_gbuf_t *gb, int64_t id, const char *cons return false; } -int cbm_gbuf_delete_unreferenced(cbm_gbuf_t *gb, const char *label, const char *const *edge_types) { - if (!gb || !label || !edge_types || !edge_types[0]) { - return CBM_NOT_FOUND; - } +/* Delete the `label` nodes no edge of `types` (NULL-terminated) touches -- + * with `fileless`, only those carrying no file; returns how many. */ +static int gbuf_delete_untouched(cbm_gbuf_t *gb, const char *label, const char *const *types, + bool fileless) { node_ptr_array_t *arr = cbm_ht_get(gb->nodes_by_label, label); if (!arr || arr->count == 0) { return 0; @@ -1568,7 +1568,8 @@ int cbm_gbuf_delete_unreferenced(cbm_gbuf_t *gb, const char *label, const char * for (int i = 0; i < arr->count; i++) { const char *qn = arr->items[i]->qualified_name; cbm_gbuf_node_t *node = qn ? cbm_ht_get(gb->node_by_qn, qn) : NULL; - if (node && !gbuf_node_touched(gb, node->id, edge_types)) { + if (node && !(fileless && node->file_path && node->file_path[0]) && + !gbuf_node_touched(gb, node->id, types)) { victims[n++] = node; } } @@ -1604,6 +1605,43 @@ int cbm_gbuf_delete_unreferenced(cbm_gbuf_t *gb, const char *label, const char * return n; } +int cbm_gbuf_delete_unreferenced(cbm_gbuf_t *gb, const char *label, const char *const *edge_types) { + if (!gb || !label || !edge_types || !edge_types[0]) { + return CBM_NOT_FOUND; + } + return gbuf_delete_untouched(gb, label, edge_types, false); +} + +typedef struct { + const char **items; + uint32_t count; +} gb_type_list_t; + +static void collect_edge_type(const char *key, void *value, void *ud) { + (void)value; + gb_type_list_t *list = ud; + list->items[list->count++] = key; +} + +int cbm_gbuf_delete_edgeless(cbm_gbuf_t *gb, const char *label) { + if (!gb || !label) { + return CBM_NOT_FOUND; + } + /* Edges are indexed by (node, type) only: "any edge" is an edge of any + * type the buffer holds (NULL-terminated by the calloc). */ + gb_type_list_t list = { + cbm_calloc(CBM_MEM_CLASS_GBUF_INDEX, + ((size_t)cbm_ht_count(gb->edges_by_type) + 1) * sizeof(*list.items)), + 0}; + if (!list.items) { + return CBM_NOT_FOUND; + } + cbm_ht_foreach(gb->edges_by_type, collect_edge_type, &list); + int n = gbuf_delete_untouched(gb, label, list.items, true); + cbm_free(CBM_MEM_CLASS_GBUF_INDEX, list.items); + return n; +} + int cbm_gbuf_load_from_db(cbm_gbuf_t *gb, const char *db_path, const char *project) { if (!gb || !db_path || !project) { return CBM_NOT_FOUND; diff --git a/src/graph_buffer/graph_buffer.h b/src/graph_buffer/graph_buffer.h index 83470f5aba..a3b5b3cd06 100644 --- a/src/graph_buffer/graph_buffer.h +++ b/src/graph_buffer/graph_buffer.h @@ -124,6 +124,13 @@ int cbm_gbuf_delete_by_file(cbm_gbuf_t *gb, const char *file_path); * is gone, as a full index never makes them. */ int cbm_gbuf_delete_unreferenced(cbm_gbuf_t *gb, const char *label, const char *const *edge_types); +/* Delete every `label` node that carries no file and that no edge of any + * type touches; returns how many. For a label made by edges of many kinds + * (Route: HTTP_CALLS, ASYNC_CALLS, HANDLES, ...): incremental indexing drops + * the file-less one whose last edge went with its files, as a full index + * never makes it. */ +int cbm_gbuf_delete_edgeless(cbm_gbuf_t *gb, const char *label); + /* Bulk-load all nodes and edges for a project from an existing SQLite DB * into this graph buffer. Returns 0 on success. */ int cbm_gbuf_load_from_db(cbm_gbuf_t *gb, const char *db_path, const char *project); diff --git a/src/operations/architecture.c b/src/operations/architecture.c index f75ea8f3b3..c17b472b5e 100644 --- a/src/operations/architecture.c +++ b/src/operations/architecture.c @@ -536,6 +536,7 @@ cbm_operation_result_t cbm_architecture_operation_execute(const char *args) { ? CBM_STORE_CORRUPT_ERROR : "project not found or not indexed"); } + (void)cbm_store_read_batch_begin(store); char *scope_path = architecture_string_arg(args, "path"); if (cbm_store_count_nodes(store, project) <= 0) { diff --git a/src/operations/compare.c b/src/operations/compare.c index 769f43df21..4bac08321a 100644 --- a/src/operations/compare.c +++ b/src/operations/compare.c @@ -89,6 +89,7 @@ static cbm_store_t *compare_open_project_store(const char *project) { cbm_project_t row = {0}; if (cbm_store_get_project(store, project, &row) == CBM_STORE_OK) { cbm_project_free_fields(&row); + (void)cbm_store_read_batch_begin(store); return store; } cbm_store_close(store); @@ -112,6 +113,7 @@ static cbm_store_t *compare_open_project_store(const char *project) { char internal_name[CBM_SZ_1K]; if (compare_primary_project_name(candidate, internal_name, sizeof(internal_name)) && strcmp(internal_name, project) == 0) { + (void)cbm_store_read_batch_begin(candidate); found = candidate; break; } diff --git a/src/operations/coverage.c b/src/operations/coverage.c index b43fe76407..cf8874b052 100644 --- a/src/operations/coverage.c +++ b/src/operations/coverage.c @@ -298,6 +298,7 @@ cbm_operation_result_t cbm_coverage_operation_execute(const char *args) { cbm_operation_arg_free(project); return result; } + (void)cbm_store_read_batch_begin(store); yyjson_doc *adoc = args ? yyjson_read(args, strlen(args), 0) : NULL; yyjson_val *aroot = adoc ? yyjson_doc_get_root(adoc) : NULL; diff --git a/src/operations/execute.c b/src/operations/execute.c index 857c5ba57c..61b699edec 100644 --- a/src/operations/execute.c +++ b/src/operations/execute.c @@ -1,6 +1,7 @@ #include "operations/operation.h" #include "operations/read.h" #include "operations/mutation.h" +#include "store/graph_mem.h" #include #include @@ -35,14 +36,16 @@ cbm_operation_result_t cbm_operation_execute(cbm_operation_context_t *context, return cbm_operation_result_copy("unknown operation", true); } const char *json = args_json ? args_json : "{}"; + cbm_operation_result_t result; if (cbm_read_operation_supported(operation)) { - return cbm_read_operation_execute(operation, json, context ? context->runtime : NULL); + result = cbm_read_operation_execute(operation, json, context ? context->runtime : NULL); + } else if (cbm_mutation_operation_supported(operation)) { + result = cbm_mutation_operation_execute(operation, json, context ? context->runtime : NULL); + } else if (!context || !context->backend) { + result = cbm_operation_result_copy("operation backend unavailable", true); + } else { + result = context->backend(context->backend_context, operation, json); } - if (cbm_mutation_operation_supported(operation)) { - return cbm_mutation_operation_execute(operation, json, context ? context->runtime : NULL); - } - if (!context || !context->backend) { - return cbm_operation_result_copy("operation backend unavailable", true); - } - return context->backend(context->backend_context, operation, json); + cbm_graph_mem_evict_expired(); + return result; } diff --git a/src/operations/file_outline.c b/src/operations/file_outline.c index 5a9308ba60..66d78f08f0 100644 --- a/src/operations/file_outline.c +++ b/src/operations/file_outline.c @@ -282,6 +282,7 @@ cbm_operation_result_t cbm_file_outline_operation_execute(const char *args_json, ? CBM_STORE_CORRUPT_ERROR : "project not found or not indexed"); } + (void)cbm_store_read_batch_begin(store); cbm_project_t info = {0}; if (cbm_store_get_project(store, project, &info) != CBM_STORE_OK) { cbm_store_close(store); diff --git a/src/operations/query.c b/src/operations/query.c index 8ca291421b..72f2c2c337 100644 --- a/src/operations/query.c +++ b/src/operations/query.c @@ -438,6 +438,7 @@ cbm_operation_result_t cbm_query_operation_execute(const char *args_json) { ? CBM_STORE_CORRUPT_ERROR : "project not found or not indexed"); } + (void)cbm_store_read_batch_begin(store); if (cbm_store_count_nodes(store, project) <= 0) { cbm_store_close(store); cbm_operation_arg_free(cursor_arg); diff --git a/src/operations/read.c b/src/operations/read.c index 0c7d232546..ebdafab303 100644 --- a/src/operations/read.c +++ b/src/operations/read.c @@ -166,6 +166,7 @@ static void add_project_entry(yyjson_mut_doc *doc, yyjson_mut_val *array, const if (!store) { return; } + (void)cbm_store_read_batch_begin(store); char project_name[1024]; if (!primary_project_name(store, project_name, sizeof(project_name))) { cbm_store_close(store); @@ -188,11 +189,12 @@ static void add_project_entry(yyjson_mut_doc *doc, yyjson_mut_val *array, const * disambiguates same-repo projects). The full git block is one status call * away for the project you actually care about. */ if (!metadata_only) { - cbm_git_context_t git = {0}; - if (root_path[0] && cbm_git_context_resolve(root_path, &git) && git.is_git && git.branch) { - yyjson_mut_obj_add_strcpy(doc, item, "branch", git.branch); + bool is_git = false; + char branch[CBM_GIT_BRANCH_MAX]; + if (root_path[0] && cbm_git_branch_resolve(root_path, &is_git, branch, sizeof(branch)) && + is_git && branch[0]) { + yyjson_mut_obj_add_strcpy(doc, item, "branch", branch); } - cbm_git_context_free(&git); } if (include_details) { yyjson_mut_obj_add_int(doc, item, "nodes", cbm_store_count_nodes(store, project_name)); diff --git a/src/operations/schema.c b/src/operations/schema.c index 9f7708c8b5..ce01e86b26 100644 --- a/src/operations/schema.c +++ b/src/operations/schema.c @@ -97,6 +97,7 @@ cbm_operation_result_t cbm_schema_operation_execute(const char *args_json) { : "project not found or not indexed", true); } + (void)cbm_store_read_batch_begin(store); if (cbm_store_count_nodes(store, project) <= 0) { cbm_store_close(store); cbm_operation_arg_free(project); diff --git a/src/operations/search.c b/src/operations/search.c index 14d60ebcda..5d69c84c89 100644 --- a/src/operations/search.c +++ b/src/operations/search.c @@ -194,6 +194,7 @@ static cbm_store_t *open_indexed_project(const char *project, cbm_store_t *store = cbm_store_host_open_query_path(db_path, open_status); if (!store) return NULL; + (void)cbm_store_read_batch_begin(store); cbm_project_t indexed = {0}; if (cbm_store_get_project(store, project, &indexed) != CBM_STORE_OK) { cbm_store_close(store); diff --git a/src/operations/snippet.c b/src/operations/snippet.c index 1c144e33d7..79c6b2532b 100644 --- a/src/operations/snippet.c +++ b/src/operations/snippet.c @@ -659,6 +659,7 @@ cbm_operation_result_t cbm_snippet_operation_execute(const char *args) { return error_result(CBM_STORE_CORRUPT_MESSAGE, CBM_STORE_CORRUPT_HINT); return error_result("project not indexed", "Run 'codebase-memory-cli index .' first."); } + (void)cbm_store_read_batch_begin(store); cbm_node_t exact = {0}; if (cbm_store_find_node_by_qn(store, project, qualified_name, &exact) == CBM_STORE_OK) { diff --git a/src/operations/source_search.c b/src/operations/source_search.c index 7ea4ea5a70..416c3730a9 100644 --- a/src/operations/source_search.c +++ b/src/operations/source_search.c @@ -7,6 +7,7 @@ #include "foundation/compat_fs.h" #include "foundation/compat_regex.h" #include "foundation/constants.h" +#include "foundation/hash_table.h" #include "foundation/log.h" #include "foundation/platform.h" #include "foundation/workspace.h" @@ -168,16 +169,26 @@ static char *source_read_file_lines(const char *path, int start, int end) { cbm_sb_t selected; cbm_sb_init(&selected); int lineno = 1; - int byte; - while ((byte = fgetc(fp)) != EOF) { - if (lineno >= start && lineno <= end) { - char ch = (char)byte; - cbm_sb_append_n(&selected, &ch, 1U); - } - if (byte == '\n') { - if (lineno >= end) - break; - lineno++; + enum { READ_FILE_LINES_CHUNK = 16384 }; + char chunk[READ_FILE_LINES_CHUNK]; + bool done = false; + size_t got; + while (!done && (got = fread(chunk, 1, sizeof(chunk), fp)) > 0) { + size_t pos = 0; + while (pos < got) { + const char *newline = memchr(chunk + pos, '\n', got - pos); + size_t span_end = newline ? (size_t)(newline - chunk) + SOURCE_SKIP_ONE : got; + if (lineno >= start && lineno <= end) { + cbm_sb_append_n(&selected, chunk + pos, span_end - pos); + } + pos = span_end; + if (newline) { + if (lineno >= end) { + done = true; + break; + } + lineno++; + } } } (void)fclose(fp); @@ -189,6 +200,55 @@ static char *source_read_file_lines(const char *path, int start, int end) { return result; } +typedef struct { + char canonical[CBM_SZ_4K]; + bool resolved; + CBMHashTable *verdicts; +} source_root_containment_t; + +static void source_root_containment_init(source_root_containment_t *c, const char *root) { + c->verdicts = NULL; + c->resolved = root && cbm_canonical_path(root, c->canonical, sizeof(c->canonical)); +#ifdef _WIN32 + if (c->resolved) { + cbm_normalize_path_sep(c->canonical); + } +#endif +} + +static void source_root_free_key(const char *key, void *value, void *userdata) { + (void)value; + (void)userdata; + cbm_free(CBM_MEM_CLASS_OTHER, (void *)key); +} + +static void source_root_containment_free(source_root_containment_t *c) { + if (c->verdicts) { + cbm_ht_foreach(c->verdicts, source_root_free_key, NULL); + cbm_ht_free(c->verdicts); + c->verdicts = NULL; + } +} + +static bool source_root_containment_check(source_root_containment_t *c, const char *path) { + if (!c->resolved || !path) { + return false; + } + void *cached = c->verdicts ? cbm_ht_get(c->verdicts, path) : NULL; + if (cached) { + return (uintptr_t)cached == 1; + } + bool inside = cbm_path_within_canonical_root(c->canonical, path); + if (!c->verdicts) { + c->verdicts = cbm_ht_create_in(CBM_MEM_CLASS_HASH_TABLE, 0); + } + char *key = c->verdicts ? cbm_mem_strdup(CBM_MEM_CLASS_OTHER, path) : NULL; + if (key) { + (void)cbm_ht_set(c->verdicts, key, (void *)(uintptr_t)(inside ? 1 : 2)); + } + return inside; +} + static bool source_utf8_is_cont(unsigned char c) { return (c & 0xC0) == 0x80; } @@ -495,7 +555,7 @@ static yyjson_mut_val *build_dedup_files_array(yyjson_mut_doc *doc, search_resul /* Attach source or context lines to a search result JSON item. */ static void attach_result_source(yyjson_mut_doc *doc, yyjson_mut_val *item, search_result_t *r, int mode, int context_lines, int source_max_lines, - const char *root_path) { + const char *root_path, source_root_containment_t *containment) { enum { MODE_FULL = 1 }; if (r->start_line <= 0 || r->end_line <= 0) { return; @@ -507,7 +567,7 @@ static void attach_result_source(yyjson_mut_doc *doc, yyjson_mut_val *item, sear * project root (a `..` segment, or a symlink/junction that discovery * followed) must not be read back into the response. Same guard the * snippet path already uses. */ - if (!cbm_path_within_root(root_path, abs_path)) { + if (!source_root_containment_check(containment, abs_path)) { return; } @@ -584,6 +644,7 @@ typedef struct { int match_limit; /* matching line numbers shown per result */ int source_max_lines; /* full-mode per-hit source window (0 = omitted) */ bool budget_hit; /* the byte ceiling forced whole rows to be dropped */ + source_root_containment_t *containment; } search_page_t; static bool search_directories_has_more(const search_page_t *page) { @@ -983,7 +1044,7 @@ static char *assemble_search_output(search_result_t *sr, grep_match_t *raw, if (mode == MODE_FULL || attach_context) { yyjson_mut_val *src = yyjson_mut_obj(doc); attach_result_source(doc, src, r, mode, context_lines, page->source_max_lines, - root_path); + root_path, page->containment); yyjson_mut_arr_add_val(row, src); } yyjson_mut_arr_add_val(results_arr, row); @@ -1279,19 +1340,88 @@ static int find_tightest_node(cbm_node_t *nodes, int count, int line) { return best; } +typedef struct { + int64_t *ids; + int *slots; + int capacity; + int count; +} source_result_index_t; + +enum { SOURCE_RESULT_INDEX_MIN_CAPACITY = 64 }; + +static uint32_t source_result_index_hash(int64_t id, int capacity) { + uint64_t mixed = (uint64_t)id * UINT64_C(0x9E3779B97F4A7C15); + return (uint32_t)(mixed >> 32) & (uint32_t)(capacity - 1); +} + +static void source_result_index_free(source_result_index_t *index) { + cbm_free(CBM_MEM_CLASS_OTHER, index->ids); + cbm_free(CBM_MEM_CLASS_OTHER, index->slots); + memset(index, 0, sizeof(*index)); +} + +static int source_result_index_find(const source_result_index_t *index, int64_t id) { + if (index->capacity == 0) { + return CBM_NOT_FOUND; + } + uint32_t at = source_result_index_hash(id, index->capacity); + while (index->slots[at] != 0) { + if (index->ids[at] == id) { + return index->slots[at] - SOURCE_SKIP_ONE; + } + at = (at + SOURCE_SKIP_ONE) & (uint32_t)(index->capacity - 1); + } + return CBM_NOT_FOUND; +} + +static void source_result_index_insert(source_result_index_t *index, int64_t id, int value) { + uint32_t at = source_result_index_hash(id, index->capacity); + while (index->slots[at] != 0) { + at = (at + SOURCE_SKIP_ONE) & (uint32_t)(index->capacity - 1); + } + index->ids[at] = id; + index->slots[at] = value + SOURCE_SKIP_ONE; + index->count++; +} + +static bool source_result_index_reserve(source_result_index_t *index) { + if ((index->count + SOURCE_SKIP_ONE) * SOURCE_PAIR_LEN <= index->capacity) { + return true; + } + int capacity = + index->capacity ? index->capacity * SOURCE_PAIR_LEN : SOURCE_RESULT_INDEX_MIN_CAPACITY; + source_result_index_t grown = { + .ids = cbm_calloc(CBM_MEM_CLASS_OTHER, (size_t)capacity * sizeof(int64_t)), + .slots = cbm_calloc(CBM_MEM_CLASS_OTHER, (size_t)capacity * sizeof(int)), + .capacity = capacity, + }; + if (!grown.ids || !grown.slots) { + source_result_index_free(&grown); + return false; + } + for (int i = 0; i < index->capacity; i++) { + if (index->slots[i] != 0) { + source_result_index_insert(&grown, index->ids[i], index->slots[i] - SOURCE_SKIP_ONE); + } + } + source_result_index_free(index); + *index = grown; + return true; +} + /* Add a grep hit to the search result set (merge into existing or create new). */ -static bool add_to_search_results(search_result_t **sr, int *sr_count, int *sr_cap, cbm_node_t *n, - int line) { - for (int j = *sr_count - 1; j >= 0; j--) { - if ((*sr)[j].node_id == n->id) { - if ((*sr)[j].match_total < INT_MAX) { - (*sr)[j].match_total++; - } - if ((*sr)[j].match_count < CBM_SZ_64) { - (*sr)[j].match_lines[(*sr)[j].match_count++] = line; - } - return true; +static bool add_to_search_results(search_result_t **sr, int *sr_count, int *sr_cap, + source_result_index_t *index, cbm_node_t *n, int line) { + int existing = source_result_index_find(index, n->id); + if (existing >= 0) { + search_result_t *r = &(*sr)[existing]; + if (r->match_total < INT_MAX) { + r->match_total++; + } + if (r->match_count < CBM_SZ_64) { + r->match_lines[r->match_count++] = line; } + return true; } if (*sr_count >= *sr_cap) { int next_capacity = *sr_cap * SOURCE_PAIR_LEN; @@ -1303,6 +1433,9 @@ static bool add_to_search_results(search_result_t **sr, int *sr_count, int *sr_c *sr = grown; *sr_cap = next_capacity; } + if (!source_result_index_reserve(index)) { + return false; + } search_result_t *r = &(*sr)[*sr_count]; r->node_id = n->id; snprintf(r->node_name, sizeof(r->node_name), "%s", n->name ? n->name : ""); @@ -1315,6 +1448,7 @@ static bool add_to_search_results(search_result_t **sr, int *sr_count, int *sr_c r->match_lines[0] = line; r->match_count = SOURCE_SKIP_ONE; r->match_total = SOURCE_SKIP_ONE; + source_result_index_insert(index, n->id, *sr_count); (*sr_count)++; return true; } @@ -1325,10 +1459,11 @@ static bool add_to_search_results(search_result_t **sr, int *sr_count, int *sr_c static bool classify_grep_hit(const grep_match_t *hit, cbm_node_t *file_nodes, int file_node_count, search_result_t **sr, int *sr_count, int *sr_cap, grep_match_t **raw, int raw_offset, int raw_limit, int *raw_count, int *raw_stored_count, - int *raw_cap, int *raw_content_truncated) { + int *raw_cap, source_result_index_t *sr_index, + int *raw_content_truncated) { int best = find_tightest_node(file_nodes, file_node_count, hit->line); if (best >= 0) { - return add_to_search_results(sr, sr_count, sr_cap, &file_nodes[best], hit->line); + return add_to_search_results(sr, sr_count, sr_cap, sr_index, &file_nodes[best], hit->line); } if (*raw_count == INT_MAX) { return false; @@ -1381,13 +1516,14 @@ static void free_file_nodes(cbm_node_t *nodes, int count) { * applies before it reads a result's source, applied to the match itself, * once per distinct file (grep groups a file's hits). A path too long for * the canonical-path buffers is one that check cannot resolve either. */ -static bool search_hit_file_within_root(const char *root_path, const char *file) { +static bool search_hit_file_within_root(source_root_containment_t *containment, + const char *root_path, const char *file) { char abs_path[CBM_SZ_4K]; int length = snprintf(abs_path, sizeof(abs_path), "%s/%s", root_path, file); if (length < 0 || (size_t)length >= sizeof(abs_path)) { return false; } - return cbm_path_within_root(root_path, abs_path); + return source_root_containment_check(containment, abs_path); } /* Parse and classify the complete grep stream without retaining one object per @@ -1408,6 +1544,17 @@ static bool scan_and_classify_grep_matches( bool current_file_inside = false; cbm_node_t *file_nodes = NULL; int file_node_count = 0; + source_root_containment_t containment; + source_root_containment_init(&containment, root_path); + source_result_index_t sr_index = {0}; + for (int i = 0; i < *sr_count; i++) { + if (!source_result_index_reserve(&sr_index)) { + source_result_index_free(&sr_index); + source_root_containment_free(&containment); + return false; + } + source_result_index_insert(&sr_index, (*sr)[i].node_id, i); + } cbm_regex_t content_regex; bool content_regex_ready = use_regex && pattern && cbm_regcomp(&content_regex, pattern, CBM_REG_EXTENDED) == CBM_REG_OK; @@ -1460,7 +1607,7 @@ static bool scan_and_classify_grep_matches( file_node_count = 0; snprintf(current_file, sizeof(current_file), "%s", file); have_current_file = true; - current_file_inside = search_hit_file_within_root(root_path, file); + current_file_inside = search_hit_file_within_root(&containment, root_path, file); if (store && current_file_inside) { (void)cbm_store_find_nodes_by_file(store, project, file, &file_nodes, &file_node_count); @@ -1490,7 +1637,7 @@ static bool scan_and_classify_grep_matches( match_start, match_end); if (!classify_grep_hit(&hit, file_nodes, file_node_count, sr, sr_count, sr_cap, raw, raw_offset, raw_limit, raw_count, raw_stored_count, raw_cap, - raw_content_truncated)) { + &sr_index, raw_content_truncated)) { ok = false; break; } @@ -1500,6 +1647,8 @@ static bool scan_and_classify_grep_matches( cbm_regfree(&content_regex); } free_file_nodes(file_nodes, file_node_count); + source_result_index_free(&sr_index); + source_root_containment_free(&containment); free(line); return ok; } @@ -1770,6 +1919,381 @@ static void search_scratch_close(search_scratch_t *scratch) { } } +/* ── In-process content scan ──────────────────────────────────────── + * search_code used to hand the scoped file list to `xargs sh grep` on POSIX and + * to a PowerShell Select-String pipeline on Windows: a process spawn per call, + * BSD grep's ~130 MB/s on macOS, and one cmdlet invocation per file on Windows + * (23-31 s per call on GitHub's windows-11-arm image, against the 30 s scan + * deadline). This scanner reads the same list and writes the records + * `grep -Hn` printed, so the classifier and everything after it are unchanged: + * - patterns: the pattern split on '\n', as `grep -f` reads its file; an + * empty pattern matches every line (grep special-cases it in -E mode too); + * - literal mode is a substring match (grep -F); regex mode runs each line + * through cbm_regex, the platform's POSIX ERE engine (grep -E); + * - a NUL byte in the first 32 KiB marks the file binary and yields no + * record, as grep prints no lines for it; after a later NUL nothing more + * is reported, as GNU grep stops line output there; + * - a record is `\n` with + * ':' (POSIX) or TAB (Windows), the separators the classifier reads; the + * line keeps a CR, which the classifier strips as before. + * Deadline, cancellation and the output limit end the scan with the causes the + * subprocess runner reported; an unreadable listed file fails the scan closed, + * as grep's error exit did. */ +enum { + SEARCH_SCAN_BINARY_PROBE = 32768, + SEARCH_SCAN_CHECK_EVERY = 4096, /* lines between deadline/cancel checks */ +}; + +typedef struct { + char **items; + size_t *lengths; + int count; + cbm_regex_t *regexes; /* regex mode: one per non-empty pattern */ + int compiled; + bool any_empty; /* an empty pattern: every line matches */ +} search_scan_patterns_t; + +static void search_scan_patterns_free(search_scan_patterns_t *patterns) { + for (int i = 0; i < patterns->count; i++) { + cbm_free(CBM_MEM_CLASS_OTHER, patterns->items[i]); + } + for (int i = 0; patterns->regexes && i < patterns->count; i++) { + if (i < patterns->compiled && patterns->lengths[i] > 0) { + cbm_regfree(&patterns->regexes[i]); + } + } + cbm_free(CBM_MEM_CLASS_OTHER, patterns->items); + cbm_free(CBM_MEM_CLASS_OTHER, patterns->lengths); + cbm_free(CBM_MEM_CLASS_OTHER, patterns->regexes); + memset(patterns, 0, sizeof(*patterns)); +} + +static bool search_scan_patterns_parse(search_scan_patterns_t *patterns, const char *pattern, + bool use_regex) { + memset(patterns, 0, sizeof(*patterns)); + int count = 1; + for (const char *p = pattern; *p; p++) { + count += *p == '\n' ? SKIP_ONE : 0; + } + patterns->items = cbm_calloc(CBM_MEM_CLASS_OTHER, (size_t)count * sizeof(char *)); + patterns->lengths = cbm_calloc(CBM_MEM_CLASS_OTHER, (size_t)count * sizeof(size_t)); + patterns->regexes = + use_regex ? cbm_calloc(CBM_MEM_CLASS_OTHER, (size_t)count * sizeof(cbm_regex_t)) : NULL; + if (!patterns->items || !patterns->lengths || (use_regex && !patterns->regexes)) { + search_scan_patterns_free(patterns); + return false; + } + const char *start = pattern; + for (int i = 0; i < count; i++) { + const char *end = strchr(start, '\n'); + size_t length = end ? (size_t)(end - start) : strlen(start); + patterns->items[i] = cbm_alloc(CBM_MEM_CLASS_OTHER, length + SKIP_ONE); + if (!patterns->items[i]) { + patterns->count = i; + search_scan_patterns_free(patterns); + return false; + } + memcpy(patterns->items[i], start, length); + patterns->items[i][length] = '\0'; + patterns->lengths[i] = length; + patterns->any_empty = patterns->any_empty || length == 0; + start = end ? end + SKIP_ONE : start + length; + } + patterns->count = count; + for (int i = 0; use_regex && i < count; i++) { + if (patterns->lengths[i] > 0 && + cbm_regcomp(&patterns->regexes[i], patterns->items[i], + CBM_REG_EXTENDED | CBM_REG_NOSUB) != CBM_REG_OK) { + patterns->compiled = i; + search_scan_patterns_free(patterns); + return false; + } + patterns->compiled = i + SKIP_ONE; + } + return true; +} + +/* First occurrence of needle in hay; memmem where the C library has it. */ +static const char *search_scan_find(const char *hay, size_t hay_length, const char *needle, + size_t needle_length) { + if (needle_length == 0) { + return hay; + } + if (needle_length > hay_length) { + return NULL; + } +#ifdef _WIN32 + const char *last = hay + (hay_length - needle_length); + for (const char *p = hay; p <= last;) { + p = memchr(p, (unsigned char)needle[0], (size_t)(last - p) + SKIP_ONE); + if (!p) { + return NULL; + } + if (memcmp(p, needle, needle_length) == 0) { + return p; + } + p++; + } + return NULL; +#else + return memmem(hay, hay_length, needle, needle_length); +#endif +} + +static size_t search_scan_count_newlines(const char *from, const char *to) { + size_t count = 0; + while (from < to) { + const char *newline = memchr(from, '\n', (size_t)(to - from)); + if (!newline) { + break; + } + count++; + from = newline + SKIP_ONE; + } + return count; +} + +typedef struct { + FILE *out; + size_t written; + size_t output_limit; + char separator; +} search_scan_sink_t; + +/* One `grep -Hn` record. False on a write error or once past the output limit. */ +static bool search_scan_emit(search_scan_sink_t *sink, const char *path, size_t path_length, + size_t line_number, const char *line, size_t line_length, + bool *over_limit) { + char number[CBM_SZ_32]; + int number_length = + snprintf(number, sizeof(number), "%c%zu%c", sink->separator, line_number, sink->separator); + if (number_length <= 0 || (size_t)number_length >= sizeof(number) || + fwrite(path, 1, path_length, sink->out) != path_length || + fwrite(number, 1, (size_t)number_length, sink->out) != (size_t)number_length || + fwrite(line, 1, line_length, sink->out) != line_length || fputc('\n', sink->out) == EOF) { + return false; + } + sink->written += path_length + (size_t)number_length + line_length + SKIP_ONE; + if (sink->output_limit > 0 && sink->written > sink->output_limit) { + *over_limit = true; + return false; + } + return true; +} + +static bool search_scan_line_matches(const search_scan_patterns_t *patterns, bool use_regex, + const char *line) { + if (patterns->any_empty) { + return true; + } + for (int i = 0; i < patterns->count; i++) { + if (use_regex ? cbm_regexec(&patterns->regexes[i], line, 0, NULL, 0) == CBM_REG_OK + : strstr(line, patterns->items[i]) != NULL) { + return true; + } + } + return false; +} + +/* Scan one file. `buffer` holds its bytes plus one spare byte. */ +static cbm_operation_command_cause_t search_scan_buffer(const cbm_operation_runtime_t *runtime, + search_scan_sink_t *sink, + const search_scan_patterns_t *patterns, + bool use_regex, const char *path, + size_t path_length, char *buffer, + size_t length, uint64_t deadline_ms) { + size_t probe = length < SEARCH_SCAN_BINARY_PROBE ? length : SEARCH_SCAN_BINARY_PROBE; + if (memchr(buffer, '\0', probe)) { + return CBM_OPERATION_COMMAND_SUCCESS; + } + const char *nul = memchr(buffer, '\0', length); + if (nul) { + /* Lines up to the last complete line before the NUL. */ + size_t keep = 0; + for (size_t i = (size_t)(nul - buffer); i > 0; i--) { + if (buffer[i - SKIP_ONE] == '\n') { + keep = i; + break; + } + } + length = keep; + } + buffer[length] = '\0'; + bool over_limit = false; + size_t line_number = 1; + size_t since_check = 0; + bool single_literal = !use_regex && !patterns->any_empty && patterns->count == 1; + size_t pos = 0; + while (pos < length) { + if (++since_check >= SEARCH_SCAN_CHECK_EVERY) { + since_check = 0; + if (cbm_operation_runtime_cancelled(runtime)) { + return CBM_OPERATION_COMMAND_CANCELLED; + } + if (cbm_now_ms() >= deadline_ms) { + return CBM_OPERATION_COMMAND_DEADLINE; + } + } + size_t line_start = pos; + if (single_literal) { + /* Jump to the next occurrence instead of visiting every line. */ + const char *hit = search_scan_find(buffer + pos, length - pos, patterns->items[0], + patterns->lengths[0]); + if (!hit) { + break; + } + line_start = (size_t)(hit - buffer); + while (line_start > pos && buffer[line_start - SKIP_ONE] != '\n') { + line_start--; + } + line_number += search_scan_count_newlines(buffer + pos, buffer + line_start); + } + char *newline = memchr(buffer + line_start, '\n', length - line_start); + size_t line_end = newline ? (size_t)(newline - buffer) : length; + /* Windows matched Select-String's Line, which has no trailing CR, so + * `foo$` matched a CRLF line there; POSIX grep matches with the CR. */ + size_t match_end = line_end; +#ifdef _WIN32 + if (match_end > line_start && buffer[match_end - SKIP_ONE] == '\r') { + match_end--; + } +#endif + char saved = buffer[match_end]; + buffer[match_end] = '\0'; + bool matched = + single_literal || search_scan_line_matches(patterns, use_regex, buffer + line_start); + buffer[match_end] = saved; + if (newline) { + *newline = '\n'; + } + if (matched && !search_scan_emit(sink, path, path_length, line_number, buffer + line_start, + line_end - line_start, &over_limit)) { + return over_limit ? CBM_OPERATION_COMMAND_OUTPUT_LIMIT : CBM_OPERATION_COMMAND_FAILURE; + } + pos = line_end + SKIP_ONE; + line_number++; + } + return CBM_OPERATION_COMMAND_SUCCESS; +} + +/* Read a whole file into *buffer (grown as needed, CBM_MEM_CLASS_EXTRACT), + * leaving one spare byte. */ +static bool search_scan_read_file(const char *path, char **buffer, size_t *capacity, + size_t *length) { + FILE *fp = cbm_fopen(path, "rb"); + if (!fp) { + return false; + } + size_t used = 0; + bool ok = true; + for (;;) { + if (*capacity - used < CBM_SZ_64K + SKIP_ONE) { + size_t grown = *capacity ? *capacity * PAIR_LEN : (size_t)CBM_SZ_64K * PAIR_LEN; + if (grown <= *capacity || grown > (size_t)INT_MAX) { + ok = false; + break; + } + char *next = cbm_realloc(CBM_MEM_CLASS_EXTRACT, *buffer, grown); + if (!next) { + ok = false; + break; + } + *buffer = next; + *capacity = grown; + } + size_t want = *capacity - used - SKIP_ONE; + size_t got = fread(*buffer + used, 1, want, fp); + used += got; + if (got < want) { + ok = !ferror(fp); + break; + } + } + (void)fclose(fp); + *length = used; + return ok; +} + +static cbm_operation_command_cause_t search_scan_in_process(const cbm_operation_runtime_t *runtime, + const char *filelist_path, + const char *pattern, bool use_regex, + FILE *out, size_t output_limit, + uint64_t deadline_ms) { +#ifdef _WIN32 + const char record_separator = '\n'; + const char field_separator = '\t'; +#else + const char record_separator = '\0'; + const char field_separator = ':'; +#endif + search_scan_patterns_t patterns; + if (!search_scan_patterns_parse(&patterns, pattern, use_regex)) { + return CBM_OPERATION_COMMAND_FAILURE; + } + char *list = NULL; + size_t list_capacity = 0; + size_t list_length = 0; + if (!search_scan_read_file(filelist_path, &list, &list_capacity, &list_length)) { + cbm_free(CBM_MEM_CLASS_EXTRACT, list); + search_scan_patterns_free(&patterns); + return CBM_OPERATION_COMMAND_FAILURE; + } + search_scan_sink_t sink = { + .out = out, .output_limit = output_limit, .separator = field_separator}; + char *buffer = NULL; + size_t capacity = 0; + cbm_operation_command_cause_t cause = CBM_OPERATION_COMMAND_SUCCESS; + size_t pos = 0; + while (cause == CBM_OPERATION_COMMAND_SUCCESS && pos < list_length) { + const char *end = memchr(list + pos, record_separator, list_length - pos); + size_t record_end = end ? (size_t)(end - list) : list_length; + list[record_end] = '\0'; + const char *path = list + pos; + size_t path_length = record_end - pos; + pos = record_end + SKIP_ONE; + if (path_length == 0) { + continue; + } + if (cbm_operation_runtime_cancelled(runtime)) { + cause = CBM_OPERATION_COMMAND_CANCELLED; + break; + } + if (cbm_now_ms() >= deadline_ms) { + cause = CBM_OPERATION_COMMAND_DEADLINE; + break; + } + size_t length = 0; + if (!search_scan_read_file(path, &buffer, &capacity, &length)) { + cause = CBM_OPERATION_COMMAND_FAILURE; + break; + } + cause = search_scan_buffer(runtime, &sink, &patterns, use_regex, path, path_length, buffer, + length, deadline_ms); + } + cbm_free(CBM_MEM_CLASS_EXTRACT, buffer); + cbm_free(CBM_MEM_CLASS_EXTRACT, list); + search_scan_patterns_free(&patterns); + return cause; +} + +#ifdef CBM_ENABLE_TEST_SEAMS +int cbm_source_search_scan_for_test(const cbm_operation_runtime_t *runtime, + const char *filelist_path, const char *pattern, bool use_regex, + const char *out_path, size_t output_limit, + uint64_t deadline_ms) { + FILE *out = cbm_fopen(out_path, "wb"); + if (!out) { + return CBM_OPERATION_COMMAND_FAILURE; + } + cbm_operation_command_cause_t cause = search_scan_in_process( + runtime, filelist_path, pattern, use_regex, out, output_limit, deadline_ms); + if (fclose(out) != 0 && cause == CBM_OPERATION_COMMAND_SUCCESS) { + cause = CBM_OPERATION_COMMAND_FAILURE; + } + return (int)cause; +} +#endif + /* Open the scratch directory, write `pattern` to the grep -f file, and leave the * file list open for write_scoped_filelist. Returns true on success; on failure * everything already created is removed before returning. */ @@ -2262,12 +2786,40 @@ cbm_operation_result_t cbm_source_search_operation_execute(const char *args, : SOURCE_SEARCH_OUTPUT_MAX; const char *scan_command = runtime && runtime->command_override ? runtime->command_override : cmd; - /* Both POSIX commands wrap grep and map its no-match status to 0, so any - * non-zero exit (a failed find/sort, an unreadable operand, a broken - * grep) is an incomplete scan and fails closed. */ - cbm_operation_command_cause_t scan_cause = cbm_operation_run_shell_command_bounded( - runtime, scan_command, output_path, scan_output_limit, scan_deadline_ms, true, - scan_deadline_latched, false, &scan_result); + /* Scoped projects already have a canonical file list, so scan it + * directly. Keep the supervised command for unindexed roots, explicit + * command overrides, and Windows patterns that need PowerShell's + * post-filter semantics. */ + bool scan_in_process = scoped && !(runtime && runtime->command_override); +#ifdef _WIN32 + scan_in_process = + scan_in_process && + (!file_pattern || cbm_search_code_file_pattern_can_prefilter(file_pattern)); +#endif + cbm_operation_command_cause_t scan_cause; + if (scan_in_process) { + FILE *scan_out = NULL; + if (scan_cancellation_latched || cbm_operation_runtime_cancelled(runtime)) { + scan_cause = CBM_OPERATION_COMMAND_CANCELLED; + } else if (scan_deadline_latched || cbm_now_ms() >= scan_deadline_ms) { + scan_cause = CBM_OPERATION_COMMAND_DEADLINE; + } else if (!(scan_out = search_scratch_file(scratch.dir, "scan", output_path, + sizeof(output_path)))) { + scan_cause = CBM_OPERATION_COMMAND_FAILURE; + } else { + scan_cause = search_scan_in_process(runtime, filelist, pattern, use_regex, scan_out, + scan_output_limit, scan_deadline_ms); + } + if (scan_out && fclose(scan_out) != 0 && scan_cause == CBM_OPERATION_COMMAND_SUCCESS) { + scan_cause = CBM_OPERATION_COMMAND_FAILURE; + } + } else { + /* The supervised fallback preserves cancellation and containment + * for recursive scans and platform-specific post-filters. */ + scan_cause = cbm_operation_run_shell_command_bounded( + runtime, scan_command, output_path, scan_output_limit, scan_deadline_ms, true, + scan_deadline_latched, false, &scan_result); + } const char *scan_message = NULL; /* COMMAND_DEADLINE renders its own text */ char limit_message[CBM_SZ_128]; FILE *fp = NULL; @@ -2413,6 +2965,9 @@ cbm_operation_result_t cbm_source_search_operation_execute(const char *args, } bool page_json = !(mode == 0 && !sc_legacy_json); bool warn_pipe = pat_has_pipe && !use_regex; + source_root_containment_t request_containment; + source_root_containment_init(&request_containment, root_path); + page.containment = &request_containment; char *result = render_search_payload(sr, raw, &page, mode, context_lines, root_path, warn_pipe, &metrics, page_json); @@ -2493,6 +3048,7 @@ cbm_operation_result_t cbm_source_search_operation_execute(const char *args, result = search_budget_floor(&page, mode, page_json); } + source_root_containment_free(&request_containment); bool result_error = result == NULL; if (!result) result = source_strdup("out of memory"); diff --git a/src/operations/source_search.h b/src/operations/source_search.h index 9431b290c0..fbe3f0e3e3 100644 --- a/src/operations/source_search.h +++ b/src/operations/source_search.h @@ -5,6 +5,7 @@ #include #include +#include cbm_operation_result_t cbm_source_search_operation_execute(const char *args_json, const cbm_operation_runtime_t *runtime); @@ -16,5 +17,11 @@ bool cbm_search_code_windows_path_matches_prefilter(const char *path, const char void cbm_search_code_build_grep_cmd(char *cmd, size_t cmd_sz, bool use_regex, bool scoped, const char *file_pattern, const char *tmpfile, const char *filelist, const char *root_path); +#ifdef CBM_ENABLE_TEST_SEAMS +int cbm_source_search_scan_for_test(const cbm_operation_runtime_t *runtime, + const char *filelist_path, const char *pattern, bool use_regex, + const char *out_path, size_t output_limit, + uint64_t deadline_ms); +#endif #endif diff --git a/src/operations/trace.c b/src/operations/trace.c index cd38507098..4e054df9de 100644 --- a/src/operations/trace.c +++ b/src/operations/trace.c @@ -359,8 +359,8 @@ static yyjson_doc *resolve_edge_types(const char *args, const char *mode, const * ordering (DEPENDS_ON), controllers (HANDLES), what an object declares * (DEFINES: an Ingress's routes) and what it deploys (DEPLOYS: a Flux * Kustomization's path, a HelmRelease's chart). */ - static const char *mode_topology[] = {"INFRA_MAPS", "USAGE", "DEPENDS_ON", - "HANDLES", "DEFINES", "DEPLOYS"}; + static const char *mode_topology[] = {"INFRA_MAPS", "USAGE", "DEPENDS_ON", "HANDLES", + "DEFINES", "DEPLOYS", "CONNECTS_TO"}; /* deploy (#2597): what a CI job, script or Make rule starts (SPAWNS: * scripts, make targets, programs), calls (CALLS: shell functions) and * puts on a cluster (DEPLOYS) — outbound from the job, inbound from a @@ -794,6 +794,7 @@ cbm_operation_result_t cbm_trace_operation_execute(const char *args) { : error_result("project not indexed", "Run 'codebase-memory-cli index .' first."); goto done; } + (void)cbm_store_read_batch_begin(store); (void)cbm_store_find_nodes_by_name(store, project, function, &nodes, &node_count); if (node_count == 0) { diff --git a/src/pipeline/pass_ansible.c b/src/pipeline/pass_ansible.c new file mode 100644 index 0000000000..6ccb2f7f44 --- /dev/null +++ b/src/pipeline/pass_ansible.c @@ -0,0 +1,1679 @@ +/* + * pass_ansible.c — the Ansible layer. + * + * PHASE 7 P7.1: composition. A playbook or a task file -- a YAML file whose + * top level is a sequence (manifests, values, CI and compose files are + * mappings) -- IMPORTS {local_name: , via: ansible, line} what its + * directives name: + * import_playbook: P the playbook P, from the file's + * directory + * include_tasks / import_tasks / + * include: T (or {file: T}) the task file T, from the file's + * directory, else from the tasks/ + * directory of the role holding it + * include_role / import_role: {name: R}, + * roles: [R | {role: R} | {name: R}] the role R: roles/R in the file's + * directory or above (a name may hold + * "/", roles/kubernetes/node), or a + * path ("./", "../") from the + * file's directory + * (the ansible.builtin. prefix allowed). The edge goes from the file to the + * target file's Module, or to the role's Folder; a target the graph does not + * hold, or one holding a Jinja expression ({{ }}), makes none. + * + * PHASE 7 P7.2: tasks. A top-level sequence is not Ansible by itself (JSON + * patches, image maps and service lists are sequences too): a file is + * Ansible when it is a playbook (an item with hosts: / import_playbook:), a + * role's tasks/ or handlers/ file, or -- to a fixpoint -- a task file an + * Ansible file includes. Each of its tasks (a playbook's pre_tasks / tasks / + * post_tasks, its handlers; a block's block / rescue / always, never the + * block) is a Function named by its `name:`, else its module, QN + * ::task/ or ::handler/ (document order), {source: ansible, + * kind, module}, which the file DEFINES. A notify CALLS {via: notify, line} + * every handler of that name or listen topic in the task's own role (the + * directory before its tasks/), else in its file (a play's handlers). The + * incremental repair declines on any change to Ansible content + * (closure_ansible_decline): a notify joins two files. + * + * PHASE 7 P7.3: applies. A kube (kubespray's library module) or k8s + * (kubernetes.core / community.kubernetes) task applies a manifest the play + * rendered: it DEPLOYS {via: ansible, module, line} the repository file the + * render read -- a template task's src (the role's templates/) or a copy + * task's (its files/; never remote_src), else the task file's directory: + * a k8s template: argument that template; + * filename == a renderer's dest (D) that renderer's src; + * {{ item.item.K }} over "{{ R.results }}" (L) + * the src of the renderer registering + * R, for each item of its loop; + * {{ item.K }} (S) for each of the task's own items, the + * src of the renderer of that dest; + * a literal filename (R) the src of each looped renderer item + * whose dest renders to it. + * A loop is a literal list, or "{{ V }}": a set_fact of the file, else the + * role's defaults/ or vars/ main.yml. Templates are searched before copies, + * each in file order. The incremental repair declines on a change to a + * role's defaults / vars main file (closure_ansible_decline), and when a + * task file would join it as a dependent -- a deleted template it applies -- + * since the repair would walk it without its role's other files + * (closure_ansible_dependent_decline). + */ +#include "foundation/constants.h" +#include "foundation/compat.h" +#include "foundation/compat_fs.h" +#include "foundation/hash_table.h" +#include "foundation/limits.h" +#include "foundation/log.h" +#include "foundation/mem_core.h" +#include "foundation/str_util.h" /* cbm_json_escape */ +#include "graph_buffer/graph_buffer.h" +#include "pipeline/pipeline.h" +#include "pipeline/pipeline_internal.h" +#include "cbm.h" +#include "helpers.h" +#include "lang_specs.h" + +#include +#include +#include +#include + +enum { AN_GROW_MIN = 16 }; + +/* A notify waiting for every handler: the task, its role and file, the name. */ +typedef struct { + int64_t task; + char *scope_role; /* "" outside a role */ + char *scope_file; + char *name; + int line; +} an_notify_t; + +typedef struct { + cbm_pipeline_ctx_t *ctx; + CBMHashTable *modules; /* file path -> its Module node */ + CBMHashTable *folders; /* directory -> its Folder node */ + const char *src; /* the file's text */ + const char *file; /* its repository path */ + char dir[CBM_SZ_1K]; /* its directory */ + char role[CBM_SZ_1K]; /* the role holding it (roles/), or "" */ + int64_t from; /* its File node */ + int edges; + char **incs; /* pass 1: the task files includes name, in file order */ + int inc_n; + int inc_cap; + char base[CBM_SZ_1K]; /* pass 2: its File QN without `.__file__` */ + char role_dir[CBM_SZ_1K]; /* the role of its tasks / handlers, or "" */ + int ordinal[PAIR_LEN]; /* tasks, handlers so far in the file */ + CBMHashTable *handlers; /* "\x1f" -> an_ids_t */ + an_notify_t *notes; + int note_n; + int note_cap; + int tasks; + int calls; + CBMHashTable *file_nodes; /* P7.3: File nodes by path (what a task applies) */ + CBMHashTable *by_rel; /* every input file's path -> its index + 1 */ + const cbm_file_info_t *files; + TSParser *parser; /* reads a role's defaults / vars */ + struct an_render *renders; /* per file: template / copy tasks */ + int render_n; + int render_cap; + struct an_apply *applies; /* per file: kube / k8s tasks */ + int apply_n; + int apply_cap; + struct an_fact *facts; /* per file: set_fact lists */ + int fact_n; + int fact_cap; + int deploys; +} an_t; + +static bool an_type(TSNode n, const char *t) { + return !ts_node_is_null(n) && strcmp(ts_node_type(n), t) == 0; +} + +/* A quoted scalar's escapes decoded in place: `''` in single quotes; + * `\"`, `\\`, `\n`, `\t` (any other `\x`: x) in double quotes. */ +static void an_unescape(char *s, bool dq) { + char *w = s; + for (const char *r = s; *r; r++) { + if (dq && r[0] == '\\' && r[SKIP_ONE]) { + r++; + *w++ = *r == 'n' ? '\n' : *r == 't' ? '\t' : *r; + } else if (!dq && r[0] == '\'' && r[SKIP_ONE] == '\'') { + r++; + *w++ = '\''; + } else { + *w++ = *r; + } + } + *w = '\0'; +} + +/* A scalar's text (quotes stripped, escapes decoded) into out; false for + * anything else. */ +static bool an_scalar(const an_t *a, TSNode n, char *out, size_t sz) { + while ((an_type(n, "flow_node") || an_type(n, "block_node") || an_type(n, "plain_scalar")) && + ts_node_named_child_count(n) == SKIP_ONE) { + n = ts_node_named_child(n, 0); + } + if (ts_node_is_null(n)) { /* a pair without its key or value */ + return false; + } + uint32_t b = ts_node_start_byte(n); + uint32_t e = ts_node_end_byte(n); + bool dq = an_type(n, "double_quote_scalar"); + bool sq = an_type(n, "single_quote_scalar"); + if (dq || sq) { + b++; + e = e > b ? e - SKIP_ONE : b; + } else if (!an_type(n, "string_scalar") && !an_type(n, "plain_scalar")) { + return false; + } + if (e - b + SKIP_ONE > sz) { + return false; + } + memcpy(out, a->src + b, e - b); + out[e - b] = '\0'; + if (dq || sq) { + an_unescape(out, dq); + } + return out[0] != '\0'; +} + +/* Is `n` an Ansible boolean that holds true -- y / yes / on / 1 / true / t, + * any case (Ansible's boolean()), a YAML boolean or a string? an_scalar + * reads strings only: `remote_src: true` is a boolean_scalar. */ +static bool an_true(const an_t *a, TSNode n) { + static const char *const TRUE_WORDS[] = {"y", "yes", "on", "1", "true", "t"}; + while ((an_type(n, "flow_node") || an_type(n, "block_node") || an_type(n, "plain_scalar")) && + ts_node_named_child_count(n) == SKIP_ONE) { + n = ts_node_named_child(n, 0); + } + char v[CBM_SZ_16]; + if (an_type(n, "boolean_scalar") || an_type(n, "integer_scalar")) { + uint32_t b = ts_node_start_byte(n); + uint32_t e = ts_node_end_byte(n); + if (e - b + SKIP_ONE > sizeof(v)) { + return false; + } + memcpy(v, a->src + b, e - b); + v[e - b] = '\0'; + } else if (!an_scalar(a, n, v, sizeof(v))) { + return false; + } + for (char *c = v; *c; c++) { + *c = (char)(*c >= 'A' && *c <= 'Z' ? *c - 'A' + 'a' : *c); + } + for (size_t i = 0; i < sizeof(TRUE_WORDS) / sizeof(TRUE_WORDS[0]); i++) { + if (strcmp(v, TRUE_WORDS[i]) == 0) { + return true; + } + } + return false; +} + +/* The key of a mapping pair, its text, the ansible.builtin. prefix dropped. */ +static bool an_key(const an_t *a, TSNode pair, char *out, size_t sz) { + static const char BUILTIN[] = "ansible.builtin."; + if (!an_scalar(a, ts_node_child_by_field_name(pair, "key", sizeof("key") - SKIP_ONE), out, + sz)) { + return false; + } + size_t bl = sizeof(BUILTIN) - SKIP_ONE; + size_t len = strlen(out); + if (len >= bl && memcmp(out, BUILTIN, bl) == 0) { + memmove(out, out + bl, len - bl + SKIP_ONE); + } + return true; +} + +/* The value of `key` in a block mapping (under its block_node), or a null node. */ +static TSNode an_get(const an_t *a, TSNode value, const char *key) { + while (an_type(value, "block_node") && ts_node_named_child_count(value) == SKIP_ONE) { + value = ts_node_named_child(value, 0); + } + for (uint32_t i = 0; an_type(value, "block_mapping") && i < ts_node_named_child_count(value); + i++) { + TSNode pair = ts_node_named_child(value, i); + char k[CBM_SZ_64]; + if (an_type(pair, "block_mapping_pair") && an_key(a, pair, k, sizeof(k)) && + strcmp(k, key) == 0) { + return ts_node_child_by_field_name(pair, "value", sizeof("value") - SKIP_ONE); + } + } + return (TSNode){0}; +} + +static void an_edge(an_t *a, const cbm_gbuf_node_t *tgt, const char *kind, TSNode pair) { + if (!tgt || a->from <= 0 || tgt->id == a->from) { + return; + } + char props[CBM_SZ_128]; + snprintf(props, sizeof(props), "{\"local_name\":\"%s\",\"via\":\"ansible\",\"line\":%d}", kind, + (int)ts_node_start_point(pair).row + SKIP_ONE); + cbm_gbuf_insert_edge(a->ctx->gbuf, a->from, tgt->id, "IMPORTS", props); + a->edges++; +} + +/* Pass 1 keeps the task files an include names: they are Ansible content + * even outside a role (an_mark_ansible). */ +static void an_include(an_t *a, const cbm_gbuf_node_t *target) { + if (!target || !target->file_path) { + return; + } + if (a->inc_n == a->inc_cap) { + int cap = a->inc_cap ? a->inc_cap * PAIR_LEN : AN_GROW_MIN; + char **grown = cbm_realloc(CBM_MEM_CLASS_OTHER, a->incs, (size_t)cap * sizeof(*grown)); + if (!grown) { + return; + } + a->incs = grown; + a->inc_cap = cap; + } + a->incs[a->inc_n++] = cbm_mem_strdup(CBM_MEM_CLASS_OTHER, target->file_path); +} + +/* A file `v` from `base` (repository-relative), its Module; NULL when none. */ +static const cbm_gbuf_node_t *an_file(const an_t *a, const char *base, const char *v) { + char path[CBM_SZ_1K]; + if (!cbm_repo_rel_join(base, v, path, sizeof(path))) { + return NULL; + } + return cbm_ht_get(a->modules, path); +} + +/* Role `name`'s Folder: a path ("./", "../") from the file's directory, else + * roles/ in that directory or above. */ +static const cbm_gbuf_node_t *an_role(const an_t *a, const char *name) { + char path[CBM_SZ_1K]; + if (name[0] == '.' || name[0] == '/') { + return cbm_repo_rel_join(a->dir, name, path, sizeof(path)) ? cbm_ht_get(a->folders, path) + : NULL; + } + char d[CBM_SZ_1K]; + snprintf(d, sizeof(d), "%s", a->dir); + for (;;) { + snprintf(path, sizeof(path), "%s%sroles/%s", d, d[0] ? "/" : "", name); + const cbm_gbuf_node_t *f = cbm_ht_get(a->folders, path); + if (f || !d[0]) { + return f; + } + char *s = strrchr(d, '/'); + if (s) { + *s = '\0'; + } else { + d[0] = '\0'; + } + } +} + +/* One mapping pair of a task or a play. */ +static void an_pair(an_t *a, TSNode pair) { + char key[CBM_SZ_64]; + char v[CBM_SZ_1K]; + if (!an_key(a, pair, key, sizeof(key))) { + return; + } + TSNode value = ts_node_child_by_field_name(pair, "value", sizeof("value") - SKIP_ONE); + if (strcmp(key, "import_playbook") == 0) { + if (an_scalar(a, value, v, sizeof(v)) && !strstr(v, "{{")) { + an_edge(a, an_file(a, a->dir, v), key, pair); + } + } else if (strcmp(key, "include_tasks") == 0 || strcmp(key, "import_tasks") == 0 || + strcmp(key, "include") == 0) { + if ((an_scalar(a, value, v, sizeof(v)) || + an_scalar(a, an_get(a, value, "file"), v, sizeof(v))) && + !strstr(v, "{{")) { + char tasks[CBM_SZ_1K]; + snprintf(tasks, sizeof(tasks), "%s/tasks", a->role); + const cbm_gbuf_node_t *t = an_file(a, a->dir, v); + const cbm_gbuf_node_t *target = t ? t : a->role[0] ? an_file(a, tasks, v) : NULL; + an_edge(a, target, key, pair); + an_include(a, target); + } + } else if (strcmp(key, "include_role") == 0 || strcmp(key, "import_role") == 0) { + if (an_scalar(a, an_get(a, value, "name"), v, sizeof(v)) && !strstr(v, "{{")) { + an_edge(a, an_role(a, v), key, pair); + } + } else if (strcmp(key, "roles") == 0) { + while (an_type(value, "block_node") && ts_node_named_child_count(value) == SKIP_ONE) { + value = ts_node_named_child(value, 0); + } + for (uint32_t i = 0; + an_type(value, "block_sequence") && i < ts_node_named_child_count(value); i++) { + TSNode item = ts_node_named_child(ts_node_named_child(value, i), 0); + if ((an_scalar(a, item, v, sizeof(v)) || + an_scalar(a, an_get(a, item, "role"), v, sizeof(v)) || + an_scalar(a, an_get(a, item, "name"), v, sizeof(v))) && + !strstr(v, "{{")) { + an_edge(a, an_role(a, v), key, ts_node_named_child(value, i)); + } + } + } +} + +/* Every mapping that is a sequence item (a play or a task, also one nested in + * block: / tasks: / handlers: ...): its pairs. */ +static void an_walk(an_t *a, TSNode root) { + TSTreeCursor cur = ts_tree_cursor_new(root); + bool more = true; + while (more) { + TSNode c = ts_tree_cursor_current_node(&cur); + if (an_type(c, "block_mapping")) { + TSNode up = ts_node_parent(ts_node_parent(c)); + for (uint32_t i = 0; + an_type(up, "block_sequence_item") && i < ts_node_named_child_count(c); i++) { + if (an_type(ts_node_named_child(c, i), "block_mapping_pair")) { + an_pair(a, ts_node_named_child(c, i)); + } + } + } + if (ts_tree_cursor_goto_first_child(&cur)) { + continue; + } + while (!ts_tree_cursor_goto_next_sibling(&cur)) { + if (!ts_tree_cursor_goto_parent(&cur)) { + more = false; + break; + } + } + } + ts_tree_cursor_delete(&cur); +} + +/* The first content line of `src` starts a sequence item: Ansible content + * (a playbook, a task list), not a mapping (manifest, values, CI, compose). */ +static bool an_is_sequence(const char *src) { + for (const char *p = src; *p;) { + const char *nl = strchr(p, '\n'); + size_t n = nl ? (size_t)(nl - p) : strlen(p); + size_t s = 0; + while (s < n && (p[s] == ' ' || p[s] == '\t' || p[s] == '\r')) { + s++; + } + bool skip = s == n || p[s] == '#' || + (n - s >= sizeof("---") - SKIP_ONE && + strncmp(p + s, "---", sizeof("---") - SKIP_ONE) == 0); + if (!skip) { + return s == 0 && n >= PAIR_LEN && p[0] == '-' && p[SKIP_ONE] == ' '; + } + p = nl ? nl + SKIP_ONE : p + n; + } + return false; +} + +/* The role holding `rel`: roles/ of the deepest "roles" component with + * a name after it, or "". */ +static void an_role_of(const char *rel, char *out, size_t sz) { + out[0] = '\0'; + for (const char *p = rel; (p = strstr(p, "roles/")) != NULL; p++) { + if (p != rel && p[-1] != '/') { + continue; + } + const char *name = p + sizeof("roles/") - SKIP_ONE; + const char *end = strchr(name, '/'); + if (end && strchr(end + SKIP_ONE, '/')) { + snprintf(out, sz, "%.*s", (int)(end - rel), rel); + } + } +} + +static char *an_read(const char *path, size_t *len) { + FILE *f = cbm_fopen(path, "rb"); + if (!f) { + return NULL; + } + (void)fseek(f, 0, SEEK_END); + long size = ftell(f); + (void)fseek(f, 0, SEEK_SET); + if (size <= 0 || size > cbm_max_file_bytes()) { + (void)fclose(f); + return NULL; + } + enum { AN_TS_PAD = 16 }; /* tree-sitter's lexer reads past the end */ + char *buf = cbm_alloc(CBM_MEM_CLASS_OTHER, (size_t)size + AN_TS_PAD); + if (!buf) { + (void)fclose(f); + return NULL; + } + size_t got = fread(buf, SKIP_ONE, (size_t)size, f); + (void)fclose(f); + memset(buf + got, 0, AN_TS_PAD); + *len = got; + return buf; +} + +/* ── Tasks (P7.2) ─────────────────────────────────────────────────── */ + +/* Task keywords: every other key of a task is its module. */ +static const char *const AN_KEYWORDS[] = {"name", + "when", + "tags", + "become", + "become_user", + "become_method", + "register", + "loop", + "with_items", + "with_dict", + "with_list", + "with_fileglob", + "with_nested", + "with_together", + "with_sequence", + "with_subelements", + "with_first_found", + "loop_control", + "notify", + "listen", + "ignore_errors", + "changed_when", + "failed_when", + "delegate_to", + "run_once", + "vars", + "environment", + "until", + "retries", + "delay", + "no_log", + "check_mode", + "diff", + "any_errors_fatal", + "throttle", + "timeout", + "args", + "action", + "local_action", + "async", + "poll", + "debugger", + "collections", + "module_defaults", + "remote_user", + "connection", + "port", + "ignore_unreachable", + "block", + "rescue", + "always", + "delegate_facts"}; + +static bool an_keyword(const char *k) { + for (size_t i = 0; i < sizeof(AN_KEYWORDS) / sizeof(AN_KEYWORDS[0]); i++) { + if (strcmp(k, AN_KEYWORDS[i]) == 0) { + return true; + } + } + return false; +} + +/* The first named child of `n` of type `t`, or a null node. */ +static TSNode an_child(TSNode n, const char *t) { + for (uint32_t i = 0; !ts_node_is_null(n) && i < ts_node_named_child_count(n); i++) { + if (an_type(ts_node_named_child(n, i), t)) { + return ts_node_named_child(n, i); + } + } + return (TSNode){0}; +} + +/* A value as a block sequence (under its block_node), or a null node. */ +static TSNode an_seq(TSNode v) { + while (an_type(v, "block_node") && ts_node_named_child_count(v) == SKIP_ONE) { + v = ts_node_named_child(v, 0); + } + return an_type(v, "block_sequence") ? v : (TSNode){0}; +} + +/* The top-level block sequence of a parsed file, or a null node. */ +static TSNode an_top(TSNode root) { + return an_seq(an_child(an_child(root, "document"), "block_node")); +} + +/* Item `i` of block sequence `seq` as a block mapping, or a null node. */ +static TSNode an_item(TSNode seq, uint32_t i) { + TSNode node = ts_node_named_child(ts_node_named_child(seq, i), 0); + return an_type(node, "block_node") ? an_child(node, "block_mapping") : (TSNode){0}; +} + +/* Does block mapping `m` hold key `key` (the ansible.builtin. prefix allowed)? */ +static bool an_has(const an_t *a, TSNode m, const char *key) { + for (uint32_t i = 0; an_type(m, "block_mapping") && i < ts_node_named_child_count(m); i++) { + char k[CBM_SZ_64]; + if (an_key(a, ts_node_named_child(m, i), k, sizeof(k)) && strcmp(k, key) == 0) { + return true; + } + } + return false; +} + +/* A playbook: a top-level item is a play (hosts:) or imports one. */ +static bool an_is_playbook(const an_t *a, TSNode top) { + for (uint32_t i = 0; !ts_node_is_null(top) && i < ts_node_named_child_count(top); i++) { + TSNode m = an_item(top, i); + if (an_has(a, m, "hosts") || an_has(a, m, "import_playbook")) { + return true; + } + } + return false; +} + +/* The role a task or handler file belongs to: the directory before the first + * tasks/ or handlers/ under the last roles/ component that has one + * (roles/kubernetes-apps/csi_driver/cinder for .../cinder/tasks/main.yml); + * "" outside a role's tasks / handlers. */ +static void an_role_dir(const char *rel, char *out, size_t sz) { + out[0] = '\0'; + for (const char *p = rel; (p = strstr(p, "roles/")) != NULL; p++) { + if (p != rel && p[-1] != '/') { + continue; + } + /* the role's name: one character at least, then its tasks/ or handlers/ */ + const char *from = p + sizeof("roles/"); + if (from > p + strlen(p)) { + break; + } + const char *t = strstr(from, "/tasks/"); + const char *h = strstr(from, "/handlers/"); + const char *end = t && (!h || t < h) ? t : h; + if (end) { + snprintf(out, sz, "%.*s", (int)(end - rel), rel); + } + } +} + +/* Ids by key ("\x1f"). */ +typedef struct { + int64_t *items; + int count; + int cap; +} an_ids_t; + +static void an_ids_add(CBMHashTable *ix, const char *key, int64_t id) { + an_ids_t *v = cbm_ht_get(ix, key); + if (!v) { + v = cbm_calloc(CBM_MEM_CLASS_OTHER, sizeof(*v)); + char *owned = cbm_mem_strdup(CBM_MEM_CLASS_OTHER, key); + if (!v || !owned) { + cbm_free(CBM_MEM_CLASS_OTHER, v); + cbm_free(CBM_MEM_CLASS_OTHER, owned); + return; + } + cbm_ht_set(ix, owned, v); + } + if (v->count > 0 && v->items[v->count - SKIP_ONE] == id) { + return; /* a handler named and listening alike */ + } + if (v->count == v->cap) { + int cap = v->cap ? v->cap * PAIR_LEN : AN_GROW_MIN; + int64_t *grown = cbm_realloc(CBM_MEM_CLASS_OTHER, v->items, (size_t)cap * sizeof(int64_t)); + if (!grown) { + return; + } + v->items = grown; + v->cap = cap; + } + v->items[v->count++] = id; +} + +/* A task list being walked: its sequence, the next item, handlers or not. */ +typedef struct { + TSNode seq; + uint32_t next; + bool handler; +} an_frame_t; + +/* How many names a notify / listen value gives (a scalar: one; a block or + * flow list: its items), its list node into `*list` (null for a scalar). */ +static uint32_t an_name_count(const an_t *a, TSNode v, TSNode *list) { + char probe[CBM_SZ_256]; + *list = (TSNode){0}; + if (an_scalar(a, v, probe, sizeof(probe))) { + return SKIP_ONE; + } + while ((an_type(v, "block_node") || an_type(v, "flow_node")) && + ts_node_named_child_count(v) == SKIP_ONE) { + v = ts_node_named_child(v, 0); + } + if (!an_type(v, "block_sequence") && !an_type(v, "flow_sequence")) { + return 0; + } + *list = v; + return ts_node_named_child_count(v); +} + +/* Name `i` of a notify / listen value (see an_name_count). */ +static bool an_name_at(const an_t *a, TSNode v, TSNode list, uint32_t i, char *out, size_t sz) { + if (ts_node_is_null(list)) { + return i == 0 && an_scalar(a, v, out, sz); + } + TSNode item = ts_node_named_child(list, i); + if (an_type(list, "block_sequence")) { + item = ts_node_named_child(item, 0); + } + return an_scalar(a, item, out, sz); +} + +/* ── Applies (P7.3) ───────────────────────────────────────────────── */ + +/* One item of a loop list: a scalar, or a mapping's scalar pairs. */ +typedef struct { + char *scalar; + char **keys; + char **vals; + int n; +} an_item_t; + +/* A loop (with_items / loop): its literal items, or its expression + * ("{{ X }}", "{{ R.results }}"). */ +typedef struct { + an_item_t *items; + int n; + char *expr; +} an_loop_t; + +/* A task rendering a file: template (src in the role's templates/) or copy + * of a repository file (its files/). */ +typedef struct an_render { + char *src; + char *dest; + char *reg; + an_loop_t loop; + bool copy; +} an_render_t; + +/* A task applying a manifest: kube / k8s, its filename (src) or template + * argument, its loop. */ +typedef struct an_apply { + int64_t task; + int line; + char *module; + char *fname; + char *tmpl; + an_loop_t loop; +} an_apply_t; + +/* A list a set_fact of the file names. */ +typedef struct an_fact { + char *name; + an_loop_t list; +} an_fact_t; + +static const char *const AN_APPLIERS[] = {"kube", "k8s", "kubernetes.core.k8s", + "community.kubernetes.k8s"}; + +static void *an_grow(void *items, int *cap, int count, size_t elem) { + if (count < *cap) { + return items; + } + int ncap = *cap ? *cap * PAIR_LEN : AN_GROW_MIN; + void *grown = cbm_realloc(CBM_MEM_CLASS_OTHER, items, (size_t)ncap * elem); + if (grown) { + *cap = ncap; + } + return grown; +} + +static char *an_dup(const char *s) { + return s ? cbm_mem_strdup(CBM_MEM_CLASS_OTHER, s) : NULL; +} + +/* A value node without its block_node / flow_node wrappers. */ +static TSNode an_bare(TSNode v) { + while ((an_type(v, "block_node") || an_type(v, "flow_node")) && + ts_node_named_child_count(v) == SKIP_ONE) { + v = ts_node_named_child(v, 0); + } + return v; +} + +/* A loop item: a scalar, or the scalar pairs of a block / flow mapping. */ +static void an_item_parse(const an_t *a, TSNode node, an_item_t *it) { + char buf[CBM_SZ_1K]; + *it = (an_item_t){0}; + if (an_scalar(a, node, buf, sizeof(buf))) { + it->scalar = an_dup(buf); + return; + } + TSNode m = an_bare(node); + if (!an_type(m, "block_mapping") && !an_type(m, "flow_mapping")) { + return; + } + int cap = 0; + int vcap = 0; + for (uint32_t i = 0; i < ts_node_named_child_count(m); i++) { + TSNode pair = ts_node_named_child(m, i); + char k[CBM_SZ_128]; + if (!an_type(pair, "block_mapping_pair") && !an_type(pair, "flow_pair")) { + continue; + } + if (!an_scalar(a, ts_node_child_by_field_name(pair, "key", sizeof("key") - SKIP_ONE), k, + sizeof(k)) || + !an_scalar(a, ts_node_child_by_field_name(pair, "value", sizeof("value") - SKIP_ONE), + buf, sizeof(buf))) { + continue; + } + char **keys = an_grow(it->keys, &cap, it->n, sizeof(char *)); + if (keys) { + it->keys = keys; + } + char **vals = keys ? an_grow(it->vals, &vcap, it->n, sizeof(char *)) : NULL; + if (!keys || !vals) { + return; + } + it->vals = vals; + it->keys[it->n] = an_dup(k); + it->vals[it->n++] = an_dup(buf); + } +} + +/* A loop value: an expression (a scalar holding {{ }}, or a list of exactly + * one -- with_items flattens one level), else a literal list of items. */ +static void an_loop_parse(const an_t *a, TSNode v, an_loop_t *out) { + char buf[CBM_SZ_1K]; + *out = (an_loop_t){0}; + if (an_scalar(a, v, buf, sizeof(buf))) { + out->expr = strstr(buf, "{{") ? an_dup(buf) : NULL; + return; + } + TSNode s = an_bare(v); + bool block = an_type(s, "block_sequence"); + if (!block && !an_type(s, "flow_sequence")) { + return; + } + int cap = 0; + for (uint32_t i = 0; i < ts_node_named_child_count(s); i++) { + TSNode item = ts_node_named_child(s, i); + if (block ? !an_type(item, "block_sequence_item") : !an_type(item, "flow_node")) { + continue; + } + if (block) { + item = ts_node_named_child(item, 0); + } + an_item_t *grown = an_grow(out->items, &cap, out->n, sizeof(an_item_t)); + if (!grown) { + return; + } + out->items = grown; + an_item_parse(a, item, &out->items[out->n++]); + } + if (out->n == SKIP_ONE && out->items[0].scalar && strstr(out->items[0].scalar, "{{")) { + out->expr = out->items[0].scalar; + out->items[0].scalar = NULL; + cbm_free(CBM_MEM_CLASS_OTHER, out->items); + out->items = NULL; + out->n = 0; + } +} + +static void an_loop_free(an_loop_t *l) { + for (int i = 0; i < l->n; i++) { + an_item_t *it = &l->items[i]; + for (int k = 0; k < it->n; k++) { + cbm_free(CBM_MEM_CLASS_OTHER, it->keys[k]); + cbm_free(CBM_MEM_CLASS_OTHER, it->vals[k]); + } + cbm_free(CBM_MEM_CLASS_OTHER, it->keys); + cbm_free(CBM_MEM_CLASS_OTHER, it->vals); + cbm_free(CBM_MEM_CLASS_OTHER, it->scalar); + } + cbm_free(CBM_MEM_CLASS_OTHER, l->items); + cbm_free(CBM_MEM_CLASS_OTHER, l->expr); + *l = (an_loop_t){0}; +} + +/* The task's loop: with_items, else loop. */ +static void an_task_loop(const an_t *a, TSNode m, an_loop_t *out) { + TSNode v = an_get(a, m, "with_items"); + an_loop_parse(a, ts_node_is_null(v) ? an_get(a, m, "loop") : v, out); +} + +/* What task `m` (Function `id`) renders, applies or sets, for an_link_applies. */ +static void an_collect(an_t *a, TSNode m, int64_t id) { + char k[CBM_SZ_128]; + char v[CBM_SZ_1K]; + for (uint32_t i = 0; i < ts_node_named_child_count(m); i++) { + TSNode pair = ts_node_named_child(m, i); + if (!an_key(a, pair, k, sizeof(k))) { + continue; + } + TSNode value = ts_node_child_by_field_name(pair, "value", sizeof("value") - SKIP_ONE); + bool copy = strcmp(k, "copy") == 0; + if (strcmp(k, "template") == 0 || copy) { + TSNode args = an_bare(value); + if (!an_type(args, "block_mapping") && !an_type(args, "flow_mapping")) { + continue; + } + if (copy && an_true(a, an_get(a, value, "remote_src"))) { + continue; + } + an_render_t *r = an_grow(a->renders, &a->render_cap, a->render_n, sizeof(*r)); + if (!r) { + continue; + } + a->renders = r; + r = &a->renders[a->render_n++]; + *r = (an_render_t){.copy = copy}; + r->src = an_scalar(a, an_get(a, value, "src"), v, sizeof(v)) ? an_dup(v) : NULL; + r->dest = an_scalar(a, an_get(a, value, "dest"), v, sizeof(v)) ? an_dup(v) : NULL; + r->reg = an_scalar(a, an_get(a, m, "register"), v, sizeof(v)) ? an_dup(v) : NULL; + an_task_loop(a, m, &r->loop); + } else if (strcmp(k, "set_fact") == 0) { + TSNode f = an_bare(value); + for (uint32_t j = 0; an_type(f, "block_mapping") && j < ts_node_named_child_count(f); + j++) { + TSNode fp = ts_node_named_child(f, j); + char name[CBM_SZ_128]; + TSNode fv = ts_node_child_by_field_name(fp, "value", sizeof("value") - SKIP_ONE); + TSNode seq = an_bare(fv); + an_fact_t *g = + (an_type(fp, "block_mapping_pair") && an_key(a, fp, name, sizeof(name)) && + (an_type(seq, "block_sequence") || an_type(seq, "flow_sequence"))) + ? an_grow(a->facts, &a->fact_cap, a->fact_n, sizeof(*g)) + : NULL; + if (g) { + a->facts = g; + a->facts[a->fact_n].name = an_dup(name); + an_loop_parse(a, fv, &a->facts[a->fact_n++].list); + } + } + } else { + for (size_t j = 0; j < sizeof(AN_APPLIERS) / sizeof(AN_APPLIERS[0]); j++) { + if (strcmp(k, AN_APPLIERS[j]) != 0) { + continue; + } + an_apply_t *p = an_grow(a->applies, &a->apply_cap, a->apply_n, sizeof(*p)); + if (!p) { + break; + } + a->applies = p; + p = &a->applies[a->apply_n++]; + *p = (an_apply_t){.task = id, + .line = (int)ts_node_start_point(m).row + SKIP_ONE, + .module = an_dup(k)}; + if (an_scalar(a, an_get(a, value, "template"), v, sizeof(v))) { + p->tmpl = an_dup(v); + } else if (an_scalar(a, an_get(a, value, "filename"), v, sizeof(v)) || + an_scalar(a, an_get(a, value, "src"), v, sizeof(v))) { + p->fname = an_dup(v); + } + an_task_loop(a, m, &p->loop); + break; + } + } + } +} + +/* The trimmed inside of a {{ }}, `n` bytes at `b`: "item" (key length 0) or + * "item.K", K an identifier (into key / kl); false for anything else. */ +static bool an_item_ref(const char *b, size_t n, const char **key, size_t *kl) { + static const char ITEM[] = "item"; + size_t il = sizeof(ITEM) - SKIP_ONE; + if (n < il || strncmp(b, ITEM, il) != 0) { + return false; + } + *key = b + il; + *kl = 0; + if (n == il) { + return true; + } + if (b[il] != '.' || n == il + SKIP_ONE) { + return false; + } + *key = b + il + SKIP_ONE; + *kl = n - il - SKIP_ONE; + for (size_t i = 0; i < *kl; i++) { + char c = (*key)[i]; + if (!((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' || + (i > 0 && c >= '0' && c <= '9'))) { + return false; + } + } + return true; +} + +/* `pattern` with each {{ item }} / {{ item.K }} replaced from `it`; false + * when one cannot be, or -- `strict` -- when any other {{ }} is left. */ +static bool an_subst(const char *pattern, const an_item_t *it, bool strict, char *out, size_t sz) { + size_t w = 0; + for (const char *p = pattern; p && *p;) { + const char *end = p[0] == '{' && p[SKIP_ONE] == '{' ? strstr(p + PAIR_LEN, "}}") : NULL; + const char *piece = p; + size_t len = SKIP_ONE; + if (end) { + const char *b = p + PAIR_LEN; + while (*b == ' ' || *b == '\t') { + b++; + } + size_t n = (size_t)(end - b); /* b stops at end's '}' at the latest */ + while (n > 0 && (b[n - SKIP_ONE] == ' ' || b[n - SKIP_ONE] == '\t')) { + n--; + } + const char *key = NULL; + size_t kl = 0; + if (an_item_ref(b, n, &key, &kl)) { + piece = kl == 0 ? it->scalar : NULL; + for (int k = 0; kl > 0 && k < it->n && !piece; k++) { + if (strlen(it->keys[k]) == kl && strncmp(it->keys[k], key, kl) == 0) { + piece = it->vals[k]; + } + } + if (!piece) { + return false; + } + len = strlen(piece); + } else if (strict) { + return false; + } else { + len = (size_t)(end + PAIR_LEN - p); + } + p = end + PAIR_LEN; + } else { + p++; + } + if (w + len >= sz) { + return false; + } + memcpy(out + w, piece, len); + w += len; + } + out[w] = '\0'; + return true; +} + +/* The repository file a template / copy `src` names: in the role's + * templates/ (files/ for a copy), else the task file's directory. */ +static const cbm_gbuf_node_t *an_src_file(const an_t *a, const char *src, bool copy) { + char base[CBM_SZ_1K]; + char path[CBM_SZ_1K]; + if (!src || strstr(src, "{{")) { + return NULL; + } + if (a->role_dir[0]) { + snprintf(base, sizeof(base), "%s/%s", a->role_dir, copy ? "files" : "templates"); + const cbm_gbuf_node_t *n = cbm_repo_rel_join(base, src, path, sizeof(path)) + ? cbm_ht_get(a->file_nodes, path) + : NULL; + if (n) { + return n; + } + } + return cbm_repo_rel_join(a->dir, src, path, sizeof(path)) ? cbm_ht_get(a->file_nodes, path) + : NULL; +} + +/* A top-level list `name` of the role's defaults/ or vars/ main.yml, into + * `out`; false when none holds one. */ +static bool an_role_list(an_t *a, const char *name, an_loop_t *out) { + static const char *const FILES[] = {"defaults/main.yml", "defaults/main.yaml", "vars/main.yml", + "vars/main.yaml"}; + bool found = false; + for (size_t f = 0; a->role_dir[0] && !found && f < sizeof(FILES) / sizeof(FILES[0]); f++) { + char rel[CBM_SZ_1K]; + snprintf(rel, sizeof(rel), "%s/%s", a->role_dir, FILES[f]); + intptr_t ix = (intptr_t)cbm_ht_get(a->by_rel, rel); + size_t len = 0; + char *src = ix > 0 ? an_read(a->files[ix - SKIP_ONE].path, &len) : NULL; + TSTree *tree = src ? ts_parser_parse_string(a->parser, NULL, src, (uint32_t)len) : NULL; + TSNode top = + tree ? an_bare(an_child(an_child(ts_tree_root_node(tree), "document"), "block_node")) + : (TSNode){0}; + const char *saved = a->src; + a->src = src; + for (uint32_t i = 0; + an_type(top, "block_mapping") && !found && i < ts_node_named_child_count(top); i++) { + TSNode pair = ts_node_named_child(top, i); + char k[CBM_SZ_128]; + TSNode v = ts_node_child_by_field_name(pair, "value", sizeof("value") - SKIP_ONE); + TSNode seq = an_bare(v); + if (an_key(a, pair, k, sizeof(k)) && strcmp(k, name) == 0 && + (an_type(seq, "block_sequence") || an_type(seq, "flow_sequence"))) { + an_loop_parse(a, v, out); + found = true; + } + } + a->src = saved; + if (tree) { + ts_tree_delete(tree); + } + cbm_free(CBM_MEM_CLASS_OTHER, src); + } + return found; +} + +/* An expression "{{ X }}" with X an identifier, then `tail` ("" or + * ".results"): X into `out`. */ +static bool an_expr_name(const char *expr, const char *tail, char *out, size_t sz) { + const char *b = expr && expr[0] == '{' && expr[SKIP_ONE] == '{' ? expr + PAIR_LEN : NULL; + size_t n = 0; + while (b && (*b == ' ' || *b == '\t')) { + b++; + } + while (b && n + SKIP_ONE < sz && + ((*b >= 'a' && *b <= 'z') || (*b >= 'A' && *b <= 'Z') || *b == '_' || + (n > 0 && *b >= '0' && *b <= '9'))) { + out[n++] = *b++; + } + out[n] = '\0'; + size_t tl = strlen(tail); + if (!n || !b || strncmp(b, tail, tl) != 0) { + return false; + } + for (b += tl; *b == ' ' || *b == '\t'; b++) {} + return strcmp(b, "}}") == 0; +} + +/* "{{ R.results }}": R into `out`. */ +static bool an_results_of(const char *expr, char *out, size_t sz) { + return an_expr_name(expr, ".results", out, sz); +} + +/* Does `s` hold a {{ item.K }} reference (one key level)? */ +static bool an_has_item_key(const char *s) { + for (const char *p = s ? strstr(s, "{{") : NULL; p; p = strstr(p + PAIR_LEN, "{{")) { + const char *b = p + PAIR_LEN; + while (*b == ' ' || *b == '\t') { + b++; + } + if (strncmp(b, "item.", sizeof("item.") - SKIP_ONE) != 0) { + continue; + } + b += sizeof("item.") - SKIP_ONE; + const char *k = b; + while ((*b >= 'a' && *b <= 'z') || (*b >= 'A' && *b <= 'Z') || *b == '_') { + b++; + } + while (b > k && (*b == ' ' || *b == '\t')) { + b++; + } + if (b > k && b[0] == '}' && b[SKIP_ONE] == '}') { + return true; + } + } + return false; +} + +/* The items a loop goes over: its literal list, or the list "{{ V }}" names + * -- a set_fact of the file, else the role's defaults / vars. `scratch` + * holds a list read from the role (the caller frees it). */ +static const an_loop_t *an_items(an_t *a, const an_loop_t *l, an_loop_t *scratch) { + if (!l->expr) { + return l->items ? l : NULL; + } + char name[CBM_SZ_128]; + if (!an_expr_name(l->expr, "", name, sizeof(name))) { + return NULL; + } + for (int i = 0; i < a->fact_n; i++) { + if (strcmp(a->facts[i].name, name) == 0) { + return &a->facts[i].list; + } + } + return an_role_list(a, name, scratch) ? scratch : NULL; +} + +static void an_deploy(an_t *a, const an_apply_t *p, const cbm_gbuf_node_t *target) { + if (!target || p->task <= 0) { + return; + } + char em[CBM_SZ_256]; + char props[CBM_SZ_512]; + cbm_json_escape(em, (int)sizeof(em), p->module); + snprintf(props, sizeof(props), "{\"via\":\"ansible\",\"module\":\"%s\",\"line\":%d}", em, + p->line); + cbm_gbuf_insert_edge(a->ctx->gbuf, p->task, target->id, "DEPLOYS", props); + a->deploys++; +} + +/* The renderer `src` of each item of `items`, substituted (strict). */ +static int an_deploy_items(an_t *a, const an_apply_t *p, const an_render_t *r, + const an_loop_t *items) { + int made = 0; + char path[CBM_SZ_1K]; + for (int i = 0; r && items && i < items->n; i++) { + const cbm_gbuf_node_t *t = + r->src && an_subst(r->src, &items->items[i], true, path, sizeof(path)) + ? an_src_file(a, path, r->copy) + : NULL; + if (t) { + an_deploy(a, p, t); + made++; + } + } + return made; +} + +/* The first renderer (templates before copies, each in file order) whose + * `dest` -- or, with `reg`, whose register -- is the very string. */ +static const an_render_t *an_render_find(const an_t *a, const char *dest, const char *reg) { + for (int pass = 0; pass < PAIR_LEN; pass++) { + for (int i = 0; i < a->render_n; i++) { + const an_render_t *r = &a->renders[i]; + const char *v = reg ? r->reg : r->dest; + if (r->copy == (pass == SKIP_ONE) && v && strcmp(v, reg ? reg : dest) == 0) { + return r; + } + } + } + return NULL; +} + +/* kube / k8s -> DEPLOYS {via: ansible, module, line} the repository file it + * applies, traced to the template (or copied file) the play rendered it from: + * a k8s template: argument; a renderer whose dest is the filename (D); for + * {{ item.item.K }} over "{{ R.results }}", each item of the renderer + * registering R (L); for {{ item.K }}, each of the task's own items through + * the renderer of that dest (S); a literal filename a looped renderer's dest + * renders to (R). */ +static void an_link_applies(an_t *a) { + for (int i = 0; i < a->apply_n; i++) { + const an_apply_t *p = &a->applies[i]; + an_loop_t scratch = {0}; + int made = 0; + if (p->tmpl) { + an_deploy(a, p, an_src_file(a, p->tmpl, false)); + continue; + } + if (!p->fname) { + continue; + } + const an_render_t *r = an_render_find(a, p->fname, NULL); + const cbm_gbuf_node_t *d = r ? an_src_file(a, r->src, r->copy) : NULL; + if (d) { + an_deploy(a, p, d); + continue; + } + char reg[CBM_SZ_128]; + if (strstr(p->fname, "item.item") && an_results_of(p->loop.expr, reg, sizeof(reg))) { + const an_render_t *g = an_render_find(a, NULL, reg); + made = g ? an_deploy_items(a, p, g, an_items(a, &g->loop, &scratch)) : 0; + } + if (!made && an_has_item_key(p->fname) && (p->loop.expr || p->loop.items)) { + an_loop_free(&scratch); + made = an_deploy_items(a, p, r, an_items(a, &p->loop, &scratch)); + } + if (!made && !strstr(p->fname, "{{ item")) { + char path[CBM_SZ_1K]; + for (int j = 0; j < a->render_n; j++) { + const an_render_t *x = &a->renders[j]; + an_loop_free(&scratch); + const an_loop_t *items = an_items(a, &x->loop, &scratch); + for (int k = 0; items && x->dest && k < items->n; k++) { + if (an_subst(x->dest, &items->items[k], false, path, sizeof(path)) && + strcmp(path, p->fname) == 0 && x->src && + an_subst(x->src, &items->items[k], true, path, sizeof(path))) { + an_deploy(a, p, an_src_file(a, path, x->copy)); + } + } + } + } + an_loop_free(&scratch); + } +} + +/* The per-file render / apply / set_fact records, freed. */ +static void an_applies_reset(an_t *a) { + for (int i = 0; i < a->render_n; i++) { + cbm_free(CBM_MEM_CLASS_OTHER, a->renders[i].src); + cbm_free(CBM_MEM_CLASS_OTHER, a->renders[i].dest); + cbm_free(CBM_MEM_CLASS_OTHER, a->renders[i].reg); + an_loop_free(&a->renders[i].loop); + } + for (int i = 0; i < a->apply_n; i++) { + cbm_free(CBM_MEM_CLASS_OTHER, a->applies[i].module); + cbm_free(CBM_MEM_CLASS_OTHER, a->applies[i].fname); + cbm_free(CBM_MEM_CLASS_OTHER, a->applies[i].tmpl); + an_loop_free(&a->applies[i].loop); + } + for (int i = 0; i < a->fact_n; i++) { + cbm_free(CBM_MEM_CLASS_OTHER, a->facts[i].name); + an_loop_free(&a->facts[i].list); + } + a->render_n = 0; + a->apply_n = 0; + a->fact_n = 0; +} + +static void an_index_handler(an_t *a, const char *name, int64_t id) { + char key[CBM_SZ_2K]; + if (a->role_dir[0]) { + snprintf(key, sizeof(key), "%s\x1f%s", a->role_dir, name); + an_ids_add(a->handlers, key, id); + } + snprintf(key, sizeof(key), "F:%s\x1f%s", a->file, name); + an_ids_add(a->handlers, key, id); +} + +static void an_note(an_t *a, int64_t task, const char *name, int line) { + if (a->note_n == a->note_cap) { + int cap = a->note_cap ? a->note_cap * PAIR_LEN : AN_GROW_MIN; + an_notify_t *grown = + cbm_realloc(CBM_MEM_CLASS_OTHER, a->notes, (size_t)cap * sizeof(*grown)); + if (!grown) { + return; + } + a->notes = grown; + a->note_cap = cap; + } + a->notes[a->note_n++] = + (an_notify_t){.task = task, + .scope_role = cbm_mem_strdup(CBM_MEM_CLASS_OTHER, a->role_dir), + .scope_file = cbm_mem_strdup(CBM_MEM_CLASS_OTHER, a->file), + .name = cbm_mem_strdup(CBM_MEM_CLASS_OTHER, name), + .line = line}; +} + +/* Task (or handler) `m`: a Function named by its `name:` (else its module), + * QN ::task/ (handler/), {source: ansible, kind, module}; the + * file DEFINES it. A handler is indexed by its name and listen topics; a + * notify waits for every handler (an_link_notifies). */ +static void an_task(an_t *a, TSNode m, bool handler) { + char module[CBM_SZ_128] = ""; + for (uint32_t i = 0; !module[0] && i < ts_node_named_child_count(m); i++) { + char k[CBM_SZ_128]; + if (an_key(a, ts_node_named_child(m, i), k, sizeof(k)) && !an_keyword(k)) { + snprintf(module, sizeof(module), "%s", k); + } + } + char action[CBM_SZ_256]; + if (!module[0] && (an_scalar(a, an_get(a, m, "action"), action, sizeof(action)) || + an_scalar(a, an_get(a, m, "local_action"), action, sizeof(action)))) { + snprintf(module, sizeof(module), "%.*s", (int)strcspn(action, " \t"), action); + } + char name[CBM_SZ_512]; + bool named = an_scalar(a, an_get(a, m, "name"), name, sizeof(name)); + if (!named) { + snprintf(name, sizeof(name), "%s", module[0] ? module : "task"); + } + const char *kind = handler ? "handler" : "task"; + int ord = ++a->ordinal[handler ? SKIP_ONE : 0]; + char qn[CBM_SZ_2K + CBM_SZ_64]; + char em[CBM_SZ_512]; + char props[CBM_SZ_1K]; + snprintf(qn, sizeof(qn), "%s::%s/%d", a->base, kind, ord); + cbm_json_escape(em, (int)sizeof(em), module); + snprintf(props, sizeof(props), "{\"source\":\"ansible\",\"kind\":\"%s\",\"module\":\"%s\"}", + kind, em); + int64_t id = cbm_gbuf_upsert_node(a->ctx->gbuf, "Function", name, qn, a->file, + (int)ts_node_start_point(m).row + SKIP_ONE, + (int)ts_node_end_point(m).row + SKIP_ONE, props); + if (id <= 0) { + return; + } + cbm_gbuf_insert_edge(a->ctx->gbuf, a->from, id, "DEFINES", "{}"); + a->tasks++; + an_collect(a, m, id); + TSNode list; + char n[CBM_SZ_256]; + if (handler) { + if (named) { + an_index_handler(a, name, id); + } + TSNode listen = an_get(a, m, "listen"); + uint32_t ln = an_name_count(a, listen, &list); + for (uint32_t i = 0; i < ln; i++) { + if (an_name_at(a, listen, list, i, n, sizeof(n))) { + an_index_handler(a, n, id); + } + } + } + for (uint32_t i = 0; i < ts_node_named_child_count(m); i++) { + TSNode pair = ts_node_named_child(m, i); + char k[CBM_SZ_64]; + if (!an_key(a, pair, k, sizeof(k)) || strcmp(k, "notify") != 0) { + continue; + } + TSNode v = ts_node_child_by_field_name(pair, "value", sizeof("value") - SKIP_ONE); + uint32_t nn = an_name_count(a, v, &list); + for (uint32_t j = 0; j < nn; j++) { + if (an_name_at(a, v, list, j, n, sizeof(n)) && !strstr(n, "{{")) { + an_note(a, id, n, (int)ts_node_start_point(pair).row + SKIP_ONE); + } + } + } +} + +/* The tasks of a task list, depth first: a `block:` item walks its block, + * rescue and always lists before the next item (Ansible's order). */ +static void an_task_list(an_t *a, TSNode seq, bool handler) { + static const char *const PARTS[] = {"always", "rescue", "block"}; /* pushed last-first */ + an_frame_t *stack = NULL; + int depth = 0; + int cap = 0; + TSNode push[sizeof(PARTS) / sizeof(PARTS[0]) + SKIP_ONE]; + int pn = 0; + push[pn++] = seq; + for (;;) { + for (int p = 0; p < pn; p++) { + if (ts_node_is_null(push[p])) { + continue; + } + if (depth == cap) { + int ncap = cap ? cap * PAIR_LEN : AN_GROW_MIN; + an_frame_t *grown = + cbm_realloc(CBM_MEM_CLASS_OTHER, stack, (size_t)ncap * sizeof(*grown)); + if (!grown) { + cbm_free(CBM_MEM_CLASS_OTHER, stack); + return; + } + stack = grown; + cap = ncap; + } + stack[depth++] = (an_frame_t){.seq = push[p], .next = 0, .handler = handler}; + } + pn = 0; + if (depth == 0) { + break; + } + an_frame_t *f = &stack[depth - SKIP_ONE]; + if (f->next >= ts_node_named_child_count(f->seq)) { + depth--; + continue; + } + TSNode m = an_item(f->seq, f->next++); + TSNode block = an_seq(an_get(a, m, "block")); + if (ts_node_is_null(m)) { + continue; + } + if (ts_node_is_null(block)) { + an_task(a, m, f->handler); + continue; + } + for (size_t p = 0; p < sizeof(PARTS) / sizeof(PARTS[0]); p++) { + push[pn++] = an_seq(an_get(a, m, PARTS[p])); + } + } + cbm_free(CBM_MEM_CLASS_OTHER, stack); +} + +/* The tasks of an Ansible file: a playbook's plays (pre_tasks, tasks, + * post_tasks, handlers), else its own items -- handlers under a handlers/ + * directory. */ +static void an_file_tasks(an_t *a, TSNode top, bool playbook) { + static const char *const SECTIONS[] = {"pre_tasks", "tasks", "post_tasks", "handlers"}; + if (!playbook) { + bool handlers = strncmp(a->file, "handlers/", sizeof("handlers/") - SKIP_ONE) == 0 || + strstr(a->file, "/handlers/") != NULL; + an_task_list(a, top, handlers); + return; + } + for (uint32_t i = 0; i < ts_node_named_child_count(top); i++) { + TSNode play = an_item(top, i); + for (size_t s = 0; !ts_node_is_null(play) && s < sizeof(SECTIONS) / sizeof(SECTIONS[0]); + s++) { + an_task_list(a, an_seq(an_get(a, play, SECTIONS[s])), + strcmp(SECTIONS[s], "handlers") == 0); + } + } +} + +/* notify -> CALLS {via: notify, line} each handler of that name or listen + * topic in the task's role, else in its file (a play's handlers). */ +static void an_link_notifies(an_t *a) { + for (int i = 0; i < a->note_n; i++) { + const an_notify_t *n = &a->notes[i]; + char key[CBM_SZ_2K]; + const an_ids_t *ids = NULL; + if (n->scope_role && n->scope_role[0]) { + snprintf(key, sizeof(key), "%s\x1f%s", n->scope_role, n->name); + ids = cbm_ht_get(a->handlers, key); + } + if (!ids) { + snprintf(key, sizeof(key), "F:%s\x1f%s", n->scope_file, n->name); + ids = cbm_ht_get(a->handlers, key); + } + char props[CBM_SZ_64]; + snprintf(props, sizeof(props), "{\"via\":\"notify\",\"line\":%d}", n->line); + for (int j = 0; ids && j < ids->count; j++) { + if (ids->items[j] != n->task) { + cbm_gbuf_insert_edge(a->ctx->gbuf, n->task, ids->items[j], "CALLS", props); + a->calls++; + } + } + } +} + +/* path -> node index of one label's nodes. */ +static CBMHashTable *an_index(cbm_pipeline_ctx_t *ctx, const char *label) { + CBMHashTable *ix = cbm_ht_create(0); + const cbm_gbuf_node_t **nodes = NULL; + int n = 0; + if (ix && cbm_gbuf_find_by_label(ctx->gbuf, label, &nodes, &n) == 0) { + for (int i = 0; i < n; i++) { + if (nodes[i]->file_path) { + cbm_ht_set(ix, nodes[i]->file_path, (void *)nodes[i]); + } + } + } + return ix; +} + +static bool an_is_yaml(const char *rel) { + const char *dot = strrchr(rel, '.'); + return dot && (strcmp(dot, ".yml") == 0 || strcmp(dot, ".yaml") == 0); +} + +bool cbm_ansible_is_sequence(const char *src) { + return an_is_sequence(src); +} + +/* A YAML file whose top level is a sequence: Ansible when it is a playbook, + * a role's tasks / handlers file, or included by an Ansible file. */ +typedef struct { + int file; /* index into files[] */ + bool playbook; + bool ansible; + int inc_lo; /* the task files it includes: incs[inc_lo, inc_hi) */ + int inc_hi; +} an_cand_t; + +typedef struct { + an_cand_t *items; + int count; + int cap; +} an_cands_t; + +static an_cand_t *an_cand_push(an_cands_t *c, int file, int inc_lo) { + if (c->count == c->cap) { + int cap = c->cap ? c->cap * PAIR_LEN : AN_GROW_MIN; + an_cand_t *grown = cbm_realloc(CBM_MEM_CLASS_OTHER, c->items, (size_t)cap * sizeof(*grown)); + if (!grown) { + return NULL; + } + c->items = grown; + c->cap = cap; + } + c->items[c->count] = (an_cand_t){.file = file, .inc_lo = inc_lo, .inc_hi = inc_lo}; + return &c->items[c->count++]; +} + +/* The per-file state both passes start from. */ +static void an_begin_file(an_t *a, const char *rel, const cbm_gbuf_node_t *fnode, const char *src) { + static const char FILE_SUFFIX[] = ".__file__"; + const char *slash = strrchr(rel, '/'); + a->src = src; + a->file = rel; + a->from = fnode->id; + snprintf(a->dir, sizeof(a->dir), "%.*s", slash ? (int)(slash - rel) : 0, rel); + an_role_of(rel, a->role, sizeof(a->role)); + an_role_dir(rel, a->role_dir, sizeof(a->role_dir)); + const char *qn = fnode->qualified_name ? fnode->qualified_name : ""; + size_t ql = strlen(qn); + size_t sl = sizeof(FILE_SUFFIX) - SKIP_ONE; + bool suffixed = ql > sl && strcmp(qn + ql - sl, FILE_SUFFIX) == 0; + snprintf(a->base, sizeof(a->base), "%.*s", (int)(suffixed ? ql - sl : ql), qn); + a->ordinal[0] = 0; + a->ordinal[SKIP_ONE] = 0; +} + +/* Which candidates are Ansible: playbooks and role tasks / handlers files, + * then -- to a fixpoint, by a worklist -- what an Ansible file includes. */ +static void an_mark_ansible(const an_t *a, an_cands_t *c, const cbm_file_info_t *files) { + CBMHashTable *by_path = cbm_ht_create(0); + int *work = cbm_calloc(CBM_MEM_CLASS_OTHER, ((size_t)c->count + SKIP_ONE) * sizeof(int)); + if (!by_path || !work) { + cbm_free(CBM_MEM_CLASS_OTHER, work); + if (by_path) { + cbm_ht_free(by_path); + } + return; + } + int top = 0; + for (int i = 0; i < c->count; i++) { + const char *rel = files[c->items[i].file].rel_path; + char role[CBM_SZ_1K]; + an_role_dir(rel, role, sizeof(role)); + cbm_ht_set(by_path, rel, (void *)(intptr_t)(i + SKIP_ONE)); + if (c->items[i].playbook || role[0]) { + c->items[i].ansible = true; + work[top++] = i; + } + } + while (top > 0) { + const an_cand_t *from = &c->items[work[--top]]; + for (int k = from->inc_lo; k < from->inc_hi; k++) { + int j = + a->incs[k] ? (int)(intptr_t)cbm_ht_get(by_path, a->incs[k]) - SKIP_ONE : -SKIP_ONE; + if (j >= 0 && !c->items[j].ansible) { + c->items[j].ansible = true; + work[top++] = j; + } + } + } + cbm_free(CBM_MEM_CLASS_OTHER, work); + cbm_ht_free(by_path); +} + +static void an_free_ids_entry(const char *key, void *value, void *ud) { + (void)ud; + an_ids_t *v = value; + if (v) { + cbm_free(CBM_MEM_CLASS_OTHER, v->items); + cbm_free(CBM_MEM_CLASS_OTHER, v); + } + cbm_free(CBM_MEM_CLASS_OTHER, (void *)key); +} + +static void an_free(an_t *a) { + if (a->modules) { + cbm_ht_free(a->modules); + } + if (a->folders) { + cbm_ht_free(a->folders); + } + for (int i = 0; i < a->inc_n; i++) { + cbm_free(CBM_MEM_CLASS_OTHER, a->incs[i]); + } + cbm_free(CBM_MEM_CLASS_OTHER, a->incs); + for (int i = 0; i < a->note_n; i++) { + cbm_free(CBM_MEM_CLASS_OTHER, a->notes[i].scope_role); + cbm_free(CBM_MEM_CLASS_OTHER, a->notes[i].scope_file); + cbm_free(CBM_MEM_CLASS_OTHER, a->notes[i].name); + } + cbm_free(CBM_MEM_CLASS_OTHER, a->notes); + if (a->handlers) { + cbm_ht_foreach(a->handlers, an_free_ids_entry, NULL); + cbm_ht_free(a->handlers); + } + an_applies_reset(a); + cbm_free(CBM_MEM_CLASS_OTHER, a->renders); + cbm_free(CBM_MEM_CLASS_OTHER, a->applies); + cbm_free(CBM_MEM_CLASS_OTHER, a->facts); + if (a->by_rel) { + cbm_ht_free(a->by_rel); + } +} + +/* Parse `files[i]` when it is a YAML file whose top level is a sequence; + * NULL otherwise. The text stays in `*src` (the caller frees it). */ +static TSTree *an_parse(TSParser *parser, const cbm_file_info_t *f, char **src) { + size_t len = 0; + *src = an_is_yaml(f->rel_path) ? an_read(f->path, &len) : NULL; + if (!*src || !an_is_sequence(*src)) { + return NULL; + } + return ts_parser_parse_string(parser, NULL, *src, (uint32_t)len); +} + +void cbm_pipeline_pass_ansible(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, + int file_count) { + an_t a = {.ctx = ctx}; + an_cands_t cands = {0}; + TSParser *parser = ts_parser_new(); + CBMHashTable *file_nodes = NULL; + if (!parser || !ts_parser_set_language(parser, cbm_ts_language(CBM_LANG_YAML))) { + if (parser) { + ts_parser_delete(parser); + } + return; + } + /* pass 1: composition (IMPORTS); which files are playbooks, what each includes */ + for (int i = 0; i < file_count; i++) { + char *src = NULL; + TSTree *tree = an_parse(parser, &files[i], &src); + if (tree && !file_nodes) { + a.modules = an_index(ctx, "Module"); + a.folders = an_index(ctx, "Folder"); + file_nodes = an_index(ctx, "File"); + } + const cbm_gbuf_node_t *fnode = + tree && file_nodes ? cbm_ht_get(file_nodes, files[i].rel_path) : NULL; + an_cand_t *c = fnode ? an_cand_push(&cands, i, a.inc_n) : NULL; + if (c) { + an_begin_file(&a, files[i].rel_path, fnode, src); + an_walk(&a, ts_tree_root_node(tree)); + c->playbook = an_is_playbook(&a, an_top(ts_tree_root_node(tree))); + c->inc_hi = a.inc_n; + } + if (tree) { + ts_tree_delete(tree); + } + cbm_free(CBM_MEM_CLASS_OTHER, src); + } + /* pass 2: the tasks of the Ansible files (what each applies), then their notifies */ + an_mark_ansible(&a, &cands, files); + a.handlers = cands.count > 0 ? cbm_ht_create(0) : NULL; + a.by_rel = a.handlers ? cbm_ht_create(0) : NULL; + for (int i = 0; a.by_rel && i < file_count; i++) { + cbm_ht_set(a.by_rel, files[i].rel_path, (void *)(intptr_t)(i + SKIP_ONE)); + } + a.files = files; + a.file_nodes = file_nodes; + a.parser = parser; + for (int c = 0; a.handlers && c < cands.count; c++) { + const cbm_file_info_t *f = &files[cands.items[c].file]; + char *src = NULL; + TSTree *tree = cands.items[c].ansible ? an_parse(parser, f, &src) : NULL; + const cbm_gbuf_node_t *fnode = tree ? cbm_ht_get(file_nodes, f->rel_path) : NULL; + if (fnode) { + an_begin_file(&a, f->rel_path, fnode, src); + an_file_tasks(&a, an_top(ts_tree_root_node(tree)), cands.items[c].playbook); + an_link_applies(&a); + an_applies_reset(&a); + } + if (tree) { + ts_tree_delete(tree); + } + cbm_free(CBM_MEM_CLASS_OTHER, src); + } + if (a.handlers) { + an_link_notifies(&a); + } + ts_parser_delete(parser); + if (file_nodes) { + cbm_ht_free(file_nodes); + } + cbm_free(CBM_MEM_CLASS_OTHER, cands.items); + an_free(&a); + if (a.edges > 0 || a.tasks > 0) { + char e[CBM_SZ_32]; + char t[CBM_SZ_32]; + char n[CBM_SZ_32]; + char d[CBM_SZ_32]; + snprintf(e, sizeof(e), "%d", a.edges); + snprintf(t, sizeof(t), "%d", a.tasks); + snprintf(n, sizeof(n), "%d", a.calls); + snprintf(d, sizeof(d), "%d", a.deploys); + cbm_log_info("pass.ansible", "imports", e, "tasks", t, "notifies", n, "deploys", d); + } +} diff --git a/src/pipeline/pass_ci.c b/src/pipeline/pass_ci.c index c8b9a44675..2fc9e09128 100644 --- a/src/pipeline/pass_ci.c +++ b/src/pipeline/pass_ci.c @@ -27,6 +27,7 @@ #include "foundation/compat.h" #include "pipeline/pipeline_internal.h" #include "cbm.h" +#include "helpers.h" /* cbm_env_address */ #include "lang_specs.h" #include @@ -718,8 +719,13 @@ static const char *ci_unquote(const char *v, size_t *len) { return v; } +static const char *ci_interp(CBMArena *a, const char *s, const ci_env_t *env); + /* One .env line s[0, n): KEY=VALUE (an `export ` prefix, quotes around the - * value removed) appended to env; a comment, a blank or another line: none. */ + * value removed) appended to env, its ${X} references expanded from the lines + * before it as compose does (`CART_ADDR=cart:${CART_PORT}`; a single-quoted + * value stays literal, an unknown variable leaves it as written); a comment, + * a blank or another line: none. */ static void ci_dotenv_line(CBMArena *a, const char *s, size_t n, ci_env_t *env) { static const char EXPORT[] = "export "; size_t b = 0; @@ -743,9 +749,12 @@ static void ci_dotenv_line(CBMArena *a, const char *s, size_t n, ci_env_t *env) v++; } size_t vl = n - v; + bool literal = v < n && s[v] == '\''; const char *val = ci_unquote(s + v, &vl); + const char *raw = cbm_arena_strndup(a, val, vl); + const char *expanded = raw && !literal ? ci_interp(a, raw, env) : NULL; env->kv[PAIR_LEN * env->n] = cbm_arena_strndup(a, s + b, kl); - env->kv[PAIR_LEN * env->n + SKIP_ONE] = cbm_arena_strndup(a, val, vl); + env->kv[PAIR_LEN * env->n + SKIP_ONE] = expanded ? expanded : raw; env->n++; } @@ -988,35 +997,60 @@ static int ci_compose_deps(const ci_t *c, TSNode svc, const char ***out) { return n; } +/* The compose service a variable's value addresses (its .env interpolated): + * a bare service name only -- the value itself is never kept. NULL: none. */ +static const char *ci_env_host(const ci_t *c, const char *name, const char *raw, + const ci_env_t *env) { + enum { CI_DNS_LABEL = 64 }; + const char *value = raw ? ci_interp(c->a, raw, env) : NULL; + char svc[CI_DNS_LABEL]; + char ns[CI_DNS_LABEL]; + return value && cbm_env_address(name, value, svc, sizeof(svc), ns, sizeof(ns)) == SKIP_ONE + ? cbm_arena_strdup(c->a, svc) + : NULL; +} + /* The variable names a service's environment sets: a mapping's keys, a * list's "NAME=value" / "NAME" items (the name only -- a value is never - * kept), with their 1-based lines. */ -static int ci_compose_env(const ci_t *c, TSNode svc, const char ***out, const int **lines) { + * kept), with their 1-based lines and the service each value addresses. */ +static int ci_compose_env(const ci_t *c, TSNode svc, const ci_env_t *dotenv, const char ***out, + const int **lines, const char ***hosts) { int n = 0; TSNode env = ci_get(c, svc, "environment"); TSNode *items = ci_items(c, env, &n); ci_pair_t *pairs = items ? NULL : ci_pairs(c, env, &n); const char **names = cbm_arena_alloc(c->a, (size_t)(n + SKIP_ONE) * sizeof(*names)); int *rows = cbm_arena_alloc(c->a, (size_t)(n + SKIP_ONE) * sizeof(*rows)); + const char **to = cbm_arena_alloc(c->a, (size_t)(n + SKIP_ONE) * sizeof(*to)); int k = 0; - for (int i = 0; names && rows && i < n; i++) { + for (int i = 0; names && rows && to && i < n; i++) { int row = 0; const char *name = NULL; + const char *raw = NULL; if (items) { const char *s = ci_scalar(c, items[i], &row); const char *eq = s ? strchr(s, '=') : NULL; name = eq ? cbm_arena_strndup(c->a, s, (size_t)(eq - s)) : s; + raw = eq ? eq + SKIP_ONE : NULL; } else { + int vrow = 0; name = pairs[i].key; row = pairs[i].row; + raw = ts_node_is_null(pairs[i].value) ? NULL : ci_scalar(c, pairs[i].value, &vrow); + } + if (name && name[0] && !raw) { + /* a name without a value takes the project environment's (.env) */ + raw = cbm_arena_sprintf(c->a, "${%s}", name); } if (name && name[0]) { names[k] = name; + to[k] = ci_env_host(c, name, raw, dotenv); rows[k++] = row + SKIP_ONE; } } *out = names; *lines = rows; + *hosts = to; return k; } @@ -1044,7 +1078,7 @@ static void ci_compose(const ci_t *c, TSNode root, const char *dir, const ci_env .line = (int)ts_node_start_point(svc).row, .end_line = (int)ts_node_end_point(svc).row + SKIP_ONE}; s.ndeps = ci_compose_deps(c, svc, &s.deps); - s.nenv = ci_compose_env(c, svc, &s.env, &s.env_lines); + s.nenv = ci_compose_env(c, svc, env, &s.env, &s.env_lines, &s.env_hosts); c->service_fn(&s, c->ud); } } diff --git a/src/pipeline/pass_configlink.c b/src/pipeline/pass_configlink.c index 5c1b416544..d4cdf3582a 100644 --- a/src/pipeline/pass_configlink.c +++ b/src/pipeline/pass_configlink.c @@ -116,15 +116,18 @@ static const cbm_gbuf_node_t **canonical_node_copy(const cbm_gbuf_node_t *const return sorted; } -/* Infrastructure files: YAML, JSON and HCL (Terraform, OpenTofu, tfvars). A - * key_symbol pair with either side in one links by exact name and only inside - * the config file's component (user decision 2026-10-10): in 17 infra - * repositories 4 of 40 sampled key_symbol edges were right, all of them exact - * and within one directory tree; the rest were substring and cross-component - * coincidences. Other config formats keep exact + substring matching. */ +/* Files whose key_symbol pairs link by exact name and only inside the config + * file's component: infrastructure files -- YAML, JSON, HCL (Terraform, + * OpenTofu, tfvars) -- (user decision 2026-10-10: in 17 infra repositories 4 + * of 40 sampled key_symbol edges were right, all of them exact and within one + * directory tree; the rest were substring and cross-component coincidences), + * and TOML and INI files (user decision 2026-10-14: all 72 of their substring + * or cross-component edges in those repositories were coincidences, such as + * `.gitleaks.toml useDefault` <- TestConfig_UseDefault). Other config formats + * (.env, .properties) keep exact + substring matching. */ static bool cl_is_infra_path(const char *path) { - static const char *const exts[] = {".yaml", ".yml", ".json", ".tf", - ".hcl", ".tofu", ".tfvars", NULL}; + static const char *const exts[] = {".yaml", ".yml", ".json", ".tf", ".hcl", ".tofu", + ".tfvars", ".toml", ".ini", ".cfg", ".conf", NULL}; const char *dot = path ? strrchr(path, '.') : NULL; const char *slash = path ? strrchr(path, '/') : NULL; if (!dot || (slash && dot < slash)) { diff --git a/src/pipeline/pass_docker.c b/src/pipeline/pass_docker.c index 4a2d83c2a0..b845211a45 100644 --- a/src/pipeline/pass_docker.c +++ b/src/pipeline/pass_docker.c @@ -52,6 +52,7 @@ #include "cbm.h" #include "helpers.h" #include "lang_specs.h" +#include "yyjson/yyjson.h" #include #include @@ -103,6 +104,7 @@ typedef struct { typedef struct { CBMHashTable *files; /* repo path -> File node */ CBMHashTable *gomain; /* repo dir -> the `main` Function of its package */ + CBMHashTable *dirs; /* every directory holding a file (not the root) */ } dk_index_t; typedef struct dk_var { @@ -549,46 +551,100 @@ static int dk_json_array(dk_t *d, TSNode arr, const char **out, int max) { return n; } +/* The logical line at byte `at` of the source, as Docker joins it: a line + * ending in `\` (blanks after it aside) goes on in the next one; + * dk_uncomment already made comment lines inside a continuation bare + * continuations. */ +static char *dk_logical_line(dk_t *d, size_t at) { + const char *s = d->src + at; + char *out = cbm_arena_alloc(d->a, strlen(s) + SKIP_ONE); + size_t n = 0; + while (out && *s) { + const char *e = strchr(s, '\n'); + size_t l = e ? (size_t)(e - s) : strlen(s); + size_t t = l; + while (t > 0 && + (s[t - SKIP_ONE] == ' ' || s[t - SKIP_ONE] == '\t' || s[t - SKIP_ONE] == '\r')) { + t--; + } + bool more = e && t > 0 && s[t - SKIP_ONE] == '\\'; + size_t keep = more ? t - SKIP_ONE : t; + memcpy(out + n, s, keep); + n += keep; + if (!more) { + break; + } + out[n++] = ' '; + s = e + SKIP_ONE; + } + if (out) { + out[n] = '\0'; + } + return out; +} + +/* A COPY / ADD in JSON form: its strings, expanded. */ +static int dk_copy_json(dk_t *d, const char *p, const char **out, int max) { + yyjson_doc *doc = yyjson_read(p, strlen(p), YYJSON_READ_STOP_WHEN_DONE); + yyjson_val *arr = doc ? yyjson_doc_get_root(doc) : NULL; + int n = 0; + yyjson_arr_iter it; + yyjson_val *v; + if (yyjson_is_arr(arr)) { + for (yyjson_arr_iter_init(arr, &it); n < max && (v = yyjson_arr_iter_next(&it));) { + const char *s = yyjson_get_str(v); + out[n++] = s ? dk_expand(d, d->local, cbm_arena_strdup(d->a, s)) : NULL; + } + } + if (doc) { + yyjson_doc_free(doc); + } + return n; +} + +/* COPY / ADD as Docker reads it, from the instruction's own text: flags + * first, then the JSON form or whitespace-separated words (a heredoc + * source is unknown). The grammar loses a JSON form opening with `/` + * (otel accounting) and, after a word with trailing blanks, takes the + * instructions that follow for more words (awesome-compose wasmedge). */ static void dk_copy(dk_t *d, TSNode ins) { static const char FROM[] = "--from="; const char *words[DK_MAX_WORDS]; int nw = 0; dk_copy_t *c = cbm_arena_alloc(d->a, sizeof(*c)); - if (c) { - *c = (dk_copy_t){.workdir = d->workdir}; + char *p = dk_logical_line(d, ts_node_start_byte(ins)); + if (!c || !p) { + return; } - for (uint32_t i = 0; i < ts_node_named_child_count(ins); i++) { - TSNode p = ts_node_named_child(ins, i); - if (dk_type(p, "path") && nw < DK_MAX_WORDS) { - const char *t = dk_text(d, p); - words[nw++] = t ? dk_expand(d, d->local, t) : NULL; - continue; - } - if (dk_type(p, "json_string_array")) { - const char *raw[DK_MAX_WORDS]; - int n = dk_json_array(d, p, raw, DK_MAX_WORDS - nw); - for (int k = 0; k < n; k++) { - words[nw++] = raw[k] ? dk_expand(d, d->local, raw[k]) : NULL; - } - continue; - } - if (!dk_type(p, "param")) { - continue; - } - const char *t = dk_text(d, p); - if (!t || strncmp(t, FROM, sizeof(FROM) - SKIP_ONE) != 0) { - continue; + *c = (dk_copy_t){.workdir = d->workdir}; + p += strcspn(p, " \t"); /* the keyword */ + for (;;) { + p += strspn(p, " \t"); + if (strncmp(p, "--", PAIR_LEN) != 0) { + break; } - const char *ref = dk_expand(d, d->local, t + sizeof(FROM) - SKIP_ONE); - if (c) { + size_t l = strcspn(p, " \t"); + if (l > sizeof(FROM) - SKIP_ONE && strncmp(p, FROM, sizeof(FROM) - SKIP_ONE) == 0) { + const char *v = + cbm_arena_strndup(d->a, p + sizeof(FROM) - SKIP_ONE, l - (sizeof(FROM) - SKIP_ONE)); + const char *ref = v ? dk_expand(d, d->local, v) : NULL; c->has_from = true; c->from = ref; + if (ref && ref[0] && !dk_is_stage(d, ref)) { + dk_link(d, ref, "copy-from", dk_line(ins)); + } } - if (ref && ref[0] && !dk_is_stage(d, ref)) { - dk_link(d, ref, "copy-from", dk_line(ins)); - } + p += l; + } + bool json = *p == '[' && (nw = dk_copy_json(d, p, words, DK_MAX_WORDS)) > 0; + while (!json && *p && nw < DK_MAX_WORDS) { + size_t l = strcspn(p, " \t"); + const char *w = cbm_arena_strndup(d->a, p, l); + words[nw++] = w && strncmp(w, "<<", PAIR_LEN) != 0 ? dk_expand(d, d->local, w) : NULL; + p += l; + p += strspn(p, " \t"); } - if (!c || nw < PAIR_LEN || d->stage >= DK_MAX_STAGES) { + if (nw < PAIR_LEN || d->stage >= DK_MAX_STAGES) { return; } c->srcs = cbm_arena_alloc(d->a, (size_t)(nw - SKIP_ONE) * sizeof(*c->srcs)); @@ -1148,6 +1204,199 @@ static void dk_envs(dk_t *d) { cbm_ht_free(seen); } +/* ── The image's files (F1: the env join) ───────────────────────────── */ + +static char *dk_read(const char *path, size_t *len); + +enum { DK_SET_CANDS = 2 }; + +static bool dk_glob(const char *s) { + return strpbrk(s, "*?[") != NULL; +} + +/* Context source `src` as a repository path under `ctx`; NULL when it climbs + * out of the repository. */ +static const char *dk_in_ctx(dk_t *d, const char *ctx, const char *src) { + while (*src == '/') { + src++; + } + const char *p = dk_norm(d, dk_join(d, ctx, src)); + return p && strncmp(p, "..", PAIR_LEN) != 0 ? p : NULL; +} + +static bool dk_exists(const dk_t *d, const char *p) { + return !p[0] || cbm_ht_get(d->ix->files, p) || (d->ix->dirs && cbm_ht_get(d->ix->dirs, p)); +} + +/* The stages the image is made of: the last, the stages each is built FROM + * and the stages each COPYs --from, transitively. */ +static void dk_image_stages(const dk_t *d, bool *in) { + in[d->stage] = true; + for (bool grew = true; grew;) { + grew = false; + for (int s = 0; s <= d->stage; s++) { + if (!in[s]) { + continue; + } + int p = d->st[s].parent; + if (p >= 0 && p < DK_MAX_STAGES && !in[p]) { + in[p] = grew = true; + } + for (const dk_copy_t *c = d->st[s].copies; c; c = c->next) { + int f = c->has_from ? dk_stage_of(d, c->from) : -1; + if (f >= 0 && f < DK_MAX_STAGES && !in[f]) { + in[f] = grew = true; + } + } + } + } +} + +/* Calls fn(d, src, ud) for each context source (no --from, no URL) the + * image's stages COPY / ADD. */ +static void dk_image_sources(dk_t *d, const bool *in, void (*fn)(dk_t *, const char *, void *), + void *ud) { + for (int s = 0; s <= d->stage; s++) { + for (const dk_copy_t *c = in[s] ? d->st[s].copies : NULL; c; c = c->next) { + for (int k = 0; !c->has_from && k < c->nsrc; k++) { + if (c->srcs[k] && !strstr(c->srcs[k], "://")) { + fn(d, c->srcs[k], ud); + } + } + } + } +} + +typedef struct { + const char *cands[DK_SET_CANDS]; + int ncand; + int hits[DK_SET_CANDS]; +} dk_ctx_vote_t; + +/* A literal source that names something below the context votes for each + * candidate context it exists under. */ +static void dk_ctx_vote(dk_t *d, const char *src, void *ud) { + dk_ctx_vote_t *v = ud; + const char *n = dk_glob(src) ? NULL : dk_in_ctx(d, "", src); + for (int i = 0; n && n[0] && i < v->ncand; i++) { + const char *p = dk_in_ctx(d, v->cands[i], src); + v->hits[i] += p && dk_exists(d, p); + } +} + +typedef struct { + const char *ctx; + const char *json; /* the array so far, without its closing ']' */ + CBMHashTable *seen; +} dk_json_list_t; + +static void dk_json_add(dk_t *d, dk_json_list_t *l, const char *item) { + if (!item || !l->json || cbm_ht_get(l->seen, item)) { + return; + } + cbm_ht_set(l->seen, item, (void *)item); + size_t cap = strlen(item) * PAIR_LEN + PAIR_LEN; + char *esc = cbm_arena_alloc(d->a, cap); + if (!esc) { + l->json = NULL; + return; + } + cbm_json_escape(esc, (int)cap, item); + l->json = cbm_arena_sprintf(d->a, "%s%s\"%s\"", l->json, l->json[SKIP_ONE] ? "," : "", esc); +} + +static void dk_file_add(dk_t *d, const char *src, void *ud) { + dk_json_list_t *l = ud; + dk_json_add(d, l, dk_in_ctx(d, l->ctx, src)); +} + +/* The context's ignore file, as repository-path patterns ("!" kept): + * .dockerignore beside the Dockerfile first, else + * /.dockerignore. */ +static void dk_ignore_add(dk_t *d, dk_json_list_t *l) { + const char *root = d->ctx->repo_path; + if (!root) { + return; + } + char path[CBM_SZ_4K]; + size_t len = 0; + snprintf(path, sizeof(path), "%s/%s.dockerignore", root, d->file); + char *src = dk_read(path, &len); + if (!src) { + snprintf(path, sizeof(path), "%s/%s%s.dockerignore", root, l->ctx, l->ctx[0] ? "/" : ""); + src = dk_read(path, &len); + } + for (char *line = src; line && *line;) { + char *end = strchr(line, '\n'); + char *next = end ? end + SKIP_ONE : line + strlen(line); + if (end) { + *end = '\0'; + } + char *s = line; + while (*s == ' ' || *s == '\t') { + s++; + } + size_t sl = strlen(s); + while (sl > 0 && + (s[sl - SKIP_ONE] == ' ' || s[sl - SKIP_ONE] == '\t' || s[sl - SKIP_ONE] == '\r')) { + s[--sl] = '\0'; + } + bool neg = s[0] == '!'; + const char *pat = neg ? s + SKIP_ONE : s; + const char *p = s[0] && s[0] != '#' ? dk_in_ctx(d, l->ctx, pat) : NULL; + if (p && strcmp(p, l->ctx) != 0) { + dk_json_add(d, l, neg ? cbm_arena_sprintf(d->a, "!%s", p) : p); + } + line = next; + } + cbm_free(CBM_MEM_CLASS_OTHER, src); +} + +/* F1: the files the image is made of -- the context sources its stages COPY + * / ADD, resolved in the build context: the Dockerfile's directory or the + * repository root, the one more of the literal sources exist under (a tie: + * the directory) -- and the context's ignore patterns, kept on the + * Dockerfile's Module (image_context, image_files, image_ignore) for the env + * join, which reads them back in an incremental run. */ +static void dk_image_set(dk_t *d) { + cbm_gbuf_node_t *mod = (cbm_gbuf_node_t *)cbm_gbuf_find_by_id(d->ctx->gbuf, d->module); + if (!mod || d->stage < 0 || d->stage >= DK_MAX_STAGES) { + return; + } + bool in[DK_MAX_STAGES] = {false}; + dk_image_stages(d, in); + dk_ctx_vote_t v = {.cands = {d->dir, ""}, .ncand = d->dir[0] ? DK_SET_CANDS : SKIP_ONE}; + dk_image_sources(d, in, dk_ctx_vote, &v); + const char *ctx = v.ncand > SKIP_ONE && v.hits[SKIP_ONE] > v.hits[0] ? "" : d->dir; + dk_json_list_t files = {.ctx = ctx, .json = "[", .seen = cbm_ht_create(0)}; + dk_json_list_t ignore = {.ctx = ctx, .json = "[", .seen = cbm_ht_create(0)}; + if (files.seen && ignore.seen) { + dk_image_sources(d, in, dk_file_add, &files); + dk_ignore_add(d, &ignore); + } + const char *props = + mod->properties_json && mod->properties_json[0] ? mod->properties_json : "{}"; + size_t pl = strlen(props); + if (files.seen && ignore.seen && files.json && files.json[SKIP_ONE] && ignore.json && + pl >= PAIR_LEN && props[pl - SKIP_ONE] == '}') { + char cesc[CBM_SZ_4K]; + cbm_json_escape(cesc, (int)sizeof(cesc), ctx); + bool empty = strspn(props + SKIP_ONE, " \t\r\n") == pl - PAIR_LEN; + const char *out = cbm_arena_sprintf( + d->a, "%.*s%s\"image_context\":\"%s\",\"image_files\":%s],\"image_ignore\":%s]}", + (int)(pl - SKIP_ONE), props, empty ? "" : ",", cesc, files.json, ignore.json); + if (out) { + (void)cbm_gbuf_node_set_properties_json(mod, out); + } + } + if (files.seen) { + cbm_ht_free(files.seen); + } + if (ignore.seen) { + cbm_ht_free(ignore.seen); + } +} + static void dk_file(dk_t *d, TSNode root) { for (uint32_t i = 0; i < ts_node_named_child_count(root); i++) { TSNode ins = ts_node_named_child(root, i); @@ -1172,6 +1421,7 @@ static void dk_file(dk_t *d, TSNode root) { } dk_spawn(d); dk_envs(d); + dk_image_set(d); } static char *dk_read(const char *path, size_t *len) { @@ -1313,9 +1563,30 @@ static void dk_compose_edge(dk_compose_t *c, int64_t src, int64_t tgt, const cha } } +/* The services of the same file a service's environment addresses: + * CONNECTS_TO {via:env, key, container} -- the variable, never its value. */ +static void dk_compose_connects(dk_compose_t *c, const cbm_compose_service_t *s, int64_t self) { + for (int i = 0; s->env_hosts && i < s->nenv; i++) { + const char *hq = s->env_hosts[i] ? dk_compose_qn(c, s->env_hosts[i]) : NULL; + const cbm_gbuf_node_t *to = hq ? cbm_gbuf_find_by_qn(c->ctx->gbuf, hq) : NULL; + if (!to) { + continue; + } + char kesc[CBM_SZ_256]; + char sesc[CBM_SZ_256]; + cbm_json_escape(kesc, (int)sizeof(kesc), s->env[i]); + cbm_json_escape(sesc, (int)sizeof(sesc), s->name); + char props[CBM_SZ_1K]; + snprintf(props, sizeof(props), "{\"via\":\"env\",\"key\":\"%s\",\"container\":\"%s\"}", + kesc, sesc); + dk_compose_edge(c, self, to->id, "CONNECTS_TO", props); + } +} + /* A compose service: a Resource {ecosystem:compose, kind:Service} the file * DEFINES; it DEPENDS_ON the services it depends on / links, USAGE - * {ref:image} its image and DEPENDS_ON {via:build} the Dockerfile it builds. */ + * {ref:image} its image, DEPENDS_ON {via:build} the Dockerfile it builds and + * CONNECTS_TO the services its environment addresses. */ static void dk_compose_service(const cbm_compose_service_t *s, void *ud) { dk_compose_t *c = ud; const char *qn = dk_compose_qn(c, s->name); @@ -1350,6 +1621,7 @@ static void dk_compose_service(const cbm_compose_service_t *s, void *ud) { const cbm_gbuf_node_t *dep = dq ? cbm_gbuf_find_by_qn(c->ctx->gbuf, dq) : NULL; dk_compose_edge(c, self->id, dep ? dep->id : 0, "DEPENDS_ON", "{\"via\":\"depends_on\"}"); } + dk_compose_connects(c, s, self->id); if (s->image) { char esc[CBM_SZ_1K]; cbm_json_escape(esc, (int)sizeof(esc), s->image); @@ -1427,17 +1699,32 @@ static bool dk_node_before(const cbm_gbuf_node_t *a, const cbm_gbuf_node_t *b) { return c < 0 || (c == 0 && a->start_line < b->start_line); } -/* The repository's File nodes by path, and the `main` of each Go main - * package by directory (the first by file and line). */ +/* Every directory above repository path `p` into `dirs`. */ +static void dk_index_dirs(CBMArena *a, CBMHashTable *dirs, const char *p) { + for (const char *s = strchr(p, '/'); s; s = strchr(s + SKIP_ONE, '/')) { + char *dir = cbm_arena_strndup(a, p, (size_t)(s - p)); + if (!dir || cbm_ht_get(dirs, dir)) { + continue; + } + cbm_ht_set(dirs, dir, dir); + } +} + +/* The repository's File nodes by path, its directories, and the `main` of + * each Go main package by directory (the first by file and line). */ static void dk_index_build(cbm_pipeline_ctx_t *ctx, CBMArena *a, dk_index_t *ix) { ix->files = cbm_ht_create(0); ix->gomain = cbm_ht_create(0); + ix->dirs = cbm_ht_create(0); const cbm_gbuf_node_t **nodes = NULL; int n = 0; if (ix->files && cbm_gbuf_find_by_label(ctx->gbuf, "File", &nodes, &n) == 0) { for (int i = 0; i < n; i++) { if (nodes[i]->file_path) { cbm_ht_set(ix->files, nodes[i]->file_path, (void *)nodes[i]); + if (ix->dirs) { + dk_index_dirs(a, ix->dirs, nodes[i]->file_path); + } } } } @@ -1963,6 +2250,9 @@ void cbm_pipeline_pass_docker(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *fi if (ix.gomain) { cbm_ht_free(ix.gomain); } + if (ix.dirs) { + cbm_ht_free(ix.dirs); + } edges += dk_link_builds(ctx, &arena); edges += dk_image_hubs(ctx, &arena); cbm_arena_destroy(&arena); diff --git a/src/pipeline/pass_envjoin.c b/src/pipeline/pass_envjoin.c new file mode 100644 index 0000000000..203aa88a88 --- /dev/null +++ b/src/pipeline/pass_envjoin.c @@ -0,0 +1,483 @@ +/* + * pass_envjoin.c — F1: which program reads the variables a container sets. + * + * P4.5 gives each variable a container sets an EnvVar of its own + * (`#env//`); a program's readers point at the + * global EnvVar of the name (CONFIGURES {strategy:env_access}). The same name + * is read by many programs, so the join connects a setter only to the + * readers inside its container's own program -- the files its image is built + * from: + * EnvVar -CONFIGURES {strategy:env_read, dockerfile}-> reader + * The image: a K8s container's literal image (USAGE {ref:image, container}) + * built from exactly one Dockerfile (Image -DEPENDS_ON {via:build}->); a + * compose service's own build edge, else its image's; a Dockerfile's own ENV. + * Its files: the Dockerfile Module's image_files (pass_docker.c) less the + * image_ignore patterns, never a test file. Measured on 17 corpora: 80 + * pairs, 80 correct; a join through the global name was in the program for + * 23 % of the pairs. + * + * Every run recomputes every such edge. An incremental repair holds only the + * changed files' nodes and edges: the unchanged files' inputs come from the + * store (cbm_envjoin_base_t, read before the K8s pass), and the patch replaces + * the stored edges wholesale. + */ +#include "foundation/constants.h" +#include "foundation/hash_table.h" +#include "foundation/log.h" +#include "foundation/mem_core.h" +#include "foundation/str_util.h" +#include "graph_buffer/graph_buffer.h" +#include "pipeline/pipeline.h" +#include "pipeline/pipeline_internal.h" +#include "yyjson/yyjson.h" + +#include +#include +#include +#include + +/* ── Docker's path patterns ─────────────────────────────────────────── */ + +/* `[...]` at p against c; *end past the class. A malformed class matches + * nothing. */ +static bool ej_class(const char *p, char c, const char **end) { + const char *q = p + SKIP_ONE; + bool neg = *q == '^' || *q == '!'; + q += neg; + bool hit = false; + for (bool first = true; *q && (first || *q != ']'); first = false) { + char lo = *q == '\\' && q[SKIP_ONE] ? *++q : *q; + q++; + char hi = lo; + if (q[0] == '-' && q[SKIP_ONE] && q[SKIP_ONE] != ']') { + q++; + hi = *q == '\\' && q[SKIP_ONE] ? *++q : *q; + q++; + } + hit = hit || (c >= lo && c <= hi); + } + if (*q != ']') { + *end = NULL; + return false; + } + *end = q + SKIP_ONE; + return hit != neg; +} + +/* Docker's pattern syntax over a whole repository path: `**` any run (`**` + * followed by `/`: whole segments only), `*` a run inside one segment, `?` + * one character of a segment, `[...]` a class, `\` an escape. */ +static bool ej_match(const char *p, const char *s) { + while (*p) { + if (p[0] == '*' && p[SKIP_ONE] == '*') { + p += PAIR_LEN; + bool segs = *p == '/'; + p += segs; + for (const char *t = s;; t++) { + if ((!segs || t == s || t[-SKIP_ONE] == '/') && ej_match(p, t)) { + return true; + } + if (!*t) { + return false; + } + } + } + if (*p == '*') { + p++; + for (const char *t = s;; t++) { + if (ej_match(p, t)) { + return true; + } + if (!*t || *t == '/') { + return false; + } + } + } + if (!*s || (*s == '/' && (*p == '?' || *p == '['))) { + return false; + } + if (*p == '[') { + const char *end = NULL; + if (!ej_class(p, *s, &end) || !end) { + return false; + } + p = end; + s++; + continue; + } + if (*p == '\\' && p[SKIP_ONE]) { + p++; + } + if (*p != '?' && *p != *s) { + return false; + } + p++; + s++; + } + return *s == '\0'; +} + +/* Pattern `p` matches path `f` or a directory above it (a matched directory + * holds all of its files). */ +static bool ej_match_up(const char *p, const char *f) { + if (ej_match(p, f)) { + return true; + } + char buf[CBM_SZ_4K]; + for (const char *s = strchr(f, '/'); s; s = strchr(s + SKIP_ONE, '/')) { + size_t n = (size_t)(s - f); + if (n >= sizeof(buf)) { + return false; + } + memcpy(buf, f, n); + buf[n] = '\0'; + if (ej_match(p, buf)) { + return true; + } + } + return false; +} + +/* ── The image's files ──────────────────────────────────────────────── */ + +typedef struct { + const char *dockerfile; /* the Dockerfile's path */ + const char **files; /* repository paths / patterns the image copies */ + int nfiles; + const char **ignore; /* ignore patterns, "!" = an exception */ + int nignore; +} ej_set_t; + +static const char **ej_strings(CBMArena *a, yyjson_val *arr, int *n) { + *n = 0; + size_t cnt = yyjson_is_arr(arr) ? yyjson_arr_size(arr) : 0; + const char **out = cnt ? cbm_arena_alloc(a, cnt * sizeof(*out)) : NULL; + yyjson_arr_iter it; + yyjson_val *v; + for (yyjson_arr_iter_init(arr, &it); out && (v = yyjson_arr_iter_next(&it));) { + const char *s = yyjson_get_str(v); + if (s) { + out[(*n)++] = cbm_arena_strdup(a, s); + } + } + return out; +} + +/* A Dockerfile Module's image set, parsed from its properties; NULL: none. */ +static ej_set_t *ej_set_parse(CBMArena *a, const char *path, const char *props) { + if (!props || !strstr(props, "\"image_files\"")) { + return NULL; + } + yyjson_doc *doc = yyjson_read(props, strlen(props), 0); + yyjson_val *root = doc ? yyjson_doc_get_root(doc) : NULL; + ej_set_t *s = root ? cbm_arena_alloc(a, sizeof(*s)) : NULL; + if (s) { + *s = (ej_set_t){.dockerfile = cbm_arena_strdup(a, path ? path : "")}; + s->files = ej_strings(a, yyjson_obj_get(root, "image_files"), &s->nfiles); + s->ignore = ej_strings(a, yyjson_obj_get(root, "image_ignore"), &s->nignore); + } + if (doc) { + yyjson_doc_free(doc); + } + return s && s->nfiles > 0 ? s : NULL; +} + +static bool ej_in_image(const ej_set_t *s, const char *f) { + bool in = false; + for (int i = 0; i < s->nfiles && !in; i++) { + const char *e = s->files[i]; + if (strpbrk(e, "*?[")) { + in = ej_match_up(e, f); + } else { + size_t el = strlen(e); + in = el == 0 || strcmp(f, e) == 0 || (strncmp(f, e, el) == 0 && f[el] == '/'); + } + } + bool ignored = false; + for (int i = 0; in && i < s->nignore; i++) { + bool neg = s->ignore[i][0] == '!'; + if (ej_match_up(s->ignore[i] + neg, f)) { + ignored = !neg; + } + } + return in && !ignored; +} + +bool cbm_envjoin_props_hold(const char *props, const char *path) { + CBMArena a; + cbm_arena_init_lazy(&a, CBM_SZ_1K); + ej_set_t *s = ej_set_parse(&a, "", props); + bool hit = false; + if (s) { + ej_set_t files = *s; + files.nignore = 0; + hit = ej_in_image(&files, path); + } + cbm_arena_destroy(&a); + return hit; +} + +/* ── The join ───────────────────────────────────────────────────────── */ + +typedef struct { + int64_t node; + const char *file; +} ej_reader_t; + +typedef struct { + int64_t image; + const char *container; /* NULL: any (compose) */ +} ej_img_t; + +typedef struct { + cbm_pipeline_ctx_t *ctx; + CBMArena a; + CBMHashTable *sets; /* Dockerfile Module id -> ej_set_t */ + CBMHashTable *image_df; /* Image id -> Dockerfile Module id, "" when not one */ + CBMHashTable *owner_df; /* compose service id -> its own build's Dockerfile id */ + CBMHashTable *readers; /* key -> ej_reader_t[] (count in nreaders) */ + CBMHashTable *nreaders; /* key -> (intptr_t) count */ + CBMHashTable *imgs; /* owner id -> ej_img_t[] (count in nimgs) */ + CBMHashTable *nimgs; /* owner id -> (intptr_t) count */ + CBMHashTable *done; /* "setter:reader" */ + int edges; +} ej_t; + +/* Appends `item` to the array table[key] (its count in counts[key]). */ +static void ej_push(CBMArena *a, CBMHashTable *table, CBMHashTable *counts, const char *key, + const void *item, size_t size) { + intptr_t n = (intptr_t)cbm_ht_get(counts, key); + const char *old = cbm_ht_get(table, key); + char *v = cbm_arena_alloc(a, (size_t)(n + SKIP_ONE) * size); + const char *k = cbm_arena_strdup(a, key); + if (!v || !k) { + return; + } + if (n) { + memcpy(v, old, (size_t)n * size); + } + memcpy(v + (size_t)n * size, item, size); + cbm_ht_set(table, k, v); + cbm_ht_set(counts, k, (void *)(n + SKIP_ONE)); +} + +static const char *ej_id(ej_t *j, int64_t id) { + return cbm_arena_sprintf(&j->a, "%lld", (long long)id); +} + +static void ej_add_set(ej_t *j, int64_t node, const char *path, const char *props) { + const char *k = ej_id(j, node); + if (!k || cbm_ht_get(j->sets, k)) { + return; + } + ej_set_t *s = ej_set_parse(&j->a, path, props); + if (s) { + cbm_ht_set(j->sets, k, s); + } +} + +static void ej_add_img(ej_t *j, int64_t owner, int64_t image, const char *props) { + const char *c = props ? strstr(props, "\"container\":\"") : NULL; + ej_img_t im = {image, NULL}; + if (c) { + c += strlen("\"container\":\""); + const char *e = strchr(c, '"'); + im.container = e ? cbm_arena_strndup(&j->a, c, (size_t)(e - c)) : NULL; + } + const char *k = ej_id(j, owner); + if (k) { + ej_push(&j->a, j->imgs, j->nimgs, k, &im, sizeof(im)); + } +} + +static void ej_add_reader(ej_t *j, const char *key, int64_t node, const char *file) { + if (!key || !file || !file[0] || cbm_is_test_path(file)) { + return; + } + ej_reader_t r = {node, cbm_arena_strdup(&j->a, file)}; + if (r.file) { + ej_push(&j->a, j->readers, j->nreaders, key, &r, sizeof(r)); + } +} + +/* Build edges: an Image's Dockerfile (when exactly one), a compose service's + * own build. */ +static void ej_index_builds(ej_t *j) { + const cbm_gbuf_edge_t **es = NULL; + int n = 0; + if (cbm_gbuf_find_edges_by_type(j->ctx->gbuf, "DEPENDS_ON", &es, &n) != 0) { + return; + } + for (int i = 0; i < n; i++) { + const char *p = es[i]->properties_json; + if (!p || !strstr(p, "\"via\":\"build\"")) { + continue; + } + const cbm_gbuf_node_t *src = cbm_gbuf_find_by_id(j->ctx->gbuf, es[i]->source_id); + if (!src || !src->label) { + continue; + } + CBMHashTable *t = strcmp(src->label, "Image") == 0 ? j->image_df + : strcmp(src->label, "Resource") == 0 ? j->owner_df + : NULL; + const char *k = t ? ej_id(j, src->id) : NULL; + const char *df = k ? ej_id(j, es[i]->target_id) : NULL; + if (!df) { + continue; + } + const char *have = cbm_ht_get(t, k); + cbm_ht_set(t, k, (void *)(have && strcmp(have, df) != 0 ? "" : df)); + } +} + +static void ej_index_gbuf(ej_t *j) { + const cbm_gbuf_node_t **mods = NULL; + int nm = 0; + if (cbm_gbuf_find_by_label(j->ctx->gbuf, "Module", &mods, &nm) == 0) { + for (int i = 0; i < nm; i++) { + ej_add_set(j, mods[i]->id, mods[i]->file_path, mods[i]->properties_json); + } + } + const cbm_gbuf_edge_t **es = NULL; + int n = 0; + if (cbm_gbuf_find_edges_by_type(j->ctx->gbuf, "USAGE", &es, &n) == 0) { + for (int i = 0; i < n; i++) { + const char *p = es[i]->properties_json; + if (p && strstr(p, "\"ref\":\"image\"")) { + ej_add_img(j, es[i]->source_id, es[i]->target_id, p); + } + } + } + if (cbm_gbuf_find_edges_by_type(j->ctx->gbuf, "CONFIGURES", &es, &n) == 0) { + for (int i = 0; i < n; i++) { + const char *p = es[i]->properties_json; + if (!p || !strstr(p, "\"strategy\":\"env_access\"")) { + continue; + } + const cbm_gbuf_node_t *src = cbm_gbuf_find_by_id(j->ctx->gbuf, es[i]->source_id); + const cbm_gbuf_node_t *env = cbm_gbuf_find_by_id(j->ctx->gbuf, es[i]->target_id); + if (src && env) { + ej_add_reader(j, env->name, src->id, src->file_path); + } + } + } + ej_index_builds(j); +} + +static void ej_index_base(ej_t *j, const cbm_envjoin_base_t *b) { + for (int i = 0; b && i < b->set_count; i++) { + ej_add_set(j, b->sets[i].node, b->sets[i].path, b->sets[i].props); + } + for (int i = 0; b && i < b->img_count; i++) { + ej_add_img(j, b->imgs[i].owner, b->imgs[i].image, b->imgs[i].props); + } + for (int i = 0; b && i < b->reader_count; i++) { + ej_add_reader(j, b->readers[i].key, b->readers[i].reader, b->readers[i].file); + } +} + +static void ej_edge(ej_t *j, int64_t setter, const ej_reader_t *r, const ej_set_t *s) { + const char *k = cbm_arena_sprintf(&j->a, "%lld:%lld", (long long)setter, (long long)r->node); + if (!k || cbm_ht_get(j->done, k) || setter == r->node) { + return; + } + cbm_ht_set(j->done, k, (void *)k); + char esc[CBM_SZ_4K]; + cbm_json_escape(esc, (int)sizeof(esc), s->dockerfile); + char props[CBM_SZ_4K + CBM_SZ_64]; + snprintf(props, sizeof(props), "{\"strategy\":\"env_read\",\"dockerfile\":\"%s\"}", esc); + if (cbm_gbuf_insert_edge(j->ctx->gbuf, setter, r->node, "CONFIGURES", props) > 0) { + j->edges++; + } +} + +/* Join one setter with the readers of its key in Dockerfile `df`'s image. */ +static void ej_join_df(ej_t *j, int64_t setter, const char *key, const char *df) { + const ej_set_t *s = df && df[0] ? cbm_ht_get(j->sets, df) : NULL; + const ej_reader_t *rs = s ? cbm_ht_get(j->readers, key) : NULL; + intptr_t n = rs ? (intptr_t)cbm_ht_get(j->nreaders, key) : 0; + for (intptr_t i = 0; i < n; i++) { + if (ej_in_image(s, rs[i].file)) { + ej_edge(j, setter, &rs[i], s); + } + } +} + +/* A scoped EnvVar: its owner, container and key from the QN. */ +static void ej_join_setter(ej_t *j, const cbm_gbuf_node_t *ev) { + const char *mark = strstr(ev->qualified_name, "#env/"); + const char *slash = mark ? strrchr(mark, '/') : NULL; + if (!mark || !slash || slash < mark + strlen("#env/")) { + return; + } + const char *owner_qn = + cbm_arena_strndup(&j->a, ev->qualified_name, (size_t)(mark - ev->qualified_name)); + const char *cstart = mark + strlen("#env/"); + const char *container = cbm_arena_strndup(&j->a, cstart, (size_t)(slash - cstart)); + const char *key = slash + SKIP_ONE; + const cbm_gbuf_node_t *owner = owner_qn ? cbm_gbuf_find_by_qn(j->ctx->gbuf, owner_qn) : NULL; + if (!owner || !container || !key[0]) { + return; + } + const char *oid = ej_id(j, owner->id); + if (oid && cbm_ht_get(j->sets, oid)) { /* a Dockerfile's own ENV */ + ej_join_df(j, ev->id, key, oid); + return; + } + const char *own = oid ? cbm_ht_get(j->owner_df, oid) : NULL; + if (own) { /* a compose service's own build */ + ej_join_df(j, ev->id, key, own); + return; + } + const ej_img_t *ims = oid ? cbm_ht_get(j->imgs, oid) : NULL; + intptr_t ni = ims ? (intptr_t)cbm_ht_get(j->nimgs, oid) : 0; + for (intptr_t i = 0; i < ni; i++) { + if (ims[i].container && strcmp(ims[i].container, container) != 0) { + continue; + } + const char *iid = ej_id(j, ims[i].image); + ej_join_df(j, ev->id, key, iid ? cbm_ht_get(j->image_df, iid) : NULL); + } +} + +int cbm_pipeline_env_join(cbm_pipeline_ctx_t *ctx) { + ej_t j = {.ctx = ctx}; + cbm_arena_init_lazy(&j.a, CBM_SZ_4K); + j.sets = cbm_ht_create(0); + j.image_df = cbm_ht_create(0); + j.owner_df = cbm_ht_create(0); + j.readers = cbm_ht_create(0); + j.nreaders = cbm_ht_create(0); + j.imgs = cbm_ht_create(0); + j.nimgs = cbm_ht_create(0); + j.done = cbm_ht_create(0); + const cbm_gbuf_node_t **evs = NULL; + int n = 0; + if (j.sets && j.image_df && j.owner_df && j.readers && j.nreaders && j.imgs && j.nimgs && + j.done) { + ej_index_gbuf(&j); + ej_index_base(&j, ctx->envjoin_base); + if (cbm_ht_count(j.sets) > 0 && + cbm_gbuf_find_by_label(ctx->gbuf, "EnvVar", &evs, &n) == 0) { + for (int i = 0; i < n; i++) { + if (evs[i]->qualified_name && strstr(evs[i]->qualified_name, "#env/")) { + ej_join_setter(&j, evs[i]); + } + } + } + } + CBMHashTable *tables[] = {j.sets, j.image_df, j.owner_df, j.readers, + j.nreaders, j.imgs, j.nimgs, j.done}; + for (size_t i = 0; i < sizeof(tables) / sizeof(tables[0]); i++) { + if (tables[i]) { + cbm_ht_free(tables[i]); + } + } + cbm_arena_destroy(&j.a); + if (j.edges > 0) { + char buf[CBM_SZ_32]; + snprintf(buf, sizeof(buf), "%d", j.edges); + cbm_log_info("pass.envjoin.edges", "edges", buf); + } + return j.edges; +} diff --git a/src/pipeline/pass_k8s.c b/src/pipeline/pass_k8s.c index 3dfa357709..1af48be43d 100644 --- a/src/pipeline/pass_k8s.c +++ b/src/pipeline/pass_k8s.c @@ -26,6 +26,7 @@ #include "foundation/str_util.h" /* cbm_json_escape */ #include "pipeline/worker_pool.h" +#include #include #include #include @@ -551,9 +552,15 @@ static bool k8s_cluster_scoped(const char *kind) { * "::": `///` (IDENTITY.md). * `obj_key` is the logical identity: the namespace it lives in, `_` for a * cluster-scoped kind, `?` when the deploy decides; `effective_ns` (when - * not NULL) is the namespace a kustomize build gives it. NULL for a QN - * without a well-formed key (the caller keeps the plain source tag). */ -static char *k8s_resource_props(const char *qn, const char *effective_ns) { + * not NULL) is the namespace a kustomize build gives it. `version` (NULL or + * "": none) is the version of its apiVersion, which the key leaves out; + * `labels` (NULL: none) its metadata.labels as "k=v,k2=v2" (extract_k8s.c: + * nothing JSON escapes in it); `ecosystem` (NULL: a manifest's) what + * declares it otherwise -- "terraform" (pass_terraform.c), plain text. NULL + * for a QN without a well-formed key (the caller keeps the plain source + * tag). */ +static char *k8s_resource_props(const char *qn, const char *effective_ns, const char *version, + const char *labels, const char *ecosystem) { enum { K8S_KEY_PARTS = 4, K8S_ESC = 6 * K8S_LABEL_LEN }; const char *key = qn ? strstr(qn, "::") : NULL; if (!key) { @@ -580,19 +587,30 @@ static char *k8s_resource_props(const char *qn, const char *effective_ns) { : effective_ns ? effective_ns : ns[0] ? ns : "?"; - char esc[K8S_KEY_PARTS + SKIP_ONE][K8S_ESC]; + char esc[K8S_KEY_PARTS + PAIR_LEN][K8S_ESC]; cbm_json_escape(esc[0], K8S_ESC, group); cbm_json_escape(esc[1], K8S_ESC, kind); cbm_json_escape(esc[2], K8S_ESC, ns); cbm_json_escape(esc[3], K8S_ESC, name); cbm_json_escape(esc[4], K8S_ESC, effective); - size_t n = K8S_KEY_PARTS * K8S_ESC + K8S_ESC + CBM_SZ_256; + cbm_json_escape(esc[5], K8S_ESC, version ? version : ""); + size_t n = + K8S_KEY_PARTS * K8S_ESC + PAIR_LEN * K8S_ESC + CBM_SZ_256 + (labels ? strlen(labels) : 0); char *props = cbm_alloc(CBM_MEM_CLASS_OTHER, n); if (!props) { return NULL; } - int w = - snprintf(props, n, "{\"source\":\"k8s\",\"group\":\"%s\",\"kind\":\"%s\"", esc[0], esc[1]); + int w = snprintf(props, n, "{\"source\":\"k8s\""); + if (ecosystem && ecosystem[0]) { + w += snprintf(props + w, n - (size_t)w, ",\"ecosystem\":\"%s\"", ecosystem); + } + w += snprintf(props + w, n - (size_t)w, ",\"group\":\"%s\",\"kind\":\"%s\"", esc[0], esc[1]); + if (esc[5][0]) { + w += snprintf(props + w, n - (size_t)w, ",\"version\":\"%s\"", esc[5]); + } + if (labels && labels[0]) { + w += snprintf(props + w, n - (size_t)w, ",\"labels\":\"%s\"", labels); + } if (ns[0]) { w += snprintf(props + w, n - (size_t)w, ",\"namespace\":\"%s\"", esc[2]); } @@ -601,6 +619,44 @@ static char *k8s_resource_props(const char *qn, const char *effective_ns) { return props; } +char *cbm_k8s_resource_props(const char *qn, const char *effective_ns, const char *version, + const char *labels, const char *ecosystem) { + return k8s_resource_props(qn, effective_ns, version, labels, ecosystem); +} + +/* The string property `key` of node `n` (as k8s_resource_props writes it: + * no escapes inside) into `out`, "" when it has none. */ +static void k8s_node_prop(const cbm_gbuf_node_t *n, const char *key, char *out, size_t out_sz) { + out[0] = '\0'; + char head[CBM_SZ_64]; + snprintf(head, sizeof(head), "\"%s\":\"", key); + const char *v = n->properties_json ? strstr(n->properties_json, head) : NULL; + if (v) { + v += strlen(head); + const char *e = strchr(v, '"'); + snprintf(out, out_sz, "%.*s", e ? (int)(e - v) : 0, v); + } +} + +/* The string property `key` of node `n` as k8s_resource_props writes it, + * any length (caller frees), NULL when it has none. */ +static char *k8s_node_prop_dup(const cbm_gbuf_node_t *n, const char *key) { + char head[CBM_SZ_64]; + snprintf(head, sizeof(head), "\"%s\":\"", key); + const char *v = n->properties_json ? strstr(n->properties_json, head) : NULL; + const char *e = v ? strchr(v + strlen(head), '"') : NULL; + if (!e) { + return NULL; + } + v += strlen(head); + char *out = cbm_alloc(CBM_MEM_CLASS_OTHER, (size_t)(e - v) + SKIP_ONE); + if (out) { + memcpy(out, v, (size_t)(e - v)); + out[e - v] = '\0'; + } + return out; +} + static void k8s_records_free(k8s_record_array_t *recs) { for (int i = 0; i < recs->pair_count; i++) { cbm_free(CBM_MEM_CLASS_OTHER, recs->pairs[i]); @@ -677,6 +733,10 @@ typedef struct { int file_n; int file_cap; bool referenced; /* included by another kustomization */ + bool component; /* `kind: Component`: patches the build including it */ + char *prefix; /* its `namePrefix:`, NULL when unset */ + char *suffix; /* its `nameSuffix:`, NULL when unset */ + int rename_line; /* the line of its namePrefix, else nameSuffix; 0: neither */ } k8s_kust_t; typedef struct { @@ -720,39 +780,83 @@ static bool k8s_grow(void **items, int *cap, int count, size_t elem) { return true; } -/* The top-level `namespace:` of a kustomization, NULL when it sets none. */ -static char *k8s_kustomize_namespace(const char *source) { - for (const char *p = source; p && *p;) { +/* The value of a kustomization's top-level `key:` line into `val`, its + * 1-based line into `*line` (when not NULL); false when it has none, or an + * empty one. */ +static bool k8s_kustomize_top_at(const char *source, const char *key, char *val, size_t val_sz, + int *line) { + char head[CBM_SZ_64]; + int hl = snprintf(head, sizeof(head), "%s:", key); + int at = SKIP_ONE; + for (const char *p = source; p && *p && hl > 0; at++) { const char *eol = strchr(p, '\n'); size_t len = eol ? (size_t)(eol - p) : strlen(p); - if (strncmp(p, "namespace:", K8S_SLEN("namespace:")) == 0) { - char line[CBM_SZ_512]; - snprintf(line, sizeof(line), "%.*s", (int)len, p); - char key[64]; - char val[K8S_LABEL_LEN]; - if (k8s_split_kv(line, key, sizeof(key), val, sizeof(val)) && val[0]) { - return cbm_mem_strdup(CBM_MEM_CLASS_OTHER, val); + if (strncmp(p, head, (size_t)hl) == 0) { + char text[CBM_SZ_512]; + snprintf(text, sizeof(text), "%.*s", (int)len, p); + char k[CBM_SZ_64]; + if (line) { + *line = at; } - return NULL; + return k8s_split_kv(text, k, sizeof(k), val, val_sz) && val[0]; } p = eol ? eol + SKIP_ONE : NULL; } - return NULL; + return false; +} + +static bool k8s_kustomize_top(const char *source, const char *key, char *val, size_t val_sz) { + return k8s_kustomize_top_at(source, key, val, val_sz, NULL); +} + +/* The top-level `namespace:` of a kustomization, NULL when it sets none. */ +static char *k8s_kustomize_namespace(const char *source) { + char val[K8S_LABEL_LEN]; + return k8s_kustomize_top(source, "namespace", val, sizeof(val)) + ? cbm_mem_strdup(CBM_MEM_CLASS_OTHER, val) + : NULL; +} + +/* A kustomize component (`kind: Component`), whose patches act on the build + * of each kustomization including it. */ +static bool k8s_kustomize_component(const char *source) { + char val[K8S_LABEL_LEN]; + return k8s_kustomize_top(source, "kind", val, sizeof(val)) && strcmp(val, "Component") == 0; +} + +/* A kustomization's `namePrefix:` / `nameSuffix:` (`key`), NULL when it + * sets none -- or one with `/` or a blank in it, which is no name part; its + * line into `*line` when that is still 0. */ +static char *k8s_kustomize_name_part(const char *source, const char *key, int *line) { + char val[K8S_LABEL_LEN]; + int at = 0; + if (!k8s_kustomize_top_at(source, key, val, sizeof(val), &at) || strpbrk(val, "/ \t")) { + return NULL; + } + if (*line == 0) { + *line = at; + } + return cbm_mem_strdup(CBM_MEM_CLASS_OTHER, val); } -static void k8s_kusts_push(k8s_kusts_t *ks, const char *path, char *ns) { +static void k8s_kusts_push(k8s_kusts_t *ks, const char *path, const char *source) { if (!k8s_grow((void **)&ks->items, &ks->cap, ks->count, sizeof(*ks->items))) { - cbm_free(CBM_MEM_CLASS_OTHER, ns); return; } - ks->items[ks->count++] = - (k8s_kust_t){.path = cbm_mem_strdup(CBM_MEM_CLASS_OTHER, path), .ns = ns}; + k8s_kust_t k = {.path = cbm_mem_strdup(CBM_MEM_CLASS_OTHER, path), + .ns = k8s_kustomize_namespace(source), + .component = k8s_kustomize_component(source)}; + k.prefix = k8s_kustomize_name_part(source, "namePrefix", &k.rename_line); + k.suffix = k8s_kustomize_name_part(source, "nameSuffix", &k.rename_line); + ks->items[ks->count++] = k; } static void k8s_kusts_free(k8s_kusts_t *ks) { for (int i = 0; i < ks->count; i++) { cbm_free(CBM_MEM_CLASS_OTHER, ks->items[i].path); cbm_free(CBM_MEM_CLASS_OTHER, ks->items[i].ns); + cbm_free(CBM_MEM_CLASS_OTHER, ks->items[i].prefix); + cbm_free(CBM_MEM_CLASS_OTHER, ks->items[i].suffix); cbm_free(CBM_MEM_CLASS_OTHER, ks->items[i].kids); cbm_free(CBM_MEM_CLASS_OTHER, (void *)ks->items[i].files); } @@ -820,6 +924,12 @@ static void k8s_kusts_wire(cbm_pipeline_ctx_t *ctx, k8s_kusts_t *ks, CBMHashTabl continue; } k8s_kust_t *kust = &ks->items[k - SKIP_ONE]; + /* a patch file (`patches`, `patchesStrategicMerge`) is no member of + * the build: its documents change objects, they are none */ + const char *props = edges[i]->properties_json; + if (props && strstr(props, "\"local_name\":\"patches")) { + continue; + } int kid = tgt->label && strcmp(tgt->label, "Folder") == 0 ? k8s_kust_in_dir(by_path, tgt->file_path) : k8s_kust_at(by_path, tgt->file_path); @@ -1104,7 +1214,14 @@ static void k8s_apply_effective_namespaces(cbm_pipeline_ctx_t *ctx, k8s_bundles_ if (!any_set) { continue; /* no build sets a namespace: the declared one stands */ } - char *props = k8s_resource_props(n->qualified_name, agree ? eff : "?"); + char version[K8S_LABEL_LEN]; + char ecosystem[K8S_LABEL_LEN]; + k8s_node_prop(n, "version", version, sizeof(version)); + k8s_node_prop(n, "ecosystem", ecosystem, sizeof(ecosystem)); + char *labels = k8s_node_prop_dup(n, "labels"); + char *props = + k8s_resource_props(n->qualified_name, agree ? eff : "?", version, labels, ecosystem); + cbm_free(CBM_MEM_CLASS_OTHER, labels); if (props) { (void)cbm_gbuf_node_set_properties_json((cbm_gbuf_node_t *)n, props); cbm_free(CBM_MEM_CLASS_OTHER, props); @@ -1124,8 +1241,10 @@ typedef struct { char *ns; /* explicit target namespace, NULL = the source's */ char *name; /* target metadata.name */ const char *via; /* static: the extractor's table */ - const char *edge; /* static: "USAGE", "INFRA_MAPS" or "DEPENDS_ON" */ + const char *edge; /* static: "USAGE", "INFRA_MAPS", "DEPENDS_ON" or "CONNECTS_TO" */ char *route; /* an exposed route "" the edge leaves from, NULL = src */ + char *key; /* CONNECTS_TO: the environment variable addressing the target */ + char *container; /* CONNECTS_TO: its container, NULL = unnamed */ } k8s_ref_t; /* A CRD and the group/kind it declares. */ @@ -1161,6 +1280,39 @@ typedef struct { int line; } k8s_env_t; +/* A container image an object runs, and the container running it. */ +typedef struct { + int64_t node; + char *image; + char *container; /* NULL: unnamed */ +} k8s_img_t; + +/* A patch a kustomization states (extract_k8s.c PATCH fact): what it names + * in the kustomization's build. */ +typedef struct { + char *kust; /* the kustomization file */ + char *path; /* the patch file (repository path), NULL: inline */ + char *kind; /* kind / group / version / name / ns named, NULL: any; */ + char *group; /* kind and name both NULL: the documents of the */ + char *version; /* patch file name them */ + char *name; + char *ns; + char *selector; /* labelSelector, NULL: none */ + int line; +} k8s_patch_t; + +/* A chart a kustomization inflates into its build (`helmCharts`). */ +typedef struct { + char *kust; /* the kustomization file */ + char *chart; /* the chart name */ + char *url; /* its repository, NULL when unset */ + char *version; /* NULL when unset */ + char *release; /* the release name, NULL when unset */ + char *ns; /* the release namespace, NULL when unset */ + char *values; /* the values file (repository path), NULL when none */ + int line; +} k8s_kchart_t; + typedef struct { k8s_ref_t *refs; int ref_n; @@ -1177,12 +1329,18 @@ typedef struct { k8s_hr_t *srcs; /* an OCIRepository (node) and the URL it fetches (chart) */ int src_n; int src_cap; - k8s_hr_t *imgs; /* an object (node) and a container image it runs (chart) */ + k8s_img_t *imgs; int img_n; int img_cap; k8s_env_t *envs; int env_n; int env_cap; + k8s_patch_t *patches; + int patch_n; + int patch_cap; + k8s_kchart_t *kcharts; + int kchart_n; + int kchart_cap; } k8s_facts_t; static void k8s_facts_free(k8s_facts_t *f) { @@ -1191,6 +1349,8 @@ static void k8s_facts_free(k8s_facts_t *f) { cbm_free(CBM_MEM_CLASS_OTHER, f->refs[i].ns); cbm_free(CBM_MEM_CLASS_OTHER, f->refs[i].name); cbm_free(CBM_MEM_CLASS_OTHER, f->refs[i].route); + cbm_free(CBM_MEM_CLASS_OTHER, f->refs[i].key); + cbm_free(CBM_MEM_CLASS_OTHER, f->refs[i].container); } for (int i = 0; i < f->crd_n; i++) { cbm_free(CBM_MEM_CLASS_OTHER, f->crds[i].group); @@ -1208,12 +1368,36 @@ static void k8s_facts_free(k8s_facts_t *f) { cbm_free(CBM_MEM_CLASS_OTHER, f->srcs[i].chart); } for (int i = 0; i < f->img_n; i++) { - cbm_free(CBM_MEM_CLASS_OTHER, f->imgs[i].chart); + cbm_free(CBM_MEM_CLASS_OTHER, f->imgs[i].image); + cbm_free(CBM_MEM_CLASS_OTHER, f->imgs[i].container); } for (int i = 0; i < f->env_n; i++) { cbm_free(CBM_MEM_CLASS_OTHER, f->envs[i].key); cbm_free(CBM_MEM_CLASS_OTHER, f->envs[i].container); } + for (int i = 0; i < f->patch_n; i++) { + k8s_patch_t *p = &f->patches[i]; + cbm_free(CBM_MEM_CLASS_OTHER, p->kust); + cbm_free(CBM_MEM_CLASS_OTHER, p->path); + cbm_free(CBM_MEM_CLASS_OTHER, p->kind); + cbm_free(CBM_MEM_CLASS_OTHER, p->group); + cbm_free(CBM_MEM_CLASS_OTHER, p->version); + cbm_free(CBM_MEM_CLASS_OTHER, p->name); + cbm_free(CBM_MEM_CLASS_OTHER, p->ns); + cbm_free(CBM_MEM_CLASS_OTHER, p->selector); + } + cbm_free(CBM_MEM_CLASS_OTHER, f->patches); + for (int i = 0; i < f->kchart_n; i++) { + k8s_kchart_t *c = &f->kcharts[i]; + cbm_free(CBM_MEM_CLASS_OTHER, c->kust); + cbm_free(CBM_MEM_CLASS_OTHER, c->chart); + cbm_free(CBM_MEM_CLASS_OTHER, c->url); + cbm_free(CBM_MEM_CLASS_OTHER, c->version); + cbm_free(CBM_MEM_CLASS_OTHER, c->release); + cbm_free(CBM_MEM_CLASS_OTHER, c->ns); + cbm_free(CBM_MEM_CLASS_OTHER, c->values); + } + cbm_free(CBM_MEM_CLASS_OTHER, f->kcharts); cbm_free(CBM_MEM_CLASS_OTHER, f->envs); cbm_free(CBM_MEM_CLASS_OTHER, f->hrs); cbm_free(CBM_MEM_CLASS_OTHER, f->srcs); @@ -1297,8 +1481,9 @@ static void k8s_facts_keep(k8s_facts_t *f, const CBMFileResult *res, const int64 } } else if (x->fact == CBM_K8S_FACT_IMAGE) { if (x->name && k8s_grow((void **)&f->imgs, &f->img_cap, f->img_n, sizeof(*f->imgs))) { - f->imgs[f->img_n++] = - (k8s_hr_t){node, cbm_mem_strdup(CBM_MEM_CLASS_OTHER, x->name), NULL}; + f->imgs[f->img_n++] = (k8s_img_t){ + node, cbm_mem_strdup(CBM_MEM_CLASS_OTHER, x->name), + x->container ? cbm_mem_strdup(CBM_MEM_CLASS_OTHER, x->container) : NULL}; } } else if (x->fact == CBM_K8S_FACT_ENV) { if (x->name && k8s_grow((void **)&f->envs, &f->env_cap, f->env_n, sizeof(*f->envs))) { @@ -1322,7 +1507,11 @@ static void k8s_facts_keep(k8s_facts_t *f, const CBMFileResult *res, const int64 .name = x->name ? cbm_mem_strdup(CBM_MEM_CLASS_OTHER, x->name) : NULL, .via = x->via, .edge = x->edge, - .route = route ? cbm_mem_strdup(CBM_MEM_CLASS_OTHER, x->route) : NULL}; + .route = route ? cbm_mem_strdup(CBM_MEM_CLASS_OTHER, x->route) : NULL, + .key = x->key ? cbm_mem_strdup(CBM_MEM_CLASS_OTHER, x->key) : NULL, + .container = x->key && x->container + ? cbm_mem_strdup(CBM_MEM_CLASS_OTHER, x->container) + : NULL}; } } } @@ -1879,12 +2068,21 @@ static void k8s_resolve_refs(cbm_pipeline_ctx_t *ctx, k8s_bundles_t *b, const k8 } } atomic_fetch_add_explicit(&g_k8s_candidates, (uint64_t)(n), memory_order_relaxed); - char props[CBM_SZ_256]; - snprintf(props, sizeof(props), "{\"%s\":\"%s\"}", - strcmp(ref->edge, "INFRA_MAPS") == 0 ? "kind" - : strcmp(ref->edge, "DEPENDS_ON") == 0 ? "via" - : "ref", - ref->via); + char props[CBM_SZ_1K]; + if (ref->key) { /* CONNECTS_TO: the variable and container, never the value */ + char kesc[CBM_SZ_256]; + char cesc[CBM_SZ_256]; + cbm_json_escape(kesc, (int)sizeof(kesc), ref->key); + cbm_json_escape(cesc, (int)sizeof(cesc), ref->container ? ref->container : "-"); + snprintf(props, sizeof(props), "{\"via\":\"%s\",\"key\":\"%s\",\"container\":\"%s\"}", + ref->via, kesc, cesc); + } else { + snprintf(props, sizeof(props), "{\"%s\":\"%s\"}", + strcmp(ref->edge, "INFRA_MAPS") == 0 ? "kind" + : strcmp(ref->edge, "DEPENDS_ON") == 0 ? "via" + : "ref", + ref->via); + } /* A name is one object per kind and namespace, so two declarations * of it in one bundle are alternatives (versioned manifests side by * side): the declaration in the reference's own file, when there is @@ -2267,7 +2465,16 @@ static void handle_k8s_manifest(cbm_pipeline_ctx_t *ctx, const char *rel_path, c continue; } - char *props = k8s_resource_props(def->qualified_name, NULL); + const char *version = NULL; /* extract_k8s.c's extra properties */ + const char *labels = NULL; + for (const char **kv = def->extra_props; kv && kv[0] && kv[SKIP_ONE]; kv += PAIR_LEN) { + if (strcmp(kv[0], "version") == 0) { + version = kv[SKIP_ONE]; + } else if (strcmp(kv[0], "labels") == 0) { + labels = kv[SKIP_ONE]; + } + } + char *props = k8s_resource_props(def->qualified_name, NULL, version, labels, NULL); int64_t node_id = cbm_gbuf_upsert_node( ctx->gbuf, "Resource", def->name, def->qualified_name, rel_path, (int)def->start_line, (int)def->end_line, props ? props : "{\"source\":\"k8s\"}"); @@ -2336,14 +2543,159 @@ static char *k8s_dep_props(const char *repository) { return props; } -/* DEPENDS_ON from `chart_id` to the shared external Chart node of `name` - * (one per name and project, so charts depending on the same chart meet). */ +/* ── External charts ─────────────────────────────────────────────── */ + +/* An external Chart is keyed by its publisher: `__chart__/`, the + * same QN in every project, so two repositories naming one published chart + * name one node. A chart whose repository does not resolve to a URL (a + * `@alias`, an empty or templated repository, a source the pass cannot read) + * keeps the per-project `.__helm_dep__.`: no identity is + * claimed for what was not stated. */ +static const char K8S_CHART_QN_PREFIX[] = "__chart__"; +static const char K8S_HELM_DEP_MARK[] = ".__helm_dep__."; + +/* A byte a stated repository URL may hold: no whitespace or control byte, + * no quote, template or shell expansion, escape or angle bracket. */ +static bool k8s_url_byte_ok(unsigned char c) { + return c > ' ' && c < 0x7f && !strchr("\"'`{}$<>\\", c); +} + +bool cbm_chart_repo_url(const char *url, char *out, size_t sz) { + if (!url || !out || sz == 0) { + return false; + } + while (*url == ' ' || *url == '\t') { + url++; + } + size_t n = strlen(url); + while (n > 0 && isspace((unsigned char)url[n - SKIP_ONE])) { + n--; + } + if (n == 0 || n + SKIP_ONE > sz || !isalpha((unsigned char)url[0])) { + return false; + } + for (size_t i = 0; i < n; i++) { + if (!k8s_url_byte_ok((unsigned char)url[i])) { + return false; + } + } + /* a query or fragment may carry a token (?token=...): like credentials + * in the authority, never put into a QN or a property */ + if (memchr(url, '?', n) || memchr(url, '#', n)) { + return false; + } + size_t s = 0; /* the scheme: [A-Za-z][A-Za-z0-9+.-]* then :// */ + while (s < n && + (isalnum((unsigned char)url[s]) || url[s] == '+' || url[s] == '-' || url[s] == '.')) { + s++; + } + if (s + K8S_SLEN("://") > n || strncmp(url + s, "://", K8S_SLEN("://")) != 0) { + return false; + } + size_t host = s + K8S_SLEN("://"); + size_t path = host; + while (path < n && url[path] != '/') { + path++; + } + /* no host, or credentials in the authority: never put into a QN */ + if (path == host || memchr(url + host, '@', path - host)) { + return false; + } + for (size_t i = 0; i < path; i++) { + out[i] = (char)tolower((unsigned char)url[i]); + } + memcpy(out + path, url + path, n - path); + out[n] = '\0'; + if (strncmp(out, "file://", K8S_SLEN("file://")) == 0) { + return false; /* a directory, not a publisher */ + } + size_t end = n; + while (end > path && out[end - SKIP_ONE] == '/') { + end--; + } + if (end - path >= K8S_SLEN(".git") && + strncmp(out + end - K8S_SLEN(".git"), ".git", K8S_SLEN(".git")) == 0) { + end -= K8S_SLEN(".git"); + while (end > path && out[end - SKIP_ONE] == '/') { + end--; + } + } + out[end] = '\0'; + return true; +} + +bool cbm_chart_ref_repo(const char *ref, char *out, size_t sz) { + if (!ref || !out || sz == 0) { + return false; + } + size_t end = strlen(ref); + while (end > 0 && ref[end - SKIP_ONE] == '/') { + end--; + } + while (end > 0 && ref[end - SKIP_ONE] != '/') { + end--; + } + while (end > 0 && ref[end - SKIP_ONE] == '/') { + end--; + } + if (end == 0 || end + SKIP_ONE > sz) { + return false; + } + memcpy(out, ref, end); + out[end] = '\0'; + return true; +} + +/* QN and node properties of the external Chart `name` (non-NULL) of + * repository `url` (NULL: unknown). */ +static void k8s_chart_identity(const char *project, const char *name, const char *url, char *qn, + size_t qsz, char *props, size_t psz) { + char norm[CBM_SZ_1K]; + if (name[0] && !strchr(name, '/') && cbm_chart_repo_url(url, norm, sizeof(norm))) { + int w = snprintf(qn, qsz, "%s%s/%s", K8S_CHART_QN_PREFIX, norm, name); + if (w > 0 && (size_t)w < qsz) { + char esc[CBM_SZ_2K]; + cbm_json_escape(esc, (int)sizeof(esc), norm); + snprintf(props, psz, "{\"source\":\"helm\",\"external\":true,\"repository\":\"%s\"}", + esc); + return; + } + } + snprintf(qn, qsz, "%s%s%s", project, K8S_HELM_DEP_MARK, name); + snprintf(props, psz, "{\"source\":\"helm\",\"external\":true}"); +} + +int64_t cbm_pipeline_external_chart(cbm_pipeline_ctx_t *ctx, const char *name, const char *url, + const char *file) { + if (!name) { + return 0; + } + char qn[CBM_SZ_2K]; + char props[CBM_SZ_4K]; + k8s_chart_identity(ctx->project_name, name, url, qn, sizeof(qn), props, sizeof(props)); + const cbm_gbuf_node_t *known = cbm_gbuf_find_by_qn(ctx->gbuf, qn); + if (known) { + return known->id; + } + return cbm_gbuf_upsert_node(ctx->gbuf, "Chart", name, qn, file, SKIP_ONE, 0, props); +} + +bool cbm_pipeline_chart_is_external(const cbm_gbuf_node_t *node) { + const char *qn = node ? node->qualified_name : NULL; + return qn && (strncmp(qn, K8S_CHART_QN_PREFIX, K8S_SLEN(K8S_CHART_QN_PREFIX)) == 0 || + strstr(qn, K8S_HELM_DEP_MARK) != NULL); +} + +/* DEPENDS_ON from `chart_id` to the external Chart node of `name` in + * `repository` (charts depending on the same published chart meet). */ static bool k8s_external_dep(cbm_pipeline_ctx_t *ctx, int64_t chart_id, const char *name, - const char *chart_rel, const char *props) { - char dep_qn[CBM_SZ_512]; - snprintf(dep_qn, sizeof(dep_qn), "%s.__helm_dep__.%s", ctx->project_name, name); - int64_t dep_id = cbm_gbuf_upsert_node(ctx->gbuf, "Chart", name, dep_qn, chart_rel, SKIP_ONE, 0, - "{\"source\":\"helm\",\"external\":true}"); + const char *repository, const char *chart_rel, const char *props) { + char dep_qn[CBM_SZ_2K]; + char node_props[CBM_SZ_4K]; + k8s_chart_identity(ctx->project_name, name, repository, dep_qn, sizeof(dep_qn), node_props, + sizeof(node_props)); + int64_t dep_id = + cbm_gbuf_upsert_node(ctx->gbuf, "Chart", name, dep_qn, chart_rel, SKIP_ONE, 0, node_props); if (dep_id <= 0) { return false; } @@ -2407,7 +2759,9 @@ static void k8s_link_local_deps(cbm_pipeline_ctx_t *ctx, k8s_local_deps_t *deps) target->id != d->chart_id) { cbm_gbuf_insert_edge(ctx->gbuf, d->chart_id, target->id, "DEPENDS_ON", d->props); } else if (d->chart_rel) { - (void)k8s_external_dep(ctx, d->chart_id, d->name, d->chart_rel, d->props); + /* a file:// directory this repository cannot resolve names no + * publisher: the per-project external Chart */ + (void)k8s_external_dep(ctx, d->chart_id, d->name, NULL, d->chart_rel, d->props); } } k8s_local_deps_free(deps); @@ -2415,8 +2769,9 @@ static void k8s_link_local_deps(cbm_pipeline_ctx_t *ctx, k8s_local_deps_t *deps) /* Emit a Chart node for a Chart.yaml and a DEPENDS_ON edge per declared * dependency (#338): to the chart in this repository for a resolvable - * `file://` dependency (once all charts are read), otherwise to a shared - * external Chart node per dependency name, the repository on the edge. */ + * `file://` dependency (once all charts are read), otherwise to the external + * Chart node of the dependency's repository and name, the repository also on + * the edge. */ static void handle_helm_chart(cbm_pipeline_ctx_t *ctx, const char *rel_path, const char *source, k8s_local_deps_t *local_deps) { cbm_helm_chart_t hc; @@ -2450,7 +2805,7 @@ static void handle_helm_chart(cbm_pipeline_ctx_t *ctx, const char *rel_path, con continue; } char *props = dep->repository[0] ? k8s_dep_props(dep->repository) : NULL; - dep_edges += k8s_external_dep(ctx, chart_id, dep->name, rel_path, props); + dep_edges += k8s_external_dep(ctx, chart_id, dep->name, dep->repository, rel_path, props); cbm_free(CBM_MEM_CLASS_OTHER, props); } cbm_helm_chart_free(&hc); @@ -2782,9 +3137,14 @@ static void k8s_prep_file(const k8s_prep_job_t *job, int i, k8s_prep_t *out) { out->source = k8s_read_file(path, &out->src_len); out->kind = out->source ? K8S_DEP : K8S_SKIP; } else if (lang == CBM_LANG_KUSTOMIZE) { - /* Its `namespace:` is the namespace of the objects it builds. */ + /* Its `namespace:` is the namespace of the objects it builds; its + * patches (PATCH facts) change some of them. */ out->source = k8s_read_file(path, &out->src_len); out->kind = out->source ? K8S_KUSTOMIZE : K8S_SKIP; + if (out->source) { + out->res = cbm_extract_file(out->source, out->src_len, CBM_LANG_KUSTOMIZE, + ctx->project_name, rel, CBM_EXTRACT_BUDGET, NULL, NULL); + } } else if (lang == CBM_LANG_YAML || lang == CBM_LANG_K8S) { /* Read source once to classify (and reuse for uncached extraction). */ out->source = k8s_read_file(path, &out->src_len); @@ -3228,7 +3588,7 @@ static void k8s_link_includes(cbm_pipeline_ctx_t *ctx, k8s_bundles_t *b, * outermost kustomizations below it -- their Kustomization node -- and the * manifests outside any, their File node; a chart dir: its Chart); a * HelmRelease -> its chart (a repository chart dir of a Git/Bucket source, - * else the shared external Chart of that name). */ + * else the external Chart of that name). */ typedef struct { cbm_pipeline_ctx_t *ctx; @@ -3264,21 +3624,17 @@ static int64_t k8s_chart_node(k8s_dep_link_t *l, const char *dir) { return 0; } -/* The shared external Chart of `name` (the one chart dependencies use). */ -static int64_t k8s_external_chart(k8s_dep_link_t *l, const char *name, const char *file) { - char qn[CBM_SZ_512]; - snprintf(qn, sizeof(qn), "%s.__helm_dep__.%s", l->ctx->project_name, name); - const cbm_gbuf_node_t *known = cbm_gbuf_find_by_qn(l->ctx->gbuf, qn); - if (known) { - return known->id; - } - return cbm_gbuf_upsert_node(l->ctx->gbuf, "Chart", name, qn, file, SKIP_ONE, 0, - "{\"source\":\"helm\",\"external\":true}"); +/* The external Chart of `name` in repository `url` (NULL: unknown), the one + * chart dependencies use. */ +static int64_t k8s_external_chart(k8s_dep_link_t *l, const char *name, const char *url, + const char *file) { + return cbm_pipeline_external_chart(l->ctx, name, url, file); } /* A HelmRelease whose chartRef names OCIRepository `oci` (its USAGE edge * {"ref":"chartRef"}, resolved with the references) deploys the chart that - * repository fetches: the shared external Chart of the URL's last segment. */ + * repository fetches: the external Chart of that OCI reference (its last + * segment the chart, the rest its repository). */ static void k8s_link_chart_refs(k8s_dep_link_t *l, const k8s_hr_t *oci) { const char *url = oci->chart; size_t end = strlen(url); @@ -3302,6 +3658,8 @@ static void k8s_link_chart_refs(k8s_dep_link_t *l, const k8s_hr_t *oci) { snprintf(props, sizeof(props), "{\"via\":\"helmrelease\",\"chart\":\"%s\",\"external\":true,\"url\":\"%s\"}", esc_name, esc_url); + char repo[CBM_SZ_1K]; + const char *repo_url = cbm_chart_ref_repo(url, repo, sizeof(repo)) ? repo : NULL; const cbm_gbuf_edge_t **in = NULL; int n = 0; if (cbm_gbuf_find_edges_by_target_type(l->ctx->gbuf, oci->node, "USAGE", &in, &n) != 0) { @@ -3312,24 +3670,69 @@ static void k8s_link_chart_refs(k8s_dep_link_t *l, const k8s_hr_t *oci) { const char *p = in[i]->properties_json; const cbm_gbuf_node_t *hr = cbm_gbuf_find_by_id(l->ctx->gbuf, in[i]->source_id); if (p && strstr(p, "\"ref\":\"chartRef\"") && hr && hr->file_path) { - k8s_deploys_edge(l, hr->id, k8s_external_chart(l, name, hr->file_path), props); + k8s_deploys_edge(l, hr->id, k8s_external_chart(l, name, repo_url, hr->file_path), + props); + } + } +} + +/* `, "":""` (escaped) at out + *w when `val` is set. */ +static void k8s_prop_add(char *out, size_t out_sz, int *w, const char *key, const char *val) { + if (!val || *w < 0 || (size_t)*w >= out_sz) { + return; + } + char esc[CBM_SZ_1K]; + cbm_json_escape(esc, (int)sizeof(esc), val); + *w += snprintf(out + *w, out_sz - (size_t)*w, ",\"%s\":\"%s\"", key, esc); +} + +/* A chart a kustomization inflates (`helmCharts`) is deployed with its + * build: DEPLOYS {via: helmcharts, external, chart, url, version, release, + * namespace, values, line} from the kustomization (its Kustomization + * Resource, else its File -- the unit a Flux / Argo deploy reaches) to the + * external Chart of that repository and name. */ +static void k8s_link_kcharts(k8s_dep_link_t *l, const k8s_facts_t *f) { + for (int i = 0; i < f->kchart_n; i++) { + const k8s_kchart_t *c = &f->kcharts[i]; + if (!c->chart || strchr(c->chart, '/') || strchr(c->chart, '$')) { + continue; + } + int64_t src = k8s_first_id(l->kusts, c->kust); + src = src > 0 ? src : k8s_first_id(l->files, c->kust); + char props[CBM_SZ_4K]; + int w = snprintf(props, sizeof(props), "{\"via\":\"helmcharts\",\"external\":true"); + k8s_prop_add(props, sizeof(props), &w, "chart", c->chart); + k8s_prop_add(props, sizeof(props), &w, "url", c->url); + k8s_prop_add(props, sizeof(props), &w, "version", c->version); + k8s_prop_add(props, sizeof(props), &w, "release", c->release); + k8s_prop_add(props, sizeof(props), &w, "namespace", c->ns); + k8s_prop_add(props, sizeof(props), &w, "values", c->values); + if (w < 0 || (size_t)w >= sizeof(props) - CBM_SZ_32) { + continue; } + snprintf(props + w, sizeof(props) - (size_t)w, ",\"line\":%d}", c->line); + k8s_deploys_edge(l, src, k8s_external_chart(l, c->chart, c->url, c->kust), props); } } -/* A container image an object runs: USAGE {ref:image} to its Image node -- - * the node a Dockerfile building or building on that image joins. */ +/* A container image an object runs: USAGE {ref:image, container} to its + * Image node -- the node a Dockerfile building or building on that image + * joins; the container names whose environment runs it. */ static void k8s_link_images(cbm_pipeline_ctx_t *ctx, const k8s_facts_t *f) { for (int i = 0; i < f->img_n; i++) { - int64_t img = cbm_pipeline_image_node(ctx, f->imgs[i].chart); + const k8s_img_t *im = &f->imgs[i]; + int64_t img = cbm_pipeline_image_node(ctx, im->image); if (img <= 0) { continue; } char esc[CBM_SZ_1K]; - cbm_json_escape(esc, (int)sizeof(esc), f->imgs[i].chart); + cbm_json_escape(esc, (int)sizeof(esc), im->image); + char cesc[CBM_SZ_512]; + cbm_json_escape(cesc, (int)sizeof(cesc), im->container ? im->container : "-"); char props[CBM_SZ_2K]; - snprintf(props, sizeof(props), "{\"ref\":\"image\",\"image\":\"%s\"}", esc); - cbm_gbuf_insert_edge(ctx->gbuf, f->imgs[i].node, img, "USAGE", props); + snprintf(props, sizeof(props), "{\"ref\":\"image\",\"image\":\"%s\",\"container\":\"%s\"}", + esc, cesc); + cbm_gbuf_insert_edge(ctx->gbuf, im->node, img, "USAGE", props); } } @@ -3346,9 +3749,569 @@ static void k8s_link_envs(cbm_pipeline_ctx_t *ctx, const k8s_facts_t *f) { } } +/* ── Kustomize patches (PHASE 5) ───────────────────────────────────── */ + +static char *k8s_dup_opt(const char *s) { + return s ? cbm_mem_strdup(CBM_MEM_CLASS_OTHER, s) : NULL; +} + +/* The repository path of `rel`, relative to directory `dir`; false when it + * leaves the repository (or is a URL). */ +static bool k8s_repo_path(const char *dir, const char *rel, char *out, size_t out_sz) { + char joined[CBM_SZ_2K]; + snprintf(joined, sizeof(joined), "%s%s%s", dir, dir[0] ? "/" : "", rel); + return k8s_norm_dir(joined, out, out_sz); +} + +static void k8s_kchart_keep(k8s_facts_t *f, const char *kust, const CBMK8sFact *x, + const char *values) { + if (!k8s_grow((void **)&f->kcharts, &f->kchart_cap, f->kchart_n, sizeof(*f->kcharts))) { + return; + } + f->kcharts[f->kchart_n++] = (k8s_kchart_t){.kust = cbm_mem_strdup(CBM_MEM_CLASS_OTHER, kust), + .chart = k8s_dup_opt(x->name), + .url = k8s_dup_opt(x->url), + .version = k8s_dup_opt(x->version), + .release = k8s_dup_opt(x->release), + .ns = k8s_dup_opt(x->ns), + .values = k8s_dup_opt(values), + .line = x->line}; +} + +/* Keep a kustomization's facts: its patches (PATCH) and the charts it + * inflates (DEPLOY via "kustomize"), each file as a repository path -- a + * patch file outside the repository (or a URL) names no file of it, a values + * file there is left out. */ +static void k8s_kustomize_keep(k8s_facts_t *f, const char *kust, const CBMFileResult *res) { + char dir[CBM_SZ_1K]; + k8s_dirname(kust, dir, sizeof(dir)); + for (int i = 0; res && i < res->k8s_facts.count; i++) { + const CBMK8sFact *x = &res->k8s_facts.items[i]; + bool chart = x->fact == CBM_K8S_FACT_DEPLOY && x->via && strcmp(x->via, "kustomize") == 0; + if (x->fact != CBM_K8S_FACT_PATCH && !chart) { + continue; + } + char path[CBM_SZ_1K]; + bool in_repo = x->path && k8s_repo_path(dir, x->path, path, sizeof(path)); + if (chart) { + k8s_kchart_keep(f, kust, x, in_repo ? path : NULL); + continue; + } + if ((x->path && !in_repo) || + !k8s_grow((void **)&f->patches, &f->patch_cap, f->patch_n, sizeof(*f->patches))) { + continue; + } + f->patches[f->patch_n++] = + (k8s_patch_t){.kust = cbm_mem_strdup(CBM_MEM_CLASS_OTHER, kust), + .path = x->path ? cbm_mem_strdup(CBM_MEM_CLASS_OTHER, path) : NULL, + .kind = k8s_dup_opt(x->kind), + .group = k8s_dup_opt(x->group), + .version = k8s_dup_opt(x->version), + .name = k8s_dup_opt(x->name), + .ns = k8s_dup_opt(x->ns), + .selector = k8s_dup_opt(x->selector), + .line = x->line}; + } +} + +static int k8s_cmp_patch(const void *a, const void *b) { + const k8s_patch_t *x = a; + const k8s_patch_t *y = b; + int c = strcmp(x->kust, y->kust); + return c ? c : (x->line > y->line) - (x->line < y->line); +} + +/* A Resource's kind and name, from its node name "Kind/name". */ +static bool k8s_kind_name(const cbm_gbuf_node_t *n, char *kind, size_t kind_sz, const char **name) { + const char *slash = n->name ? strchr(n->name, '/') : NULL; + if (!slash) { + return false; + } + snprintf(kind, kind_sz, "%.*s", (int)(slash - n->name), n->name); + *name = slash + SKIP_ONE; + return true; +} + +/* What a patch names: kind / group / version / name / ns (NULL: any). A + * name compares as text: a target's is a regular expression in kustomize, + * which an object name (DNS characters, `.` matching itself) meets as text + * alone -- a pattern (`web-.*`) names nothing here. */ +typedef struct { + const char *kind; + const char *group; + const char *version; + const char *name; + const char *ns; + const char *selector; +} k8s_want_t; + +/* Does label list `labels` ("k=v,k2=v2") carry `item` ("k=v") whole, or -- + * `item` ending in "=" -- any value of that key? */ +static bool k8s_labels_have(const char *labels, const char *item) { + size_t il = strlen(item); + bool any_value = il > 0 && item[il - SKIP_ONE] == '='; + for (const char *p = labels; p && *p;) { + size_t l = strcspn(p, ","); + if (any_value ? (l >= il && strncmp(p, item, il) == 0) + : (l == il && strncmp(p, item, il) == 0)) { + return true; + } + p = p[l] ? p + l + SKIP_ONE : NULL; + } + return false; +} + +/* Does Resource `t` meet label selector `sel`: every term `k=v` / `k==v` + * (carried), `k!=v` (not carried), `k` (the key present), `!k` (absent)? A + * set-based term (`k in (a,b)`) names no key and meets nothing. */ +static bool k8s_selector_meets(const cbm_gbuf_node_t *t, const char *sel) { + char *labels = k8s_node_prop_dup(t, "labels"); + bool ok = true; + for (const char *p = sel; ok && p && *p;) { + size_t l = strcspn(p, ","); + char term[CBM_SZ_512]; + snprintf(term, sizeof(term), "%.*s", (int)l, p); + char *s = term; + while (*s == ' ') { + s++; + } + for (char *e = s + strlen(s); e > s && e[-SKIP_ONE] == ' '; e--) { + e[-SKIP_ONE] = '\0'; + } + char item[CBM_SZ_512]; + char *eq = strstr(s, "!="); + if (eq) { + snprintf(item, sizeof(item), "%.*s=%s", (int)(eq - s), s, eq + PAIR_LEN); + ok = !k8s_labels_have(labels, item); + } else if ((eq = strchr(s, '='))) { + const char *v = eq[SKIP_ONE] == '=' ? eq + PAIR_LEN : eq + SKIP_ONE; + snprintf(item, sizeof(item), "%.*s=%s", (int)(eq - s), s, v); + ok = k8s_labels_have(labels, item); + } else if (s[0] == '!') { + snprintf(item, sizeof(item), "%s=", s + SKIP_ONE); + ok = !k8s_labels_have(labels, item); + } else { + snprintf(item, sizeof(item), "%s=", s); + ok = s[0] && k8s_labels_have(labels, item); + } + p = p[l] ? p + l + SKIP_ONE : NULL; + } + cbm_free(CBM_MEM_CLASS_OTHER, labels); + return ok; +} + +/* Is Resource `t` what `w` names? A namespace meets the target's declared + * one or its effective one. */ +static bool k8s_patch_meets(const cbm_gbuf_node_t *t, const k8s_want_t *w) { + char tkind[K8S_LABEL_LEN]; + const char *tname = NULL; + if (!t->label || strcmp(t->label, "Resource") != 0 || + !k8s_kind_name(t, tkind, sizeof(tkind), &tname)) { + return false; + } + if (w->kind && strcmp(w->kind, tkind) != 0) { + return false; + } + if (w->name && strcmp(w->name, tname) != 0) { + return false; + } + char prop[K8S_LABEL_LEN]; + if (w->group) { + k8s_node_prop(t, "group", prop, sizeof(prop)); + if (strcmp(w->group, prop) != 0) { + return false; + } + } + if (w->version) { + k8s_node_prop(t, "version", prop, sizeof(prop)); + if (strcmp(w->version, prop) != 0) { + return false; + } + } + if (w->ns) { + char decl[K8S_LABEL_LEN]; + k8s_declared_ns(t->qualified_name, decl, sizeof(decl)); + k8s_effective_ns(t, prop, sizeof(prop)); + if (strcmp(w->ns, decl) != 0 && strcmp(w->ns, prop) != 0) { + return false; + } + } + return !w->selector || k8s_selector_meets(t, w->selector); +} + +static void k8s_patch_edge(cbm_pipeline_ctx_t *ctx, int64_t src, int64_t tgt, + const k8s_patch_t *p) { + if (src <= 0 || tgt <= 0 || src == tgt) { + return; + } + char ek[CBM_SZ_2K]; + char ep[CBM_SZ_2K]; + cbm_json_escape(ek, (int)sizeof(ek), p->kust); + cbm_json_escape(ep, (int)sizeof(ep), p->path ? p->path : "inline"); + char props[CBM_SZ_4K + CBM_SZ_1K]; + snprintf(props, sizeof(props), + "{\"via\":\"patch\",\"kustomization\":\"%s\",\"patch\":\"%s\",\"line\":%d}", ek, ep, + p->line); + cbm_gbuf_insert_edge(ctx->gbuf, src, tgt, "OVERRIDE", props); +} + +/* The build a patch acts in: its manifests, their Resources by name + * (`names`: "Kind/name" -> ids, this build's only -- a repository-wide name + * would visit every copy of a name for every patch, quadratic in copies) and + * every Resource by file. */ +typedef struct { + cbm_pipeline_ctx_t *ctx; + const k8s_paths_t *files; + CBMHashTable *names; + CBMHashTable *by_file; +} k8s_build_t; + +/* Draw `p` from `src` to each object of the build `w` names. A named kind + * is looked up by name, anything else walks the build. */ +static void k8s_patch_targets(const k8s_build_t *b, const k8s_patch_t *p, int64_t src, + const k8s_want_t *w) { + cbm_gbuf_t *gb = b->ctx->gbuf; + if (w->kind && w->name) { + char key[CBM_SZ_1K]; + snprintf(key, sizeof(key), "%s/%s", w->kind, w->name); + const k8s_ids_t *ids = cbm_ht_get(b->names, key); + for (int i = 0; ids && i < ids->count; i++) { + const cbm_gbuf_node_t *t = cbm_gbuf_find_by_id(gb, ids->items[i]); + atomic_fetch_add_explicit(&g_k8s_candidates, SKIP_ONE, memory_order_relaxed); + if (t && k8s_patch_meets(t, w)) { + k8s_patch_edge(b->ctx, src, t->id, p); + } + } + return; + } + for (int f = 0; f < b->files->count; f++) { + const k8s_ids_t *objs = cbm_ht_get(b->by_file, b->files->items[f]); + for (int o = 0; objs && o < objs->count; o++) { + const cbm_gbuf_node_t *t = cbm_gbuf_find_by_id(gb, objs->items[o]); + atomic_fetch_add_explicit(&g_k8s_candidates, SKIP_ONE, memory_order_relaxed); + if (t && k8s_patch_meets(t, w)) { + k8s_patch_edge(b->ctx, src, t->id, p); + } + } + } +} + +/* The File node of `path` (a kustomization, a patch file: the node kustomize + * IMPORTS leave from), 0 when none. */ +static int64_t k8s_file_of(cbm_pipeline_ctx_t *ctx, const char *path) { + const cbm_gbuf_node_t *n = cbm_pipeline_file_node(ctx->gbuf, ctx->project_name, path); + return n ? n->id : 0; +} + +/* Patch `p` in build `b`: a path patch without a target -- each of its + * documents to the object of the same group, kind and name (and namespace, + * when it declares one); otherwise from its documents -- a file of none (a + * JSON6902 operation list): from its File; inline: from the kustomization's + * File `kust_file` -- to what it names. */ +static void k8s_patch_in_build(const k8s_build_t *b, const k8s_patch_t *p, int64_t kust_file) { + const k8s_ids_t *docs = p->path ? cbm_ht_get(b->by_file, p->path) : NULL; + if (!p->kind && !p->name && !p->selector) { + for (int d = 0; docs && d < docs->count; d++) { + const cbm_gbuf_node_t *doc = cbm_gbuf_find_by_id(b->ctx->gbuf, docs->items[d]); + char kind[K8S_LABEL_LEN]; + char group[K8S_LABEL_LEN]; + char ns[K8S_LABEL_LEN]; + const char *name = NULL; + if (!doc || !k8s_kind_name(doc, kind, sizeof(kind), &name)) { + continue; + } + k8s_node_prop(doc, "group", group, sizeof(group)); + k8s_declared_ns(doc->qualified_name, ns, sizeof(ns)); + k8s_want_t w = {kind, group[0] ? group : NULL, NULL, name, ns[0] ? ns : NULL, NULL}; + k8s_patch_targets(b, p, doc->id, &w); + } + return; + } + k8s_want_t w = {p->kind, p->group, p->version, p->name, p->ns, p->selector}; + if (docs && docs->count > 0) { + for (int d = 0; d < docs->count; d++) { + k8s_patch_targets(b, p, docs->items[d], &w); + } + } else { + k8s_patch_targets(b, p, p->path ? k8s_file_of(b->ctx, p->path) : kust_file, &w); + } +} + +/* The builds the patches of kustomization `k` act in, into `out`: its own; + * a component's are those of each kustomization including it (`inc`: the + * includers of every kustomization, by index). */ +static void k8s_patch_builds(const k8s_kusts_t *ks, const k8s_ints_t *inc, int k, k8s_ints_t *out) { + out->count = 0; + if (!ks->items[k].component) { + if (k8s_grow((void **)&out->items, &out->cap, out->count, sizeof(int))) { + out->items[out->count++] = k; + } + return; + } + for (int i = 0; i < inc[k].count; i++) { + if (k8s_grow((void **)&out->items, &out->cap, out->count, sizeof(int))) { + out->items[out->count++] = inc[k].items[i]; + } + } +} + +/* Kustomize patches -> OVERRIDE {via: patch, kustomization, patch, line}, + * child -> base: each patch against the build of the kustomization stating + * it, a component's against the build of each kustomization including it + * (the component's own resources are part of that build). */ +static void k8s_link_patches(cbm_pipeline_ctx_t *ctx, const k8s_kusts_t *ks, k8s_facts_t *f) { + if (f->patch_n == 0 || ks->count == 0) { + return; + } + CBMHashTable *by_path = cbm_ht_create(0); + CBMHashTable *by_file = cbm_ht_create(0); + int *seen = cbm_calloc(CBM_MEM_CLASS_OTHER, (size_t)ks->count * sizeof(int)); + k8s_ints_t *inc = cbm_calloc(CBM_MEM_CLASS_OTHER, (size_t)ks->count * sizeof(*inc)); + const cbm_gbuf_node_t **res = NULL; + int res_n = 0; + if (by_path && by_file && seen && inc && + cbm_gbuf_find_by_label(ctx->gbuf, "Resource", &res, &res_n) == 0) { + for (int i = 0; i < res_n; i++) { + if (res[i]->file_path) { + k8s_index_add(by_file, res[i]->file_path, res[i]->id); + } + } + for (int i = 0; i < ks->count; i++) { + cbm_ht_set(by_path, ks->items[i].path, (void *)(intptr_t)(i + SKIP_ONE)); + for (int c = 0; c < ks->items[i].kid_n; c++) { + k8s_ints_t *v = &inc[ks->items[i].kids[c]]; + if (k8s_grow((void **)&v->items, &v->cap, v->count, sizeof(int))) { + v->items[v->count++] = i; + } + } + } + k8s_qsort(f->patches, (size_t)f->patch_n, sizeof(*f->patches), k8s_cmp_patch); + k8s_ints_t builds = {0}; + int mark = 0; + for (int i = 0; i < f->patch_n;) { + int j = i; + while (j < f->patch_n && strcmp(f->patches[j].kust, f->patches[i].kust) == 0) { + j++; + } + int k = k8s_kust_at(by_path, f->patches[i].kust) - SKIP_ONE; + int64_t kust_file = 0; + if (k >= 0) { + k8s_patch_builds(ks, inc, k, &builds); + kust_file = k8s_file_of(ctx, ks->items[k].path); + } + for (int bi = 0; k >= 0 && bi < builds.count; bi++) { + k8s_paths_t files = {0}; + k8s_kust_closure(ks, builds.items[bi], seen, ++mark, &files); + k8s_paths_sort_unique(&files); + CBMHashTable *names = cbm_ht_create(0); + for (int x = 0; names && x < files.count; x++) { + const k8s_ids_t *objs = cbm_ht_get(by_file, files.items[x]); + for (int o = 0; objs && o < objs->count; o++) { + const cbm_gbuf_node_t *n = cbm_gbuf_find_by_id(ctx->gbuf, objs->items[o]); + if (n && n->name) { + k8s_index_add(names, n->name, n->id); + } + } + } + k8s_build_t b = {ctx, &files, names, by_file}; + for (int q = i; names && q < j; q++) { + k8s_patch_in_build(&b, &f->patches[q], kust_file); + } + if (names) { + cbm_ht_foreach(names, k8s_free_ids_entry, NULL); + cbm_ht_free(names); + } + cbm_free(CBM_MEM_CLASS_OTHER, (void *)files.items); + } + i = j; + } + cbm_free(CBM_MEM_CLASS_OTHER, builds.items); + } + for (int i = 0; inc && i < ks->count; i++) { + cbm_free(CBM_MEM_CLASS_OTHER, inc[i].items); + } + cbm_free(CBM_MEM_CLASS_OTHER, inc); + cbm_free(CBM_MEM_CLASS_OTHER, seen); + if (by_file) { + cbm_ht_foreach(by_file, k8s_free_ids_entry, NULL); + cbm_ht_free(by_file); + } + if (by_path) { + cbm_ht_free(by_path); + } +} + +/* kustomize's prefix / suffix transformers leave these names alone + * (api/internal/builtins/PrefixTransformer.go prefixFieldSpecsToSkip, the + * same list in SuffixTransformer.go). */ +static bool k8s_rename_skips(const char *group, const char *kind) { + return strcmp(kind, "CustomResourceDefinition") == 0 || strcmp(kind, "Namespace") == 0 || + (strcmp(kind, "APIService") == 0 && strcmp(group, "apiregistration.k8s.io") == 0); +} + +/* A kustomization on the way down a renaming build: the prefix and suffix + * the levels above and its own put around its objects' names, and the + * namespace the outermost level setting one gives them (NULL: their own). */ +typedef struct { + int kust; + const char *ns; + char pre[K8S_LABEL_LEN]; + char suf[K8S_LABEL_LEN]; +} k8s_rename_frame_t; + +/* The variant of object `obj` the renaming build of `root` deploys: a + * Resource `Kind/` declared by the kustomization (`base`: + * its File QN without `.__file__`), ecosystem kustomize; the kustomization + * DEFINES it and it OVERRIDEs {via: namePrefix | nameSuffix, kustomization, + * line} the object. */ +static void k8s_rename_variant(cbm_pipeline_ctx_t *ctx, const k8s_kust_t *root, const char *base, + int64_t kust_file, const k8s_rename_frame_t *f, + const cbm_gbuf_node_t *obj) { + char kind[K8S_LABEL_LEN]; + char group[K8S_LABEL_LEN]; + char version[K8S_LABEL_LEN]; + char declared[K8S_LABEL_LEN]; + const char *name = NULL; + if (!k8s_kind_name(obj, kind, sizeof(kind), &name)) { + return; + } + k8s_node_prop(obj, "group", group, sizeof(group)); + if (!group[0] || k8s_rename_skips(group, kind)) { + return; + } + k8s_node_prop(obj, "version", version, sizeof(version)); + k8s_declared_ns(obj->qualified_name, declared, sizeof(declared)); + const char *ns = k8s_cluster_scoped(kind) || !f->ns ? declared : f->ns; + char deployed[K8S_LABEL_LEN]; + int dl = snprintf(deployed, sizeof(deployed), "%s%s%s", f->pre, name, f->suf); + char qn[CBM_SZ_2K]; + char rname[CBM_SZ_256]; + if (dl <= 0 || (size_t)dl >= sizeof(deployed) || + snprintf(qn, sizeof(qn), "%s::%s/%s/%s/%s", base, group, kind, ns[0] ? ns : "-", + deployed) >= (int)sizeof(qn) || + snprintf(rname, sizeof(rname), "%s/%s", kind, deployed) >= (int)sizeof(rname)) { + return; + } + char *props = k8s_resource_props(qn, NULL, version, NULL, "kustomize"); + int64_t id = props ? cbm_gbuf_upsert_node(ctx->gbuf, "Resource", rname, qn, root->path, + root->rename_line, root->rename_line, props) + : 0; + cbm_free(CBM_MEM_CLASS_OTHER, props); + if (id <= 0 || id == obj->id) { + return; + } + if (kust_file > 0) { + cbm_gbuf_insert_edge(ctx->gbuf, kust_file, id, "DEFINES", "{}"); + } + char ek[CBM_SZ_2K]; + cbm_json_escape(ek, (int)sizeof(ek), root->path); + char eprops[CBM_SZ_2K + CBM_SZ_256]; + snprintf(eprops, sizeof(eprops), "{\"via\":\"%s\",\"kustomization\":\"%s\",\"line\":%d}", + root->prefix ? "namePrefix" : "nameSuffix", ek, root->rename_line); + cbm_gbuf_insert_edge(ctx->gbuf, id, obj->id, "OVERRIDE", eprops); +} + +/* Every object the build of renaming kustomization `root` reaches gets its + * variant: down the kustomizations it includes, each level's prefix inside + * the ones above, its namespace where none above sets one (a component's + * own transforms are not followed). `by_file`: Resource ids by manifest. */ +static void k8s_rename_build(cbm_pipeline_ctx_t *ctx, const k8s_kusts_t *ks, int root, + CBMHashTable *by_file, int *seen, int mark) { + static const char FILE_SUFFIX[] = ".__file__"; + const k8s_kust_t *r = &ks->items[root]; + const cbm_gbuf_node_t *kf = cbm_pipeline_file_node(ctx->gbuf, ctx->project_name, r->path); + size_t ql = kf && kf->qualified_name ? strlen(kf->qualified_name) : 0; + size_t sl = sizeof(FILE_SUFFIX) - SKIP_ONE; + char base[CBM_SZ_1K]; + if (ql <= sl || strcmp(kf->qualified_name + ql - sl, FILE_SUFFIX) != 0 || + ql - sl >= sizeof(base)) { + return; + } + snprintf(base, sizeof(base), "%.*s", (int)(ql - sl), kf->qualified_name); + k8s_rename_frame_t *stack = NULL; + int depth = 0; + int cap = 0; + if (!k8s_grow((void **)&stack, &cap, depth, sizeof(*stack))) { + return; + } + stack[depth] = (k8s_rename_frame_t){.kust = root, .ns = r->ns}; + snprintf(stack[depth].pre, sizeof(stack[depth].pre), "%s", r->prefix ? r->prefix : ""); + snprintf(stack[depth].suf, sizeof(stack[depth].suf), "%s", r->suffix ? r->suffix : ""); + depth++; + while (depth > 0) { + k8s_rename_frame_t f = stack[--depth]; + if (seen[f.kust] == mark) { + continue; + } + seen[f.kust] = mark; + const k8s_kust_t *k = &ks->items[f.kust]; + for (int i = 0; i < k->file_n; i++) { + const k8s_ids_t *objs = cbm_ht_get(by_file, k->files[i]); + for (int o = 0; objs && o < objs->count; o++) { + const cbm_gbuf_node_t *n = cbm_gbuf_find_by_id(ctx->gbuf, objs->items[o]); + if (n) { + k8s_rename_variant(ctx, r, base, kf->id, &f, n); + } + } + } + for (int i = 0; i < k->kid_n; i++) { + const k8s_kust_t *kid = &ks->items[k->kids[i]]; + const char *kp = kid->component || !kid->prefix ? "" : kid->prefix; + const char *kx = kid->component || !kid->suffix ? "" : kid->suffix; + if (!k8s_grow((void **)&stack, &cap, depth, sizeof(*stack))) { + break; + } + k8s_rename_frame_t *c = &stack[depth]; + *c = (k8s_rename_frame_t){.kust = k->kids[i], + .ns = f.ns || kid->component ? f.ns : kid->ns}; + if (snprintf(c->pre, sizeof(c->pre), "%s%s", f.pre, kp) < (int)sizeof(c->pre) && + snprintf(c->suf, sizeof(c->suf), "%s%s", kx, f.suf) < (int)sizeof(c->suf)) { + depth++; + } + } + } + cbm_free(CBM_MEM_CLASS_OTHER, stack); +} + +/* Kustomize namePrefix / nameSuffix -> a variant Resource per object a + * kustomization renames, with its deployed name, OVERRIDE -> the object. + * Minted after the joins: references inside a build use the declared names + * (kustomize rewrites them), so nothing joins a variant. */ +static void k8s_link_renames(cbm_pipeline_ctx_t *ctx, const k8s_kusts_t *ks) { + bool any = false; + for (int i = 0; i < ks->count && !any; i++) { + any = !ks->items[i].component && (ks->items[i].prefix || ks->items[i].suffix); + } + if (!any) { + return; + } + CBMHashTable *by_file = cbm_ht_create(0); + int *seen = cbm_calloc(CBM_MEM_CLASS_OTHER, (size_t)ks->count * sizeof(int)); + const cbm_gbuf_node_t **res = NULL; + int res_n = 0; + if (by_file && seen && cbm_gbuf_find_by_label(ctx->gbuf, "Resource", &res, &res_n) == 0) { + for (int i = 0; i < res_n; i++) { + if (res[i]->file_path) { + k8s_index_add(by_file, res[i]->file_path, res[i]->id); + } + } + int mark = 0; + for (int i = 0; i < ks->count; i++) { + if (!ks->items[i].component && (ks->items[i].prefix || ks->items[i].suffix)) { + k8s_rename_build(ctx, ks, i, by_file, seen, ++mark); + } + } + } + cbm_free(CBM_MEM_CLASS_OTHER, seen); + if (by_file) { + cbm_ht_foreach(by_file, k8s_free_ids_entry, NULL); + cbm_ht_free(by_file); + } +} + static void k8s_link_deploys(cbm_pipeline_ctx_t *ctx, k8s_kusts_t *ks, k8s_bundles_t *b, const k8s_facts_t *f) { - if (f->dep_n == 0 && f->hr_n == 0 && f->src_n == 0) { + if (f->dep_n == 0 && f->hr_n == 0 && f->src_n == 0 && f->kchart_n == 0) { return; } k8s_dep_link_t l = {.ctx = ctx, @@ -3415,13 +4378,18 @@ static void k8s_link_deploys(cbm_pipeline_ctx_t *ctx, k8s_kusts_t *ks, k8s_bundl k8s_deploys_edge(&l, hr->node, k8s_chart_node(&l, dir), props); } } else if (src && src->file_path && !strchr(hr->chart, '/') && !strchr(hr->chart, '$')) { - k8s_deploys_edge(&l, hr->node, k8s_external_chart(&l, hr->chart, src->file_path), + /* the source's URL is not read (a HelmRepository's spec.url): + * the per-project external Chart */ + k8s_deploys_edge(&l, hr->node, k8s_external_chart(&l, hr->chart, NULL, src->file_path), props); } } for (int s = 0; l.files && l.charts && s < f->src_n; s++) { k8s_link_chart_refs(&l, &f->srcs[s]); } + if (l.files && l.kusts) { + k8s_link_kcharts(&l, f); + } cbm_free(CBM_MEM_CLASS_OTHER, (void *)members.items); cbm_free(CBM_MEM_CLASS_OTHER, (void *)units.items); cbm_free(CBM_MEM_CLASS_OTHER, (void *)manifests.items); @@ -3497,7 +4465,8 @@ int cbm_pipeline_pass_k8s(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, helm_count++; break; case K8S_KUSTOMIZE: - k8s_kusts_push(&kusts, rel, k8s_kustomize_namespace(p->source)); + k8s_kusts_push(&kusts, rel, p->source); + k8s_kustomize_keep(&facts, rel, p->res); break; case K8S_MANIFEST: handle_k8s_manifest(ctx, rel, p->source, p->res, &recs, &facts); @@ -3525,6 +4494,10 @@ int cbm_pipeline_pass_k8s(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, } cbm_free(CBM_MEM_CLASS_OTHER, prep); + /* Kubernetes objects Terraform declares (pass_terraform.c), before the + * bundles and the joins: references meet them like manifest objects. */ + cbm_pipeline_tf_k8s_objects(ctx, files, file_count); + /* Connect Services to the workloads their selectors target (INFRA_MAPS), * inside the bundles that deploy them together. */ k8s_bundles_build(ctx, &kusts, &bundles); @@ -3536,6 +4509,8 @@ int cbm_pipeline_pass_k8s(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, k8s_link_deploys(ctx, &kusts, &bundles, &facts); k8s_link_images(ctx, &facts); k8s_link_envs(ctx, &facts); + k8s_link_patches(ctx, &kusts, &facts); + k8s_link_renames(ctx, &kusts); k8s_facts_free(&facts); k8s_records_free(&recs); /* Link `file://` chart dependencies now that every Chart.yaml is read, @@ -3547,7 +4522,10 @@ int cbm_pipeline_pass_k8s(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, /* Shell and Make deploy commands, against the nodes linked above. */ cbm_pipeline_pass_shell(ctx, files, file_count); cbm_pipeline_pass_terraform(ctx, files, file_count); + cbm_pipeline_pass_ansible(ctx, files, file_count); cbm_pipeline_pass_docker(ctx, files, file_count); + /* The variables a container sets, to the readers in its own image. */ + (void)cbm_pipeline_env_join(ctx); cbm_log_info("pass.done", "pass", "k8s", "manifests", itoa_k8s(manifest_count)); (void)helm_count; diff --git a/src/pipeline/pass_operator.c b/src/pipeline/pass_operator.c index 85fd106d8d..239cacd85c 100644 --- a/src/pipeline/pass_operator.c +++ b/src/pipeline/pass_operator.c @@ -1640,7 +1640,7 @@ static CBMHashTable *gc_chart_index(const cbm_gbuf_t *gb) { const char *qn = nodes[i]->qualified_name; const char *slash = fp ? strrchr(fp, '/') : NULL; const char *base = slash ? slash + SKIP_ONE : fp; - if (!fp || !qn || strstr(qn, ".__helm_dep__.") || + if (!fp || !qn || cbm_pipeline_chart_is_external(nodes[i]) || (strcmp(base, "Chart.yaml") != 0 && strcmp(base, "Chart.yml") != 0) || !slash) { continue; } diff --git a/src/pipeline/pass_route_nodes.c b/src/pipeline/pass_route_nodes.c index 3bc472f6aa..9f0c57ddbf 100644 --- a/src/pipeline/pass_route_nodes.c +++ b/src/pipeline/pass_route_nodes.c @@ -78,6 +78,17 @@ static void route_edge_visitor(const cbm_gbuf_edge_t *edge, void *userdata) { if (strcmp(edge->type, "HTTP_CALLS") != 0 && strcmp(edge->type, "ASYNC_CALLS") != 0) { return; } + /* A script's request (pass_shell.c) names its host and already ends at + * its Route, which keeps the host for one outside the repository: a + * path-only Route made from its url_path would meet a handler it never + * calls. A code client's edge names none and refreshes its Route here. */ + char host_buf[CBM_SZ_256]; + if (json_extract(edge->properties_json, "host", host_buf, sizeof(host_buf))) { + const cbm_gbuf_node_t *target = cbm_gbuf_find_by_id(ctx->gb, edge->target_id); + if (target && target->label && strcmp(target->label, "Route") == 0) { + return; + } + } /* Extract url_path from properties */ char url_buf[CBM_SZ_512]; diff --git a/src/pipeline/pass_shell.c b/src/pipeline/pass_shell.c index cce3916727..8e6f82ce91 100644 --- a/src/pipeline/pass_shell.c +++ b/src/pipeline/pass_shell.c @@ -11,8 +11,8 @@ * kubectl ... -k D the Kustomization of D * kustomize build D | ... | kubectl apply -f - (also kubectl kustomize D) * helm install|upgrade [NAME] CHART the Chart of a chart dir, else the - * shared external Chart (repo/name, - * oci://.../name) + * external Chart of that name + * (repo/name, oci://.../name) * helm template [NAME] CHART | ... | kubectl apply -f - * find D -name G -exec kubectl apply -f {} \; D's manifests matching G * @@ -60,6 +60,7 @@ #include "cbm.h" #include "lang_specs.h" +#include #include #include #include @@ -215,7 +216,7 @@ static bool sh_index_build(sh_index_t *ix, cbm_pipeline_ctx_t *ctx) { const char *base = fp ? sh_basename(fp) : ""; const char *qn = nodes[i]->qualified_name; if ((strcmp(base, "Chart.yaml") == 0 || strcmp(base, "Chart.yml") == 0) && qn && - !strstr(qn, ".__helm_dep__.")) { + !cbm_pipeline_chart_is_external(nodes[i])) { cbm_ht_set(ix->charts, fp, (void *)(intptr_t)nodes[i]->id); } } @@ -378,6 +379,7 @@ typedef struct { bool spawn; /* SPAWNS the script `tgt` / the Process `ext`, not DEPLOYS */ bool make; /* SPAWNS the rule of target `ext` that makefile `path` reads */ bool build; /* the image `ext` is built from the Dockerfile `path` */ + bool http; /* HTTP_CALLS the URL `ext` (scheme://host/path) with method `path` */ } sh_hit_t; typedef struct { @@ -2040,6 +2042,187 @@ static void sh_spawn(sh_t *s, TSNode cmd, const sh_argv_t *a) { sh_hit_push(s, cmd, (sh_hit_t){.via = s->exec ? "exec" : "shell", .ext = name, .spawn = true}); } +/* ── HTTP requests (F3) ─────────────────────────────────────────────── */ + +/* `url` (http / https) as scheme://host[:port]/path -- the user info, the + * query and the fragment dropped, so no credential or token reaches the + * graph; NULL when it is not one, or its host is unknown. */ +static const char *sh_url_norm(CBMArena *a, const char *url) { + const char *scheme_end = strstr(url, "://"); + if (!scheme_end || (strncmp(url, "http://", strlen("http://")) != 0 && + strncmp(url, "https://", strlen("https://")) != 0)) { + return NULL; + } + const char *auth = scheme_end + strlen("://"); + size_t al = strcspn(auth, "/?#"); + const char *at = NULL; + for (size_t i = 0; i < al; i++) { + if (auth[i] == '@') { + at = auth + i; + } + } + const char *host = at ? at + SKIP_ONE : auth; + size_t hl = (size_t)(auth + al - host); + const char *h = hl ? cbm_arena_strndup(a, host, hl) : NULL; + if (!h || strpbrk(h, "$`{}\"' \t\\")) { + return NULL; + } + const char *path = auth + al; + size_t pl = strcspn(path, "?#"); + return cbm_arena_sprintf(a, "%.*s%.*s%.*s", (int)(auth - url), url, (int)hl, host, + pl ? (int)pl : SKIP_ONE, pl ? path : "/"); +} + +/* Skip one expansion at p (`${...}`, `$(...)`, `$NAME`, a backquoted + * command); returns the byte after it. */ +static const char *sh_url_skip_expansion(const char *p) { + if (*p == '`') { + const char *e = strchr(p + SKIP_ONE, '`'); + return e ? e + SKIP_ONE : p + strlen(p); + } + char open = p[SKIP_ONE]; + if (open == '{' || open == '(') { + char close = open == '{' ? '}' : ')'; + int depth = 0; + for (const char *q = p + SKIP_ONE; *q; q++) { + depth += *q == open; + depth -= *q == close; + if (depth == 0) { + return q + SKIP_ONE; + } + } + return p + strlen(p); + } + const char *q = p + SKIP_ONE; + while (isalnum((unsigned char)*q) || *q == '_') { + q++; + } + return q; +} + +/* An http(s) URL word the script states with a literal host but variable + * parts after it ("https://host/releases/download/${VER}/x.tgz"), + * from its source text as the shell reads the word -- up to an unquoted + * blank or operator, as tree-sitter-bash may split an unquoted word with + * expansions into several arguments: quotes dropped, each expansion of the + * path a `{}` as in a Route path; then sh_url_norm. NULL when the host is + * not literal. */ +static const char *sh_url_template(sh_t *s, const sh_argv_t *a, int i) { + sh_str_t out = {0}; + char quote = 0; + for (const char *p = s->src + a->span[PAIR_LEN * i]; *p;) { + if (!quote && (strchr(" \t\r\n|;&<>)", *p) || (*p == '\\' && p[SKIP_ONE] == '\n'))) { + break; + } + if ((*p == '"' || *p == '\'') && (!quote || quote == *p)) { + quote = quote ? 0 : *p; + p++; + } else if ((*p == '$' || *p == '`') && quote != '\'') { + sh_str_add(s->a, &out, "{}", PAIR_LEN); + p = sh_url_skip_expansion(p); + } else { + sh_str_add(s->a, &out, p, SKIP_ONE); + p++; + } + } + const char *raw = sh_str_done(&out); + return raw ? sh_url_norm(s->a, raw) : NULL; /* a `{}` host is no literal one */ +} + +static bool sh_ieq(const char *a, const char *b) { + for (; *a && *b; a++, b++) { + if (toupper((unsigned char)*a) != toupper((unsigned char)*b)) { + return false; + } + } + return *a == *b; +} + +/* Options whose value is a URL that is not the request: a proxy, a referer. */ +static bool sh_http_url_option(const char *o) { + static const char *const OPTS[] = {"-x", "--proxy", "--preproxy", "-e", + "--referer", "--doh-url", "--proxy1.0", NULL}; + return sh_in(o, OPTS); +} + +/* The method curl / wget sends: -X / --request / --method; a body (-d, -F, + * --data*, --json, --form*, --post-data / --post-file) POST, -T PUT, -I HEAD, + * -G back to GET; default GET. */ +static const char *sh_http_method(const sh_argv_t *a, int from) { + static const char *const METHODS[] = {"GET", "POST", "PUT", "PATCH", + "DELETE", "HEAD", "OPTIONS", NULL}; + const char *method = "GET"; + bool get = false; + for (int i = from; i < a->n; i++) { + const char *w = a->v[i]; + if (!w) { + continue; + } + const char *named = NULL; + if (sh_eq(w, "-X") || sh_eq(w, "--request") || sh_eq(w, "--method")) { + named = i + SKIP_ONE < a->n ? a->v[i + SKIP_ONE] : NULL; + } else if (sh_starts(w, "-X") && w[PAIR_LEN]) { + named = w + PAIR_LEN; + } else if (sh_starts(w, "--request=") || sh_starts(w, "--method=")) { + named = strchr(w, '=') + SKIP_ONE; + } + for (int m = 0; named && METHODS[m]; m++) { + if (sh_ieq(named, METHODS[m])) { + return METHODS[m]; + } + } + if (sh_eq(w, "-G") || sh_eq(w, "--get")) { + get = true; + } else if (sh_eq(w, "-I") || sh_eq(w, "--head")) { + method = "HEAD"; + } else if (sh_eq(w, "-T") || sh_starts(w, "--upload-file")) { + method = "PUT"; + } else if (sh_eq(w, "-d") || sh_eq(w, "-F") || sh_starts(w, "--data") || + sh_starts(w, "--json") || sh_starts(w, "--form") || + sh_starts(w, "--post-data") || sh_starts(w, "--post-file")) { + method = strcmp(method, "GET") == 0 ? "POST" : method; + } + } + return get && strcmp(method, "POST") == 0 ? "GET" : method; +} + +/* curl / wget with a URL the script states (constant words): an HTTP + * request to it, from the command's function. */ +static void sh_http(sh_t *s, TSNode cmd, const sh_argv_t *a) { + int p = sh_program(s, a); + const char *prog = p >= 0 && a->v[p] ? sh_basename(a->v[p]) : NULL; + if (!prog || (strcmp(prog, "curl") != 0 && strcmp(prog, "wget") != 0)) { + return; + } + const char *via = strcmp(prog, "curl") == 0 ? "curl" : "wget"; + const char *method = sh_http_method(a, p + SKIP_ONE); + for (int i = p + SKIP_ONE; i < a->n; i++) { + /* arguments that touch in the source are one shell word, which + * tree-sitter-bash may split around an expansion */ + int j = i; + while (j + SKIP_ONE < a->n && + a->span[PAIR_LEN * (j + SKIP_ONE)] == a->span[PAIR_LEN * j + SKIP_ONE]) { + j++; + } + bool url_option = + i > p + SKIP_ONE && a->v[i - SKIP_ONE] && sh_http_url_option(a->v[i - SKIP_ONE]); + sh_str_t word = {0}; + for (int k = i; k <= j; k++) { + sh_str_add(s->a, &word, a->v[k], a->v[k] ? strlen(a->v[k]) : 0); + } + const char *w = sh_str_done(&word); + /* a `$` left in an evaluated word is an expansion the grammar took + * for text (`$OS-$ARCH.tgz`): no value, so the template */ + const char *url = url_option ? NULL + : w && !strpbrk(w, "$`") ? sh_url_norm(s->a, w) + : sh_url_template(s, a, i); + if (url) { + sh_hit_push(s, cmd, (sh_hit_t){.via = via, .ext = url, .path = method, .http = true}); + } + i = j; + } +} + /* ── Statements ────────────────────────────────────────────────────── */ static void sh_walk(sh_t *s, TSNode n); @@ -2344,6 +2527,7 @@ static void sh_command(sh_t *s, TSNode cmd) { sh_deploy_command(s, cmd, &a, NULL, &out); sh_spawn(s, cmd, &a); sh_build(s, cmd, &a); + sh_http(s, cmd, &a); } sh_argv_free(&a); } @@ -2392,6 +2576,7 @@ static void sh_pipe_run(sh_t *s, TSNode n, sh_src_t *src) { } sh_spawn(s, cmd, &a); sh_build(s, cmd, &a); + sh_http(s, cmd, &a); sh_argv_free(&a); sh_redirects(s, el); } @@ -3384,16 +3569,11 @@ static int64_t sh_source(cbm_pipeline_ctx_t *ctx, const sh_span_t *spans, int n, return fn ? fn->id : 0; } -/* The shared external Chart of `name` (the one chart dependencies use). */ +/* The external Chart of `name` a helm command names: the hit keeps no + * repository URL (a `repo/name` alias, an oci:// reference's last segment), + * so the per-project one. */ static int64_t sh_external_chart(cbm_pipeline_ctx_t *ctx, const char *name, const char *file) { - char qn[CBM_SZ_512]; - snprintf(qn, sizeof(qn), "%s.__helm_dep__.%s", ctx->project_name, name); - const cbm_gbuf_node_t *known = cbm_gbuf_find_by_qn(ctx->gbuf, qn); - if (known) { - return known->id; - } - return cbm_gbuf_upsert_node(ctx->gbuf, "Chart", name, qn, file, SKIP_ONE, 0, - "{\"source\":\"helm\",\"external\":true}"); + return cbm_pipeline_external_chart(ctx, name, NULL, file); } /* A hit's CI context as JSON members (`,"ci":..,"job":..`); "" for none. */ @@ -3479,11 +3659,106 @@ static int sh_link_build(cbm_pipeline_ctx_t *ctx, const sh_hit_t *x, CBMHashTabl return SKIP_ONE; } +/* A host that is a service of the repository: one DNS label or a + * `..svc...` name whose first label names a Kubernetes Service of + * it. Its Route is the path alone, where that service's handler of the path + * meets it, as code clients do; any other host -- a download, a + * port-forwarded localhost -- stays in the Route, so it never meets a + * handler it does not reach. (Not a compose service: the Docker pass makes + * those after this one in a full run, while a repair holds them already; + * the QN tells them apart, as a repair's proxies carry no properties.) */ +static bool sh_http_service(cbm_pipeline_ctx_t *ctx, const char *host) { + const char *first_dot = strchr(host, '.'); + const char *svc = first_dot ? strchr(first_dot + SKIP_ONE, '.') : NULL; + bool dns_form = !first_dot || (svc && sh_starts(svc, ".svc") && + (svc[strlen(".svc")] == '\0' || svc[strlen(".svc")] == '.')); + if (!dns_form || strcmp(host, "localhost") == 0) { + return false; + } + char name[CBM_SZ_256]; + snprintf(name, sizeof(name), "Service/%.*s", + (int)(first_dot ? (size_t)(first_dot - host) : strlen(host)), host); + const cbm_gbuf_node_t **nodes = NULL; + int n = 0; + if (cbm_gbuf_find_by_name(ctx->gbuf, name, &nodes, &n) != 0) { + return false; + } + for (int i = 0; i < n; i++) { + if (nodes[i]->label && strcmp(nodes[i]->label, "Resource") == 0 && + nodes[i]->qualified_name && !strstr(nodes[i]->qualified_name, "::compose/")) { + return true; + } + } + return false; +} + +/* An HTTP request a script makes: HTTP_CALLS {via: curl | wget, method, + * url_path, host} to its Route (sh_http_service). */ +static int sh_link_http(cbm_pipeline_ctx_t *ctx, int64_t src, const sh_hit_t *x) { + const char *auth = strstr(x->ext, "://"); + if (!auth || src <= 0) { + return 0; + } + auth += strlen("://"); + size_t al = strcspn(auth, "/"); + char host[CBM_SZ_256]; + size_t hl = auth[0] == '[' ? strcspn(auth, "]") + SKIP_ONE : strcspn(auth, ":/"); + if (hl == 0 || hl > al || hl >= sizeof(host)) { + return 0; + } + for (size_t i = 0; i < hl; i++) { + host[i] = (char)tolower((unsigned char)auth[i]); + } + host[hl] = '\0'; + const char *path = auth + al; + char cpath[CBM_SZ_2K]; + cbm_route_canon_path(path, cpath, sizeof(cpath)); + bool local = sh_http_service(ctx, host); + char qn[CBM_ROUTE_QN_SIZE]; + snprintf(qn, sizeof(qn), "__route__%s__%.*s%s", x->path, local ? 0 : (int)(auth + al - x->ext), + x->ext, cpath); + char authority[CBM_SZ_256]; /* the host with its port */ + if (al >= sizeof(authority)) { + return 0; + } + for (size_t i = 0; i < al; i++) { + authority[i] = (char)tolower((unsigned char)auth[i]); + } + authority[al] = '\0'; + char ehost[CBM_SZ_512]; + char epath[CBM_SZ_4K]; + cbm_json_escape(ehost, (int)sizeof(ehost), authority); + cbm_json_escape(epath, (int)sizeof(epath), path); + char rprops[CBM_SZ_1K]; + if (local) { + snprintf(rprops, sizeof(rprops), "{\"method\":\"%s\"}", x->path); + } else { + snprintf(rprops, sizeof(rprops), "{\"method\":\"%s\",\"host\":\"%s\",\"external\":true}", + x->path, ehost); + } + int64_t route = + cbm_gbuf_upsert_node(ctx->gbuf, "Route", local ? path : x->ext, qn, "", 0, 0, rprops); + if (route <= 0 || route == src) { + return 0; + } + char ci[CBM_SZ_1K]; + sh_ci_props(x, ci, sizeof(ci)); + char props[CBM_SZ_4K + CBM_SZ_1K]; + snprintf(props, sizeof(props), + "{\"via\":\"%s\",\"method\":\"%s\",\"url_path\":\"%s\",\"host\":\"%s\"%s,\"line\":%d}", + x->via, x->path, epath, ehost, ci, x->line); + cbm_gbuf_insert_edge(ctx->gbuf, src, route, "HTTP_CALLS", props); + return SKIP_ONE; +} + static int sh_link_hit(cbm_pipeline_ctx_t *ctx, const sh_hit_t *x, int64_t src, CBMHashTable *modules) { if (x->spawn) { return src > 0 ? sh_link_spawn(ctx, src, x, modules) : 0; } + if (x->http) { + return sh_link_http(ctx, src, x); + } int64_t tgt = x->tgt > 0 ? x->tgt : sh_external_chart(ctx, x->ext, x->file); if (src <= 0 || tgt <= 0 || src == tgt) { return 0; diff --git a/src/pipeline/pass_terraform.c b/src/pipeline/pass_terraform.c index 309f6c99c6..6328311c17 100644 --- a/src/pipeline/pass_terraform.c +++ b/src/pipeline/pass_terraform.c @@ -7,8 +7,9 @@ * chart = "./charts/x" / "${path.module}/charts/x" the Chart of that * directory, relative to * the module's directory - * repository = "", chart = "" the shared external - * chart = "oci://registry/.../" Chart of that name + * repository = "", chart = "" the external Chart of + * chart = "oci://registry/.../" that name (an oci:// + * reference: of its URL) * Values are evaluated from string literals and ${path.module}; any other * expression is unknown, and an unknown chart is no edge. Run at the end of * the K8s pass, over the same files. @@ -67,9 +68,10 @@ static bool tf_add(char *out, size_t sz, size_t *len, const char *p, size_t n) { return true; } -/* A string value: a literal, or a template of literals and ${path.module}. - * false when any part is something else. */ -static bool tf_string(const tf_t *t, TSNode n, char *out, size_t sz) { +/* A string value of file text `src` in module directory `mdir`: a literal, or + * a template of literals and ${path.module}. false when any part is something + * else. */ +static bool tf_string(const char *src, const char *mdir, TSNode n, char *out, size_t sz) { size_t len = 0; out[0] = '\0'; while (tf_type(n, "expression") || tf_type(n, "literal_value") || tf_type(n, "template_expr") || @@ -93,7 +95,7 @@ static bool tf_string(const tf_t *t, TSNode n, char *out, size_t sz) { continue; } if (tf_type(c, "template_literal")) { - if (!tf_add(out, sz, &len, t->src + b, e - b)) { + if (!tf_add(out, sz, &len, src + b, e - b)) { return false; } continue; @@ -115,12 +117,12 @@ static bool tf_string(const tf_t *t, TSNode n, char *out, size_t sz) { uint32_t xe = ts_node_end_byte(expr); static const char PATH_MODULE[] = "path.module"; if (xe - xb != sizeof(PATH_MODULE) - SKIP_ONE || - strncmp(t->src + xb, PATH_MODULE, xe - xb) != 0) { + strncmp(src + xb, PATH_MODULE, xe - xb) != 0) { return false; } /* an absolute marker: the module directory, from the repo root */ if (!tf_add(out, sz, &len, "\x01", SKIP_ONE) || - !tf_add(out, sz, &len, t->mdir, strlen(t->mdir))) { + !tf_add(out, sz, &len, mdir, strlen(mdir))) { return false; } } @@ -181,6 +183,10 @@ static bool tf_repo_path(const char *mdir, const char *v, char *out, size_t sz) return true; } +bool cbm_repo_rel_join(const char *dir, const char *v, char *out, size_t sz) { + return tf_repo_path(dir, v, out, sz); +} + /* Chart.yaml path -> its (local) Chart node. */ static CBMHashTable *tf_chart_index(cbm_pipeline_ctx_t *ctx) { CBMHashTable *ix = cbm_ht_create(0); @@ -193,7 +199,9 @@ static CBMHashTable *tf_chart_index(cbm_pipeline_ctx_t *ctx) { const char *fp = nodes[i]->file_path; const char *qn = nodes[i]->qualified_name; const char *base = fp ? tf_basename(fp) : ""; - if (fp && qn && !strstr(qn, ".__helm_dep__.") && + /* an external chart a Chart.yaml depends on carries that file's + * path too: only the chart it declares is the directory's */ + if (fp && qn && !cbm_pipeline_chart_is_external(nodes[i]) && (strcmp(base, "Chart.yaml") == 0 || strcmp(base, "Chart.yml") == 0)) { cbm_ht_set(ix, fp, (void *)(intptr_t)nodes[i]->id); } @@ -215,18 +223,6 @@ static int64_t tf_local_chart(const tf_t *t, const char *dir) { return 0; } -/* The shared external Chart of `name` (the one chart dependencies use). */ -static int64_t tf_external_chart(cbm_pipeline_ctx_t *ctx, const char *name, const char *file) { - char qn[CBM_SZ_512]; - snprintf(qn, sizeof(qn), "%s.__helm_dep__.%s", ctx->project_name, name); - const cbm_gbuf_node_t *known = cbm_gbuf_find_by_qn(ctx->gbuf, qn); - if (known) { - return known->id; - } - return cbm_gbuf_upsert_node(ctx->gbuf, "Chart", name, qn, file, SKIP_ONE, 0, - "{\"source\":\"helm\",\"external\":true}"); -} - /* The node of block `resource.helm_release.` in the file, by the QN * its definition got -- not among every block of that name: modules that * share a release name (`this`) would make each lookup scan all of them. */ @@ -277,7 +273,7 @@ static void tf_helm_release(tf_t *t, TSNode block, const char *name) { uint32_t ke = ts_node_end_byte(key); size_t kl = ke - kb; if (kl == strlen("chart") && strncmp(t->src + kb, "chart", kl) == 0) { - has_chart = tf_string(t, val, chart, sizeof(chart)); + has_chart = tf_string(t->src, t->mdir, val, chart, sizeof(chart)); } else if (kl == strlen("repository") && strncmp(t->src + kb, "repository", kl) == 0) { has_repo = true; } @@ -288,6 +284,9 @@ static void tf_helm_release(tf_t *t, TSNode block, const char *name) { int64_t src = tf_block_node(t, name); int64_t tgt = 0; bool external = false; + /* the repository of an oci:// reference; "" when unknown (a + * `repository =` value is not read) */ + char repo[CBM_SZ_1K] = ""; char path[CBM_SZ_1K] = ""; static const char OCI[] = "oci://"; if (strncmp(chart, OCI, sizeof(OCI) - SKIP_ONE) == 0) { @@ -295,6 +294,10 @@ static void tf_helm_release(tf_t *t, TSNode block, const char *name) { if (tf_plain_name(base)) { snprintf(path, sizeof(path), "%s", base); external = true; + /* the reference names its repository: all but the chart */ + if (!cbm_chart_ref_repo(chart, repo, sizeof(repo))) { + repo[0] = '\0'; + } } } else if (has_repo) { if (tf_plain_name(chart)) { @@ -305,7 +308,7 @@ static void tf_helm_release(tf_t *t, TSNode block, const char *name) { tgt = tf_local_chart(t, path); } if (external) { - tgt = tf_external_chart(t->ctx, path, t->file); + tgt = cbm_pipeline_external_chart(t->ctx, path, repo[0] ? repo : NULL, t->file); } if (src <= 0 || tgt <= 0) { return; @@ -393,6 +396,1311 @@ static bool tf_is_module_file(const char *rel) { return dot && (strcmp(dot, ".tf") == 0 || strcmp(dot, ".tofu") == 0); } +/* A variable-definitions file (.tfvars; a .tfvars.json is JSON, not read here). */ +static bool tf_is_tfvars(const char *rel) { + const char *dot = strrchr(tf_basename(rel), '.'); + return dot && strcmp(dot, ".tfvars") == 0; +} + +/* The body of a config file's root node. */ +static TSNode tf_root_body(TSNode root) { + for (uint32_t i = 0; i < ts_node_named_child_count(root); i++) { + if (tf_type(ts_node_named_child(root, i), "body")) { + return ts_node_named_child(root, i); + } + } + return (TSNode){0}; +} + +/* ── References (PHASE 6 P6.1 - P6.3) ─────────────────────────────── */ + +/* A Terraform module is a directory: every .tf / .tofu file of it. A + * reference resolves inside its module only -- var.X to `variable "X"`, + * local.X to the locals block declaring X, module.X to `module "X"`, + * data.T.N to `data "T" "N"`, T.N to `resource "T" "N"` (T.N declared there); + * anything else (each, count, self, path, terraform, a for-expression name, a + * builtin function) to nothing. Edge: USAGE {ref, via: terraform, line} from + * the top-level block holding the reference to the declaring block; an entry + * of the block's depends_on list orders it after the object: DEPENDS_ON, the + * same properties. + * + * A module call with a local source (`module "X" { source = "./net" }`; the + * directory inside the repository, holding .tf / .tofu files) IMPORTS + * {local_name: X, via: terraform, line} that directory's Folder, and + * module.X.Y (also module.X[i].Y, module.X[*].Y) USAGEs {ref: output}, beside + * module.X, the `output "Y"` of it. + * + * A .tfvars file sets variables of the module in its own directory -- or, + * when that declares none of its names, of the nearest ancestor directory's + * module that does (applied by a -var-file the repository does not state: + * the edge says `inferred`): each assignment to a declared variable + * OVERRIDEs {via: tfvars, line} its `variable` block. */ + +typedef struct { + char *file; /* the referencing file */ + char *src; /* the holding top-level block's definition name */ + char *key; /* what it names: variable.X, local.X, module.X, data.T.N, resource.T.N */ + char *out; /* module.X.Y: Y, the output it reads; else NULL */ + const char *ref; /* static: var, local, module, data, resource */ + int line; + bool dep; /* stands in the holder's depends_on list */ +} tf_ref_t; + +/* A module call with a local source. */ +typedef struct { + char *file; /* the calling file */ + char *name; /* its block: module.X */ + char *dir; /* the source directory, from the repository root */ + int line; +} tf_call_t; + +typedef struct { + cbm_pipeline_ctx_t *ctx; + CBMArena *arena; + CBMHashTable *decls; /* key (as tf_ref_t.key) -> declaring file (arena) */ + CBMHashTable *calls; /* module.X -> its local source directory (arena) */ + tf_ref_t *refs; + int ref_n; + int ref_cap; + tf_call_t *mcalls; + int call_n; + int call_cap; + const char *dir; /* the module's directory */ + const char *src; /* the file's text */ + const char *file; /* its repository path (arena) */ + bool dep; /* walking a depends_on list */ +} tf_mod_t; + +static const char TF_MODULE_PREFIX[] = "module."; + +static const char *tf_text(CBMArena *a, const char *src, TSNode n) { + uint32_t b = ts_node_start_byte(n); + return cbm_arena_strndup(a, src + b, ts_node_end_byte(n) - b); +} + +/* A block label: a string_lit's text, or an identifier (`resource aws_x y`). */ +static const char *tf_block_label(CBMArena *a, const char *src, TSNode n) { + if (tf_type(n, "identifier")) { + return tf_text(a, src, n); + } + for (uint32_t i = 0; tf_type(n, "string_lit") && i < ts_node_named_child_count(n); i++) { + TSNode c = ts_node_named_child(n, i); + if (tf_type(c, "template_literal")) { + return tf_text(a, src, c); + } + } + return NULL; +} + +/* A top-level block's definition name: its keyword and labels joined by + * "." (variable.X, resource.T.N, locals) -- the name extract_defs gives it. */ +static const char *tf_block_name(CBMArena *a, const char *src, TSNode block, TSNode *body) { + const char *name = NULL; + *body = (TSNode){0}; + for (uint32_t i = 0; i < ts_node_named_child_count(block); i++) { + TSNode c = ts_node_named_child(block, i); + if (tf_type(c, "body")) { + *body = c; + } else if (i == 0 && tf_type(c, "identifier")) { + name = tf_text(a, src, c); + } else if (name) { + const char *label = tf_block_label(a, src, c); + name = label ? cbm_arena_sprintf(a, "%s.%s", name, label) : name; + } + } + return name; +} + +/* Room for one more element in a growing array. */ +static bool tf_grow(void **items, int *cap, int n, size_t size) { + if (n < *cap) { + return true; + } + int ncap = *cap ? *cap * PAIR_LEN : CBM_SZ_64; + void *grown = cbm_realloc(CBM_MEM_CLASS_OTHER, *items, (size_t)ncap * size); + if (!grown) { + return false; + } + *items = grown; + *cap = ncap; + return true; +} + +static void tf_ref_push(tf_mod_t *m, const char *src_block, const char *ref, const char *key, + const char *out, int line) { + if (tf_grow((void **)&m->refs, &m->ref_cap, m->ref_n, sizeof(*m->refs))) { + m->refs[m->ref_n++] = (tf_ref_t){ + (char *)m->file, (char *)src_block, (char *)key, (char *)out, ref, line, m->dep}; + } +} + +/* The identifier of a get_attr node. */ +static const char *tf_attr_name(CBMArena *a, const char *src, TSNode g) { + for (uint32_t c = 0; c < ts_node_named_child_count(g); c++) { + if (tf_type(ts_node_named_child(g, c), "identifier")) { + return tf_text(a, src, ts_node_named_child(g, c)); + } + } + return NULL; +} + +/* The k-th attribute name of the traversal `v` starts. Its siblings, once the + * grammar's hidden expression rule is flattened, are get_attr / index / + * splat nodes: an index ([0], ["a"]) is passed over, a splat's own + * attributes ([*].y, .*.y) count. NULL past the attributes. */ +static const char *tf_attr(CBMArena *a, const char *src, TSNode v, int k) { + int i = 0; + for (TSNode s = ts_node_next_named_sibling(v); !ts_node_is_null(s); + s = ts_node_next_named_sibling(s)) { + if (tf_type(s, "index")) { + continue; + } + if (tf_type(s, "splat")) { + TSNode sp = ts_node_named_child_count(s) > 0 ? ts_node_named_child(s, 0) : s; + for (uint32_t c = 0; c < ts_node_named_child_count(sp); c++) { + TSNode g = ts_node_named_child(sp, c); + if (tf_type(g, "get_attr") && i++ == k) { + return tf_attr_name(a, src, g); + } + } + continue; + } + if (!tf_type(s, "get_attr")) { + return NULL; + } + if (i++ == k) { + return tf_attr_name(a, src, s); + } + } + return NULL; +} + +/* One traversal `root.attr...` inside block `src_block`. */ +static void tf_traversal(tf_mod_t *m, const char *src_block, TSNode v) { + if (ts_node_named_child_count(v) == 0) { + return; + } + CBMArena *a = m->arena; + const char *root = tf_text(a, m->src, ts_node_named_child(v, 0)); + const char *a1 = tf_attr(a, m->src, v, 0); + if (!root || !a1) { + return; + } + int line = (int)ts_node_start_point(v).row + SKIP_ONE; + if (strcmp(root, "var") == 0) { + tf_ref_push(m, src_block, "var", cbm_arena_sprintf(a, "variable.%s", a1), NULL, line); + } else if (strcmp(root, "local") == 0) { + tf_ref_push(m, src_block, "local", cbm_arena_sprintf(a, "local.%s", a1), NULL, line); + } else if (strcmp(root, "module") == 0) { + tf_ref_push(m, src_block, "module", cbm_arena_sprintf(a, "module.%s", a1), + tf_attr(a, m->src, v, SKIP_ONE), line); + } else if (strcmp(root, "data") == 0) { + const char *a2 = tf_attr(a, m->src, v, SKIP_ONE); + if (a2) { + tf_ref_push(m, src_block, "data", cbm_arena_sprintf(a, "data.%s.%s", a1, a2), NULL, + line); + } + } else if (strchr(root, '_')) { + tf_ref_push(m, src_block, "resource", cbm_arena_sprintf(a, "resource.%s.%s", root, a1), + NULL, line); + } +} + +/* Every traversal under `n` (iterative: deep expressions). */ +static void tf_walk_refs(tf_mod_t *m, const char *src_block, TSNode n) { + TSTreeCursor cur = ts_tree_cursor_new(n); + bool more = true; + while (more) { + TSNode c = ts_tree_cursor_current_node(&cur); + if (tf_type(c, "variable_expr")) { + tf_traversal(m, src_block, c); + } + if (ts_tree_cursor_goto_first_child(&cur)) { + continue; + } + while (!ts_tree_cursor_goto_next_sibling(&cur)) { + if (!ts_tree_cursor_goto_parent(&cur)) { + more = false; + break; + } + } + } + ts_tree_cursor_delete(&cur); +} + +/* Node `n`'s text (in file text `src`) is `word`. */ +static bool tf_text_is(const char *src, TSNode n, const char *word) { + if (ts_node_is_null(n)) { + return false; + } + uint32_t b = ts_node_start_byte(n); + size_t len = strlen(word); + return ts_node_end_byte(n) - b == len && memcmp(src + b, word, len) == 0; +} + +static bool tf_is_word(const tf_mod_t *m, TSNode n, const char *word) { + return tf_text_is(m->src, n, word); +} + +/* The references of one top-level block's body, those of its depends_on + * list marked as such. */ +static void tf_walk_block(tf_mod_t *m, const char *src_block, TSNode body) { + for (uint32_t i = 0; i < ts_node_named_child_count(body); i++) { + TSNode c = ts_node_named_child(body, i); + m->dep = tf_type(c, "attribute") && ts_node_named_child_count(c) > 0 && + tf_is_word(m, ts_node_named_child(c, 0), "depends_on"); + tf_walk_refs(m, src_block, c); + } + m->dep = false; +} + +/* `module "X" { source = "./dir" }`: the call, its source resolved from the + * module's directory. A local source is "./..." or "../..."; any other -- a + * registry address, git, http, a bucket -- lies outside the repository. */ +static void tf_mod_call(tf_mod_t *m, const char *name, TSNode block, TSNode body) { + for (uint32_t i = 0; i < ts_node_named_child_count(body); i++) { + TSNode at = ts_node_named_child(body, i); + if (!tf_type(at, "attribute") || ts_node_named_child_count(at) < PAIR_LEN || + !tf_is_word(m, ts_node_named_child(at, 0), "source")) { + continue; + } + char v[CBM_SZ_1K]; + char dir[CBM_SZ_1K]; + if (!tf_string(m->src, m->dir, ts_node_named_child(at, SKIP_ONE), v, sizeof(v)) || + (strncmp(v, "./", sizeof("./") - SKIP_ONE) != 0 && + strncmp(v, "../", sizeof("../") - SKIP_ONE) != 0) || + !tf_repo_path(m->dir, v, dir, sizeof(dir)) || + !tf_grow((void **)&m->mcalls, &m->call_cap, m->call_n, sizeof(*m->mcalls))) { + return; + } + char *d = cbm_arena_strdup(m->arena, dir); + if (d) { + m->mcalls[m->call_n++] = (tf_call_t){(char *)m->file, (char *)name, d, + (int)ts_node_start_point(block).row + SKIP_ONE}; + cbm_ht_set(m->calls, name, d); + } + return; + } +} + +/* One file of a module: its declarations (blocks, locals entries), module + * calls and the references of each top-level block. */ +static void tf_mod_file(tf_mod_t *m, TSNode root) { + CBMArena *a = m->arena; + TSNode body = tf_root_body(root); + for (uint32_t i = 0; !ts_node_is_null(body) && i < ts_node_named_child_count(body); i++) { + TSNode b = ts_node_named_child(body, i); + TSNode bb; + const char *name = tf_type(b, "block") ? tf_block_name(a, m->src, b, &bb) : NULL; + if (!name) { + continue; + } + cbm_ht_set(m->decls, name, (void *)m->file); /* `{}` declares too */ + if (ts_node_is_null(bb)) { + continue; + } + if (strcmp(name, "locals") == 0) { + for (uint32_t j = 0; j < ts_node_named_child_count(bb); j++) { + TSNode at = ts_node_named_child(bb, j); + if (tf_type(at, "attribute") && ts_node_named_child_count(at) > 0) { + const char *key = tf_text(a, m->src, ts_node_named_child(at, 0)); + cbm_ht_set(m->decls, cbm_arena_sprintf(a, "local.%s", key), (void *)m->file); + } + } + } else if (strncmp(name, TF_MODULE_PREFIX, sizeof(TF_MODULE_PREFIX) - SKIP_ONE) == 0) { + tf_mod_call(m, name, b, bb); + } + tf_walk_block(m, name, bb); + } +} + +/* The node of definition `name` in `file`, by the QN extract_defs gave it. */ +static int64_t tf_def_node(tf_mod_t *m, const char *file, const char *name) { + const char *qn = cbm_fqn_compute(m->arena, m->ctx->project_name, file, name); + const cbm_gbuf_node_t *n = qn ? cbm_gbuf_find_by_qn(m->ctx->gbuf, qn) : NULL; + atomic_fetch_add_explicit(&g_tf_candidates, SKIP_ONE, memory_order_relaxed); + return n && n->file_path && strcmp(n->file_path, file) == 0 ? n->id : 0; +} + +/* Order paths by directory, then name: a directory's files are contiguous. */ +static int tf_cmp_dir_path(const void *x, const void *y) { + const char *p = *(const char *const *)x; + const char *q = *(const char *const *)y; + const char *ps = strrchr(p, '/'); + const char *qs = strrchr(q, '/'); + size_t pl = ps ? (size_t)(ps - p) : 0; + size_t ql = qs ? (size_t)(qs - q) : 0; + size_t n = pl < ql ? pl : ql; + int c = strncmp(p, q, n); + if (c == 0 && pl != ql) { + c = pl < ql ? -1 : 1; + } + return c ? c : strcmp(p, q); +} + +static bool tf_same_dir(const char *p, const char *q) { + const char *ps = strrchr(p, '/'); + const char *qs = strrchr(q, '/'); + size_t pl = ps ? (size_t)(ps - p) : 0; + size_t ql = qs ? (size_t)(qs - q) : 0; + return pl == ql && strncmp(p, q, pl) == 0; +} + +/* One reference edge; 1 when made. */ +static int tf_ref_edge(tf_mod_t *m, int64_t src, int64_t tgt, const char *type, const char *ref, + int line) { + if (src <= 0 || tgt <= 0 || src == tgt) { + return 0; + } + char props[CBM_SZ_128]; + snprintf(props, sizeof(props), "{\"ref\":\"%s\",\"via\":\"terraform\",\"line\":%d}", ref, line); + cbm_gbuf_insert_edge(m->ctx->gbuf, src, tgt, type, props); + return SKIP_ONE; +} + +/* Resolve the module's references against its declarations, module.X.Y + * against the outputs of X's source directory (`mods`: directory -> its + * module), and its local module calls to their directories' Folders. */ +static int tf_mod_link(tf_mod_t *m, CBMHashTable *mods, CBMHashTable *folders) { + int edges = 0; + for (int i = 0; i < m->ref_n; i++) { + const tf_ref_t *r = &m->refs[i]; + const char *tfile = cbm_ht_get(m->decls, r->key); + const char *oname = NULL; + const char *ofile = NULL; + if (r->out) { + const char *cdir = cbm_ht_get(m->calls, r->key); + const tf_mod_t *callee = cdir ? cbm_ht_get(mods, cdir) : NULL; + oname = callee ? cbm_arena_sprintf(m->arena, "output.%s", r->out) : NULL; + ofile = oname ? cbm_ht_get(callee->decls, oname) : NULL; + } + if (!tfile && !ofile) { + continue; + } + int64_t src = tf_def_node(m, r->file, r->src); + if (tfile) { + bool local = strcmp(r->ref, "local") == 0; + edges += tf_ref_edge(m, src, tf_def_node(m, tfile, local ? "locals" : r->key), + r->dep ? "DEPENDS_ON" : "USAGE", r->ref, r->line); + } + if (ofile) { + edges += tf_ref_edge(m, src, tf_def_node(m, ofile, oname), "USAGE", "output", r->line); + } + } + for (int i = 0; i < m->call_n; i++) { + const tf_call_t *c = &m->mcalls[i]; + const cbm_gbuf_node_t *folder = NULL; + if (cbm_ht_get(mods, c->dir)) { + /* the repository root has no Folder: its Project stands for it */ + folder = c->dir[0] ? (folders ? cbm_ht_get(folders, c->dir) : NULL) + : cbm_gbuf_find_by_qn(m->ctx->gbuf, m->ctx->project_name); + } + if (folder && !c->dir[0] && (!folder->label || strcmp(folder->label, "Project") != 0)) { + folder = NULL; + } + int64_t src = folder ? tf_def_node(m, c->file, c->name) : 0; + if (src <= 0) { + continue; + } + char props[CBM_SZ_256]; + snprintf(props, sizeof(props), "{\"local_name\":\"%s\",\"via\":\"terraform\",\"line\":%d}", + c->name + sizeof(TF_MODULE_PREFIX) - SKIP_ONE, c->line); + cbm_gbuf_insert_edge(m->ctx->gbuf, src, folder->id, "IMPORTS", props); + edges++; + } + return edges; +} + +static void tf_mod_free(tf_mod_t *m) { + if (m->decls) { + cbm_ht_free(m->decls); + } + if (m->calls) { + cbm_ht_free(m->calls); + } + cbm_free(CBM_MEM_CLASS_OTHER, m->refs); + cbm_free(CBM_MEM_CLASS_OTHER, m->mcalls); + cbm_free(CBM_MEM_CLASS_OTHER, m); +} + +/* One module directory, its files `paths[0..n)`: parse each (read from the + * repository) into its declarations, calls and references. */ +static tf_mod_t *tf_mod_parse(cbm_pipeline_ctx_t *ctx, CBMArena *arena, TSParser *parser, + const char *const *paths, int n) { + tf_mod_t *m = cbm_calloc(CBM_MEM_CLASS_OTHER, sizeof(*m)); + if (!m) { + return NULL; + } + const char *slash = strrchr(paths[0], '/'); + *m = (tf_mod_t){.ctx = ctx, + .arena = arena, + .decls = cbm_ht_create(0), + .calls = cbm_ht_create(0), + .dir = + cbm_arena_strndup(arena, paths[0], slash ? (size_t)(slash - paths[0]) : 0)}; + if (!m->decls || !m->calls || !m->dir) { + tf_mod_free(m); + return NULL; + } + for (int i = 0; i < n; i++) { + char path[CBM_SZ_2K]; + snprintf(path, sizeof(path), "%s/%s", ctx->repo_path, paths[i]); + size_t len = 0; + char *src = tf_read(path, &len); + TSTree *tree = src ? ts_parser_parse_string(parser, NULL, src, (uint32_t)len) : NULL; + if (tree) { + m->src = src; + m->file = cbm_arena_strdup(arena, paths[i]); + tf_mod_file(m, ts_tree_root_node(tree)); + ts_tree_delete(tree); + } + m->src = NULL; + cbm_free(CBM_MEM_CLASS_OTHER, src); + } + return m; +} + +/* path -> node, over the nodes of one label (Folder, File). */ +static CBMHashTable *tf_label_index(cbm_pipeline_ctx_t *ctx, const char *label) { + CBMHashTable *ix = cbm_ht_create(0); + const cbm_gbuf_node_t **nodes = NULL; + int n = 0; + if (ix && cbm_gbuf_find_by_label(ctx->gbuf, label, &nodes, &n) == 0) { + for (int i = 0; i < n; i++) { + if (nodes[i]->file_path) { + cbm_ht_set(ix, nodes[i]->file_path, (void *)nodes[i]); + } + } + } + return ix; +} + +/* The module of the first of `d` and its ancestors (d is cut in place) + * declaring one of `vnames[0..n)`; NULL when none does. */ +static tf_mod_t *tf_tfvars_module(CBMHashTable *mods, char *d, const char *const *vnames, int n) { + for (;;) { + tf_mod_t *m = cbm_ht_get(mods, d); + for (int k = 0; m && k < n; k++) { + if (vnames[k] && cbm_ht_get(m->decls, vnames[k])) { + return m; + } + } + if (!d[0]) { + return NULL; + } + char *s = strrchr(d, '/'); + if (s) { + *s = '\0'; + } else { + d[0] = '\0'; + } + } +} + +/* A .tfvars file sets variables of a module: the one in its own directory + * when that declares one of its names, else the nearest ancestor directory's + * that does (a file there is applied by a -var-file the repository does not + * state: the edge says `inferred`). Each top-level assignment whose name the + * module declares OVERRIDEs {via: tfvars, line}, from the file's Variable of + * that name, the `variable` block. */ +static int tf_link_tfvars(CBMArena *arena, TSParser *parser, CBMHashTable *mods, + const char *repo_path, const char *rel) { + char path[CBM_SZ_2K]; + snprintf(path, sizeof(path), "%s/%s", repo_path, rel); + size_t len = 0; + char *src = tf_read(path, &len); + TSTree *tree = src ? ts_parser_parse_string(parser, NULL, src, (uint32_t)len) : NULL; + TSNode body = tree ? tf_root_body(ts_tree_root_node(tree)) : (TSNode){0}; + uint32_t nc = ts_node_is_null(body) ? 0 : ts_node_named_child_count(body); + const char **vnames = nc ? cbm_arena_alloc(arena, (size_t)nc * sizeof(*vnames)) : NULL; + TSNode *attrs = nc ? cbm_arena_alloc(arena, (size_t)nc * sizeof(*attrs)) : NULL; + int n = 0; + for (uint32_t i = 0; vnames && attrs && i < nc; i++) { + TSNode at = ts_node_named_child(body, i); + if (tf_type(at, "attribute") && ts_node_named_child_count(at) > 0) { + const char *key = tf_text(arena, src, ts_node_named_child(at, 0)); + vnames[n] = key ? cbm_arena_sprintf(arena, "variable.%s", key) : NULL; + attrs[n++] = at; + } + } + const char *slash = strrchr(rel, '/'); + char d[CBM_SZ_1K]; + snprintf(d, sizeof(d), "%.*s", slash ? (int)(slash - rel) : 0, rel); + size_t own = strlen(d); + tf_mod_t *m = n > 0 ? tf_tfvars_module(mods, d, vnames, n) : NULL; + int edges = 0; + for (int k = 0; m && k < n; k++) { + const char *vfile = vnames[k] ? cbm_ht_get(m->decls, vnames[k]) : NULL; + const char *key = vfile ? vnames[k] + sizeof("variable.") - SKIP_ONE : NULL; + int64_t from = key ? tf_def_node(m, rel, key) : 0; + int64_t to = from > 0 ? tf_def_node(m, vfile, vnames[k]) : 0; + if (to > 0) { + char props[CBM_SZ_128]; + snprintf(props, sizeof(props), "{\"via\":\"tfvars\",%s\"line\":%d}", + strlen(d) == own ? "" : "\"inferred\":true,", + (int)ts_node_start_point(attrs[k]).row + SKIP_ONE); + cbm_gbuf_insert_edge(m->ctx->gbuf, from, to, "OVERRIDE", props); + edges++; + } + } + if (tree) { + ts_tree_delete(tree); + } + cbm_free(CBM_MEM_CLASS_OTHER, src); + return edges; +} + +/* Every module directory, `paths[0..pn)` sorted by directory: parse all, + * then link each -- a reference may read an output of another directory -- + * and the .tfvars files `vpaths[0..vn)` to the module whose variables they set. */ +static int tf_link_repo(cbm_pipeline_ctx_t *ctx, TSParser *parser, const char *const *paths, int pn, + const char *const *vpaths, int vn) { + CBMArena arena; + cbm_arena_init_lazy(&arena, CBM_SZ_4K); + CBMHashTable *mods = cbm_ht_create(0); + tf_mod_t **list = cbm_calloc(CBM_MEM_CLASS_OTHER, (size_t)pn * sizeof(*list)); + int nm = 0; + int calls = 0; + for (int i = 0; mods && list && i < pn;) { + int j = i; + while (j < pn && tf_same_dir(paths[i], paths[j])) { + j++; + } + tf_mod_t *m = tf_mod_parse(ctx, &arena, parser, paths + i, j - i); + if (m) { + list[nm++] = m; + cbm_ht_set(mods, m->dir, m); + calls += m->call_n; + } + i = j; + } + CBMHashTable *folders = calls > 0 ? tf_label_index(ctx, "Folder") : NULL; + int edges = 0; + for (int k = 0; k < nm; k++) { + edges += tf_mod_link(list[k], mods, folders); + } + for (int k = 0; mods && k < vn; k++) { + edges += tf_link_tfvars(&arena, parser, mods, ctx->repo_path, vpaths[k]); + } + for (int k = 0; k < nm; k++) { + tf_mod_free(list[k]); + } + if (folders) { + cbm_ht_free(folders); + } + if (mods) { + cbm_ht_free(mods); + } + cbm_free(CBM_MEM_CLASS_OTHER, list); + cbm_arena_destroy(&arena); + return edges; +} + +/* When one of the run's files is a .tf / .tofu / .tfvars file, every module + * directory of the repository (the graph's File nodes) is resolved whole: on + * an incremental run an unchanged file's reference may meet what a changed + * one declares -- in its own directory (a variable, a locals entry), in a + * module it calls (an output), or a .tfvars file a variable it sets. */ +static int tf_link_refs(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, int file_count, + TSParser *parser) { + bool any = false; + for (int i = 0; i < file_count && !any; i++) { + any = (files[i].language == CBM_LANG_HCL && tf_is_module_file(files[i].rel_path)) || + tf_is_tfvars(files[i].rel_path); + } + const cbm_gbuf_node_t **fnodes = NULL; + int fn = 0; + if (!any || cbm_gbuf_find_by_label(ctx->gbuf, "File", &fnodes, &fn) != 0 || fn <= 0) { + return 0; + } + const char **paths = cbm_calloc(CBM_MEM_CLASS_OTHER, (size_t)fn * sizeof(char *)); + const char **vpaths = cbm_calloc(CBM_MEM_CLASS_OTHER, (size_t)fn * sizeof(char *)); + int pn = 0; + int vn = 0; + for (int i = 0; paths && vpaths && i < fn; i++) { + const char *rel = fnodes[i]->file_path; + if (rel && tf_is_module_file(rel)) { + paths[pn++] = rel; + } else if (rel && tf_is_tfvars(rel)) { + vpaths[vn++] = rel; + } + } + int edges = 0; + if (pn > 0 && ts_parser_set_language(parser, cbm_ts_language(CBM_LANG_HCL))) { + qsort((void *)paths, (size_t)pn, sizeof(*paths), tf_cmp_dir_path); + qsort((void *)vpaths, (size_t)vn, sizeof(*vpaths), tf_cmp_dir_path); + edges = tf_link_repo(ctx, parser, paths, pn, vpaths, vn); + } + cbm_free(CBM_MEM_CLASS_OTHER, (void *)paths); + cbm_free(CBM_MEM_CLASS_OTHER, (void *)vpaths); + return edges; +} + +/* ── Terragrunt (PHASE 6 P6.4) ─────────────────────────────────────── */ + +/* A Terragrunt unit is a terragrunt.hcl file. Its expressions are evaluated + * in the unit's context -- also those of a file it includes, which Terragrunt + * evaluates as part of the unit: + * include "N" { path = E } IMPORTS {local_name: N, via: terragrunt, + * line} the file E names, from the block + * read_terragrunt_config(E) IMPORTS {via: terragrunt, read: , line} the file E names: + * from the unit's top-level block holding + * the call or, for a call in an included + * file, from the unit's include block that + * brings that file in + * dependency "N" { config_path = E } DEPENDS_ON {local_name: N, via: terragrunt, + * line} the terragrunt.hcl of directory E + * terraform { source = "./..." } IMPORTS {via: terragrunt, line} the Folder + * E is a string template of literals and find_in_parent_folders(name) (the + * first file of that name in the unit's parent directory or above), + * dirname(x) and get_terragrunt_dir(); anything else is unknown, and an + * unknown path is no edge. A target must be a file (a directory) the graph + * holds. */ + +static const char TG_UNIT[] = "terragrunt.hcl"; +static const char TG_INCLUDE[] = "include."; +static const char TG_DEPENDENCY[] = "dependency."; + +typedef struct { + cbm_pipeline_ctx_t *ctx; + CBMArena *arena; + CBMHashTable *files; /* repository path -> File node */ + CBMHashTable *folders; /* directory -> Folder node */ + const char *unit; /* the unit's path */ + const char *udir; /* its directory */ + const char *src; /* the text of the file being read */ + int edges; +} tg_t; + +/* The directory part of `p`, in place; an anchored value keeps its anchor. */ +static void tg_dirname(char *p) { + char *s = strrchr(p, '/'); + if (s) { + *s = '\0'; + } else { + p[p[0] == '\x01' ? SKIP_ONE : 0] = '\0'; + } +} + +/* find_in_parent_folders(name): the first `name` in the unit's parent + * directory or above, anchored at the repository root. */ +static bool tg_find_up(const tg_t *g, const char *name, char *out, size_t sz) { + char d[CBM_SZ_1K]; + snprintf(d, sizeof(d), "%s", g->udir); + while (d[0]) { + tg_dirname(d); + char cand[CBM_SZ_2K]; + snprintf(cand, sizeof(cand), "%s%s%s", d, d[0] ? "/" : "", name); + if (cbm_ht_get(g->files, cand)) { + snprintf(out, sz, "\x01%s", cand); + return true; + } + } + return false; +} + +static bool tg_eval(const tg_t *g, TSNode n, char *out, size_t sz); + +/* The argument expressions of a call (up to `max`); their number. */ +static int tg_args(TSNode call, TSNode *args, int max) { + int k = 0; + for (uint32_t i = 0; i < ts_node_named_child_count(call); i++) { + TSNode c = ts_node_named_child(call, i); + for (uint32_t j = 0; tf_type(c, "function_arguments") && j < ts_node_named_child_count(c); + j++) { + if (tf_type(ts_node_named_child(c, j), "expression")) { + if (k < max) { + args[k] = ts_node_named_child(c, j); + } + k++; + } + } + } + return k; +} + +/* A call of a function the evaluator knows. */ +static bool tg_call(const tg_t *g, TSNode call, char *out, size_t sz) { + TSNode fn = ts_node_named_child(call, 0); + TSNode args[PAIR_LEN]; + int na = tg_args(call, args, PAIR_LEN); + char a[CBM_SZ_1K]; + if (tf_text_is(g->src, fn, "find_in_parent_folders")) { + return na == SKIP_ONE && tg_eval(g, args[0], a, sizeof(a)) && a[0] != '\x01' && + tg_find_up(g, a, out, sz); + } + if (tf_text_is(g->src, fn, "dirname")) { + if (na != SKIP_ONE || !tg_eval(g, args[0], out, sz)) { + return false; + } + tg_dirname(out); + return true; + } + if (tf_text_is(g->src, fn, "get_terragrunt_dir") && na == 0) { + snprintf(out, sz, "\x01%s", g->udir); + return true; + } + return false; +} + +/* An expression's value: a string template of literals and known calls; + * false when any part is something else. */ +static bool tg_eval(const tg_t *g, TSNode n, char *out, size_t sz) { + while ( + (tf_type(n, "expression") || tf_type(n, "literal_value") || tf_type(n, "template_expr")) && + ts_node_named_child_count(n) == SKIP_ONE) { + n = ts_node_named_child(n, 0); + } + if (tf_type(n, "function_call")) { + return tg_call(g, n, out, sz); + } + if (!tf_type(n, "string_lit") && !tf_type(n, "quoted_template")) { + return false; + } + size_t len = 0; + out[0] = '\0'; + for (uint32_t i = 0; i < ts_node_named_child_count(n); i++) { + TSNode c = ts_node_named_child(n, i); + if (tf_type(c, "quoted_template_start") || tf_type(c, "quoted_template_end")) { + continue; + } + uint32_t b = ts_node_start_byte(c); + if (tf_type(c, "template_literal")) { + if (!tf_add(out, sz, &len, g->src + b, ts_node_end_byte(c) - b)) { + return false; + } + continue; + } + TSNode expr = {0}; + for (uint32_t j = 0; + tf_type(c, "template_interpolation") && j < ts_node_named_child_count(c); j++) { + if (tf_type(ts_node_named_child(c, j), "expression")) { + expr = ts_node_named_child(c, j); + } + } + char part[CBM_SZ_1K]; + if (ts_node_is_null(expr) || !tg_eval(g, expr, part, sizeof(part)) || + !tf_add(out, sz, &len, part, strlen(part))) { + return false; + } + } + return true; +} + +/* The expression of attribute `key` in a block body; a null node when none. */ +static TSNode tg_attr(const tg_t *g, TSNode body, const char *key) { + for (uint32_t i = 0; !ts_node_is_null(body) && i < ts_node_named_child_count(body); i++) { + TSNode at = ts_node_named_child(body, i); + if (tf_type(at, "attribute") && ts_node_named_child_count(at) >= PAIR_LEN && + tf_text_is(g->src, ts_node_named_child(at, 0), key)) { + return ts_node_named_child(at, SKIP_ONE); + } + } + return (TSNode){0}; +} + +/* The node of definition `name` in `file`, by the QN extract_defs gave it. */ +static int64_t tg_def_node(const tg_t *g, const char *file, const char *name) { + const char *qn = cbm_fqn_compute(g->arena, g->ctx->project_name, file, name); + const cbm_gbuf_node_t *n = qn ? cbm_gbuf_find_by_qn(g->ctx->gbuf, qn) : NULL; + atomic_fetch_add_explicit(&g_tf_candidates, SKIP_ONE, memory_order_relaxed); + return n && n->file_path && strcmp(n->file_path, file) == 0 ? n->id : 0; +} + +static void tg_edge(tg_t *g, int64_t src, const cbm_gbuf_node_t *tgt, const char *type, + const char *props) { + if (src > 0 && tgt && tgt->id != src) { + cbm_gbuf_insert_edge(g->ctx->gbuf, src, tgt->id, type, props); + g->edges++; + } +} + +/* Expression `e` as the repository path of a graph file (relative values + * from `base`); NULL when unknown or not in the graph. */ +static const cbm_gbuf_node_t *tg_file(const tg_t *g, TSNode e, const char *base, char *path, + size_t sz) { + char v[CBM_SZ_1K]; + if (ts_node_is_null(e) || !tg_eval(g, e, v, sizeof(v)) || !tf_repo_path(base, v, path, sz)) { + return NULL; + } + return cbm_ht_get(g->files, path); +} + +/* Every read_terragrunt_config(E) of file `site` (its body `body`): IMPORTS + * the file E names, relative paths from `base`, from `from` -- or, when 0, + * from the top-level block (a top-level attribute: the file) holding it. */ +static void tg_reads(tg_t *g, TSNode body, const char *site, const char *base, int64_t from) { + for (uint32_t i = 0; !ts_node_is_null(body) && i < ts_node_named_child_count(body); i++) { + TSNode b = ts_node_named_child(body, i); + int64_t src = from; + if (!src) { + TSNode bb; + const char *name = tf_type(b, "block") ? tf_block_name(g->arena, g->src, b, &bb) : NULL; + const cbm_gbuf_node_t *file = cbm_ht_get(g->files, site); + src = name ? tg_def_node(g, site, name) : file ? file->id : 0; + } + TSTreeCursor cur = ts_tree_cursor_new(b); + bool more = true; + while (more) { + TSNode c = ts_tree_cursor_current_node(&cur); + TSNode args[SKIP_ONE]; + if (tf_type(c, "function_call") && + tf_text_is(g->src, ts_node_named_child(c, 0), "read_terragrunt_config") && + tg_args(c, args, SKIP_ONE) >= SKIP_ONE) { + char path[CBM_SZ_1K]; + const cbm_gbuf_node_t *tgt = tg_file(g, args[0], base, path, sizeof(path)); + char props[CBM_SZ_2K]; + snprintf(props, sizeof(props), + "{\"via\":\"terragrunt\",\"read\":\"%s\",\"line\":%d}", site, + (int)ts_node_start_point(c).row + SKIP_ONE); + tg_edge(g, src, tgt, "IMPORTS", props); + } + if (ts_tree_cursor_goto_first_child(&cur)) { + continue; + } + while (!ts_tree_cursor_goto_next_sibling(&cur)) { + if (!ts_tree_cursor_goto_parent(&cur)) { + more = false; + break; + } + } + } + ts_tree_cursor_delete(&cur); + } +} + +/* One block of the unit: an include, a dependency, the terraform block. An + * include's file (its path into `inc`, its block into `inc_node`) is read + * afterwards for the calls it holds. */ +static void tg_block(tg_t *g, TSNode b, char *inc, size_t inc_sz, int64_t *inc_node) { + TSNode bb; + const char *name = tf_type(b, "block") ? tf_block_name(g->arena, g->src, b, &bb) : NULL; + if (!name || ts_node_is_null(bb)) { + return; + } + int line = (int)ts_node_start_point(b).row + SKIP_ONE; + char path[CBM_SZ_1K]; + char props[CBM_SZ_256]; + if (strncmp(name, TG_INCLUDE, sizeof(TG_INCLUDE) - SKIP_ONE) == 0) { + const cbm_gbuf_node_t *tgt = + tg_file(g, tg_attr(g, bb, "path"), g->udir, path, sizeof(path)); + int64_t src = tgt ? tg_def_node(g, g->unit, name) : 0; + snprintf(props, sizeof(props), "{\"local_name\":\"%s\",\"via\":\"terragrunt\",\"line\":%d}", + name + sizeof(TG_INCLUDE) - SKIP_ONE, line); + tg_edge(g, src, tgt, "IMPORTS", props); + if (src > 0) { + snprintf(inc, inc_sz, "%s", path); + *inc_node = src; + } + } else if (strncmp(name, TG_DEPENDENCY, sizeof(TG_DEPENDENCY) - SKIP_ONE) == 0) { + char v[CBM_SZ_1K]; + char dir[CBM_SZ_1K]; + TSNode e = tg_attr(g, bb, "config_path"); + if (!ts_node_is_null(e) && tg_eval(g, e, v, sizeof(v)) && + tf_repo_path(g->udir, v, dir, sizeof(dir))) { + snprintf(path, sizeof(path), "%s%s%s", dir, dir[0] ? "/" : "", TG_UNIT); + const cbm_gbuf_node_t *tgt = cbm_ht_get(g->files, path); + snprintf(props, sizeof(props), + "{\"local_name\":\"%s\",\"via\":\"terragrunt\",\"line\":%d}", + name + sizeof(TG_DEPENDENCY) - SKIP_ONE, line); + tg_edge(g, tgt ? tg_def_node(g, g->unit, name) : 0, tgt, "DEPENDS_ON", props); + } + } else if (strcmp(name, "terraform") == 0) { + char v[CBM_SZ_1K]; + TSNode e = tg_attr(g, bb, "source"); + if (!ts_node_is_null(e) && tg_eval(g, e, v, sizeof(v)) && + (strncmp(v, "./", sizeof("./") - SKIP_ONE) == 0 || + strncmp(v, "../", sizeof("../") - SKIP_ONE) == 0) && + tf_repo_path(g->udir, v, path, sizeof(path))) { + const cbm_gbuf_node_t *tgt = g->folders ? cbm_ht_get(g->folders, path) : NULL; + snprintf(props, sizeof(props), "{\"via\":\"terragrunt\",\"line\":%d}", line); + tg_edge(g, tgt ? tg_def_node(g, g->unit, name) : 0, tgt, "IMPORTS", props); + } + } +} + +enum { TG_MAX_INCLUDES = 32 }; + +/* One unit: its blocks and reads, then the reads of the files it includes. */ +static void tg_unit(tg_t *g, TSParser *parser, const char *rel) { + const char *slash = strrchr(rel, '/'); + g->unit = rel; + g->udir = cbm_arena_strndup(g->arena, rel, slash ? (size_t)(slash - rel) : 0); + char path[CBM_SZ_2K]; + snprintf(path, sizeof(path), "%s/%s", g->ctx->repo_path, rel); + size_t len = 0; + char *src = g->udir ? tf_read(path, &len) : NULL; + TSTree *tree = src ? ts_parser_parse_string(parser, NULL, src, (uint32_t)len) : NULL; + char inc[TG_MAX_INCLUDES][CBM_SZ_1K]; + int64_t inc_node[TG_MAX_INCLUDES]; + int ni = 0; + if (tree) { + g->src = src; + TSNode body = tf_root_body(ts_tree_root_node(tree)); + for (uint32_t i = 0; !ts_node_is_null(body) && i < ts_node_named_child_count(body); i++) { + inc_node[ni] = 0; + tg_block(g, ts_node_named_child(body, i), inc[ni], sizeof(inc[ni]), &inc_node[ni]); + if (ni < TG_MAX_INCLUDES - SKIP_ONE && inc_node[ni] > 0) { + ni++; /* past the cap, the last slot is overwritten */ + } + } + tg_reads(g, body, rel, g->udir, 0); + ts_tree_delete(tree); + } + cbm_free(CBM_MEM_CLASS_OTHER, src); + for (int k = 0; k < ni; k++) { + snprintf(path, sizeof(path), "%s/%s", g->ctx->repo_path, inc[k]); + src = tf_read(path, &len); + tree = src ? ts_parser_parse_string(parser, NULL, src, (uint32_t)len) : NULL; + if (tree) { + g->src = src; + char base[CBM_SZ_1K]; + snprintf(base, sizeof(base), "%s", inc[k]); + tg_dirname(base); + tg_reads(g, tf_root_body(ts_tree_root_node(tree)), inc[k], base, inc_node[k]); + ts_tree_delete(tree); + } + cbm_free(CBM_MEM_CLASS_OTHER, src); + } + g->src = NULL; +} + +static int tg_cmp_str(const void *x, const void *y) { + return strcmp(*(const char *const *)x, *(const char *const *)y); +} + +/* Every Terragrunt unit of the repository, when the run holds a .hcl file + * (a unit, or a file units include or read). */ +static int tg_link_units(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, int file_count, + TSParser *parser) { + bool any = false; + for (int i = 0; i < file_count && !any; i++) { + const char *dot = strrchr(tf_basename(files[i].rel_path), '.'); + any = dot && strcmp(dot, ".hcl") == 0; + } + const cbm_gbuf_node_t **fnodes = NULL; + int fn = 0; + if (!any || cbm_gbuf_find_by_label(ctx->gbuf, "File", &fnodes, &fn) != 0 || fn <= 0) { + return 0; + } + CBMArena arena; + cbm_arena_init_lazy(&arena, CBM_SZ_4K); + tg_t g = {.ctx = ctx, .arena = &arena, .files = cbm_ht_create(0)}; + const char **units = cbm_calloc(CBM_MEM_CLASS_OTHER, (size_t)fn * sizeof(char *)); + int nu = 0; + for (int i = 0; g.files && units && i < fn; i++) { + const char *rel = fnodes[i]->file_path; + if (rel) { + cbm_ht_set(g.files, rel, (void *)fnodes[i]); + if (strcmp(tf_basename(rel), TG_UNIT) == 0) { + units[nu++] = rel; + } + } + } + if (nu > 0 && ts_parser_set_language(parser, cbm_ts_language(CBM_LANG_HCL))) { + qsort((void *)units, (size_t)nu, sizeof(*units), tg_cmp_str); + g.folders = tf_label_index(ctx, "Folder"); + for (int i = 0; i < nu; i++) { + tg_unit(&g, parser, units[i]); + } + } + if (g.folders) { + cbm_ht_free(g.folders); + } + if (g.files) { + cbm_ht_free(g.files); + } + cbm_free(CBM_MEM_CLASS_OTHER, (void *)units); + cbm_arena_destroy(&arena); + return g.edges; +} + +/* ── Kubernetes objects Terraform declares (PHASE 6 P6.5) ──────────── */ + +/* resource "kubernetes_[_vN]" "N" declares the object of the kind its + * type names -- a kubernetes_manifest the apiVersion / kind of its manifest + * -- with metadata name and namespace from string literals. It is a + * Resource of the K8s identity scheme (IDENTITY.md: ::/ + * //), with a manifest object's properties plus + * `ecosystem: terraform`; its file DEFINES it and the block DEPLOYS + * {via: terraform, line} it. A computed name (var., local., an + * interpolation) is unknown: no Resource. Types acting on an object declared + * elsewhere (owner references, readiness waits, label / annotation patches) + * declare none. Run before the K8s joins: references meet these objects like + * manifest ones. */ + +typedef struct { + const char *type; /* after "kubernetes_", without a _vN suffix */ + const char *group; + const char *kind; +} tf_k8s_kind_t; + +static const tf_k8s_kind_t TF_K8S_KINDS[] = { + {"namespace", "core", "Namespace"}, + {"service", "core", "Service"}, + {"service_account", "core", "ServiceAccount"}, + {"default_service_account", "core", "ServiceAccount"}, + {"config_map", "core", "ConfigMap"}, + {"secret", "core", "Secret"}, + {"persistent_volume_claim", "core", "PersistentVolumeClaim"}, + {"persistent_volume", "core", "PersistentVolume"}, + {"pod", "core", "Pod"}, + {"limit_range", "core", "LimitRange"}, + {"resource_quota", "core", "ResourceQuota"}, + {"endpoints", "core", "Endpoints"}, + {"replication_controller", "core", "ReplicationController"}, + {"deployment", "apps", "Deployment"}, + {"stateful_set", "apps", "StatefulSet"}, + {"daemonset", "apps", "DaemonSet"}, + {"daemon_set", "apps", "DaemonSet"}, + {"job", "batch", "Job"}, + {"cron_job", "batch", "CronJob"}, + {"cluster_role", "rbac.authorization.k8s.io", "ClusterRole"}, + {"cluster_role_binding", "rbac.authorization.k8s.io", "ClusterRoleBinding"}, + {"role", "rbac.authorization.k8s.io", "Role"}, + {"role_binding", "rbac.authorization.k8s.io", "RoleBinding"}, + {"network_policy", "networking.k8s.io", "NetworkPolicy"}, + {"ingress", "networking.k8s.io", "Ingress"}, + {"ingress_class", "networking.k8s.io", "IngressClass"}, + {"storage_class", "storage.k8s.io", "StorageClass"}, + {"csi_driver", "storage.k8s.io", "CSIDriver"}, + {"priority_class", "scheduling.k8s.io", "PriorityClass"}, + {"horizontal_pod_autoscaler", "autoscaling", "HorizontalPodAutoscaler"}, + {"pod_disruption_budget", "policy", "PodDisruptionBudget"}, + {"runtime_class", "node.k8s.io", "RuntimeClass"}, + {"mutating_webhook_configuration", "admissionregistration.k8s.io", + "MutatingWebhookConfiguration"}, + {"validating_webhook_configuration", "admissionregistration.k8s.io", + "ValidatingWebhookConfiguration"}, + {"api_service", "apiregistration.k8s.io", "APIService"}, + {"certificate_signing_request", "certificates.k8s.io", "CertificateSigningRequest"}, +}; + +/* "kubernetes_[_vN[beta|alphaM]]": into out; false for another + * provider's type. */ +static bool tf_k8s_type(const char *rtype, char *out, size_t sz) { + static const char PREFIX[] = "kubernetes_"; + if (strncmp(rtype, PREFIX, sizeof(PREFIX) - SKIP_ONE) != 0) { + return false; + } + snprintf(out, sz, "%s", rtype + sizeof(PREFIX) - SKIP_ONE); + char *v = strrchr(out, '_'); + if (v && v[SKIP_ONE] == 'v' && v[PAIR_LEN] >= '0' && v[PAIR_LEN] <= '9') { + *v = '\0'; + } + return out[0] != '\0'; +} + +/* A string literal's text (no interpolation, no ${path.module}); false else. */ +static bool tf_literal(const char *src, TSNode n, char *out, size_t sz) { + return tf_string(src, "", n, out, sz) && !strchr(out, '\x01'); +} + +/* The value of `key` in an object expression (keys bare or quoted). */ +static TSNode tf_object_get(const char *src, TSNode expr, const char *key) { + TSNode n = expr; + while ((tf_type(n, "expression") || tf_type(n, "collection_value")) && + ts_node_named_child_count(n) > 0) { + n = ts_node_named_child(n, 0); + } + for (uint32_t i = 0; tf_type(n, "object") && i < ts_node_named_child_count(n); i++) { + TSNode el = ts_node_named_child(n, i); + if (!tf_type(el, "object_elem")) { + continue; + } + TSNode k = ts_node_child_by_field_name(el, "key", sizeof("key") - SKIP_ONE); + char text[CBM_SZ_64]; + bool named = tf_literal(src, k, text, sizeof(text)); + if (!named) { + TSNode id = k; + while ((tf_type(id, "expression") || tf_type(id, "variable_expr")) && + ts_node_named_child_count(id) > 0) { + id = ts_node_named_child(id, 0); + } + named = tf_type(id, "identifier") && tf_text_is(src, id, key); + text[0] = '\0'; + } + if ((named && !text[0]) || (text[0] && strcmp(text, key) == 0)) { + return ts_node_child_by_field_name(el, "val", sizeof("val") - SKIP_ONE); + } + } + return (TSNode){0}; +} + +/* The expression of attribute `key` in a block body, or a null node. */ +static TSNode tf_body_attr(const char *src, TSNode body, const char *key) { + for (uint32_t i = 0; !ts_node_is_null(body) && i < ts_node_named_child_count(body); i++) { + TSNode at = ts_node_named_child(body, i); + if (tf_type(at, "attribute") && ts_node_named_child_count(at) >= PAIR_LEN && + tf_text_is(src, ts_node_named_child(at, 0), key)) { + return ts_node_named_child(at, SKIP_ONE); + } + } + return (TSNode){0}; +} + +/* The body of the nested block `name` in a block body, or a null node. */ +static TSNode tf_body_block(const char *src, TSNode body, const char *name) { + for (uint32_t i = 0; !ts_node_is_null(body) && i < ts_node_named_child_count(body); i++) { + TSNode b = ts_node_named_child(body, i); + if (!tf_type(b, "block") || ts_node_named_child_count(b) == 0 || + !tf_text_is(src, ts_node_named_child(b, 0), name)) { + continue; + } + for (uint32_t j = 0; j < ts_node_named_child_count(b); j++) { + if (tf_type(ts_node_named_child(b, j), "body")) { + return ts_node_named_child(b, j); + } + } + } + return (TSNode){0}; +} + +/* One top-level block of a .tf file: the object it declares, if any. */ +static void tf_k8s_block(cbm_pipeline_ctx_t *ctx, CBMArena *a, const char *src, const char *rel, + const char *base, int64_t file_id, TSNode block) { + TSNode body; + const char *bname = tf_block_name(a, src, block, &body); + static const char RES[] = "resource."; + char rtype[CBM_SZ_128]; + char type[CBM_SZ_128]; + if (!bname || ts_node_is_null(body) || strncmp(bname, RES, sizeof(RES) - SKIP_ONE) != 0) { + return; + } + const char *dot = strchr(bname + sizeof(RES) - SKIP_ONE, '.'); + snprintf(rtype, sizeof(rtype), "%.*s", dot ? (int)(dot - bname - (sizeof(RES) - SKIP_ONE)) : 0, + bname + sizeof(RES) - SKIP_ONE); + if (!tf_k8s_type(rtype, type, sizeof(type))) { + return; + } + char group[CBM_SZ_256] = ""; + char kind[CBM_SZ_128] = ""; + char version[CBM_SZ_64] = ""; + char name[CBM_SZ_256] = ""; + char ns[CBM_SZ_256] = ""; + if (strcmp(type, "manifest") == 0) { + TSNode m = tf_body_attr(src, body, "manifest"); + char av[CBM_SZ_256]; + TSNode meta = tf_object_get(src, m, "metadata"); + if (!tf_literal(src, tf_object_get(src, m, "kind"), kind, sizeof(kind)) || + !tf_literal(src, tf_object_get(src, m, "apiVersion"), av, sizeof(av)) || + !tf_literal(src, tf_object_get(src, meta, "name"), name, sizeof(name))) { + return; + } + const char *slash = strchr(av, '/'); + if (slash) { + snprintf(group, sizeof(group), "%.*s", (int)(slash - av), av); + } else { + snprintf(group, sizeof(group), "core"); + } + snprintf(version, sizeof(version), "%s", slash ? slash + SKIP_ONE : av); + if (!tf_literal(src, tf_object_get(src, meta, "namespace"), ns, sizeof(ns))) { + ns[0] = '\0'; + } + } else { + const tf_k8s_kind_t *k = NULL; + for (size_t i = 0; i < sizeof(TF_K8S_KINDS) / sizeof(TF_K8S_KINDS[0]) && !k; i++) { + k = strcmp(TF_K8S_KINDS[i].type, type) == 0 ? &TF_K8S_KINDS[i] : NULL; + } + TSNode meta = tf_body_block(src, body, "metadata"); + if (!k) { + return; + } + snprintf(group, sizeof(group), "%s", k->group); + snprintf(kind, sizeof(kind), "%s", k->kind); + if (strcmp(type, "default_service_account") == 0) { + snprintf(name, sizeof(name), "default"); + } else if (!tf_literal(src, tf_body_attr(src, meta, "name"), name, sizeof(name))) { + return; + } + if (!tf_literal(src, tf_body_attr(src, meta, "namespace"), ns, sizeof(ns))) { + ns[0] = '\0'; + } + } + if (strchr(name, '/') || strchr(ns, '/') || strchr(kind, '/') || !name[0] || !kind[0]) { + return; /* the identity key splits on "/" */ + } + const char *qn = + cbm_arena_sprintf(a, "%s::%s/%s/%s/%s", base, group, kind, ns[0] ? ns : "-", name); + const char *rname = cbm_arena_sprintf(a, "%s/%s", kind, name); + char *props = qn ? cbm_k8s_resource_props(qn, NULL, version, NULL, "terraform") : NULL; + int64_t id = props ? cbm_gbuf_upsert_node(ctx->gbuf, "Resource", rname, qn, rel, + (int)ts_node_start_point(block).row + SKIP_ONE, + (int)ts_node_end_point(block).row + SKIP_ONE, props) + : 0; + cbm_free(CBM_MEM_CLASS_OTHER, props); + if (id <= 0) { + return; + } + if (file_id > 0) { + cbm_gbuf_insert_edge(ctx->gbuf, file_id, id, "DEFINES", "{}"); + } + const char *bqn = cbm_fqn_compute(a, ctx->project_name, rel, bname); + const cbm_gbuf_node_t *bnode = bqn ? cbm_gbuf_find_by_qn(ctx->gbuf, bqn) : NULL; + if (bnode && bnode->id != id) { + char eprops[CBM_SZ_64]; + snprintf(eprops, sizeof(eprops), "{\"via\":\"terraform\",\"line\":%d}", + (int)ts_node_start_point(block).row + SKIP_ONE); + cbm_gbuf_insert_edge(ctx->gbuf, bnode->id, id, "DEPLOYS", eprops); + } +} + +void cbm_pipeline_tf_k8s_objects(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, + int file_count) { + TSParser *parser = NULL; + CBMHashTable *file_nodes = NULL; + for (int i = 0; i < file_count; i++) { + const char *rel = files[i].rel_path; + if (files[i].language != CBM_LANG_HCL || !tf_is_module_file(rel)) { + continue; + } + size_t len = 0; + char *src = tf_read(files[i].path, &len); + if (!src || !strstr(src, "kubernetes_")) { + cbm_free(CBM_MEM_CLASS_OTHER, src); + continue; + } + if (!parser) { + parser = ts_parser_new(); + file_nodes = tf_label_index(ctx, "File"); + if (!parser || !ts_parser_set_language(parser, cbm_ts_language(CBM_LANG_HCL))) { + cbm_free(CBM_MEM_CLASS_OTHER, src); + break; + } + } + /* the file base: the File node's QN without ".__file__" (IDENTITY.md) */ + static const char FILE_SUFFIX[] = ".__file__"; + const cbm_gbuf_node_t *file = file_nodes ? cbm_ht_get(file_nodes, rel) : NULL; + size_t fl = file && file->qualified_name ? strlen(file->qualified_name) : 0; + size_t sl = sizeof(FILE_SUFFIX) - SKIP_ONE; + CBMArena arena; + cbm_arena_init_lazy(&arena, CBM_SZ_4K); + const char *base = fl > sl && strcmp(file->qualified_name + fl - sl, FILE_SUFFIX) == 0 + ? cbm_arena_strndup(&arena, file->qualified_name, fl - sl) + : NULL; + TSTree *tree = base ? ts_parser_parse_string(parser, NULL, src, (uint32_t)len) : NULL; + TSNode body = tree ? tf_root_body(ts_tree_root_node(tree)) : (TSNode){0}; + for (uint32_t j = 0; !ts_node_is_null(body) && j < ts_node_named_child_count(body); j++) { + TSNode b = ts_node_named_child(body, j); + if (tf_type(b, "block")) { + tf_k8s_block(ctx, &arena, src, rel, base, file->id, b); + } + } + cbm_arena_destroy(&arena); + if (tree) { + ts_tree_delete(tree); + } + cbm_free(CBM_MEM_CLASS_OTHER, src); + } + if (parser) { + ts_parser_delete(parser); + } + if (file_nodes) { + cbm_ht_free(file_nodes); + } +} + void cbm_pipeline_pass_terraform(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, int file_count) { TSParser *parser = NULL; @@ -434,6 +1742,11 @@ void cbm_pipeline_pass_terraform(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t } cbm_free(CBM_MEM_CLASS_OTHER, src); } + if (!parser) { + parser = ts_parser_new(); + } + int refs = parser ? tf_link_refs(ctx, files, file_count, parser) : 0; + int units = parser ? tg_link_units(ctx, files, file_count, parser) : 0; if (parser) { ts_parser_delete(parser); } @@ -441,9 +1754,13 @@ void cbm_pipeline_pass_terraform(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t cbm_ht_free(charts); } cbm_arena_destroy(&arena); - if (edges > 0) { + if (edges > 0 || refs > 0 || units > 0) { char buf[CBM_SZ_32]; + char rbuf[CBM_SZ_32]; + char ubuf[CBM_SZ_32]; snprintf(buf, sizeof(buf), "%d", edges); - cbm_log_info("pass.terraform.deploys", "edges", buf); + snprintf(rbuf, sizeof(rbuf), "%d", refs); + snprintf(ubuf, sizeof(ubuf), "%d", units); + cbm_log_info("pass.terraform.edges", "deploys", buf, "refs", rbuf, "terragrunt", ubuf); } } diff --git a/src/pipeline/pipeline.c b/src/pipeline/pipeline.c index 2830c9bd94..b6bba61a86 100644 --- a/src/pipeline/pipeline.c +++ b/src/pipeline/pipeline.c @@ -262,6 +262,15 @@ struct cbm_pipeline { * "invalid_existing_db" (#1864). */ bool final_existed; bool existing_generation; + /* The copy into the stage is deferred until routing needs it: an + * unchanged repository is proven current on the live generation + * (live_db_path, read-only) and never copies, seals or renames a + * database. backup_succeeded records the copy once it ran; live_noop + * marks a run that ended on the live proof with nothing staged. */ + bool stage_backup_pending; + bool backup_succeeded; + bool live_noop; + const char *live_db_path; /* borrowed from cbm_pipeline_run */ /* ADR (project_summaries) captured before a full-reindex DB delete, so it * can be restored after the rebuild. NULL when no ADR existed. Issue #516. */ @@ -850,7 +859,10 @@ cbm_pipeline_t *cbm_pipeline_new(const char *repo_path, const char *db_path, p->repo_path = strdup(repo_path); p->db_path = db_path ? strdup(db_path) : NULL; p->project_name = cbm_project_name_from_path(repo_path); - (void)cbm_git_context_resolve(repo_path, &p->git_ctx); + /* The Git context is resolved once, when the run starts + * (pipeline_refresh_git_context): a context resolved here was always + * replaced before any use, at the cost of seven git subprocesses per + * index call. Until then the branch QN names the working tree. */ p->branch_qn = cbm_git_context_branch_qn(p->project_name, &p->git_ctx); p->requested_mode = mode; p->mode = mode; @@ -2732,12 +2744,69 @@ static int capture_existing_adr(cbm_pipeline_t *p, const char *db_path) { return 0; } +static int remove_stage_files(const char *stage_path); + +/* Copy the live generation into this run's stage: every route except the + * live no-op works on that copy. A failed copy behaves as the eager copy it + * replaces did: the stage files go (the name and its lock stay this run's, + * the rebuilt generation is renamed over it later) and the run has no + * previous generation, so it rebuilds in full. */ +static void materialize_stage_backup(cbm_pipeline_t *p) { + p->stage_backup_pending = false; + p->backup_succeeded = cbm_store_backup_path(p->live_db_path, p->db_path) == CBM_STORE_OK; + if (!p->backup_succeeded) { + cbm_log_warn("pipeline.stage", "action", "backup_failed_full_rebuild", "path", + p->live_db_path); + (void)remove_stage_files(p->db_path); + } + p->existing_generation = p->backup_succeeded; +} + +/* The live generation is provably current: read-only, it passes the shallow + * integrity and format checks the staged route applies to its copy, and the + * exact semantic-manifest test the staged route would answer no-op with. A + * copy of it would route to that no-op and publish the same database again, + * so the run ends here with nothing copied, sealed or renamed. Any doubt + * answers false and the run copies and routes exactly as before. */ +static bool live_generation_current(cbm_pipeline_t *p, const cbm_file_hash_t *baseline_manifest, + int baseline_count) { + struct stat db_st; + if (!p->live_db_path || stat(p->live_db_path, &db_st) != 0) { + return false; + } + cbm_store_t *store = cbm_store_open_path_query(p->live_db_path); + if (!store) { + return false; + } + int fmt = 0; + bool current = cbm_store_check_integrity(store); + if (current) { + cbm_store_get_format_version(store, &fmt); + current = + fmt == CBM_INDEX_FORMAT_VERSION && + cbm_pipeline_incremental_generation_exact(p, store, baseline_manifest, baseline_count); + } + cbm_store_close(store); + return current; +} + /* Route an existing generation. Full rebuilds never delete the live DB here: * publication owns the eventual atomic replacement after every pass and * metadata write has succeeded. */ static int try_incremental_or_delete_db(cbm_pipeline_t *p, cbm_file_info_t *files, int file_count, const cbm_file_hash_t *baseline_manifest, int baseline_count, bool force_full_on_mismatch) { + if (p->stage_backup_pending) { + if (live_generation_current(p, baseline_manifest, baseline_count)) { + p->stage_backup_pending = false; + p->live_noop = true; + cbm_log_info("pipeline.route", "path", "incremental_manifest", "generation", "live"); + cbm_log_info("incremental.noop", "reason", "semantic_manifest_equal", "generation", + "live"); + return 0; + } + materialize_stage_backup(p); + } if (!p->existing_generation) { /* Nothing to be incremental against: a first index, or a * destination that could not be copied (already reported as @@ -2840,12 +2909,15 @@ static bool promote_mode_to_existing_coverage(cbm_pipeline_t *p) { if (!p || !p->project_name) { return false; } - char *db_path = resolve_db_path(p); + /* Until routing copies it, the stage is an empty placeholder: the + * coverage the copy would carry is read from the live generation. */ + char *resolved = p->stage_backup_pending ? NULL : resolve_db_path(p); + const char *db_path = p->stage_backup_pending ? p->live_db_path : resolved; if (!db_path) { return false; } cbm_store_t *store = cbm_store_open_path_query(db_path); - free(db_path); + free(resolved); if (!store) { return false; } @@ -4586,6 +4658,29 @@ static void sweep_orphan_stages(const char *final_path) { free(dir_path); } +/* End a run whose exact no-op was proven on the live generation. Nothing was + * staged, so nothing is sealed or renamed and the live database stays the + * current generation. The checks publication makes on the run's sources and + * destination still decide success, and the artifact is refreshed from the + * live database exactly as it would be from a republished one. */ +static int finish_live_noop(cbm_pipeline_t *p, const char *final_path) { + cbm_userconfig_t *fresh_config = cbm_userconfig_load_with_source(p->repo_path); + bool source_matches = cbm_pipeline_project_source_matches(p, fresh_config); + cbm_userconfig_free(fresh_config); + struct stat final_st; + int rc = 0; + if (!source_matches) { + cbm_log_error("pipeline.err", "phase", "test_declarations_source_changed"); + rc = CBM_PIPELINE_ABORT_PRESERVE_DB; + } else if (stat(final_path, &final_st) != 0) { + /* The destination vanished during the run: someone else owns it. */ + rc = CBM_PIPELINE_ABORT_PRESERVE_DB; + } else { + rc = export_after_publish(p, final_path); + } + return rc; +} + int cbm_pipeline_run(cbm_pipeline_t *p) { /* Per-index attribution: peaks and phase totals are about THIS index, not * the process history, so they start clean here. The first mark opens @@ -4617,21 +4712,12 @@ int cbm_pipeline_run(cbm_pipeline_t *p) { return CBM_NOT_FOUND; } - bool backup_succeeded = false; - if (final_existed) { - backup_succeeded = cbm_store_backup_path(final_path, staging_path) == CBM_STORE_OK; - if (!backup_succeeded) { - cbm_log_warn("pipeline.stage", "action", "backup_failed_full_rebuild", "path", - final_path); - /* The copy is gone but the NAME stays ours: the rebuilt - * generation is renamed over it by the inner finalize and then - * published from it below, so its lock is held to the end. */ - remove_stage_files(staging_path); - } - } - + /* The existing generation is copied into the stage only when routing + * needs a copy (materialize_stage_backup): an exact no-op is proven on + * the live generation and copies nothing -- on a kernel-scale index the + * copy alone was a 19 GB write per unchanged re-index. */ p->final_existed = final_existed; - p->existing_generation = final_existed && backup_succeeded; + p->existing_generation = final_existed; char *configured_db_path = p->db_path; p->db_path = strdup(staging_path); if (!p->db_path) { @@ -4643,11 +4729,20 @@ int cbm_pipeline_run(cbm_pipeline_t *p) { } p->metrics.staging_ms = cbm_now_ms() - staged_at; uint64_t indexed_at = cbm_now_ms(); + p->stage_backup_pending = final_existed; + p->backup_succeeded = false; + p->live_noop = false; + p->live_db_path = final_path; cbm_log_debug("pipeline.setup", "step", "staged"); int rc = cbm_pipeline_run_staged(p); p->metrics.index_ms = cbm_now_ms() - indexed_at; free(p->db_path); p->db_path = configured_db_path; + bool backup_succeeded = p->backup_succeeded; + bool live_noop = p->live_noop; + p->stage_backup_pending = false; + p->live_noop = false; + p->live_db_path = NULL; /* Report WHY the run stopped. Everything below happens before the final * rename, so the live database is still the previous generation and an @@ -4661,12 +4756,17 @@ int cbm_pipeline_run(cbm_pipeline_t *p) { pipeline_release_test_config(p); return rc; } - if (check_cancel(p)) { + /* A cancelled run and a live no-op both end here without publishing: + * the stage placeholder is discarded, and only the no-op goes on to + * finish on the live generation. */ + bool cancelled = check_cancel(p); + if (cancelled || live_noop) { cleanup_staging_db(staging_path); free(staging_path); + rc = cancelled ? CBM_PIPELINE_ABORT_PRESERVE_DB : finish_live_noop(p, final_path); free(final_path); pipeline_release_test_config(p); - return CBM_PIPELINE_ABORT_PRESERVE_DB; + return rc; } uint64_t published_at = cbm_now_ms(); if (seal_staging_db(staging_path) != 0) { diff --git a/src/pipeline/pipeline_delta.c b/src/pipeline/pipeline_delta.c index 366d4db4a8..c01cdd8800 100644 --- a/src/pipeline/pipeline_delta.c +++ b/src/pipeline/pipeline_delta.c @@ -151,7 +151,12 @@ int cbm_delta_snapshot_inbound(cbm_store_t *store, const char *project, const ch int step_rc; while ((step_rc = sqlite3_step(stmt)) == SQLITE_ROW) { const char *type = (const char *)sqlite3_column_text(stmt, 2); - if (delta_edge_type_is_recomputed(type)) { + const char *eprops = (const char *)sqlite3_column_text(stmt, 3); + /* the env join's edges come from file-owned EnvVars, but every + * run recomputes them whole (pass_envjoin.c) */ + bool joined = type && strcmp(type, CBM_ENVJOIN_EDGE_TYPE) == 0 && eprops && + strstr(eprops, "\"strategy\":\"env_read\""); + if (delta_edge_type_is_recomputed(type) || joined) { continue; } if (count >= cap) { @@ -247,6 +252,38 @@ int cbm_delta_purge(cbm_store_t *store, const char *project, const char *const * return 0; } +/* Run a node DELETE binding ?1 = project, ?2 = label, ?3.. = the first + * `type_count` of `types`; the number of nodes deleted, or -1. */ +static int delta_delete_nodes(sqlite3 *db, const char *sql, const char *project, const char *label, + const char *const *types, int type_count) { + sqlite3_stmt *stmt = NULL; + if (sqlite3_prepare_v2(db, sql, CBM_NOT_FOUND, &stmt, NULL) != SQLITE_OK) { + return CBM_NOT_FOUND; + } + sqlite3_bind_text(stmt, 1, project, CBM_NOT_FOUND, SQLITE_TRANSIENT); + sqlite3_bind_text(stmt, 2, label, CBM_NOT_FOUND, SQLITE_TRANSIENT); + for (int i = 0; i < type_count; i++) { + sqlite3_bind_text(stmt, i + 3, types[i], CBM_NOT_FOUND, SQLITE_TRANSIENT); + } + int rc = sqlite3_step(stmt); + sqlite3_finalize(stmt); + return rc == SQLITE_DONE ? sqlite3_changes(db) : CBM_NOT_FOUND; +} + +int cbm_delta_drop_edgeless(cbm_store_t *store, const char *project, const char *label) { + sqlite3 *db = cbm_store_get_db(store); + if (!db) { + return CBM_NOT_FOUND; + } + /* Each NOT EXISTS is served by an (id, type) index's id prefix. */ + static const char SQL[] = + "DELETE FROM nodes WHERE project = ?1 AND label = ?2" + " AND (file_path IS NULL OR file_path = '')" + " AND NOT EXISTS (SELECT 1 FROM edges e WHERE e.target_id = nodes.id)" + " AND NOT EXISTS (SELECT 1 FROM edges e WHERE e.source_id = nodes.id)"; + return delta_delete_nodes(db, SQL, project, label, NULL, 0); +} + int cbm_delta_drop_unreferenced(cbm_store_t *store, const char *project, const char *label, const char *const *edge_types) { sqlite3 *db = cbm_store_get_db(store); @@ -269,18 +306,7 @@ int cbm_delta_drop_unreferenced(cbm_store_t *store, const char *project, const c if (t == 0 || len <= 0 || (size_t)len >= sizeof(sql)) { return CBM_NOT_FOUND; } - sqlite3_stmt *stmt = NULL; - if (sqlite3_prepare_v2(db, sql, CBM_NOT_FOUND, &stmt, NULL) != SQLITE_OK) { - return CBM_NOT_FOUND; - } - sqlite3_bind_text(stmt, 1, project, CBM_NOT_FOUND, SQLITE_TRANSIENT); - sqlite3_bind_text(stmt, 2, label, CBM_NOT_FOUND, SQLITE_TRANSIENT); - for (int i = 0; i < t; i++) { - sqlite3_bind_text(stmt, i + 3, edge_types[i], CBM_NOT_FOUND, SQLITE_TRANSIENT); - } - int rc = sqlite3_step(stmt); - sqlite3_finalize(stmt); - return rc == SQLITE_DONE ? sqlite3_changes(db) : CBM_NOT_FOUND; + return delta_delete_nodes(db, sql, project, label, edge_types, t); } /* The first column of `sql` (?1 = the project) as arena strings in *out. */ @@ -340,6 +366,157 @@ int cbm_delta_image_components(cbm_store_t *store, const char *project, CBMArena a, out); } +/* Calls fn for each row of `sql` (?1 = project); the row count, or -1. */ +static int delta_each_row(cbm_store_t *store, const char *project, const char *sql, + void (*fn)(sqlite3_stmt *, void *), void *ud) { + sqlite3 *db = cbm_store_get_db(store); + sqlite3_stmt *stmt = NULL; + if (!db || sqlite3_prepare_v2(db, sql, CBM_NOT_FOUND, &stmt, NULL) != SQLITE_OK) { + return CBM_NOT_FOUND; + } + sqlite3_bind_text(stmt, 1, project, CBM_NOT_FOUND, SQLITE_TRANSIENT); + int n = 0; + int rc; + while ((rc = sqlite3_step(stmt)) == SQLITE_ROW) { + fn(stmt, ud); + n++; + } + sqlite3_finalize(stmt); + return rc == SQLITE_DONE ? n : CBM_NOT_FOUND; +} + +/* Room for one more of `size` bytes in arena array *items (count n). */ +static bool delta_room(CBMArena *a, void **items, int n, int *cap, size_t size) { + if (n < *cap) { + return true; + } + int ncap = *cap ? *cap * PAIR_LEN : CBM_SZ_64; + void *grown = cbm_arena_alloc(a, (size_t)ncap * size); + if (!grown) { + return false; + } + if (n) { + memcpy(grown, *items, (size_t)n * size); + } + *items = grown; + *cap = ncap; + return true; +} + +typedef struct { + cbm_envjoin_base_t *b; + int cap; + bool failed; +} delta_ej_t; + +static const char *delta_col(CBMArena *a, sqlite3_stmt *st, int i) { + const char *s = (const char *)sqlite3_column_text(st, i); + return cbm_arena_strdup(a, s ? s : ""); +} + +static void delta_ej_set(sqlite3_stmt *st, void *ud) { + delta_ej_t *e = ud; + cbm_envjoin_base_t *b = e->b; + if (!delta_room(&b->arena, (void **)&b->sets, b->set_count, &e->cap, sizeof(*b->sets))) { + e->failed = true; + return; + } + b->sets[b->set_count++] = + (cbm_envjoin_set_row_t){sqlite3_column_int64(st, 0), delta_col(&b->arena, st, SKIP_ONE), + delta_col(&b->arena, st, PAIR_LEN)}; +} + +static void delta_ej_img(sqlite3_stmt *st, void *ud) { + delta_ej_t *e = ud; + cbm_envjoin_base_t *b = e->b; + if (!delta_room(&b->arena, (void **)&b->imgs, b->img_count, &e->cap, sizeof(*b->imgs))) { + e->failed = true; + return; + } + b->imgs[b->img_count++] = + (cbm_envjoin_img_row_t){sqlite3_column_int64(st, 0), sqlite3_column_int64(st, SKIP_ONE), + delta_col(&b->arena, st, PAIR_LEN)}; +} + +static void delta_ej_reader(sqlite3_stmt *st, void *ud) { + delta_ej_t *e = ud; + cbm_envjoin_base_t *b = e->b; + if (!delta_room(&b->arena, (void **)&b->readers, b->reader_count, &e->cap, + sizeof(*b->readers))) { + e->failed = true; + return; + } + b->readers[b->reader_count++] = + (cbm_envjoin_reader_row_t){sqlite3_column_int64(st, 0), delta_col(&b->arena, st, SKIP_ONE), + delta_col(&b->arena, st, PAIR_LEN)}; +} + +int cbm_delta_envjoin_base(cbm_store_t *store, const char *project, cbm_envjoin_base_t *out) { + *out = (cbm_envjoin_base_t){0}; + cbm_arena_init_lazy(&out->arena, CBM_SZ_4K); + delta_ej_t sets = {.b = out}; + delta_ej_t imgs = {.b = out}; + delta_ej_t readers = {.b = out}; + if (delta_each_row(store, project, + "SELECT id, file_path, properties FROM nodes WHERE project = ?1" + " AND label = 'Module' AND instr(properties, '\"image_files\"') > 0" + " ORDER BY id", + delta_ej_set, &sets) < 0 || + delta_each_row(store, project, + "SELECT source_id, target_id, properties FROM edges WHERE project = ?1" + " AND type = 'USAGE' AND instr(properties, '\"ref\":\"image\"') > 0" + " ORDER BY id", + delta_ej_img, &imgs) < 0 || + delta_each_row(store, project, + "SELECT e.source_id, s.file_path, t.name FROM edges e" + " JOIN nodes s ON s.id = e.source_id JOIN nodes t ON t.id = e.target_id" + " WHERE e.project = ?1 AND e.type = 'CONFIGURES'" + " AND instr(e.properties, '\"strategy\":\"env_access\"') > 0" + " ORDER BY e.id", + delta_ej_reader, &readers) < 0 || + sets.failed || imgs.failed || readers.failed) { + cbm_envjoin_base_free(out); + return CBM_NOT_FOUND; + } + return 0; +} + +typedef struct { + const char *const *paths; + int path_count; + bool hit; +} delta_holds_t; + +static void delta_holds_row(sqlite3_stmt *st, void *ud) { + delta_holds_t *h = ud; + const char *props = (const char *)sqlite3_column_text(st, 0); + for (int i = 0; props && !h->hit && i < h->path_count; i++) { + h->hit = cbm_envjoin_props_hold(props, h->paths[i]); + } +} + +int cbm_delta_image_holds_any(cbm_store_t *store, const char *project, const char *const *paths, + int path_count) { + delta_holds_t h = {paths, path_count, false}; + if (path_count <= 0) { + return 0; + } + if (delta_each_row(store, project, + "SELECT properties FROM nodes WHERE project = ?1 AND label = 'Module'" + " AND instr(properties, '\"image_files\"') > 0", + delta_holds_row, &h) < 0) { + return CBM_NOT_FOUND; + } + return h.hit ? SKIP_ONE : 0; +} + +void cbm_envjoin_base_free(cbm_envjoin_base_t *b) { + if (b) { + cbm_arena_destroy(&b->arena); + *b = (cbm_envjoin_base_t){0}; + } +} + int cbm_delta_drop_edges_from_label(cbm_store_t *store, const char *project, const char *type, const char *label) { sqlite3 *db = cbm_store_get_db(store); @@ -640,15 +817,18 @@ static void delta_patch_node(const cbm_gbuf_node_t *node, void *userdata) { ctx->nodes++; } -/* An edge of the class recomputed wholesale on every run (an Image's build - * edges): the staging graph dropped its old set, so it is written even - * between pre-existing nodes. */ +/* An edge of a class recomputed wholesale on every run (an Image's build + * edges, the env join's edges): the staging graph dropped its old set, so it + * is written even between pre-existing nodes. */ static bool delta_edge_recomputed(const delta_patch_ctx_t *ctx, const cbm_gbuf_edge_t *edge) { - if (!ctx->gbuf || strcmp(edge->type, CBM_BUILD_EDGE_TYPE) != 0) { + const char *want = strcmp(edge->type, CBM_BUILD_EDGE_TYPE) == 0 ? CBM_BUILD_EDGE_SOURCE + : strcmp(edge->type, CBM_ENVJOIN_EDGE_TYPE) == 0 ? CBM_ENVJOIN_EDGE_SOURCE + : NULL; + if (!ctx->gbuf || !want) { return false; } const cbm_gbuf_node_t *src = cbm_gbuf_find_by_id(ctx->gbuf, edge->source_id); - return src && src->label && strcmp(src->label, CBM_BUILD_EDGE_SOURCE) == 0; + return src && src->label && strcmp(src->label, want) == 0; } static void delta_patch_edge(const cbm_gbuf_edge_t *edge, void *userdata) { diff --git a/src/pipeline/pipeline_incremental.c b/src/pipeline/pipeline_incremental.c index c5f4ffafb5..519dac7499 100644 --- a/src/pipeline/pipeline_incremental.c +++ b/src/pipeline/pipeline_incremental.c @@ -32,6 +32,7 @@ enum { INCR_RING_BUF = 4, INCR_RING_MASK = 3, INCR_TS_BUF = 24 }; #include "foundation/compat.h" #include "foundation/compat_fs.h" #include "foundation/compat_thread.h" +#include "foundation/limits.h" /* cbm_max_file_bytes */ #include "foundation/platform.h" #include "foundation/sha256.h" @@ -404,8 +405,15 @@ static int semantic_manifest_control_entry(semantic_manifest_builder_t *builder, } bool root_control = (!rel_dir || !rel_dir[0]) && (strcmp(name, ".cbmignore") == 0 || strcmp(name, ".codebase-memory.json") == 0); - bool control = - strcmp(name, ".gitignore") == 0 || root_control || semantic_manifest_package_control(name); + /* A build context's ignore file (or BuildKit's .dockerignore) + * shapes which files an image holds (pass_docker.c image_ignore), and + * discovery indexes neither. */ + static const char DOCKERIGNORE[] = ".dockerignore"; + size_t nl = strlen(name); + bool dockerignore = nl >= sizeof(DOCKERIGNORE) - SKIP_ONE && + strcmp(name + nl - (sizeof(DOCKERIGNORE) - SKIP_ONE), DOCKERIGNORE) == 0; + bool control = strcmp(name, ".gitignore") == 0 || root_control || + semantic_manifest_package_control(name) || dockerignore; return info.is_regular && control ? semantic_manifest_add(builder, project, rel_path, abs_path) : 0; } @@ -2509,6 +2517,157 @@ static const char *closure_k8s_decline(cbm_store_t *store, const char *project, return NULL; } +/* Terraform / OpenTofu / Terragrunt inputs reach past their own file + * (pass_terraform.c): the files of a module directory resolve each other's + * references, a module call's source decides which outputs another file of + * the directory reads, a .tfvars file sets the variables beside it, and a + * Terragrunt unit includes and reads other .hcl files. The repair recomputes + * the changed files' edges only, so a delta touching any of them -- an edge + * held by two unchanged files may hang on it -- declines to the full + * rebuild. Returns the reason, or NULL. */ +static const char *closure_terraform_decline(char **paths, int n) { + static const char *const exts[] = {".tf", ".tofu", ".tfvars", ".hcl"}; + for (int i = 0; i < n; i++) { + const char *slash = strrchr(paths[i], '/'); + const char *dot = strrchr(slash ? slash + SKIP_ONE : paths[i], '.'); + for (size_t k = 0; dot && k < sizeof(exts) / sizeof(exts[0]); k++) { + if (strcmp(dot, exts[k]) == 0) { + return "terraform_input_changed"; + } + } + } + return NULL; +} + +/* A role's defaults/ or vars/ main.yml (.yaml): the lists a kube task's loop + * reads (pass_ansible.c, an_role_list) -- a roles/ component before. */ +static bool closure_ansible_role_vars(const char *rel) { + static const char *const TAILS[] = {"/defaults/main.yml", "/defaults/main.yaml", + "/vars/main.yml", "/vars/main.yaml"}; + static const char ROLES[] = "roles/"; + size_t n = strlen(rel); + for (size_t k = 0; k < sizeof(TAILS) / sizeof(TAILS[0]); k++) { + size_t tl = strlen(TAILS[k]); + if (n <= tl || strcmp(rel + n - tl, TAILS[k]) != 0) { + continue; + } + const char *end = rel + n - tl; /* the role directory: rel up to end */ + for (const char *p = rel; (p = strstr(p, ROLES)) != NULL && p < end; p++) { + if ((p == rel || p[-SKIP_ONE] == '/') && p + sizeof(ROLES) - SKIP_ONE < end) { + return true; + } + } + } + return false; +} + +/* Does `rel` hold Ansible tasks (a Function {source: ansible})? -1 on a + * lookup error. */ +static int closure_holds_ansible_tasks(cbm_store_t *store, const char *project, const char *rel) { + cbm_node_t *nodes = NULL; + int count = 0; + if (cbm_store_find_nodes_by_file(store, project, rel, &nodes, &count) != CBM_STORE_OK) { + return -SKIP_ONE; + } + int held = 0; + for (int j = 0; j < count && !held; j++) { + held = nodes[j].label && strcmp(nodes[j].label, "Function") == 0 && + nodes[j].properties_json && + strstr(nodes[j].properties_json, "\"source\":\"ansible\"") != NULL; + } + cbm_store_free_nodes(nodes, count); + return held; +} + +/* Ansible (pass_ansible.c): a task's notify CALLS the handlers of its role + * in another file, whether a file is Ansible at all can hang on another + * file's include, and what a kube task DEPLOYS on the lists of its role's + * defaults / vars. The repair recomputes the changed files only, so a + * changed or deleted file that held Ansible tasks, a changed YAML file whose + * top level is a sequence (it may be Ansible now), or a changed or deleted + * role defaults / vars main file declines to the full rebuild. Returns the + * reason, or NULL. */ +static const char *closure_ansible_decline(cbm_store_t *store, const char *project, + const char *repo_path, char **paths, int n_changed, + int n_deleted) { + for (int i = 0; i < n_changed + n_deleted; i++) { + const char *rel = paths[i]; + if (closure_ansible_role_vars(rel)) { + return "ansible_role_vars_changed"; + } + int held = closure_holds_ansible_tasks(store, project, rel); + if (held != 0) { + return held < 0 ? "ansible_lookup_failed" : "ansible_tasks_changed"; + } + const char *dot = strrchr(rel, '.'); + if (i >= n_changed || !dot || (strcmp(dot, ".yaml") != 0 && strcmp(dot, ".yml") != 0)) { + continue; + } + char path[CBM_SZ_4K]; + snprintf(path, sizeof(path), "%s/%s", repo_path, rel); + FILE *f = cbm_fopen(path, "rb"); + if (!f) { + continue; + } + (void)fseek(f, 0, SEEK_END); + long size = ftell(f); + (void)fseek(f, 0, SEEK_SET); + char *text = size > 0 && size <= cbm_max_file_bytes() + ? cbm_alloc(CBM_MEM_CLASS_OTHER, (size_t)size + SKIP_ONE) + : NULL; + size_t got = text ? fread(text, SKIP_ONE, (size_t)size, f) : 0; + (void)fclose(f); + if (text) { + text[got] = '\0'; + } + bool sequence = text && cbm_ansible_is_sequence(text); + cbm_free(CBM_MEM_CLASS_OTHER, text); + if (sequence) { + return "ansible_content_changed"; + } + } + return NULL; +} + +/* A file holding Ansible tasks that the repair would re-walk as a dependent + * (an edge of it reaches a changed or deleted file: a deleted template its + * task DEPLOYS) is walked without the files its edges are drawn from -- + * its role's defaults / vars, handlers, included task files -- which are + * outside the closure: it would lose those edges, or keep a dropped one's + * fallback undrawn. Declines to the full rebuild. Returns the reason, or + * NULL. */ +static const char *closure_ansible_dependent_decline(cbm_store_t *store, const char *project, + char **dependents, int count) { + for (int i = 0; i < count; i++) { + int held = closure_holds_ansible_tasks(store, project, dependents[i]); + if (held != 0) { + return held < 0 ? "ansible_lookup_failed" : "ansible_dependent"; + } + } + return NULL; +} + +/* The env join (pass_envjoin.c) reads each image's files from its + * Dockerfile's Module, stored with the ignore patterns of its context: an + * ignore file changing, or a deletion of a file an image copies (which + * context a Dockerfile's sources resolve in can follow it), declines to the + * full rebuild. Adding a file declines already. Returns the reason, or NULL. */ +static const char *closure_envjoin_decline(cbm_store_t *store, const char *project, char **paths, + int n_changed, int n_deleted) { + for (int i = 0; i < n_changed + n_deleted; i++) { + const char *slash = strrchr(paths[i], '/'); + const char *base = slash ? slash + SKIP_ONE : paths[i]; + size_t bl = strlen(base); + size_t sl = strlen(".dockerignore"); + if (bl >= sl && strcmp(base + bl - sl, ".dockerignore") == 0) { + return "dockerignore_changed"; + } + } + int held = cbm_delta_image_holds_any(store, project, (const char *const *)paths + n_changed, + n_deleted); + return held > 0 ? "image_file_deleted" : held < 0 ? "image_lookup_failed" : NULL; +} + /* rel's directory prefix of len bytes as the plain Folder QN, in the arena. */ static char *closure_prefix_folder_qn(CBMArena *a, const char *project, const char *rel, size_t len) { @@ -2709,11 +2868,24 @@ static int closure_try_plan(cbm_pipeline_t *p, cbm_store_t *store, const char *p if (decline) { goto done; } + decline = closure_terraform_decline(changed_paths, n_changed + n_deleted); + if (decline) { + goto done; + } + decline = closure_ansible_decline(store, project, cbm_pipeline_repo_path(p), changed_paths, + n_changed, n_deleted); + if (decline) { + goto done; + } decline = closure_stem_mark_decline(project, files, file_count, changed_paths + n_changed, n_deleted); if (decline) { goto done; } + decline = closure_envjoin_decline(store, project, changed_paths, n_changed, n_deleted); + if (decline) { + goto done; + } /* Load the previous generation's surfaces and probe the changed files' * fresh ones. Missing rows fail closed. */ @@ -2872,6 +3044,10 @@ static int closure_try_plan(cbm_pipeline_t *p, cbm_store_t *store, const char *p decline = "doc_scope_changed"; goto done; } + decline = closure_ansible_dependent_decline(store, project, dependents, dependent_count); + if (decline) { + goto done; + } /* Closure = changed ∪ dependents. Every member must be in the current * discovery (a dependent outside it cannot be re-resolved). */ @@ -3014,6 +3190,9 @@ static int run_closure_delta(cbm_pipeline_t *p, const char *db_path, const char * collisions of the whole discovered set, as in a full run. */ cbm_stem_marks_t stem_marks; cbm_stem_marks_init(&stem_marks, project, files, file_count); + /* The env join's inputs in the files the repair leaves alone. */ + cbm_envjoin_base_t envjoin_base = {0}; + cbm_arena_init_lazy(&envjoin_base.arena, CBM_SZ_4K); cbm_doclink_scope_t *doc_base = NULL; int doc_base_count = 0; cbm_doc_link_row_t *doc_fresh = NULL; @@ -3076,7 +3255,7 @@ static int run_closure_delta(cbm_pipeline_t *p, const char *db_path, const char goto out; } int64_t max_db_id = cbm_delta_preseed(staging, project, gbuf); - if (max_db_id < 0) { + if (max_db_id < 0 || cbm_delta_envjoin_base(staging, project, &envjoin_base) != 0) { goto out; } registry = cbm_registry_new(); @@ -3231,6 +3410,7 @@ static int run_closure_delta(cbm_pipeline_t *p, const char *db_path, const char .build_decl_files = decl_files, .build_decl_count = decl_count, .stem_marks = &stem_marks, + .envjoin_base = &envjoin_base, }; for (int i = 0; i < ci; i++) { char *file_qn = cbm_pipeline_fqn_compute(project, changed_files[i].rel_path, "__file__"); @@ -3300,9 +3480,12 @@ static int run_closure_delta(cbm_pipeline_t *p, const char *db_path, const char /* An Image's build edges (Image -> Dockerfile, pass_docker.c) belong to * the skaffold / compose file declaring them, not to either end: the K8s * pass just recomputed them from every build file, so they replace the - * staging set wholesale. */ + * staging set wholesale. So do the env join's edges (pass_envjoin.c), + * recomputed from this repair's files and the stored rest. */ if (cbm_delta_drop_edges_from_label(staging, project, CBM_BUILD_EDGE_TYPE, - CBM_BUILD_EDGE_SOURCE) < 0) { + CBM_BUILD_EDGE_SOURCE) < 0 || + cbm_delta_drop_edges_from_label(staging, project, CBM_ENVJOIN_EDGE_TYPE, + CBM_ENVJOIN_EDGE_SOURCE) < 0) { goto out; } cbm_clock_gettime(CLOCK_MONOTONIC, &t); @@ -3312,9 +3495,11 @@ static int run_closure_delta(cbm_pipeline_t *p, const char *db_path, const char cbm_log_info("delta.patch_done", "elapsed_ms", itoa_buf((int)elapsed_ms(t))); /* The proxy buffer holds no edges, so only the patched staging graph can * tell a Process node no SPAWNS reaches any more (its last spawner was - * purged), or an Image nothing builds on, builds or runs. */ + * purged), an Image nothing builds on, builds or runs, or a file-less + * Route no caller or handler is left on. */ if (cbm_delta_drop_unreferenced(staging, project, "Process", SH_PROCESS_EDGES) < 0 || - cbm_delta_drop_unreferenced(staging, project, "Image", DK_IMAGE_EDGES) < 0) { + cbm_delta_drop_unreferenced(staging, project, "Image", DK_IMAGE_EDGES) < 0 || + cbm_delta_drop_edgeless(staging, project, "Route") < 0) { goto out; } @@ -3497,6 +3682,7 @@ static int run_closure_delta(cbm_pipeline_t *p, const char *db_path, const char cbm_delta_free_snapshot(snapshot, snapshot_count); cbm_arena_destroy(&decl_arena); cbm_stem_marks_free(&stem_marks); + cbm_envjoin_base_free(&envjoin_base); surface_name_set_free(stale_surface_set); cbm_doclinks_free_scopes(doc_base, doc_base_count); cbm_doclinks_free_rows(doc_fresh, doc_fresh_count); @@ -3557,6 +3743,62 @@ static bool incr_doc_links_current(cbm_store_t *store, const char *project) { return current; } +/* The exact no-op test: the stored generation's metadata is current for this + * mode, its doc-link layer is usable, and its recorded semantic manifest + * equals the baseline byte for byte. Read-only. The two out flags report the + * halves the staged route also needs for its own routing. */ +static bool incr_generation_exact(cbm_pipeline_t *p, cbm_store_t *store, const char *project, + const cbm_file_hash_t *stored, int stored_count, + const cbm_file_hash_t *baseline_manifest, int baseline_count, + bool *metadata_current_out, bool *doc_links_current_out) { + cbm_coverage_meta_t meta = {0}; + int meta_rc = cbm_store_coverage_meta_get(store, project, &meta); + const char *mode_name = incr_mode_name(cbm_pipeline_get_mode(p)); + bool metadata_current = + meta_rc == CBM_STORE_OK && meta.coverage_version == CBM_SEMANTIC_INDEX_VERSION && + meta.hash_records_complete && meta.index_mode && strcmp(meta.index_mode, mode_name) == 0; + /* A generation whose doc-link layer is missing or failed is not + * current even with identical inputs: index_status tells the user to + * re-run, and the re-run has to rebuild. */ + bool doc_links_current = metadata_current && incr_doc_links_current(store, project); + bool exact = doc_links_current && cbm_pipeline_semantic_manifests_equal( + stored, stored_count, baseline_manifest, baseline_count); + cbm_store_coverage_meta_clear(&meta); + *metadata_current_out = metadata_current; + *doc_links_current_out = doc_links_current; + return exact; +} + +bool cbm_pipeline_incremental_generation_exact(cbm_pipeline_t *p, cbm_store_t *store, + const cbm_file_hash_t *baseline_manifest, + int baseline_count) { +#if defined(CBM_INCREMENTAL_TEST_API) && CBM_INCREMENTAL_TEST_API + /* A forced legacy-partial run must reach the staged route that takes + * the one-shot flag; answering here would consume the run without it. */ + if (atomic_load(&g_incr_test_force_legacy_partial)) { + return false; + } +#endif + const char *project = cbm_pipeline_project_name(p); + cbm_file_hash_t *stored = NULL; + int stored_count = 0; + if (!store || + cbm_store_get_file_hashes(store, project, &stored, &stored_count) != CBM_STORE_OK) { + return false; + } + bool metadata_current = false; + bool doc_links_current = false; + bool exact = incr_generation_exact(p, store, project, stored, stored_count, baseline_manifest, + baseline_count, &metadata_current, &doc_links_current); + cbm_store_free_file_hashes(stored, stored_count); +#if defined(CBM_INCREMENTAL_TEST_API) && CBM_INCREMENTAL_TEST_API + if (exact) { + incr_test_set_last_route(CBM_INCREMENTAL_ROUTE_NOOP); + } +#endif + return exact; +} + int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_file_info_t *files, int file_count, const cbm_file_hash_t *baseline_manifest, int baseline_count, bool force_full_on_mismatch) { @@ -3602,21 +3844,11 @@ int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_fil * rebuilds the complete graph. This avoids partial LSP/package/config * convergence gaps while retaining the common unchanged-repo fast path. */ if (!force_legacy_partial) { - cbm_coverage_meta_t meta = {0}; - int meta_rc = cbm_store_coverage_meta_get(store, project, &meta); - const char *mode_name = incr_mode_name(cbm_pipeline_get_mode(p)); - bool metadata_current = meta_rc == CBM_STORE_OK && - meta.coverage_version == CBM_SEMANTIC_INDEX_VERSION && - meta.hash_records_complete && meta.index_mode && - strcmp(meta.index_mode, mode_name) == 0; - /* A generation whose doc-link layer is missing or failed is not - * current even with identical inputs: index_status tells the user to - * re-run, and the re-run has to rebuild. */ - bool doc_links_current = metadata_current && incr_doc_links_current(store, project); - bool exact = doc_links_current && - cbm_pipeline_semantic_manifests_equal(stored, stored_count, baseline_manifest, - baseline_count); - cbm_store_coverage_meta_clear(&meta); + bool metadata_current = false; + bool doc_links_current = false; + bool exact = + incr_generation_exact(p, store, project, stored, stored_count, baseline_manifest, + baseline_count, &metadata_current, &doc_links_current); if (exact) { cbm_store_free_file_hashes(stored, stored_count); cbm_store_close(store); @@ -3831,12 +4063,15 @@ int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_fil /* An Image's build edges (Image -> Dockerfile, pass_docker.c) belong to * the skaffold / compose file declaring them, not to either end: the K8s * pass recomputes them all from every build file, so none is kept (nor - * captured below). */ + * captured below). Neither is an env join edge (pass_envjoin.c): the + * join recomputes them all over the whole loaded graph. */ CBMArena decl_arena; /* the files that declared image builds before */ cbm_arena_init_lazy(&decl_arena, CBM_SZ_4K); const char **decl_files = NULL; int decl_count = incr_build_decl_files(existing, &decl_arena, &decl_files); (void)cbm_gbuf_delete_edges_from_label(existing, CBM_BUILD_EDGE_TYPE, CBM_BUILD_EDGE_SOURCE); + (void)cbm_gbuf_delete_edges_from_label(existing, CBM_ENVJOIN_EDGE_TYPE, + CBM_ENVJOIN_EDGE_SOURCE); /* Snapshot inbound cross-file edges into changed files BEFORE purging, so * the cascade delete doesn't permanently drop edges whose source lives in @@ -3943,9 +4178,12 @@ int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_fil * pass_shell.c), an Image only through its DEPENDS_ON / USAGE edges * (pass_docker.c); neither carries a file: when the last file naming * one is gone, nothing else removes it -- a full index never makes - * it. This buffer holds the whole graph. */ + * it. Nor a file-less Route (a call site's, pass_calls.c, + * pass_shell.c) without its last caller and handler. This buffer + * holds the whole graph. */ (void)cbm_gbuf_delete_unreferenced(existing, "Process", SH_PROCESS_EDGES); (void)cbm_gbuf_delete_unreferenced(existing, "Image", DK_IMAGE_EDGES); + (void)cbm_gbuf_delete_edgeless(existing, "Route"); phase_rc = run_postpasses(&ctx, changed_files, ci, project, true); } diff --git a/src/pipeline/pipeline_internal.h b/src/pipeline/pipeline_internal.h index bfbeccfc70..3c3ad2a8a7 100644 --- a/src/pipeline/pipeline_internal.h +++ b/src/pipeline/pipeline_internal.h @@ -364,6 +364,37 @@ typedef struct { void cbm_pkg_entries_init(cbm_pkg_entries_t *e); void cbm_pkg_entries_free(cbm_pkg_entries_t *e); +/* The env join's inputs held by the unchanged files of an incremental + * repair (pass_envjoin.c), read from the store before the K8s pass; ids are + * the stored rows, i.e. the repair buffer's proxies. */ +typedef struct { + int64_t node; /* a Dockerfile Module */ + const char *path; /* its file */ + const char *props; /* its properties (image_files, image_ignore) */ +} cbm_envjoin_set_row_t; + +typedef struct { + int64_t owner; /* USAGE {ref:image} source */ + int64_t image; /* its Image */ + const char *props; +} cbm_envjoin_img_row_t; + +typedef struct { + int64_t reader; /* CONFIGURES {strategy:env_access} source */ + const char *file; + const char *key; /* the global EnvVar's name */ +} cbm_envjoin_reader_row_t; + +typedef struct { + CBMArena arena; /* owns every string and array */ + cbm_envjoin_set_row_t *sets; + int set_count; + cbm_envjoin_img_row_t *imgs; + int img_count; + cbm_envjoin_reader_row_t *readers; + int reader_count; +} cbm_envjoin_base_t; + /* Shared context passed to each pass function. * Derived from cbm_pipeline_t fields during run. */ typedef struct { @@ -413,6 +444,10 @@ typedef struct { * `qn#ext`. Borrowed for the extraction phase; NULL elsewhere. */ const cbm_stem_marks_t *stem_marks; + /* An incremental repair: the env join's inputs held by the unchanged + * files (pass_envjoin.c). Borrowed; NULL in a full run. */ + const cbm_envjoin_base_t *envjoin_base; + /* Sequential cross-LSP registry arena. The lsp_cross pass builds its * shared per-language registries here; resolved_calls entries may BORROW * strings owned by these registries, and the later calls pass still @@ -1245,7 +1280,8 @@ typedef void (*cbm_build_site_fn)(const cbm_build_site_t *site, void *ud); * (repository path; NULL: no build, or unknown), the service names its * depends_on and links list, and the names of the environment variables its * environment sets (map keys / list items' names -- never a value), with - * their 1-based lines. */ + * their 1-based lines and the service name each value addresses (a bare + * name, `name:port`, a URL's host; NULL: none -- cbm_env_address). */ typedef struct { const char *name; const char *image; @@ -1254,6 +1290,7 @@ typedef struct { int ndeps; const char **env; const int *env_lines; + const char **env_hosts; int nenv; int line; int end_line; @@ -1276,16 +1313,69 @@ bool cbm_build_path(const char *rel); void cbm_build_sites(CBMArena *a, const char *rel, const char *src, size_t len, const char *dotenv, cbm_build_site_fn fn, void *ud); -/* Terraform / OpenTofu modules (pass_terraform.c): a helm_release DEPLOYS its - * chart. Run at the end of the K8s pass, over the same files. */ +/* Terraform / OpenTofu / Terragrunt (pass_terraform.c): references, module + * calls, tfvars, Terragrunt units; a helm_release DEPLOYS its chart. Run at + * the end of the K8s pass, over the same files. */ void cbm_pipeline_pass_terraform(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, int file_count); +/* Kubernetes objects Terraform declares (pass_terraform.c): a + * kubernetes_ resource with a literal name is a Resource of the K8s + * identity scheme, ecosystem terraform. Run by the K8s pass before its joins. */ +void cbm_pipeline_tf_k8s_objects(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, + int file_count); + +/* A Resource's properties from its identity QN (pass_k8s.c, + * k8s_resource_props); `ecosystem` NULL for a manifest's. Caller frees. */ +char *cbm_k8s_resource_props(const char *qn, const char *effective_ns, const char *version, + const char *labels, const char *ecosystem); + +/* Ansible (pass_ansible.c): a playbook or task file IMPORTS the playbooks, + * task files and roles its directives name; its tasks and handlers are + * Functions, a notify CALLS the handlers it names, a kube / k8s task + * DEPLOYS the template it applies. Run at the end of the K8s pass, over the + * same files. */ +void cbm_pipeline_pass_ansible(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, + int file_count); + +/* Does a YAML text's first content line start a sequence item (pass_ansible.c)? + * The incremental planner's test for Ansible content. */ +bool cbm_ansible_is_sequence(const char *src); + +/* `v` (a path; "./" and "../" allowed) from repository directory `dir`, as a + * repository path into out; false past the root or when it does not fit + * (pass_terraform.c). */ +bool cbm_repo_rel_join(const char *dir, const char *v, char *out, size_t sz); + /* The Image node of image reference `ref` (its normalised repository: * docker.io for no registry host, library/ for an official image; tag and * digest dropped), made when new; 0 for scratch or no image. */ int64_t cbm_pipeline_image_node(cbm_pipeline_ctx_t *ctx, const char *ref); +/* A chart repository URL as an identity (pass_k8s.c): `url` trimmed, its + * scheme and host lower-cased, trailing `/` and `.git` dropped. False for no + * URL: no scheme or host, a file:// directory, credentials in the authority, + * a query or fragment (`?`, `#`: a token may ride there), or a byte no + * stated URL holds (whitespace, quote, template, `$`). A + * repository URL is never cut: oci://host/x with chart x is oci://host/x/x. */ +bool cbm_chart_repo_url(const char *url, char *out, size_t sz); + +/* The repository of chart reference `ref` (a Flux OCIRepository url, a + * Terraform `chart = "oci://..."`): all before its last segment, the chart. + * False when none is left. */ +bool cbm_chart_ref_repo(const char *ref, char *out, size_t sz); + +/* The external Chart node of chart `name` from repository `url` (pass_k8s.c), + * made when new and owned by `file`: `__chart__/` {source, + * external, repository} -- one QN in every project -- when `url` resolves + * (cbm_chart_repo_url), else `.__helm_dep__.`. */ +int64_t cbm_pipeline_external_chart(cbm_pipeline_ctx_t *ctx, const char *name, const char *url, + const char *file); + +/* Is Chart node `node` an external chart (either identity) rather than one + * a Chart.yaml of this repository declares? */ +bool cbm_pipeline_chart_is_external(const cbm_gbuf_node_t *node); + /* Environment variable `key` that `owner` sets (pass_docker.c): a K8s or * compose Resource for its container `container`, a Dockerfile Module for * its image (container NULL). Makes the container-scoped EnvVar @@ -1304,6 +1394,16 @@ int64_t cbm_pipeline_env_setter(cbm_pipeline_ctx_t *ctx, const cbm_gbuf_node_t * void cbm_pipeline_pass_docker(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *files, int file_count); +/* The env join (pass_envjoin.c): a container's EnvVar CONFIGURES {strategy: + * env_read} each reader of its name inside the files the container's image + * is built from. Recomputes every such edge; after the Docker pass. Returns + * how many. */ +int cbm_pipeline_env_join(cbm_pipeline_ctx_t *ctx); + +/* Whether a Dockerfile Module's properties (image_files) copy `path` into + * the image, ignore patterns aside. */ +bool cbm_envjoin_props_hold(const char *props, const char *path); + /* Pre-dump pass: markdown → file REFERENCES_FILE linking. */ int cbm_pipeline_pass_doclinks(cbm_pipeline_ctx_t *ctx); @@ -1397,6 +1497,13 @@ int cbm_pipeline_run_incremental(cbm_pipeline_t *p, const char *db_path, cbm_fil int file_count, const cbm_file_hash_t *baseline_manifest, int baseline_count, bool force_full_on_mismatch); +/* The exact no-op test cbm_pipeline_run_incremental routes by, read-only on + * an already opened generation: true when that generation is current for the + * pipeline's mode and records exactly the baseline manifest. */ +bool cbm_pipeline_incremental_generation_exact(cbm_pipeline_t *p, cbm_store_t *store, + const cbm_file_hash_t *baseline_manifest, + int baseline_count); + /* Exact semantic inputs for no-op/forced-full routing. The manifest contains * every discovered source plus repository controls actually consumed by * discovery, package mapping, path aliases, and extension overrides. */ @@ -1454,8 +1561,43 @@ int cbm_pipeline_build_fresh_semantic_manifest(cbm_pipeline_t *p, const char *pr * spelled `proj.pkg.foo/`, or, where the directory is a Java/Go package, * the file's Module is `proj.pkg.foo#sh`. The Module the Folder used to * swallow is a node again. An index built before holds neither, so it - * rebuilds once on upgrade. */ -enum { CBM_SEMANTIC_INDEX_VERSION = 8 }; + * rebuilds once on upgrade. + * 9: a container's variable reaches its program: the EnvVar a container + * sets CONFIGURES {strategy:env_read} the readers of its name inside the + * files the container's image is built from (pass_envjoin.c); a K8s + * image reference says its container (USAGE {ref:image, container}) and + * a Dockerfile's Module its image's files (image_context, image_files, + * image_ignore). A workload or compose service CONNECTS_TO the Service + * its environment addresses (extract_k8s.c, pass_docker.c). A script's + * curl / wget HTTP_CALLS the Route of the URL it states (pass_shell.c). + * A kustomize patch OVERRIDEs the objects of its build it changes, its + * file an IMPORTS of the kustomization; a Resource keeps the version of + * its apiVersion and its labels; a kustomization DEPLOYS the charts it inflates + * (helmCharts); an object a namePrefix / nameSuffix renames has a + * variant Resource {ecosystem:kustomize} with its deployed name, which + * OVERRIDEs {via:namePrefix|nameSuffix} it (extract_k8s.c, + * pass_k8s.c). A Terraform reference + * USAGEs {ref, via:terraform}, from its block, the block of its module + * (directory) declaring it (one in depends_on: DEPENDS_ON), instead of + * the generic name matcher's USAGE / CALLS; a bare block label names + * its block like a quoted one; a module call with a local source + * IMPORTS its directory's Folder (the root: the Project) and module.X.Y + * USAGEs {ref:output} the output there; a .tfvars assignment OVERRIDEs + * {via:tfvars} the variable block of its own directory (or, inferred, + * of the nearest ancestor module declaring its names); a Terragrunt + * unit's include / read_terragrunt_config IMPORTS {via:terragrunt} the + * file it names, a dependency DEPENDS_ON the other unit; a + * kubernetes_* resource with a literal name is a Resource + * {ecosystem:terraform} its block DEPLOYS + * (pass_terraform.c, extract_unified.c, extract_defs.c); an Ansible + * playbook / task file IMPORTS {via:ansible} the playbooks, task files + * and roles its directives name, its tasks and handlers are Functions + * {source:ansible}, a notify CALLS {via:notify} the handlers it names, + * a kube / k8s task DEPLOYS {via:ansible} the template it applies + * (pass_ansible.c). An external Chart is keyed by its publisher URL. + * An index built before holds none of them, so it rebuilds once on + * upgrade. */ +enum { CBM_SEMANTIC_INDEX_VERSION = 9 }; typedef struct { cbm_gbuf_t *gbuf; @@ -1551,6 +1693,10 @@ int cbm_delta_purge(cbm_store_t *store, const char *project, const char *const * * on failure. */ int cbm_delta_drop_unreferenced(cbm_store_t *store, const char *project, const char *label, const char *const *edge_types); +/* Delete the project's `label` nodes that carry no file and that no edge of + * any type touches (a Route whose last caller and handler were purged); + * returns how many, or -1 on failure. */ +int cbm_delta_drop_edgeless(cbm_store_t *store, const char *project, const char *label); /* An Image's build edges (Image -DEPENDS_ON {via:build}-> Dockerfile, * pass_docker.c) belong to the build configuration declaring them, not to * either end: every run recomputes them whole from all build files, the @@ -1558,6 +1704,22 @@ int cbm_delta_drop_unreferenced(cbm_store_t *store, const char *project, const c * between pre-existing nodes. */ #define CBM_BUILD_EDGE_TYPE "DEPENDS_ON" #define CBM_BUILD_EDGE_SOURCE "Image" +/* The env join's edges (EnvVar -CONFIGURES {strategy:env_read}-> reader, + * pass_envjoin.c) likewise: no other producer draws CONFIGURES from an + * EnvVar, every run recomputes them whole. */ +#define CBM_ENVJOIN_EDGE_TYPE "CONFIGURES" +#define CBM_ENVJOIN_EDGE_SOURCE "EnvVar" + +/* The env join's inputs the store holds (the unchanged files of a closure + * repair, read after the purge), into `out` (its arena owns them); 0, or -1 + * on failure. cbm_envjoin_base_free releases it. */ +int cbm_delta_envjoin_base(cbm_store_t *store, const char *project, cbm_envjoin_base_t *out); +void cbm_envjoin_base_free(cbm_envjoin_base_t *b); +/* 1 when a stored image copies one of `paths` (a deletion then changes + * which context the Dockerfile's sources resolve in), 0 when none, -1 on + * failure. */ +int cbm_delta_image_holds_any(cbm_store_t *store, const char *project, const char *const *paths, + int path_count); /* The files whose File node the shell pass marked as running a container * build ({"declares_build":true}), into arena `a`; returns how many, or -1 diff --git a/src/store/graph_mem.c b/src/store/graph_mem.c new file mode 100644 index 0000000000..e046e3801d --- /dev/null +++ b/src/store/graph_mem.c @@ -0,0 +1,837 @@ +/* + * graph_mem.c — read-only in-memory adjacency over a published store's edges. + * See graph_mem.h for the identity, lifetime and budget contract. + */ +#include "store/graph_mem.h" +#include "foundation/constants.h" +#include "foundation/log.h" +#include "foundation/mem.h" +#include "foundation/mem_core.h" +#include "foundation/platform.h" + +#include +#include +#include +#include + +enum { + GM_MAX_TYPES = 64, /* one bit each in cbm_graph_mem_types_t */ + GM_VERSION_MAX = 96, /* generation + file state, see store_graph_mem */ + GM_CAP_DIVISOR = 4, /* all layers together: at most budget / 4 */ + GM_SET_MIN_CAP = 64, /* BFS visited set: initial slots (power of two) */ + GM_SET_LOAD_NUM = 7, /* grow the visited set past 7/10 load */ + GM_SET_LOAD_DEN = 10, + GM_GROWTH = 2, + GM_COL_TARGET = 1, + GM_COL_TYPE = 2, +}; + +/* Ids and edge counts must fit the uint32 CSR with room for "+1" offsets. */ +#define GM_ID_LIMIT ((int64_t)INT32_MAX - 1) +#define GM_HASH_MUL 2654435761U + +struct cbm_graph_mem { + /* Registry state, guarded by the registry mutex. */ + struct cbm_graph_mem *next; + char *path; + char version[GM_VERSION_MAX]; + int refs; + bool ready; /* built; false while its builder runs */ + bool detached; /* unlinked from the registry; freed at the last release */ + uint64_t last_use; + uint64_t idle_since_ms; /* when the last reference went away */ + size_t bytes; /* reserved estimate while building, actual once ready */ + + /* The graph, immutable once ready. Offsets cover node ids 0..n_ids-1: + * the out-edges of id u are out_nbr/out_type[out_off[u] .. out_off[u+1]). */ + uint32_t n_ids; + uint32_t n_edges; + uint32_t *out_off; + uint32_t *out_nbr; /* target of each out-edge */ + uint8_t *out_type; + uint32_t *in_off; + uint32_t *in_nbr; /* source of each in-edge */ + uint8_t *in_type; + int type_count; + char *types[GM_MAX_TYPES]; + size_t type_len[GM_MAX_TYPES]; +}; + +static cbm_graph_mem_t *g_layers; /* linked (not detached) layers */ +static size_t g_total_bytes; /* every live layer, linked or detached */ +static uint64_t g_use_clock; /* logical LRU clock */ +static uint64_t g_builds; + +/* A static SQLite mutex needs no init and works on every platform SQLite + * builds on (SQLITE_THREADSAFE=1). store.c owns APP1. */ +static sqlite3_mutex *gm_registry_mutex(void) { + return sqlite3_mutex_alloc(SQLITE_MUTEX_STATIC_APP2); +} + +#ifdef CBM_ENABLE_TEST_SEAMS +static uint64_t (*g_clock_seam)(void); +static size_t g_cached_bytes_seam; +void cbm_graph_mem_set_clock_for_testing(uint64_t (*now_ms)(void)) { + g_clock_seam = now_ms; +} +void cbm_graph_mem_set_cached_bytes_for_testing(size_t bytes) { + g_cached_bytes_seam = bytes; +} +static uint64_t gm_now_ms(void) { + return g_clock_seam ? g_clock_seam() : cbm_now_ms(); +} +#else +static uint64_t gm_now_ms(void) { + return cbm_now_ms(); +} +#endif + +static void *gm_alloc(size_t bytes) { + return cbm_alloc(CBM_MEM_CLASS_GRAPH_CACHE, bytes ? bytes : 1); +} + +static void gm_free_ptr(void *p) { + cbm_free(CBM_MEM_CLASS_GRAPH_CACHE, p); +} + +static void gm_free_graph(cbm_graph_mem_t *g) { + gm_free_ptr(g->out_off); + gm_free_ptr(g->out_nbr); + gm_free_ptr(g->out_type); + gm_free_ptr(g->in_off); + gm_free_ptr(g->in_nbr); + gm_free_ptr(g->in_type); + g->out_off = g->out_nbr = g->in_off = g->in_nbr = NULL; + g->out_type = g->in_type = NULL; + for (int i = 0; i < g->type_count; i++) { + gm_free_ptr(g->types[i]); + g->types[i] = NULL; + } + g->type_count = 0; +} + +static void gm_destroy(cbm_graph_mem_t *g) { + gm_free_graph(g); + gm_free_ptr(g->path); + gm_free_ptr(g); +} + +/* ── Registry (caller holds the mutex) ───────────────────────────── */ + +static cbm_graph_mem_t *gm_find(const char *path, const char *version) { + for (cbm_graph_mem_t *g = g_layers; g; g = g->next) { + if (strcmp(g->path, path) == 0 && strcmp(g->version, version) == 0) { + return g; + } + } + return NULL; +} + +static void gm_unlink(cbm_graph_mem_t *g) { + for (cbm_graph_mem_t **p = &g_layers; *p; p = &(*p)->next) { + if (*p == g) { + *p = g->next; + g->next = NULL; + g->detached = true; + return; + } + } +} + +/* Unlink a layer; free it now when nobody holds it, else at its last release. */ +static void gm_retire(cbm_graph_mem_t *g) { + gm_unlink(g); + if (g->refs == 0) { + g_total_bytes -= g->bytes; + gm_destroy(g); + } +} + +/* Older versions of a path are never asked for again by a new store (one + * opened later reads the current version), so they go now. */ +static void gm_retire_other_versions(const char *path, const char *version) { + cbm_graph_mem_t *g = g_layers; + while (g) { + cbm_graph_mem_t *next = g->next; + if (strcmp(g->path, path) == 0 && strcmp(g->version, version) != 0) { + gm_retire(g); + } + g = next; + } +} + +/* Make room for `bytes` under the cap by evicting idle layers, oldest first. + * False when it cannot fit: the cap is unset (no budget) or every remaining + * byte is held by a request. */ +static bool gm_make_room(size_t bytes) { + size_t budget = cbm_mem_budget(); + size_t cap = budget / GM_CAP_DIVISOR; + if (cap == 0 || bytes > cap) { + return false; + } + while (g_total_bytes + bytes > cap) { + cbm_graph_mem_t *victim = NULL; + for (cbm_graph_mem_t *g = g_layers; g; g = g->next) { + if (g->ready && g->refs == 0 && (!victim || g->last_use < victim->last_use)) { + victim = g; + } + } + if (!victim) { + return false; + } + gm_retire(victim); + } + /* The cap bounds the layers; the process as a whole must still fit. */ + return cbm_mem_charged() + bytes <= budget; +} + +/* Retire every layer no request has held for CBM_GRAPH_MEM_IDLE_EVICT_MS. + * Returns the bytes given back. */ +static size_t gm_evict_expired_locked(uint64_t now) { + size_t freed = 0; + cbm_graph_mem_t *g = g_layers; + while (g) { + cbm_graph_mem_t *next = g->next; + if (g->ready && g->refs == 0 && now >= g->idle_since_ms && + now - g->idle_since_ms >= CBM_GRAPH_MEM_IDLE_EVICT_MS) { + freed += g->bytes; + gm_retire(g); + } + g = next; + } + return freed; +} + +static void gm_log_evicted(size_t freed) { + if (freed > 0) { + char bytes[CBM_SZ_32]; + (void)snprintf(bytes, sizeof(bytes), "%zu", freed); + cbm_log_info("graph_mem.evict", "reason", "idle", "bytes", bytes); + } +} + +/* ── Build ──────────────────────────────────────────────────────── */ + +static bool gm_bounds(sqlite3 *db, int64_t *max_node, int64_t *max_edge) { + sqlite3_stmt *st = NULL; + if (sqlite3_prepare_v2(db, "SELECT (SELECT max(id) FROM nodes), (SELECT max(id) FROM edges);", + -1, &st, NULL) != SQLITE_OK) { + sqlite3_finalize(st); + return false; + } + bool ok = sqlite3_step(st) == SQLITE_ROW; + if (ok) { + *max_node = sqlite3_column_int64(st, 0); + *max_edge = sqlite3_column_int64(st, 1); + } + sqlite3_finalize(st); + return ok && *max_node >= 0 && *max_edge >= 0 && *max_node < GM_ID_LIMIT && + *max_edge < GM_ID_LIMIT; +} + +/* Upper bound of a layer's bytes: edge ids are unique positive rowids, so the + * edge count never exceeds max(edges.id). */ +static size_t gm_estimate(int64_t max_node, int64_t max_edge) { + size_t offsets = ((size_t)max_node + 2) * sizeof(uint32_t); + size_t per_edge = sizeof(uint32_t) + sizeof(uint8_t); + return GM_GROWTH * offsets + GM_GROWTH * (size_t)max_edge * per_edge; +} + +static int gm_intern(cbm_graph_mem_t *g, const unsigned char *text, size_t len, int *last) { + if (*last >= 0 && g->type_len[*last] == len && memcmp(g->types[*last], text, len) == 0) { + return *last; + } + for (int i = 0; i < g->type_count; i++) { + if (g->type_len[i] == len && memcmp(g->types[i], text, len) == 0) { + *last = i; + return i; + } + } + if (g->type_count >= GM_MAX_TYPES) { + return -1; + } + char *copy = gm_alloc(len + 1); + if (!copy) { + return -1; + } + memcpy(copy, text, len); + copy[len] = '\0'; + g->types[g->type_count] = copy; + g->type_len[g->type_count] = len; + *last = g->type_count; + return g->type_count++; +} + +static bool gm_grow_u32(uint32_t **arr, size_t *cap, size_t need) { + if (need <= *cap) { + return true; + } + size_t next = *cap ? *cap : 1; + while (next < need) { + next *= GM_GROWTH; + } + uint32_t *grown = cbm_realloc(CBM_MEM_CLASS_GRAPH_CACHE, *arr, next * sizeof(uint32_t)); + if (!grown) { + return false; + } + *arr = grown; + *cap = next; + return true; +} + +/* The out-CSR, filled straight from the scan: rows arrive ordered by + * source_id, so each node's list is contiguous and its start offset is the + * edge count when its first row (or a later id's) shows up. */ +typedef struct { + size_t off_cap; + size_t edge_cap; + uint32_t next_start; /* smallest id whose start offset is not set yet */ + uint32_t max_id; /* largest endpoint seen */ + int64_t prev_src; + int last_type; +} gm_scan_t; + +static bool gm_scan_row(cbm_graph_mem_t *g, gm_scan_t *sc, sqlite3_stmt *st) { + int64_t src = sqlite3_column_int64(st, 0); + int64_t dst = sqlite3_column_int64(st, GM_COL_TARGET); + const unsigned char *type = sqlite3_column_text(st, GM_COL_TYPE); + if (src < 0 || dst < 0 || src >= GM_ID_LIMIT || dst >= GM_ID_LIMIT || src < sc->prev_src || + !type || g->n_edges >= (uint32_t)GM_ID_LIMIT) { + return false; + } + int type_id = gm_intern(g, type, (size_t)sqlite3_column_bytes(st, GM_COL_TYPE), &sc->last_type); + if (type_id < 0 || !gm_grow_u32(&g->out_off, &sc->off_cap, (size_t)src + GM_GROWTH)) { + return false; + } + while (sc->next_start <= (uint32_t)src) { + g->out_off[sc->next_start++] = g->n_edges; + } + if (g->n_edges >= sc->edge_cap) { + size_t next = sc->edge_cap ? sc->edge_cap * GM_GROWTH : 1; + uint32_t *nbr = cbm_realloc(CBM_MEM_CLASS_GRAPH_CACHE, g->out_nbr, next * sizeof(uint32_t)); + if (!nbr) { + return false; + } + g->out_nbr = nbr; + uint8_t *types = cbm_realloc(CBM_MEM_CLASS_GRAPH_CACHE, g->out_type, next); + if (!types) { + return false; + } + g->out_type = types; + sc->edge_cap = next; + } + g->out_nbr[g->n_edges] = (uint32_t)dst; + g->out_type[g->n_edges] = (uint8_t)type_id; + g->n_edges++; + sc->prev_src = src; + if ((uint32_t)src > sc->max_id) { + sc->max_id = (uint32_t)src; + } + if ((uint32_t)dst > sc->max_id) { + sc->max_id = (uint32_t)dst; + } + return true; +} + +/* The in-CSR from the out-CSR: count, prefix-sum, then place each edge using + * in_off[v] as v's cursor, which leaves in_off shifted by one id — shift it + * back. Iterating sources in id order keeps every in-list sorted by source. */ +static bool gm_build_inbound(cbm_graph_mem_t *g) { + size_t n_ids = g->n_ids; + g->in_off = cbm_calloc(CBM_MEM_CLASS_GRAPH_CACHE, (n_ids + 1) * sizeof(uint32_t)); + g->in_nbr = gm_alloc((size_t)g->n_edges * sizeof(uint32_t)); + g->in_type = gm_alloc(g->n_edges); + if (!g->in_off || !g->in_nbr || !g->in_type) { + return false; + } + for (uint32_t e = 0; e < g->n_edges; e++) { + g->in_off[g->out_nbr[e] + 1]++; + } + for (size_t v = 0; v < n_ids; v++) { + g->in_off[v + 1] += g->in_off[v]; + } + for (uint32_t u = 0; u < g->n_ids; u++) { + for (uint32_t e = g->out_off[u]; e < g->out_off[u + 1]; e++) { + uint32_t pos = g->in_off[g->out_nbr[e]]++; + g->in_nbr[pos] = u; + g->in_type[pos] = g->out_type[e]; + } + } + for (size_t v = n_ids; v > 0; v--) { + g->in_off[v] = g->in_off[v - 1]; + } + g->in_off[0] = 0; + return true; +} + +static bool gm_build(cbm_graph_mem_t *g, sqlite3 *db, int64_t max_node, int64_t max_edge) { + gm_scan_t sc = {.prev_src = 0, .last_type = -1, .max_id = (uint32_t)max_node}; + sc.edge_cap = (size_t)max_edge; + g->out_nbr = gm_alloc(sc.edge_cap * sizeof(uint32_t)); + g->out_type = gm_alloc(sc.edge_cap); + if (!g->out_nbr || !g->out_type || + !gm_grow_u32(&g->out_off, &sc.off_cap, (size_t)max_node + GM_GROWTH)) { + return false; + } + /* ORDER BY the covering UNIQUE index's leading columns: a sequential walk + * of sqlite_autoindex_edges_1, no sort and no table access. */ + sqlite3_stmt *st = NULL; + if (sqlite3_prepare_v2(db, + "SELECT source_id, target_id, type FROM edges " + "ORDER BY source_id, target_id, type;", + -1, &st, NULL) != SQLITE_OK) { + sqlite3_finalize(st); + return false; + } + int rc; + bool ok = true; + while ((rc = sqlite3_step(st)) == SQLITE_ROW) { + if (!gm_scan_row(g, &sc, st)) { + ok = false; + break; + } + } + sqlite3_finalize(st); + if (!ok || rc != SQLITE_DONE) { + return false; + } + g->n_ids = sc.max_id + 1; + if (!gm_grow_u32(&g->out_off, &sc.off_cap, (size_t)g->n_ids + 1)) { + return false; + } + while (sc.next_start <= g->n_ids) { + g->out_off[sc.next_start++] = g->n_edges; + } + /* max(edges.id) over-counts after deletions; give the slack back. */ + if (g->n_edges < sc.edge_cap) { + size_t keep = g->n_edges ? g->n_edges : 1; + uint32_t *nbr = cbm_realloc(CBM_MEM_CLASS_GRAPH_CACHE, g->out_nbr, keep * sizeof(uint32_t)); + uint8_t *types = nbr ? cbm_realloc(CBM_MEM_CLASS_GRAPH_CACHE, g->out_type, keep) : NULL; + if (nbr) { + g->out_nbr = nbr; + } + if (types) { + g->out_type = types; + } + } + return gm_build_inbound(g); +} + +static size_t gm_actual_bytes(const cbm_graph_mem_t *g) { + size_t offsets = ((size_t)g->n_ids + 1) * sizeof(uint32_t); + size_t per_edge = sizeof(uint32_t) + sizeof(uint8_t); + size_t bytes = GM_GROWTH * offsets + GM_GROWTH * (size_t)g->n_edges * per_edge; + for (int i = 0; i < g->type_count; i++) { + bytes += g->type_len[i] + 1; + } + return bytes + sizeof(*g) + strlen(g->path) + 1; +} + +/* ── Acquire / release ──────────────────────────────────────────── */ + +/* A ready hit takes a reference; a hit still being built answers NULL (the + * caller uses SQL rather than waiting on another thread's build). */ +static cbm_graph_mem_t *gm_take(cbm_graph_mem_t *hit) { + if (!hit->ready) { + return NULL; + } + hit->refs++; + hit->last_use = ++g_use_clock; + return hit; +} + +static cbm_graph_mem_t *gm_new_entry(const char *db_path, const char *version, size_t est) { + cbm_graph_mem_t *g = cbm_calloc(CBM_MEM_CLASS_GRAPH_CACHE, sizeof(*g)); + if (!g) { + return NULL; + } + g->path = cbm_mem_strdup(CBM_MEM_CLASS_GRAPH_CACHE, db_path); + if (!g->path) { + gm_free_ptr(g); + return NULL; + } + memcpy(g->version, version, strlen(version) + 1); + g->refs = 1; + g->bytes = est; + return g; +} + +cbm_graph_mem_t *cbm_graph_mem_acquire(sqlite3 *db, const char *db_path, const char *version) { + if (!db || !db_path || !version || !version[0] || strlen(version) >= GM_VERSION_MAX) { + return NULL; + } + uint64_t now = gm_now_ms(); + sqlite3_mutex *mu = gm_registry_mutex(); + sqlite3_mutex_enter(mu); + size_t freed = gm_evict_expired_locked(now); + cbm_graph_mem_t *hit = gm_find(db_path, version); + cbm_graph_mem_t *taken = hit ? gm_take(hit) : NULL; + sqlite3_mutex_leave(mu); + gm_log_evicted(freed); + if (hit) { + return taken; + } + + int64_t max_node = 0; + int64_t max_edge = 0; + if (!gm_bounds(db, &max_node, &max_edge)) { + return NULL; + } + size_t est = gm_estimate(max_node, max_edge); + + sqlite3_mutex_enter(mu); + hit = gm_find(db_path, version); + if (hit) { + taken = gm_take(hit); + sqlite3_mutex_leave(mu); + return taken; + } + gm_retire_other_versions(db_path, version); + bool fits = gm_make_room(est); + cbm_graph_mem_t *g = fits ? gm_new_entry(db_path, version, est) : NULL; + if (g) { + g->next = g_layers; + g_layers = g; + g_total_bytes += est; + } + sqlite3_mutex_leave(mu); + if (!g) { + if (!fits) { + char need[CBM_SZ_32]; + (void)snprintf(need, sizeof(need), "%zu", est); + cbm_log_info("graph_mem.skip", "reason", "budget", "bytes", need); + } + return NULL; + } + + /* The build runs outside the mutex: other keys stay servable, and a + * request for this key meanwhile takes the SQL path (gm_take). */ + uint64_t started_ms = cbm_now_ms(); + bool ok = gm_build(g, db, max_node, max_edge); + if (!ok) { + gm_free_graph(g); /* all-or-nothing: no partial layer survives */ + } + size_t actual = ok ? gm_actual_bytes(g) : 0; + if (ok) { + char edges[CBM_SZ_32]; + char bytes[CBM_SZ_32]; + char elapsed[CBM_SZ_32]; + (void)snprintf(edges, sizeof(edges), "%u", g->n_edges); + (void)snprintf(bytes, sizeof(bytes), "%zu", actual); + (void)snprintf(elapsed, sizeof(elapsed), "%llu", + (unsigned long long)(cbm_now_ms() - started_ms)); + cbm_log_info("graph_mem.build", "edges", edges, "bytes", bytes, "elapsed_ms", elapsed); + } + + sqlite3_mutex_enter(mu); + g_total_bytes = g_total_bytes - g->bytes + actual; + g->bytes = actual; + if (ok) { + g->ready = true; + g->last_use = ++g_use_clock; + g_builds++; + } else { + g->refs = 0; + gm_retire(g); + g = NULL; + } + sqlite3_mutex_leave(mu); + return g; +} + +void cbm_graph_mem_release(cbm_graph_mem_t *g) { + if (!g) { + return; + } + uint64_t now = gm_now_ms(); + sqlite3_mutex *mu = gm_registry_mutex(); + sqlite3_mutex_enter(mu); + g->refs--; + if (g->refs == 0) { + g->idle_since_ms = now; + } + if (g->refs == 0 && g->detached) { + g_total_bytes -= g->bytes; + gm_destroy(g); + } + sqlite3_mutex_leave(mu); +} + +void cbm_graph_mem_evict_expired(void) { + uint64_t now = gm_now_ms(); + sqlite3_mutex *mu = gm_registry_mutex(); + sqlite3_mutex_enter(mu); + size_t freed = gm_evict_expired_locked(now); + sqlite3_mutex_leave(mu); + gm_log_evicted(freed); +} + +/* ── Queries ────────────────────────────────────────────────────── */ + +cbm_graph_mem_types_t cbm_graph_mem_type_mask(const cbm_graph_mem_t *g, const char *const *types, + int count) { + cbm_graph_mem_types_t mask = 0; + for (int i = 0; g && types && i < count; i++) { + if (!types[i]) { + continue; + } + size_t len = strlen(types[i]); + for (int t = 0; t < g->type_count; t++) { + if (g->type_len[t] == len && memcmp(g->types[t], types[i], len) == 0) { + mask |= (cbm_graph_mem_types_t)1 << t; + break; + } + } + } + return mask; +} + +int cbm_graph_mem_degree(const cbm_graph_mem_t *g, int64_t id, bool inbound, + cbm_graph_mem_types_t mask) { + if (!g || id < 0 || id >= (int64_t)g->n_ids) { + return 0; + } + const uint32_t *off = inbound ? g->in_off : g->out_off; + const uint8_t *type = inbound ? g->in_type : g->out_type; + int n = 0; + for (uint32_t e = off[id]; e < off[id + 1]; e++) { + n += (int)((mask >> type[e]) & 1U); + } + return n; +} + +size_t cbm_graph_mem_bytes(const cbm_graph_mem_t *g) { + return g ? g->bytes : 0; +} + +/* Visited set for one BFS: open addressing over id + 1 (0 marks a free slot). */ +typedef struct { + uint32_t *slots; + size_t cap; + size_t count; +} gm_set_t; + +static size_t gm_set_home(uint32_t id, size_t mask) { + return (size_t)(id * GM_HASH_MUL) & mask; +} + +static bool gm_set_grow(gm_set_t *s) { + size_t cap = s->cap ? s->cap * GM_GROWTH : GM_SET_MIN_CAP; + uint32_t *slots = cbm_calloc(CBM_MEM_CLASS_GRAPH_CACHE, cap * sizeof(uint32_t)); + if (!slots) { + return false; + } + for (size_t i = 0; i < s->cap; i++) { + uint32_t key = s->slots[i]; + if (key) { + size_t j = gm_set_home(key - 1, cap - 1); + while (slots[j]) { + j = (j + 1) & (cap - 1); + } + slots[j] = key; + } + } + gm_free_ptr(s->slots); + s->slots = slots; + s->cap = cap; + return true; +} + +/* 1 = added, 0 = already present, -1 = out of memory. */ +static int gm_set_add(gm_set_t *s, uint32_t id) { + if ((s->count + 1) * GM_SET_LOAD_DEN > s->cap * GM_SET_LOAD_NUM && !gm_set_grow(s)) { + return -1; + } + size_t mask = s->cap - 1; + for (size_t i = gm_set_home(id, mask);; i = (i + 1) & mask) { + if (s->slots[i] == 0) { + s->slots[i] = id + 1; + s->count++; + return 1; + } + if (s->slots[i] == id + 1) { + return 0; + } + } +} + +typedef struct { + uint32_t *ids; + size_t count; + size_t cap; +} gm_list_t; + +static bool gm_list_push(gm_list_t *l, uint32_t id) { + if (!gm_grow_u32(&l->ids, &l->cap, l->count + 1)) { + return false; + } + l->ids[l->count++] = id; + return true; +} + +typedef struct { + cbm_graph_mem_hop_t *rows; + size_t count; + size_t cap; +} gm_hops_t; + +static bool gm_hops_push(gm_hops_t *h, uint32_t id, int hop) { + if (h->count == h->cap) { + size_t next = h->cap ? h->cap * GM_GROWTH : GM_SET_MIN_CAP; + cbm_graph_mem_hop_t *rows = + cbm_realloc(CBM_MEM_CLASS_GRAPH_CACHE, h->rows, next * sizeof(cbm_graph_mem_hop_t)); + if (!rows) { + return false; + } + h->rows = rows; + h->cap = next; + } + h->rows[h->count].id = id; + h->rows[h->count].hop = hop; + h->count++; + return true; +} + +static int gm_hop_id_cmp(const void *pa, const void *pb) { + const cbm_graph_mem_hop_t *a = pa; + const cbm_graph_mem_hop_t *b = pb; + return (a->id > b->id) - (a->id < b->id); +} + +/* One BFS level: expand every frontier node over the masked edges. A node is + * first reached at its shortest distance; the start is never re-entered (its + * neighbours were all reached at hop 1), only recorded once at the length of + * the shortest cycle back to it — the CTE's (start, hop>0) row. */ +typedef struct { + const uint32_t *off; + const uint32_t *nbr; + const uint8_t *type; + cbm_graph_mem_types_t mask; + uint32_t start; + bool start_back; + gm_set_t seen; + gm_hops_t out; +} gm_walk_t; + +static bool gm_expand_level(gm_walk_t *w, const gm_list_t *frontier, gm_list_t *next, int hop) { + for (size_t f = 0; f < frontier->count; f++) { + uint32_t u = frontier->ids[f]; + for (uint32_t e = w->off[u]; e < w->off[u + 1]; e++) { + if (((w->mask >> w->type[e]) & 1U) == 0) { + continue; + } + uint32_t v = w->nbr[e]; + if (v == w->start) { + if (!w->start_back) { + w->start_back = true; + if (!gm_hops_push(&w->out, v, hop)) { + return false; + } + } + continue; + } + int added = gm_set_add(&w->seen, v); + if (added < 0) { + return false; + } + if (added > 0 && (!gm_hops_push(&w->out, v, hop) || !gm_list_push(next, v))) { + return false; + } + } + } + return true; +} + +bool cbm_graph_mem_bfs(const cbm_graph_mem_t *g, int64_t start, bool inbound, + cbm_graph_mem_types_t mask, int max_depth, cbm_graph_mem_hop_t **out, + int *count) { + *out = NULL; + *count = 0; + if (!g || max_depth < 1 || mask == 0 || start < 0 || start >= (int64_t)g->n_ids) { + return true; /* nothing reachable: the CTE yields only the anchor */ + } + gm_walk_t w = { + .off = inbound ? g->in_off : g->out_off, + .nbr = inbound ? g->in_nbr : g->out_nbr, + .type = inbound ? g->in_type : g->out_type, + .mask = mask, + .start = (uint32_t)start, + }; + gm_list_t frontier = {0}; + gm_list_t next = {0}; + bool ok = gm_list_push(&frontier, w.start); + for (int hop = 1; ok && hop <= max_depth && frontier.count > 0; hop++) { + size_t level_begin = w.out.count; + next.count = 0; + ok = gm_expand_level(&w, &frontier, &next, hop); + if (ok && w.out.count - level_begin > 1) { + qsort(w.out.rows + level_begin, w.out.count - level_begin, sizeof(cbm_graph_mem_hop_t), + gm_hop_id_cmp); + } + gm_list_t swap = frontier; + frontier = next; + next = swap; + } + gm_free_ptr(frontier.ids); + gm_free_ptr(next.ids); + gm_free_ptr(w.seen.slots); + if (!ok || w.out.count > (size_t)INT32_MAX) { + gm_free_ptr(w.out.rows); + return false; + } + *out = w.out.rows; + *count = (int)w.out.count; + return true; +} + +void cbm_graph_mem_hops_free(cbm_graph_mem_hop_t *hops) { + gm_free_ptr(hops); +} + +/* ── Registry introspection ─────────────────────────────────────── */ + +int cbm_graph_mem_cached_count(void) { + sqlite3_mutex *mu = gm_registry_mutex(); + sqlite3_mutex_enter(mu); + int n = 0; + for (cbm_graph_mem_t *g = g_layers; g; g = g->next) { + n++; + } + sqlite3_mutex_leave(mu); + return n; +} + +size_t cbm_graph_mem_cached_bytes(void) { +#ifdef CBM_ENABLE_TEST_SEAMS + if (g_cached_bytes_seam) { + return g_cached_bytes_seam; + } +#endif + sqlite3_mutex *mu = gm_registry_mutex(); + sqlite3_mutex_enter(mu); + size_t bytes = g_total_bytes; + sqlite3_mutex_leave(mu); + return bytes; +} + +uint64_t cbm_graph_mem_build_count(void) { + sqlite3_mutex *mu = gm_registry_mutex(); + sqlite3_mutex_enter(mu); + uint64_t n = g_builds; + sqlite3_mutex_leave(mu); + return n; +} + +void cbm_graph_mem_drop_idle(void) { + sqlite3_mutex *mu = gm_registry_mutex(); + sqlite3_mutex_enter(mu); + cbm_graph_mem_t *g = g_layers; + while (g) { + cbm_graph_mem_t *next = g->next; + if (g->ready && g->refs == 0) { + gm_retire(g); + } + g = next; + } + sqlite3_mutex_leave(mu); +} diff --git a/src/store/graph_mem.h b/src/store/graph_mem.h new file mode 100644 index 0000000000..7e8d30a392 --- /dev/null +++ b/src/store/graph_mem.h @@ -0,0 +1,122 @@ +/* + * graph_mem.h — read-only in-memory adjacency over a published store's edges. + * + * A graph walk on a kernel-scale index (8.5M nodes, 16.9M edges) spends its + * time in SQLite B-tree seeks: trace_path's recursive CTE took 6.6 s and a + * degree-filtered search_graph 3.9 s, where the same questions on django take + * 20-100 ms. This layer reads the edges ONCE, in one sequential scan of the + * covering UNIQUE(source_id, target_id, type, ...) index, into a typed CSR + * (node id -> out/in neighbours, each with its interned edge type) and answers + * those walks from RAM. Node rows still come from SQLite; only the edge + * structure is held. + * + * Identity: a layer is keyed by (db path, version), where the caller promises + * that one version names one edge set. store.c builds the version from the + * store generation (every publish stamps a new one) plus the rollback-journal + * file state (the header change counter every commit bumps — it also catches + * writers that mutate a published DB in place without a new generation), and + * admits only read-only query stores of a sealed, non-legacy DB. + * + * Lifetime: the daemon closes its query stores after every request, so layers + * live in a process-wide registry, shared by refcount across request threads. + * Every byte is allocated through the memory core (CBM_MEM_CLASS_GRAPH_CACHE) + * and the registry is bounded: the layers together stay under a quarter of + * the memory budget, unheld ones are evicted oldest-first to make room, and a + * layer that still would not fit (or that would push the process over its + * budget) is not built — the caller keeps the SQL path. A build is + * all-or-nothing: a failed or refused build leaves no partial layer behind. + * A layer nobody held for CBM_GRAPH_MEM_IDLE_EVICT_MS is evicted on the next + * request (no timer thread), and the daemon subtracts the resident bytes from + * what it hands index workers (cbm_graph_mem_cached_bytes). + */ +#ifndef CBM_GRAPH_MEM_H +#define CBM_GRAPH_MEM_H + +#include +#include +#include + +struct sqlite3; + +typedef struct cbm_graph_mem cbm_graph_mem_t; + +/* A set of interned edge types: bit i = the layer's type i. */ +typedef uint64_t cbm_graph_mem_types_t; + +/* One reached node: its id and its shortest distance from the start. */ +typedef struct { + int64_t id; + int hop; +} cbm_graph_mem_hop_t; + +/* The layer for (db_path, version): the cached one, or one built now from + * `db`'s edges table. The caller must hold one read transaction across the + * version read and this call, so the scan sees that version's edges. + * NULL means "use the SQL path": no budget, over the cap, another thread is + * building this key right now, a schema or id range the layer cannot hold, or + * a failed scan. A non-NULL layer is held until cbm_graph_mem_release. */ +cbm_graph_mem_t *cbm_graph_mem_acquire(struct sqlite3 *db, const char *db_path, + const char *version); +void cbm_graph_mem_release(cbm_graph_mem_t *g); + +/* The type set naming `types` (exact byte match, like SQL `type IN (...)`). + * NULL entries and types absent from the graph contribute nothing. */ +cbm_graph_mem_types_t cbm_graph_mem_type_mask(const cbm_graph_mem_t *g, const char *const *types, + int count); + +/* Shortest-walk BFS with the exact semantics of store_bfs's recursive CTE: + * every node reached by a walk of 1..max_depth edges whose types are in + * `mask`, at its MINIMAL such length. The start itself appears only when a + * cycle leads back to it (at the cycle's length). Rows come sorted by + * (hop, id); ids absent from the nodes table are NOT filtered here. + * Returns false only on allocation failure. *out is freed with + * cbm_graph_mem_hops_free. */ +bool cbm_graph_mem_bfs(const cbm_graph_mem_t *g, int64_t start, bool inbound, + cbm_graph_mem_types_t mask, int max_depth, cbm_graph_mem_hop_t **out, + int *count); +void cbm_graph_mem_hops_free(cbm_graph_mem_hop_t *hops); + +/* Number of edges with a type in `mask` ending at (inbound) or leaving + * (outbound) node `id` — SQL's COUNT(*) over target_id/source_id = id. */ +int cbm_graph_mem_degree(const cbm_graph_mem_t *g, int64_t id, bool inbound, + cbm_graph_mem_types_t mask); + +/* Bytes this layer holds (all through the memory core). */ +size_t cbm_graph_mem_bytes(const cbm_graph_mem_t *g); + +/* ── Idle eviction ──────────────────────────────────────────────── */ + +/* How long a layer may sit unheld before it goes. Ten minutes outlasts the + * gaps between an agent's graph calls within one task (seconds to a few + * minutes), so a working session keeps its layer, while a session that moved + * on stops pinning it (237 MB for the kernel). A wrong guess costs one rebuild + * — about 1.2-1.5 s on the 17 GB kernel index, tens of ms on ordinary repos — + * against ten minutes of held memory. */ +enum { CBM_GRAPH_MEM_IDLE_EVICT_MS = 10 * 60 * 1000 }; + +/* Evict every layer unheld for CBM_GRAPH_MEM_IDLE_EVICT_MS. The daemon calls + * it at the end of every request and acquire runs it too: a daemon that stops + * asking stops paying, with no timer thread. */ +void cbm_graph_mem_evict_expired(void); + +/* ── Registry introspection (tests, diagnostics) ─────────────────── */ + +/* Layers currently held by the registry, and their total bytes — every + * resident layer, held or not, which is what index-worker budgets give up. */ +int cbm_graph_mem_cached_count(void); +size_t cbm_graph_mem_cached_bytes(void); +/* Builds completed since process start: a test's proof that a call hit the + * cache (unchanged) or built (incremented). */ +uint64_t cbm_graph_mem_build_count(void); +/* Drop every unheld layer (refs == 0), however recently used. */ +void cbm_graph_mem_drop_idle(void); + +#ifdef CBM_ENABLE_TEST_SEAMS +/* The clock idle eviction reads; NULL restores cbm_now_ms. Tests move time + * instead of waiting for it. */ +void cbm_graph_mem_set_clock_for_testing(uint64_t (*now_ms)(void)); +/* Pin cbm_graph_mem_cached_bytes to `bytes`; 0 restores the live reading. */ +void cbm_graph_mem_set_cached_bytes_for_testing(size_t bytes); +#endif + +#endif /* CBM_GRAPH_MEM_H */ diff --git a/src/store/store.c b/src/store/store.c index 4e4e2ceca7..7ff9a84d03 100644 --- a/src/store/store.c +++ b/src/store/store.c @@ -9,6 +9,7 @@ // for ISO timestamp #include +#include #include #include "foundation/constants.h" #include "foundation/arena.h" @@ -54,9 +55,13 @@ enum { ST_GLOB_MIN_LEN = 3, ST_GLOB_SKIP = 2, ST_MAX_LANG = 10, - ST_SEARCH_MAX_BINDS = 32, /* increased: LIKE pre-filter adds binds per pattern */ - ST_LIKE_POOL_MAX = 12, /* max malloc'd LIKE strings alive during one search */ - ST_LIKE_HINT_MAX = 2, /* max LIKE hints extracted per regex pattern */ + ST_SEARCH_MAX_BINDS = 48, /* LIKE pre-filters + anchored-name range bounds add binds */ + ST_LIKE_POOL_MAX = 12, /* max malloc'd LIKE strings alive during one search */ + ST_LIKE_HINT_MAX = 2, /* max LIKE hints extracted per regex pattern */ + ST_NAME_RANGE_PREFIX = 64, /* longest anchored name prefix seeked on */ + ST_NAME_RANGE_LETTERS = 3, /* case-insensitive: letters expanded into case variants */ + ST_NAME_RANGE_MAX = 8, /* 2^ST_NAME_RANGE_LETTERS name ranges (UNION ALL arms) */ + ST_NAME_RANGE_ARM_SQL = 96, /* per-arm SQL beyond the select list and WHERE */ ST_MAX_PKGS = 64, ST_INIT_CAP_4 = 4, ST_HEADER_PREFIX = 3, @@ -69,10 +74,21 @@ enum { ST_HANDLER_PROP_LEN = 9, ST_BFS_CTE_ROW_MULTIPLIER = 8, ST_BFS_MAX_CTE_ROWS = 4096, + /* SQLite database header (fileformat2 §1.3) */ + ST_DB_HEADER_LEN = 100, + ST_HDR_WRITE_VERSION = 18, /* 1 = rollback journal, 2 = WAL */ + ST_HDR_READ_VERSION = 19, + ST_HDR_ROLLBACK = 1, + ST_HDR_CHANGE_COUNTER = 24, + ST_HDR_PAGE_COUNT = 28, + ST_BE_SHIFT_8 = 8, + ST_BE_SHIFT_16 = 16, + ST_BE_SHIFT_24 = 24, }; #define SLEN(s) (sizeof(s) - 1) #include "store/store.h" +#include "store/graph_mem.h" #include #include "foundation/compat_fs.h" @@ -121,6 +137,16 @@ struct cbm_store { const void *progress_owner; /* exclusive native progress guard token */ const char *db_path; /* heap-allocated, or NULL for :memory: */ char errbuf[CBM_SZ_512]; + bool read_batch; /* a read batch's transaction is open */ + uint64_t read_batch_started_ms; /* when it was last (re)begun */ + /* In-memory adjacency (graph_mem.h). Only a read-only query store of a + * sealed (rollback-journal) DB qualifies, keyed by its generation and + * file state (store_rollback_file_state). Acquired on the first walk, + * released at close. */ + bool query_store; + bool graph_mem_tried; + cbm_graph_mem_t *graph_mem; + cbm_graph_mem_types_t graph_mem_degree_mask; /* search's in_deg/out_deg types */ /* Prepared statements (lazily initialized, cached for lifetime) */ sqlite3_stmt *stmt_upsert_node; @@ -207,10 +233,60 @@ static char *heap_strdup(const char *s) { } /* Prepare a statement (cached). If already prepared, reset+clear. */ +/* ── Read batch ──────────────────────────────────────────────────── + * A read tool issues thousands of small statements. In autocommit mode each + * one takes and drops the database file lock and stats the journal and WAL + * files — most of a whole-graph count(*)'s time. A read batch runs them in one + * deferred read transaction, renewed every ST_READ_BATCH_MS at the next + * statement: a writer never waits behind the batch much longer than that, and + * every statement still reads data committed at most that long ago (while the + * batch holds a rollback-journal database's shared lock, no commit happens). + * Committing a read transaction is allowed with read statements still + * stepping; their lock is then held until they finish, as before. */ +enum { ST_READ_BATCH_MS = 100 }; + +bool cbm_store_read_batch_begin(cbm_store_t *s) { + if (!s || !s->db || s->read_batch || !sqlite3_get_autocommit(s->db)) { + return false; + } + if (sqlite3_exec(s->db, "BEGIN DEFERRED;", NULL, NULL, NULL) != SQLITE_OK) { + return false; + } + s->read_batch = true; + s->read_batch_started_ms = cbm_now_ms(); + return true; +} + +void cbm_store_read_batch_end(cbm_store_t *s) { + if (!s || !s->read_batch) { + return; + } + s->read_batch = false; + if (s->db && !sqlite3_get_autocommit(s->db)) { + (void)sqlite3_exec(s->db, "COMMIT;", NULL, NULL, NULL); + } +} + +void cbm_store_read_batch_expire_for_test(cbm_store_t *s) { + if (s && s->read_batch) { + uint64_t now = cbm_now_ms(); + s->read_batch_started_ms = now > ST_READ_BATCH_MS ? now - ST_READ_BATCH_MS : 0; + } +} + +static void read_batch_renew(cbm_store_t *s) { + if (!s->read_batch || cbm_now_ms() - s->read_batch_started_ms < ST_READ_BATCH_MS) { + return; + } + cbm_store_read_batch_end(s); + (void)cbm_store_read_batch_begin(s); +} + static sqlite3_stmt *prepare_cached(cbm_store_t *s, sqlite3_stmt **slot, const char *sql) { if (!s || !s->db) { return NULL; } + read_batch_renew(s); if (*slot) { sqlite3_reset(*slot); sqlite3_clear_bindings(*slot); @@ -1132,6 +1208,7 @@ cbm_store_t *cbm_store_open_path_query(const char *db_path) { } s->db_path = heap_strdup(db_path); + s->query_store = true; /* Security: block ATTACH/DETACH to prevent file creation via SQL injection. */ sqlite3_set_authorizer(s->db, store_authorizer, NULL); @@ -1363,6 +1440,7 @@ void cbm_store_close(cbm_store_t *s) { if (!s) { return; } + cbm_store_read_batch_end(s); /* Checkpoint WAL before close to prevent orphan WAL accumulation. * Best-effort — silently skips if concurrent reader holds a lock. */ @@ -1409,6 +1487,7 @@ void cbm_store_close(cbm_store_t *s) { /* Use sqlite3_close_v2 — auto-deallocates when last statement finalizes. * Prevents ASan false-positive leaks from sqlite3 internal state. */ sqlite3_close_v2(s->db); + cbm_graph_mem_release(s->graph_mem); safe_str_free(&s->db_path); free(s); } @@ -3056,11 +3135,12 @@ static void scan_node(sqlite3_stmt *stmt, cbm_node_t *n) { n->properties_json = heap_strdup((const char *)sqlite3_column_text(stmt, ST_COL_8)); } +static const char st_find_node_by_id_sql[] = + "SELECT id, project, label, name, qualified_name, file_path, " + "start_line, end_line, properties FROM nodes WHERE id = ?1;"; + int cbm_store_find_node_by_id(cbm_store_t *s, int64_t id, cbm_node_t *out) { - sqlite3_stmt *stmt = - prepare_cached(s, &s->stmt_find_node_by_id, - "SELECT id, project, label, name, qualified_name, file_path, " - "start_line, end_line, properties FROM nodes WHERE id = ?1;"); + sqlite3_stmt *stmt = prepare_cached(s, &s->stmt_find_node_by_id, st_find_node_by_id_sql); if (!stmt) { return CBM_STORE_ERR; } @@ -3295,6 +3375,280 @@ int cbm_store_find_nodes_by_file(cbm_store_t *s, const char *project, const char project, file_path, out, count); } +/* ── Streamed node scan ────────────────────────────────────────────── + * cbm_store_find_nodes / cbm_store_find_nodes_by_label materialise every row + * at once: 8.5 M kernel nodes with all their strings are 4.9 GB before the + * caller sees the first one. The scan below hands the same rows out a page at + * a time, and resets each page's statement before returning it, so no read + * lock outlives a page: a writer waits behind one page, as behind one eager + * lookup, never behind the caller's whole query. + * + * Pages continue by keyset along the index the eager query walks, so they + * concatenate to the eager order: + * all nodes idx_nodes_file (project, file_path): the NULL file_path rows by + * id (index order puts NULL first), then by (file_path, id). A + * page continues with the rest of its last row's file (by id), + * then with the files after it: two exact index seeks, where a + * (file_path, id) row-value bound seeks the file only and would + * rescan a large file from its start on every page; + * one label idx_nodes_label (project, label): by id. + * open checks that plan with EXPLAIN QUERY PLAN on the eager query itself. + * Any other plan (a changed schema, planner statistics) falls back to the + * eager scan handed out in batches: the same rows, without the saving. */ +#define NODE_SCAN_COLS \ + "SELECT id, project, label, name, qualified_name, file_path, start_line, end_line, " \ + "properties FROM nodes " + +typedef enum { + NODE_SCAN_LABEL, /* label = ?2 AND id >= ?3 */ + NODE_SCAN_NULL_FILE, /* file_path IS NULL AND id >= ?2 */ + NODE_SCAN_FILE_FIRST, /* file_path IS NOT NULL: the first file page */ + NODE_SCAN_FILE_SAME, /* file_path = ?2 AND id > ?3: the rest of the last file */ + NODE_SCAN_FILE_AFTER, /* file_path > ?2: the files after it */ + NODE_SCAN_STMTS, +} node_scan_stmt_t; + +enum { NODE_SCAN_COL_FILE = 5 }; /* file_path in NODE_SCAN_COLS */ + +static const char *const node_scan_sql[NODE_SCAN_STMTS] = { + NODE_SCAN_COLS "WHERE project = ?1 AND label = ?2 AND id >= ?3 ORDER BY id LIMIT ?4;", + NODE_SCAN_COLS "WHERE project = ?1 AND file_path IS NULL AND id >= ?2 ORDER BY id LIMIT ?3;", + NODE_SCAN_COLS "WHERE project = ?1 AND file_path IS NOT NULL ORDER BY file_path, id LIMIT ?2;", + NODE_SCAN_COLS "WHERE project = ?1 AND file_path = ?2 AND id > ?3 ORDER BY id LIMIT ?4;", + NODE_SCAN_COLS "WHERE project = ?1 AND file_path > ?2 ORDER BY file_path, id LIMIT ?3;", +}; + +struct cbm_store_node_scan { + cbm_store_t *store; + char *project; + char *label; + bool paged; + node_scan_stmt_t phase; /* LABEL, NULL_FILE, FILE_SAME or FILE_AFTER */ + bool keyed; /* last_id (and for files last_file) hold the previous page's end */ + int64_t last_id; + sqlite3_value *last_file; + sqlite3_stmt *stmts[NODE_SCAN_STMTS]; + bool done; + bool eager_failed; /* fallback: the eager scan itself failed; next reports it once */ + cbm_node_t *eager; /* fallback: the eager scan's rows, handed out in order */ + int eager_count; + int eager_next; +}; + +#ifdef CBM_ENABLE_TEST_SEAMS +static atomic_int node_scan_test_fail_countdown = ATOMIC_VAR_INIT(CBM_NOT_FOUND); + +void cbm_store_node_scan_test_fail_page_after(int successful_pages) { + atomic_store(&node_scan_test_fail_countdown, successful_pages); +} +#endif + +/* True when SQLite runs `sql` as the single index search `plan`. */ +static bool node_scan_plan_is(cbm_store_t *s, const char *sql, const char *plan) { + char eqp[CBM_SZ_512]; + snprintf(eqp, sizeof(eqp), "EXPLAIN QUERY PLAN %s", sql); + sqlite3_stmt *stmt = NULL; + if (sqlite3_prepare_v2(s->db, eqp, CBM_NOT_FOUND, &stmt, NULL) != SQLITE_OK) { + sqlite3_finalize(stmt); + return false; + } + int rows = 0; + bool match = false; + while (sqlite3_step(stmt) == SQLITE_ROW) { + const char *detail = (const char *)sqlite3_column_text(stmt, ST_COL_3); + match = rows == 0 && detail && strcmp(detail, plan) == 0; + rows++; + } + sqlite3_finalize(stmt); + return match && rows == SKIP_ONE; +} + +cbm_store_node_scan_t *cbm_store_node_scan_open(cbm_store_t *s, const char *project, + const char *label) { + if (!s || !s->db) { + return NULL; + } + cbm_store_node_scan_t *scan = cbm_calloc(CBM_MEM_CLASS_STORE, sizeof(*scan)); + if (!scan) { + return NULL; + } + scan->store = s; + scan->project = cbm_mem_strdup(CBM_MEM_CLASS_STORE, project); + scan->label = cbm_mem_strdup(CBM_MEM_CLASS_STORE, label); + if ((project && !scan->project) || (label && !scan->label)) { + cbm_store_node_scan_close(scan); + return NULL; + } + scan->phase = label ? NODE_SCAN_LABEL : NODE_SCAN_NULL_FILE; /* NULL file_path rows first */ + if (label) { + scan->paged = + node_scan_plan_is(s, NODE_SCAN_COLS "WHERE project = ?1 AND label = ?2;", + "SEARCH nodes USING INDEX idx_nodes_label (project=? AND label=?)"); + } else { + scan->paged = node_scan_plan_is(s, NODE_SCAN_COLS "WHERE project = ?1;", + "SEARCH nodes USING INDEX idx_nodes_file (project=?)"); + } + if (!scan->paged) { + int rc = label ? cbm_store_find_nodes_by_label(s, project, label, &scan->eager, + &scan->eager_count) + : cbm_store_find_nodes(s, project, &scan->eager, &scan->eager_count); + scan->eager_failed = rc != CBM_STORE_OK; + } + return scan; +} + +bool cbm_store_node_scan_paged(const cbm_store_node_scan_t *scan) { + return scan && scan->paged; +} + +/* The current phase has no rows left. The NULL file_path rows hand over to + * the files, the rest of a file to the files after it; a label's rows and + * the files after the last one end the scan. */ +static void node_scan_phase_done(cbm_store_node_scan_t *scan) { + if (scan->phase == NODE_SCAN_NULL_FILE) { + scan->keyed = false; + scan->phase = NODE_SCAN_FILE_AFTER; + } else if (scan->phase == NODE_SCAN_FILE_SAME) { + scan->phase = NODE_SCAN_FILE_AFTER; /* after last_file */ + } else { + scan->done = true; + } +} + +static void node_scan_bind(cbm_store_node_scan_t *scan, sqlite3_stmt *stmt, node_scan_stmt_t which, + int max) { + int col = SKIP_ONE; + bind_text(stmt, col++, scan->project); + if (which == NODE_SCAN_LABEL) { + bind_text(stmt, col++, scan->label); + } + if (which == NODE_SCAN_FILE_SAME || which == NODE_SCAN_FILE_AFTER) { + sqlite3_bind_value(stmt, col++, scan->last_file); + } + if (which == NODE_SCAN_LABEL || which == NODE_SCAN_NULL_FILE) { + sqlite3_bind_int64(stmt, col++, scan->keyed ? scan->last_id + SKIP_ONE : INT64_MIN); + } else if (which == NODE_SCAN_FILE_SAME) { + sqlite3_bind_int64(stmt, col++, scan->last_id); + } + sqlite3_bind_int(stmt, col, max); +} + +/* One page of the current phase into out[0..max). A short page ends the + * phase; the statement is reset before returning either way. */ +static int node_scan_page(cbm_store_node_scan_t *scan, cbm_node_t *out, int max, int *got) { + *got = 0; + cbm_store_t *s = scan->store; + node_scan_stmt_t which = scan->phase; + if (which == NODE_SCAN_FILE_AFTER && !scan->keyed) { + which = NODE_SCAN_FILE_FIRST; + } + bool after_id = + which == NODE_SCAN_LABEL || which == NODE_SCAN_NULL_FILE || which == NODE_SCAN_FILE_SAME; + if (after_id && scan->keyed && scan->last_id == INT64_MAX) { + node_scan_phase_done(scan); /* nothing can follow the largest id */ + return CBM_STORE_OK; + } +#ifdef CBM_ENABLE_TEST_SEAMS + if (graph_compare_test_countdown_fires(&node_scan_test_fail_countdown)) { + snprintf(s->errbuf, sizeof(s->errbuf), "node scan aborted: injected page failure"); + return CBM_STORE_ERR; + } +#endif + read_batch_renew(s); + sqlite3_stmt **slot = &scan->stmts[which]; + if (!*slot && + sqlite3_prepare_v2(s->db, node_scan_sql[which], CBM_NOT_FOUND, slot, NULL) != SQLITE_OK) { + store_set_error_sqlite(s, "node scan prepare"); + return CBM_STORE_ERR; + } + sqlite3_stmt *stmt = *slot; + node_scan_bind(scan, stmt, which, max); + + int n = 0; + int rc = SQLITE_ROW; + while (n < max && (rc = sqlite3_step(stmt)) == SQLITE_ROW) { + scan_node(stmt, &out[n]); + n++; + } + bool failed = n < max && rc != SQLITE_DONE; + if (n == max) { /* the statement still sits on the page's last row */ + scan->keyed = true; + scan->last_id = out[n - SKIP_ONE].id; + if (which == NODE_SCAN_FILE_FIRST || which == NODE_SCAN_FILE_AFTER) { + /* a new last file: continue with the rest of it */ + sqlite3_value_free(scan->last_file); + scan->last_file = sqlite3_value_dup(sqlite3_column_value(stmt, NODE_SCAN_COL_FILE)); + failed = !scan->last_file; + scan->phase = NODE_SCAN_FILE_SAME; + } + } + sqlite3_reset(stmt); + if (failed) { + store_set_error_sqlite(s, "node scan aborted"); + for (int i = 0; i < n; i++) { + cbm_node_free_fields(&out[i]); + } + return CBM_STORE_ERR; + } + if (n < max) { + node_scan_phase_done(scan); + } + *got = n; + return CBM_STORE_OK; +} + +int cbm_store_node_scan_next(cbm_store_node_scan_t *scan, cbm_node_t *out, int max, int *count) { + *count = 0; + if (!scan || max <= 0) { + return CBM_STORE_OK; + } + if (scan->eager_failed) { + scan->eager_failed = false; /* reported once; the scan then reads as exhausted */ + return CBM_STORE_ERR; + } + if (!scan->paged) { + int left = scan->eager_count - scan->eager_next; + int n = left < max ? left : max; + if (n > 0) { + memcpy(out, scan->eager + scan->eager_next, (size_t)n * sizeof(cbm_node_t)); + memset(scan->eager + scan->eager_next, 0, (size_t)n * sizeof(cbm_node_t)); + scan->eager_next += n; + } + *count = n; + return CBM_STORE_OK; + } + int n = 0; + while (n < max && !scan->done) { + int got = 0; + if (node_scan_page(scan, out + n, max - n, &got) != CBM_STORE_OK) { + /* A failed page yields no rows from this call and ends the scan; + * cbm_store_error() names the failure. */ + for (int i = 0; i < n; i++) { + cbm_node_free_fields(&out[i]); + } + scan->done = true; + return CBM_STORE_ERR; + } + n += got; + } + *count = n; + return CBM_STORE_OK; +} + +void cbm_store_node_scan_close(cbm_store_node_scan_t *scan) { + if (!scan) { + return; + } + for (int i = 0; i < NODE_SCAN_STMTS; i++) { + sqlite3_finalize(scan->stmts[i]); + } + sqlite3_value_free(scan->last_file); + cbm_store_free_nodes(scan->eager, scan->eager_count); /* rows not handed out */ + cbm_free(CBM_MEM_CLASS_STORE, scan->project); + cbm_free(CBM_MEM_CLASS_STORE, scan->label); + cbm_free(CBM_MEM_CLASS_STORE, scan); +} + typedef struct { cbm_store_cancel_fn cancel; void *context; @@ -6302,6 +6656,100 @@ int cbm_store_restore_from(cbm_store_t *dst, cbm_store_t *src) { return CBM_STORE_OK; } +/* ── In-memory adjacency ─────────────────────────────────────────── */ + +/* The edge types search_graph's in_deg/out_deg count (cbm_store_search). */ +static const char *const st_degree_types[] = {"CALLS", "USAGE", "CALL_REFERENCE", "INHERITS", + "IMPLEMENTS"}; + +/* cbm_adj_degree(id, inbound): the in-memory twin of search's two + * correlated COUNT(*) subqueries over st_degree_types. */ +static void sqlite_graph_mem_degree(sqlite3_context *ctx, int argc, sqlite3_value **argv) { + (void)argc; + const cbm_store_t *s = sqlite3_user_data(ctx); + sqlite3_result_int(ctx, cbm_graph_mem_degree(s->graph_mem, sqlite3_value_int64(argv[0]), + sqlite3_value_int(argv[SKIP_ONE]) != 0, + s->graph_mem_degree_mask)); +} + +static uint32_t st_be32(const unsigned char *p) { + return ((uint32_t)p[0] << ST_BE_SHIFT_24) | ((uint32_t)p[SKIP_ONE] << ST_BE_SHIFT_16) | + ((uint32_t)p[ST_COL_2] << ST_BE_SHIFT_8) | (uint32_t)p[ST_COL_3]; +} + +/* The committed state of a rollback-journal DB: its header's change counter + * and page count, read through SQLite's own file handle while this + * connection's read transaction holds the SHARED lock, so no commit can land + * mid-read. Every rollback-mode commit bumps the counter (it is how SQLite + * tells other connections their page cache is stale), and switching into or + * out of WAL is itself such a commit. That makes the state exact where the + * store generation is not: the cross-repo pass writes CROSS_* edges straight + * into other projects' published DBs without stamping a generation. A + * WAL-mode file is refused — a WAL commit need not touch the header — which + * is the state of a published DB after any in-place writer switched it to + * WAL; it keeps the SQL path until the next publish seals it again. */ +static bool store_rollback_file_state(cbm_store_t *s, char *out, size_t out_sz) { + sqlite3_file *file = NULL; + unsigned char hdr[ST_DB_HEADER_LEN]; + if (sqlite3_file_control(s->db, "main", SQLITE_FCNTL_FILE_POINTER, (void *)&file) != + SQLITE_OK || + !file || !file->pMethods || + file->pMethods->xRead(file, hdr, (int)sizeof(hdr), 0) != SQLITE_OK) { + return false; + } + if (hdr[ST_HDR_WRITE_VERSION] != ST_HDR_ROLLBACK || + hdr[ST_HDR_READ_VERSION] != ST_HDR_ROLLBACK) { + return false; + } + snprintf(out, out_sz, "c%u.p%u", st_be32(hdr + ST_HDR_CHANGE_COUNTER), + st_be32(hdr + ST_HDR_PAGE_COUNT)); + return true; +} + +/* This store's adjacency layer, or NULL for the SQL path; tried once per + * store. The layer's version is the store generation plus the file state, and + * both — like a miss's edge scan — are read inside one read transaction, so a + * layer holds exactly the edges of the version it is keyed to. */ +static cbm_graph_mem_t *store_graph_mem(cbm_store_t *s) { + if (s->graph_mem_tried) { + return s->graph_mem; + } + s->graph_mem_tried = true; + if (!s->query_store || !s->db || !s->db_path) { + return NULL; + } + bool own_txn = sqlite3_get_autocommit(s->db) != 0; + if (own_txn && sqlite3_exec(s->db, "BEGIN DEFERRED;", NULL, NULL, NULL) != SQLITE_OK) { + return NULL; + } + char generation[CBM_SZ_128]; + char state[CBM_SZ_64]; + char version[CBM_SZ_256]; + /* The generation read is the transaction's first read: the SHARED lock + * the file-state read relies on is held from here on. */ + if (cbm_store_generation(s, generation, sizeof(generation)) == CBM_STORE_OK && + strcmp(generation, "legacy") != 0 && store_rollback_file_state(s, state, sizeof(state))) { + snprintf(version, sizeof(version), "%s/%s", generation, state); + s->graph_mem = cbm_graph_mem_acquire(s->db, s->db_path, version); + } + if (own_txn) { + (void)sqlite3_exec(s->db, "COMMIT;", NULL, NULL, NULL); + } + if (s->graph_mem && + sqlite3_create_function(s->db, "cbm_adj_degree", ST_COL_2, + SQLITE_UTF8 | SQLITE_DETERMINISTIC | SQLITE_DIRECTONLY, s, + sqlite_graph_mem_degree, NULL, NULL) != SQLITE_OK) { + cbm_graph_mem_release(s->graph_mem); + s->graph_mem = NULL; + } + if (s->graph_mem) { + s->graph_mem_degree_mask = + cbm_graph_mem_type_mask(s->graph_mem, st_degree_types, + (int)(sizeof(st_degree_types) / sizeof(st_degree_types[0]))); + } + return s->graph_mem; +} + /* ── Search ─────────────────────────────────────────────────────── */ /* Convert a glob pattern to SQL LIKE pattern. */ @@ -6405,6 +6853,36 @@ int cbm_extract_like_hints(const char *pattern, char **out, int max_out) { return count; } +/* Patterns run as POSIX ERE without REG_NEWLINE, so '^' anchors at the start of + * the name only, and a pattern without '|' is one concatenation: every match + * begins with the literal run after the '^'. A following '*', '?', '{' may + * repeat the run's last character zero times and '+' may be combined with them + * (glibc reads "a+?" as "(a+)?"), so that character is dropped; so is the one + * before a backslash, which no rule here needs to interpret. */ +int cbm_regex_anchored_prefix(const char *pattern, char *out, int out_sz) { + if (!pattern || !out || out_sz <= 1 || pattern[0] != '^' || strchr(pattern, '|')) { + if (out && out_sz > 0) { + out[0] = '\0'; + } + return 0; + } + int len = 0; + const char *p = pattern + SKIP_ONE; + while (*p && len < out_sz - SKIP_ONE) { + unsigned char c = (unsigned char)*p; + if (c < ' ' || c > '~' || strchr(".[]()*+?{}|^$\\", c)) { + break; + } + out[len++] = (char)c; + p++; + } + if (len > 0 && *p && strchr("*+?{\\", *p)) { + len--; + } + out[len] = '\0'; + return len; +} + /* ── ensureCaseInsensitive / stripCaseFlag ────────────────────── */ const char *cbm_ensure_case_insensitive(const char *pattern) { @@ -6701,6 +7179,151 @@ static int search_build_where(const cbm_search_params_t *params, char *where, in return nparams; } +/* Anchored name patterns ("^schedule$", "^kmalloc"): every matching name begins + * with the pattern's literal prefix (cbm_regex_anchored_prefix), so it lies in a + * half-open idx_nodes_name range [prefix, prefix with its last byte + 1). Under + * iregexp the prefix matches in any letter case: the process never calls + * setlocale, so the regex runs in the C locale and folds ASCII letters only, and + * each of the first ST_NAME_RANGE_LETTERS letters doubles the ranges. The ranges + * are ADDED to the full WHERE (LIKE hints and regex stay), so the rows are those + * of the plain query — which instead scans every name of the project. */ +typedef struct { + char lo[ST_NAME_RANGE_MAX][ST_NAME_RANGE_PREFIX]; + char hi[ST_NAME_RANGE_MAX][ST_NAME_RANGE_PREFIX]; + int count; +} search_name_ranges_t; + +static bool name_range_letter(char c) { + return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z'); +} + +static char name_range_case(char c, bool lower) { + if (lower && c >= 'A' && c <= 'Z') { + return (char)(c - 'A' + 'a'); + } + if (!lower && c >= 'a' && c <= 'z') { + return (char)(c - 'a' + 'A'); + } + return c; +} + +static int search_name_ranges(const cbm_search_params_t *p, search_name_ranges_t *r) { + r->count = 0; + /* Without the project the index cannot seek the name; degree filters wrap + * the query in a subselect this rewrite leaves alone. */ + if (!p->project || !p->name_pattern || p->min_degree >= 0 || p->max_degree >= 0) { + return 0; + } + char prefix[ST_NAME_RANGE_PREFIX]; + int len = cbm_regex_anchored_prefix(p->name_pattern, prefix, (int)sizeof(prefix)); + if (len == 0) { + return 0; + } + /* A pattern the regex function rejects fails the plain query once a row + * reaches the regex; fewer rows reach it here, so it keeps the plain query. */ + int flags = CBM_REG_EXTENDED | CBM_REG_NOSUB | (p->case_sensitive ? 0 : CBM_REG_ICASE); + cbm_regex_t re; + if (cbm_regcomp(&re, p->name_pattern, flags) != 0) { + return 0; + } + cbm_regfree(&re); + int letters = 0; + if (!p->case_sensitive) { + int k = 0; + for (; k < len; k++) { + if (name_range_letter(prefix[k])) { + if (letters == ST_NAME_RANGE_LETTERS) { + break; + } + letters++; + } + } + len = k; + } + int variants = SKIP_ONE << letters; + for (int v = 0; v < variants; v++) { + int bit = letters - SKIP_ONE; + for (int i = 0; i < len; i++) { + char c = prefix[i]; + if (!p->case_sensitive && name_range_letter(c)) { + /* Variant 0 is all upper case: ranges in ascending byte order. */ + c = name_range_case(c, ((v >> bit) & SKIP_ONE) != 0); + bit--; + } + r->lo[v][i] = c; + r->hi[v][i] = c; + } + r->lo[v][len] = '\0'; + r->hi[v][len] = '\0'; + r->hi[v][len - SKIP_ONE]++; /* printable ASCII: never wraps */ + } + r->count = variants; + return variants; +} + +/* Main and count SQL over the name ranges: one UNION ALL arm per range, each + * the plain WHERE plus its range, so every arm seeks idx_nodes_name. The arms + * are disjoint and each yields (name, id) order, so ORDER BY 4, 1 (n.name, + * n.id) merges them without a sort and the per-row degree subqueries run only + * for rows the page emits; the count sums the per-range counts. On false + * (allocation, bind slots) nothing is bound and the plain query runs. */ +/* Append to buf[0..cap) at *len; false (nothing written past cap) when it + * does not fit. */ +static bool search_sql_append(char *buf, size_t cap, size_t *len, const char *fmt, ...) { + if (*len >= cap) { + return false; + } + va_list args; + va_start(args, fmt); + int wrote = vsnprintf(buf + *len, cap - *len, fmt, args); + va_end(args); + if (wrote < 0 || (size_t)wrote >= cap - *len) { + return false; + } + *len += (size_t)wrote; + return true; +} + +static bool search_build_ranged_sql(const char *select_cols, const char *where, + const search_name_ranges_t *r, int limit, int offset, + search_bind_t *binds, int *bind_idx, char **out_sql, + char **out_count_sql) { + if (*bind_idx + (r->count * ST_COL_2) > ST_SEARCH_MAX_BINDS) { + return false; + } + size_t cap = (strlen(select_cols) + strlen(where) + ST_NAME_RANGE_ARM_SQL) * (size_t)r->count + + ST_NAME_RANGE_ARM_SQL; + char *sql = cbm_alloc(CBM_MEM_CLASS_STORE, cap); + char *cnt = cbm_alloc(CBM_MEM_CLASS_STORE, cap); + int first_bind = *bind_idx; + size_t sl = 0; + size_t cl = 0; + bool ok = sql && cnt && search_sql_append(cnt, cap, &cl, "SELECT "); + for (int i = 0; ok && i < r->count; i++) { + int lo = *bind_idx + SKIP_ONE; + where_bind_text(binds, bind_idx, r->lo[i]); + where_bind_text(binds, bind_idx, r->hi[i]); + ok = search_sql_append(sql, cap, &sl, + "%s%s FROM nodes n WHERE (%s) AND n.name >= ?%d AND n.name < ?%d", + i > 0 ? " UNION ALL " : "", select_cols, where, lo, lo + SKIP_ONE) && + search_sql_append(cnt, cap, &cl, + "%s(SELECT COUNT(*) FROM nodes n WHERE (%s) AND n.name >= ?%d AND " + "n.name < ?%d)", + i > 0 ? " + " : "", where, lo, lo + SKIP_ONE); + } + ok = ok && search_sql_append(sql, cap, &sl, " ORDER BY 4, 1 LIMIT %d OFFSET %d", limit, offset); + if (!ok) { + /* Nothing is bound: the caller runs the plain query. */ + *bind_idx = first_bind; + cbm_free(CBM_MEM_CLASS_STORE, sql); + cbm_free(CBM_MEM_CLASS_STORE, cnt); + return false; + } + *out_sql = sql; + *out_count_sql = cnt; + return true; +} + int cbm_store_search(cbm_store_t *s, const cbm_search_params_t *params, cbm_search_output_t *out) { memset(out, 0, sizeof(*out)); if (!s || !s->db) { @@ -6719,6 +7342,16 @@ int cbm_store_search(cbm_store_t *s, const cbm_search_params_t *params, cbm_sear "(SELECT COUNT(*) FROM edges e WHERE e.source_id = n.id AND " "e.type IN ('CALLS', 'USAGE', 'CALL_REFERENCE', 'INHERITS', " "'IMPLEMENTS')) AS out_deg "; + /* A degree filter evaluates both counts for EVERY candidate row (787k + * Functions on the kernel: two B-tree seeks each, 1.9 s per pass). The + * adjacency layer answers them from RAM; the query is otherwise the same, + * so filter, order, page and total are unchanged. */ + bool has_degree_filter = (params->min_degree >= 0 || params->max_degree >= 0); + if (has_degree_filter && store_graph_mem(s)) { + select_cols = "SELECT n.id, n.project, n.label, n.name, n.qualified_name, " + "n.file_path, n.start_line, n.end_line, n.properties, " + "cbm_adj_degree(n.id, 1) AS in_deg, cbm_adj_degree(n.id, 0) AS out_deg "; + } char where[CBM_SZ_2K] = ""; search_bind_t binds[ST_SEARCH_MAX_BINDS]; @@ -6735,7 +7368,6 @@ int cbm_store_search(cbm_store_t *s, const cbm_search_params_t *params, cbm_sear } /* Degree filters */ - bool has_degree_filter = (params->min_degree >= 0 || params->max_degree >= 0); search_apply_degree_filter(sql, sizeof(sql), params); /* Count query — stripped of per-row edge subqueries for the common (no-degree-filter) @@ -6761,9 +7393,21 @@ int cbm_store_search(cbm_store_t *s, const cbm_search_params_t *params, cbm_sear id_col, limit, offset); strncat(sql, order_limit, sizeof(sql) - strlen(sql) - 1); + /* Anchored name pattern: the same rows through idx_nodes_name seeks. */ + search_name_ranges_t name_ranges; + char *ranged_sql = NULL; + char *ranged_count_sql = NULL; + if (nparams > 0 && search_name_ranges(params, &name_ranges) > 0) { + (void)search_build_ranged_sql(select_cols, where, &name_ranges, limit, offset, binds, + &bind_idx, &ranged_sql, &ranged_count_sql); + } + const char *main_sql = ranged_sql ? ranged_sql : sql; + const char *total_sql = ranged_count_sql ? ranged_count_sql : count_sql; + /* Execute count query */ sqlite3_stmt *cnt_stmt = NULL; - int rc = sqlite3_prepare_v2(s->db, count_sql, CBM_NOT_FOUND, &cnt_stmt, NULL); + int rc = sqlite3_prepare_v2(s->db, total_sql, CBM_NOT_FOUND, &cnt_stmt, NULL); + cbm_free(CBM_MEM_CLASS_STORE, ranged_count_sql); /* prepare_v2 copies the text */ if (rc == SQLITE_OK) { for (int i = 0; i < bind_idx; i++) { bind_text(cnt_stmt, i + SKIP_ONE, binds[i].text); @@ -6776,7 +7420,8 @@ int cbm_store_search(cbm_store_t *s, const cbm_search_params_t *params, cbm_sear /* Execute main query */ sqlite3_stmt *main_stmt = NULL; - rc = sqlite3_prepare_v2(s->db, sql, CBM_NOT_FOUND, &main_stmt, NULL); + rc = sqlite3_prepare_v2(s->db, main_sql, CBM_NOT_FOUND, &main_stmt, NULL); + cbm_free(CBM_MEM_CLASS_STORE, ranged_sql); if (rc != SQLITE_OK) { store_set_error_sqlite(s, "search prepare"); like_pool_free(&like_pool); @@ -7011,43 +7656,47 @@ static int bfs_cte_row_limit_for_depth(int max_results, int max_depth) { return (int)scaled; } -static int store_bfs(cbm_store_t *s, int64_t start_id, const char *direction, - const char **edge_types, int edge_type_count, int max_depth, int max_results, - bool trail, int max_edges, cbm_traverse_result_t *out) { - memset(out, 0, sizeof(*out)); - if (max_results < 0) { - max_results = 0; - } - - cbm_node_t root = {0}; - int rc = cbm_store_find_node_by_id(s, start_id, &root); - if (rc != CBM_STORE_OK) { - return rc; - } - out->root = root; +/* One store_bfs walk, shared by its SQL and in-memory paths. */ +typedef struct { + int64_t start_id; + bool inbound; + const char *types_clause; + const char **edge_types; + int edge_type_count; + int max_depth; + int max_results; + int probe_limit; /* max_results + 1: saturation stays observable */ + bool trail; +} bfs_query_t; - char types_clause[CBM_SZ_512]; - bfs_build_types_clause(edge_type_count, types_clause, (int)sizeof(types_clause)); +typedef struct { + cbm_node_hop_t *visited; + int count; + int cap; +} bfs_rows_t; - /* Build recursive CTE for BFS */ - char sql[CBM_SZ_4K]; - const char *join_cond; - const char *next_id; - bool is_inbound = (direction != NULL) && (strcmp(direction, "inbound") == 0); +static void bfs_rows_init(bfs_rows_t *rows) { + rows->cap = ST_INIT_CAP_16; + rows->count = 0; + rows->visited = malloc(rows->cap * sizeof(cbm_node_hop_t)); +} - if (is_inbound) { - join_cond = "e.target_id = bfs.node_id"; - next_id = "e.source_id"; - } else { - join_cond = "e.source_id = bfs.node_id"; - next_id = "e.target_id"; +static cbm_node_hop_t *bfs_rows_next(bfs_rows_t *rows) { + if (rows->count >= rows->cap) { + rows->cap *= ST_GROWTH; + rows->visited = safe_realloc(rows->visited, rows->cap * sizeof(cbm_node_hop_t)); } + return &rows->visited[rows->count++]; +} - /* One extra row makes max_results saturation observable on both branches. */ - int probe_limit = max_results < INT_MAX ? max_results + SKIP_ONE : max_results; - int cte_row_limit = 0; - if (trail) { - cte_row_limit = bfs_cte_row_limit_for_depth(max_results, max_depth); +/* The recursive CTE: trail (relationship-unique, for Cypher) or shortest-path. */ +static int bfs_visit_sql(cbm_store_t *s, const bfs_query_t *q, bfs_rows_t *rows, int *cte_rows, + int *cte_row_limit) { + char sql[CBM_SZ_4K]; + const char *join_cond = q->inbound ? "e.target_id = bfs.node_id" : "e.source_id = bfs.node_id"; + const char *next_id = q->inbound ? "e.source_id" : "e.target_id"; + if (q->trail) { + *cte_row_limit = bfs_cte_row_limit_for_depth(q->max_results, q->max_depth); snprintf(sql, sizeof(sql), "WITH RECURSIVE bfs(node_id, hop, edge_path) AS (" " SELECT %lld, 0, ''" @@ -7072,8 +7721,8 @@ static int store_bfs(cbm_store_t *s, int64_t start_id, const char *direction, "(SELECT count(*) FROM bfs) " "FROM bfs JOIN nodes n ON n.id = bfs.node_id " "WHERE bfs.hop > 0 ORDER BY bfs.hop, n.id LIMIT %d;", - (long long)start_id, next_id, join_cond, types_clause, max_depth, - cte_row_limit + SKIP_ONE, probe_limit); + (long long)q->start_id, next_id, join_cond, q->types_clause, q->max_depth, + *cte_row_limit + SKIP_ONE, q->probe_limit); } else { snprintf(sql, sizeof(sql), /* SHORTEST-PATH semantics: the UNION dedupes (node, hop) pairs. @@ -7096,52 +7745,130 @@ static int store_bfs(cbm_store_t *s, int64_t start_id, const char *direction, "GROUP BY n.id " "ORDER BY hop, n.id " "LIMIT %d;", - (long long)start_id, next_id, join_cond, types_clause, max_depth, probe_limit); + (long long)q->start_id, next_id, join_cond, q->types_clause, q->max_depth, + q->probe_limit); } sqlite3_stmt *stmt = NULL; - rc = sqlite3_prepare_v2(s->db, sql, CBM_NOT_FOUND, &stmt, NULL); - if (rc != SQLITE_OK) { + if (sqlite3_prepare_v2(s->db, sql, CBM_NOT_FOUND, &stmt, NULL) != SQLITE_OK) { store_set_error_sqlite(s, "bfs prepare"); return CBM_STORE_ERR; } /* Bind edge type parameters */ - if (edge_type_count > 0) { - for (int i = 0; i < edge_type_count; i++) { - bind_text(stmt, i + SKIP_ONE, edge_types[i]); + if (q->edge_type_count > 0) { + for (int i = 0; i < q->edge_type_count; i++) { + bind_text(stmt, i + SKIP_ONE, q->edge_types[i]); } } else { bind_text(stmt, SKIP_ONE, "CALLS"); } - int cap = ST_INIT_CAP_16; - int n = 0; - int cte_rows = 0; - cbm_node_hop_t *visited = malloc(cap * sizeof(cbm_node_hop_t)); - + bfs_rows_init(rows); int scan_rc15; while ((scan_rc15 = sqlite3_step(stmt)) == SQLITE_ROW) { - if (n >= cap) { - cap *= ST_GROWTH; - visited = safe_realloc(visited, cap * sizeof(cbm_node_hop_t)); + cbm_node_hop_t *row = bfs_rows_next(rows); + scan_node(stmt, &row->node); + row->hop = sqlite3_column_int(stmt, ST_COL_9); + if (q->trail) { + *cte_rows = sqlite3_column_int(stmt, ST_COL_10); } - scan_node(stmt, &visited[n].node); - visited[n].hop = sqlite3_column_int(stmt, ST_COL_9); - if (trail) { - cte_rows = sqlite3_column_int(stmt, ST_COL_10); - } - n++; } if (scan_rc15 != SQLITE_DONE) { /* SCANCHK:15:stmt */ store_set_error_sqlite(s, "row scan aborted"); sqlite3_finalize(stmt); - out->visited = visited; - out->visited_count = n; return CBM_STORE_ERR; } - sqlite3_finalize(stmt); + return CBM_STORE_OK; +} + +/* The shortest-path walk from the adjacency layer: the same rows as the CTE's + * JOIN nodes + ORDER BY hop, n.id + LIMIT, because cbm_graph_mem_bfs yields + * every reached id at its minimal hop in (hop, id) order and the node rows are + * read here in that order. An id with no node row is skipped, as the JOIN + * skips it. */ +static int bfs_visit_mem(cbm_store_t *s, const cbm_graph_mem_t *gm, const bfs_query_t *q, + bfs_rows_t *rows) { + static const char *const calls_only[] = {"CALLS"}; + bool typed = q->edge_type_count > 0; + cbm_graph_mem_types_t mask = cbm_graph_mem_type_mask(gm, typed ? q->edge_types : calls_only, + typed ? q->edge_type_count : SKIP_ONE); + cbm_graph_mem_hop_t *hops = NULL; + int hop_count = 0; + if (!cbm_graph_mem_bfs(gm, q->start_id, q->inbound, mask, q->max_depth, &hops, &hop_count)) { + store_set_error(s, "bfs: adjacency walk out of memory"); + return CBM_STORE_ERR; + } + bfs_rows_init(rows); + int rc = CBM_STORE_OK; + for (int i = 0; i < hop_count && rows->count < q->probe_limit && rc == CBM_STORE_OK; i++) { + sqlite3_stmt *stmt = prepare_cached(s, &s->stmt_find_node_by_id, st_find_node_by_id_sql); + if (!stmt) { + rc = CBM_STORE_ERR; + break; + } + sqlite3_bind_int64(stmt, SKIP_ONE, hops[i].id); + int step = sqlite3_step(stmt); + if (step == SQLITE_ROW) { + cbm_node_hop_t *row = bfs_rows_next(rows); + scan_node(stmt, &row->node); + row->hop = hops[i].hop; + } else if (step != SQLITE_DONE) { + store_set_error_sqlite(s, "bfs node row"); + rc = CBM_STORE_ERR; + } + sqlite3_reset(stmt); + } + cbm_graph_mem_hops_free(hops); + return rc; +} + +static int store_bfs(cbm_store_t *s, int64_t start_id, const char *direction, + const char **edge_types, int edge_type_count, int max_depth, int max_results, + bool trail, int max_edges, cbm_traverse_result_t *out) { + memset(out, 0, sizeof(*out)); + if (max_results < 0) { + max_results = 0; + } + + cbm_node_t root = {0}; + int rc = cbm_store_find_node_by_id(s, start_id, &root); + if (rc != CBM_STORE_OK) { + return rc; + } + out->root = root; + + char types_clause[CBM_SZ_512]; + bfs_build_types_clause(edge_type_count, types_clause, (int)sizeof(types_clause)); + + bfs_query_t q = { + .start_id = start_id, + .inbound = (direction != NULL) && (strcmp(direction, "inbound") == 0), + .types_clause = types_clause, + .edge_types = edge_types, + .edge_type_count = edge_type_count, + .max_depth = max_depth, + .max_results = max_results, + /* One extra row makes max_results saturation observable on both branches. */ + .probe_limit = max_results < INT_MAX ? max_results + SKIP_ONE : max_results, + .trail = trail, + }; + bfs_rows_t rows = {0}; + int cte_rows = 0; + int cte_row_limit = 0; + /* A shortest-path walk is answered from the adjacency layer when this + * store has one; trail mode keeps its relationship-unique CTE. */ + cbm_graph_mem_t *gm = trail ? NULL : store_graph_mem(s); + rc = gm ? bfs_visit_mem(s, gm, &q, &rows) + : bfs_visit_sql(s, &q, &rows, &cte_rows, &cte_row_limit); + cbm_node_hop_t *visited = rows.visited; + int n = rows.count; + if (rc != CBM_STORE_OK) { + out->visited = visited; + out->visited_count = n; + return rc; + } if (trail && cte_rows > cte_row_limit) { out->truncated = true; @@ -7629,6 +8356,25 @@ int cbm_store_count_edges_scoped(cbm_store_t *s, const char *project, const char return n; } +/* Per-label and per-type counts. GROUP BY over the (project, label) index + * decodes every entry to compare its group key; stepping through the distinct + * values with MIN(col) > previous seeks and counting each value's index range + * is ~1.6x faster (kernel, 8.5M nodes: 0.26 -> 0.16 s; 16.9M edges: 0.54 -> + * 0.32 s). The values reach ORDER BY in the ascending order GROUP BY emitted + * them in, so tied counts keep their order. label and type are NOT NULL. */ +static const char ST_LABEL_COUNTS_SQL[] = + "WITH RECURSIVE v(x) AS (SELECT MIN(label) FROM nodes WHERE project = ?1 UNION ALL " + "SELECT (SELECT MIN(label) FROM nodes WHERE project = ?1 AND label > v.x) FROM v " + "WHERE v.x IS NOT NULL) " + "SELECT x, (SELECT COUNT(*) FROM nodes WHERE project = ?1 AND label = v.x) AS c " + "FROM v WHERE x IS NOT NULL ORDER BY c DESC;"; +static const char ST_TYPE_COUNTS_SQL[] = + "WITH RECURSIVE v(x) AS (SELECT MIN(type) FROM edges WHERE project = ?1 UNION ALL " + "SELECT (SELECT MIN(type) FROM edges WHERE project = ?1 AND type > v.x) FROM v " + "WHERE v.x IS NOT NULL) " + "SELECT x, (SELECT COUNT(*) FROM edges WHERE project = ?1 AND type = v.x) AS c " + "FROM v WHERE x IS NOT NULL ORDER BY c DESC;"; + /* with_props=false skips the per-label/per-type JSON property-key discovery: * those json_each() scans walk EVERY row of each label/type (minutes-scale on * multi-million-node graphs) and get_architecture only needs the counts. */ @@ -7641,8 +8387,7 @@ static int get_schema_impl(cbm_store_t *s, const char *project, cbm_schema_info_ /* Node labels */ { - const char *sql = "SELECT label, COUNT(*) FROM nodes WHERE project = ?1 GROUP BY label " - "ORDER BY COUNT(*) DESC;"; + const char *sql = ST_LABEL_COUNTS_SQL; sqlite3_stmt *stmt = NULL; if (sqlite3_prepare_v2(s->db, sql, CBM_NOT_FOUND, &stmt, NULL) != SQLITE_OK || !stmt) { if (stmt) { @@ -7714,8 +8459,7 @@ static int get_schema_impl(cbm_store_t *s, const char *project, cbm_schema_info_ /* Edge types */ { - const char *sql = "SELECT type, COUNT(*) FROM edges WHERE project = ?1 GROUP BY type ORDER " - "BY COUNT(*) DESC;"; + const char *sql = ST_TYPE_COUNTS_SQL; sqlite3_stmt *stmt = NULL; if (sqlite3_prepare_v2(s->db, sql, CBM_NOT_FOUND, &stmt, NULL) != SQLITE_OK || !stmt) { if (stmt) { @@ -8686,15 +9430,24 @@ static int arch_packages(cbm_store_t *s, const char *project, const char *path, char like[CBM_SZ_512]; bool scoped = arch_path_prepare(path, norm, sizeof(norm), like, sizeof(like)); char sqlbuf[ST_SQL_BUF]; + /* GROUP BY +n.name, not n.name: with a bare column the planner walks + * idx_nodes_name over EVERY node of the project to get groups in name order, + * with a table lookup per node to test the label (6.3 s on the 8.5M-node + * kernel graph, for 5 Package nodes). The unary plus keeps the value and its + * BINARY collation but not the index, so the rows come from the + * idx_nodes_label seek. The ORDER BY sorter keeps tied counts in group + * emission order, which was name order from the index; a sorted GROUP BY + * follows the ORDER BY direction instead, so the name tie-break is explicit. */ const char *base = "SELECT n.name, COUNT(*) as cnt FROM nodes n " "WHERE n.project=?1 AND n.label='Package'"; if (scoped) { snprintf(sqlbuf, sizeof(sqlbuf), "%s AND (n.file_path = ?2 OR n.file_path LIKE ?3) " - "GROUP BY n.name ORDER BY cnt DESC LIMIT 15", + "GROUP BY +n.name ORDER BY cnt DESC, +n.name LIMIT 15", base); } else { - snprintf(sqlbuf, sizeof(sqlbuf), "%s GROUP BY n.name ORDER BY cnt DESC LIMIT 15", base); + snprintf(sqlbuf, sizeof(sqlbuf), "%s GROUP BY +n.name ORDER BY cnt DESC, +n.name LIMIT 15", + base); } sqlite3_stmt *stmt = NULL; if (sqlite3_prepare_v2(s->db, sqlbuf, CBM_NOT_FOUND, &stmt, NULL) != SQLITE_OK) { diff --git a/src/store/store.h b/src/store/store.h index 58bb63cedd..ca9569ae41 100644 --- a/src/store/store.h +++ b/src/store/store.h @@ -478,6 +478,16 @@ int cbm_store_commit(cbm_store_t *s); /* Rollback the current transaction. */ int cbm_store_rollback(cbm_store_t *s); +/* Read batch: run the reads that follow in one deferred read transaction, + * renewed every 100 ms so a writer is never held back longer, instead of one + * lock round trip per statement. Begin returns false (and does nothing) when + * a transaction is already open. For read-only work only; end it before any + * write, transaction or read scope on the same store. Close ends it. */ +bool cbm_store_read_batch_begin(cbm_store_t *s); +void cbm_store_read_batch_end(cbm_store_t *s); +/* Test seam: make an open read batch due for renewal at its next statement. */ +void cbm_store_read_batch_expire_for_test(cbm_store_t *s); + /* Stable request read scope. Borrows an exclusively owned, single-thread * store and callback context. Refuses a transaction, busy statement or owned * progress handler; idle cached statements are allowed. Saves query_only and @@ -615,6 +625,29 @@ int cbm_store_find_nodes_by_label(cbm_store_t *s, const char *project, const cha int cbm_store_find_nodes_by_file(cbm_store_t *s, const char *project, const char *file_path, cbm_node_t **out, int *count); +/* Streamed node scan: the rows of cbm_store_find_nodes (label NULL) or + * cbm_store_find_nodes_by_label, in the same order, a batch at a time. Pages + * are keyset queries along the eager query's index, and no read lock is held + * between calls. open returns NULL on error. next fills out[0..max) and sets + * *count; a count of 0 means the scan is exhausted. The caller frees each + * returned node with cbm_node_free_fields. A failed read returns + * CBM_STORE_ERR with no rows from that call and ends the scan; + * cbm_store_error() names the failure. paged is false when the eager query's + * plan is not the one the pages reproduce: the scan then hands out the eager + * scan's rows (same order, no memory saving). Close the scan before closing + * the store. */ +typedef struct cbm_store_node_scan cbm_store_node_scan_t; +cbm_store_node_scan_t *cbm_store_node_scan_open(cbm_store_t *s, const char *project, + const char *label); +int cbm_store_node_scan_next(cbm_store_node_scan_t *scan, cbm_node_t *out, int max, int *count); +bool cbm_store_node_scan_paged(const cbm_store_node_scan_t *scan); +void cbm_store_node_scan_close(cbm_store_node_scan_t *scan); +#ifdef CBM_ENABLE_TEST_SEAMS +/* One-shot fault: after this many successful pages (of any scan), the next + * page fails like a SQLite step error; -1 disables the seam. */ +void cbm_store_node_scan_test_fail_page_after(int successful_pages); +#endif + /* Return a stable, paginated outline for one exact repository-relative file. * File/folder/container nodes are excluded. labels may be NULL when * label_count is zero; otherwise labels are exact-match filters. The query is @@ -1215,6 +1248,13 @@ char *cbm_glob_to_like(const char *pattern); * Each out[i] is malloc'd — caller must free each string. */ int cbm_extract_like_hints(const char *pattern, char **out, int max_out); +/* Literal prefix every match of an anchored POSIX ERE must begin with: the + * plain printable-ASCII run after a leading '^', minus its last character when + * a quantifier or escape follows it. Returns its length (written NUL-terminated + * to out); 0 when the pattern is not anchored, contains '|', or starts with an + * operator. Case is kept as written. */ +int cbm_regex_anchored_prefix(const char *pattern, char *out, int out_sz); + /* Prepend (?i) to a regex pattern if not already present. * Returns a static buffer — do NOT free. */ const char *cbm_ensure_case_insensitive(const char *pattern); diff --git a/tests/fixtures/complexity/HCL/main.tf b/tests/fixtures/complexity/HCL/main.tf index 14b399a79c..74cf28398b 100644 --- a/tests/fixtures/complexity/HCL/main.tf +++ b/tests/fixtures/complexity/HCL/main.tf @@ -1,10 +1,16 @@ +locals { + ns = "${var.prefix}-apps" +} + resource "helm_release" "this" { - name = "app" - chart = "${path.module}/charts/app" + name = "app" + namespace = local.ns + chart = "${path.module}/charts/app" } resource "helm_release" "ingress" { name = "ingress" repository = "https://kubernetes.github.io/ingress-nginx" chart = "ingress-nginx" + depends_on = [helm_release.this] } diff --git a/tests/fixtures/complexity/HCL/variables.tf b/tests/fixtures/complexity/HCL/variables.tf new file mode 100644 index 0000000000..cf328746c9 --- /dev/null +++ b/tests/fixtures/complexity/HCL/variables.tf @@ -0,0 +1,3 @@ +variable "prefix" { + default = "cx" +} diff --git a/tests/fixtures/complexity/Kubernetes/kustomization.yaml b/tests/fixtures/complexity/Kubernetes/kustomization.yaml index aea332b3f3..c1ea5df8cf 100644 --- a/tests/fixtures/complexity/Kubernetes/kustomization.yaml +++ b/tests/fixtures/complexity/Kubernetes/kustomization.yaml @@ -6,3 +6,19 @@ resources: - crd.yaml - ocirepository.yaml - helmrelease.yaml +patches: + - path: patch.yaml + - path: patch-more.yaml + - patch: |- + apiVersion: v1 + kind: Service + metadata: + name: web + spec: + type: NodePort + - patch: |- + - op: add + path: /metadata/labels/tier + value: front + target: + kind: Deployment diff --git a/tests/fixtures/complexity/Kubernetes/patch-more.yaml b/tests/fixtures/complexity/Kubernetes/patch-more.yaml new file mode 100644 index 0000000000..73e395b9bc --- /dev/null +++ b/tests/fixtures/complexity/Kubernetes/patch-more.yaml @@ -0,0 +1,27 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: web-config +data: + mode: patched +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: web + annotations: + patched: "true" +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: web + annotations: + patched: "true" +--- +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: web +spec: + maxReplicas: 9 diff --git a/tests/fixtures/complexity/Kubernetes/patch.yaml b/tests/fixtures/complexity/Kubernetes/patch.yaml new file mode 100644 index 0000000000..a3ed04101e --- /dev/null +++ b/tests/fixtures/complexity/Kubernetes/patch.yaml @@ -0,0 +1,6 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: web +spec: + replicas: 3 diff --git a/tests/repro/repro_call_argument_matrix_a.c b/tests/repro/repro_call_argument_matrix_a.c index 5399bddab4..f4be433313 100644 --- a/tests/repro/repro_call_argument_matrix_a.c +++ b/tests/repro/repro_call_argument_matrix_a.c @@ -823,12 +823,16 @@ SEMANTIC_CASE(scss, SCSS, "main.scss", "@function bare($handler) {\n @return $handler;\n}\n", "$handler", "call_expression", 2, 2); -NATIVE_SCOPE_CASE(hcl, HCL, "main.tf", "locals {\n watched = [1]\n result = length(watched)\n}\n", - "function_call", +/* A Terraform / OpenTofu module file (.tf, .tofu) leaves its references to + * the module resolver (pass_terraform.c) and extracts no generic calls or + * usages (probe_terraform_module_files_extract_no_generic_refs); the generic + * HCL behaviour checked here is that of every other HCL file. */ +NATIVE_SCOPE_CASE(hcl, HCL, "main.hcl", + "locals {\n watched = [1]\n result = length(watched)\n}\n", "function_call", "HCL exposes built-in function applications but cannot declare user routines, " "so local-expression references use exact global counts", "length", NULL, "watched", NULL, NULL, - "locals {\n watched = [1]\n result = watched\n}\n", "bare.tf", NULL, "watched", + "locals {\n watched = [1]\n result = watched\n}\n", "bare.hcl", NULL, "watched", NULL, 1, "variable_expr", "watched", 1, 1, 1, 0); /* This known-valid PL/pgSQL shape exercises the grammar-real `invocation` node. diff --git a/tests/test_configlink.c b/tests/test_configlink.c index 5cdce7eaab..cf6d139661 100644 --- a/tests/test_configlink.c +++ b/tests/test_configlink.c @@ -85,7 +85,8 @@ static void rm_rf(const char *path) { /* ── Strategy 1: Config Key → Code Symbol ───────────────────────── */ /* Go: TestConfigKeySymbol_ExactMatch - * config.toml has max_connections, main.go has getMaxConnections() */ + * config.toml has max_connections, main.go has maxConnections(). (A TOML key + * links by exact name only, see configlink_key_symbol_toml_ini_exact_scoped.) */ TEST(configlink_key_symbol_exact_match) { cbm_gbuf_t *gb = cbm_gbuf_new("test", "/tmp/test"); @@ -94,9 +95,9 @@ TEST(configlink_key_symbol_exact_match) { "test.config.max_connections", "config.toml", 0, 0, NULL); ASSERT_GT(cfg_id, 0); - /* Code Function node: getMaxConnections from main.go */ - int64_t func_id = cbm_gbuf_upsert_node(gb, "Function", "getMaxConnections", - "test.main.getMaxConnections", "main.go", 0, 0, NULL); + /* Code Function node: maxConnections from main.go */ + int64_t func_id = cbm_gbuf_upsert_node(gb, "Function", "maxConnections", + "test.main.maxConnections", "main.go", 0, 0, NULL); ASSERT_GT(func_id, 0); /* Run configlink */ @@ -113,12 +114,14 @@ TEST(configlink_key_symbol_exact_match) { } /* Go: TestConfigKeySymbol_SubstringMatch - * config.toml has request_timeout, main.go has getRequestTimeoutSeconds() */ + * app.properties has request_timeout, main.go has getRequestTimeoutSeconds(). + * (Formats outside the exact + component rule -- .properties, .env -- keep + * substring matching.) */ TEST(configlink_key_symbol_substring_match) { cbm_gbuf_t *gb = cbm_gbuf_new("test", "/tmp/test"); - cbm_gbuf_upsert_node(gb, "Variable", "request_timeout", "test.config.request_timeout", - "config.toml", 0, 0, NULL); + cbm_gbuf_upsert_node(gb, "Variable", "request_timeout", "test.app.request_timeout", + "app.properties", 0, 0, NULL); cbm_gbuf_upsert_node(gb, "Function", "getRequestTimeoutSeconds", "test.main.getRequestTimeoutSeconds", "main.go", 0, 0, NULL); @@ -281,6 +284,48 @@ TEST(configlink_key_symbol_terraform_variable) { PASS(); } +/* TOML and INI keys (.toml, .ini, .cfg, .conf) link like infrastructure keys: + * by exact name and only inside the config file's component. In 17 infra + * repositories all 72 TOML/INI key_symbol edges were substring or + * cross-component coincidences (`.gitleaks.toml useDefault` <- + * TestConfig_UseDefault, `grafana.ini admin_user` <- GenAdminUser). Each + * format's key below reaches only svc-a's exact `requestTimeout`: not svc-a's + * substring `getRequestTimeoutSeconds`, not svc-b's exact `requestTimeout`. */ +TEST(configlink_key_symbol_toml_ini_exact_scoped) { + static const char *const files[] = {"svc-a/config.toml", "svc-a/settings.ini", + "svc-a/setup.cfg", "svc-a/app.conf"}; + enum { FILES = 4 }; + cbm_gbuf_t *gb = cbm_gbuf_new("test", "/tmp/test"); + int64_t cfg[FILES]; + for (int i = 0; i < FILES; i++) { + char qn[CBM_SZ_128]; + snprintf(qn, sizeof(qn), "test.cfg%d.request_timeout", i); + cfg[i] = cbm_gbuf_upsert_node(gb, "Variable", "request_timeout", qn, files[i], 0, 0, NULL); + } + int64_t inside = + cbm_gbuf_upsert_node(gb, "Function", "requestTimeout", "test.svc_a.main.requestTimeout", + "svc-a/main.go", 0, 0, NULL); + cbm_gbuf_upsert_node(gb, "Function", "getRequestTimeoutSeconds", + "test.svc_a.client.getRequestTimeoutSeconds", "svc-a/client.go", 0, 0, + NULL); + cbm_gbuf_upsert_node(gb, "Function", "requestTimeout", "test.svc_b.main.requestTimeout", + "svc-b/main.go", 0, 0, NULL); + run_configlink(gb, "test", NULL); + const cbm_gbuf_edge_t **edges = NULL; + int count = 0; + cbm_gbuf_find_edges_by_type(gb, "CONFIGURES", &edges, &count); + ASSERT_EQ(count, FILES); + for (int i = 0; i < FILES; i++) { + bool found = false; + for (int e = 0; e < count; e++) { + found = found || (edges[e]->source_id == inside && edges[e]->target_id == cfg[i]); + } + ASSERT_TRUE(found); + } + cbm_gbuf_free(gb); + PASS(); +} + /* No count of config keys decides what links: the collector stopped at 4,096. */ TEST(configlink_key_symbol_no_config_cap) { enum { KEYS = 4200 }; @@ -469,9 +514,11 @@ enum { CL_DET_CANDIDATES = 9000 }; /* > the 8192 code-entry cap */ static cbm_gbuf_t *build_oversized_corpus(bool reverse) { cbm_gbuf_t *gb = cbm_gbuf_new("test", "/tmp/test"); - /* One config key every candidate can match on: "max_connections". */ - cbm_gbuf_upsert_node(gb, "Variable", "max_connections", "test.config.max_connections", - "config.toml", 0, 0, NULL); + /* One config key every candidate can match on by substring: + * "max_connections", in a format that keeps substring matching (TOML and + * INI keys link by exact name only). */ + cbm_gbuf_upsert_node(gb, "Variable", "max_connections", "test.app.max_connections", + "app.properties", 0, 0, NULL); for (int k = 0; k < CL_DET_CANDIDATES; k++) { int i = reverse ? (CL_DET_CANDIDATES - 1 - k) : k; @@ -529,6 +576,7 @@ SUITE(configlink) { RUN_TEST(configlink_key_symbol_config_dir_component); RUN_TEST(configlink_key_symbol_no_config_cap); RUN_TEST(configlink_key_symbol_terraform_variable); + RUN_TEST(configlink_key_symbol_toml_ini_exact_scoped); /* Strategy 2: Dependency → Import */ RUN_TEST(configlink_dep_import_package_json); diff --git a/tests/test_cypher.c b/tests/test_cypher.c index fde427de1a..5f2f3eb15a 100644 --- a/tests/test_cypher.c +++ b/tests/test_cypher.c @@ -1388,11 +1388,12 @@ TEST(cypher_return_distinct_counts_distinct_rows_against_cap_issue1364) { ASSERT_TRUE(r.truncated); cbm_cypher_result_free(&r); - /* An oversized user LIMIT cannot bypass the engine projection budget. */ + /* Unbounded projections stay within the engine budget. An explicit LIMIT + * is a query clause and may exceed the caller's default row budget. */ const char *bounded_queries[] = { - "MATCH (n) RETURN n.name LIMIT 1000000", - "MATCH (n) RETURN DISTINCT n.name LIMIT 1000000", - "MATCH (n) RETURN * LIMIT 1000000", + "MATCH (n) RETURN n.name", + "MATCH (n) RETURN DISTINCT n.name", + "MATCH (n) RETURN *", }; for (size_t i = 0; i < sizeof(bounded_queries) / sizeof(bounded_queries[0]); i++) { memset(&r, 0, sizeof(r)); @@ -5647,6 +5648,518 @@ TEST(cypher_fifty_union_branches) { PASS(); } +/* ── ORDER BY, DISTINCT and grouping against a reference ─────────── + * These were bubble sorts and linear duplicate/group scans — seconds to + * minutes on a 30k-row result. The results must stay exactly the old ones: a + * stable sort of the unsorted rows, first-occurrence DISTINCT, first-seen + * group order. Each case derives that reference from the engine's own + * unsorted rows, on a graph built for ties, duplicates and large groups. */ + +enum { REF_NODES = 400, REF_NAMES = 97 }; + +static cbm_store_t *setup_reference_store(void) { + cbm_store_t *s = cbm_store_open_memory(); + cbm_store_upsert_project(s, "test", "/tmp/test"); + int64_t ids[REF_NODES]; + for (int i = 0; i < REF_NODES; i++) { + char name[32]; + char qn[32]; + char file[32]; + snprintf(name, sizeof(name), "fn_%02d", (i * 7) % REF_NAMES); /* repeated names */ + snprintf(qn, sizeof(qn), "test.n%d", i); + snprintf(file, sizeof(file), "f%02d.c", i % 13); + cbm_node_t n = {.project = "test", + .label = "Function", + .name = name, + .qualified_name = qn, + .file_path = file, + .start_line = (i * 31) % 50, /* tied lines */ + .end_line = 60}; + ids[i] = cbm_store_upsert_node(s, &n); + } + /* Every node calls both hubs (n0, n1: ~400 callers, 97 distinct names, + * past the hashed COUNT(DISTINCT) threshold) and one more node. */ + for (int i = 0; i < REF_NODES; i++) { + int targets[3] = {0, 1, (i * i + 3) % REF_NODES}; + for (int t = 0; t < 3; t++) { + if (targets[t] != i) { + cbm_edge_t e = {.project = "test", + .source_id = ids[i], + .target_id = ids[targets[t]], + .type = "CALLS"}; + cbm_store_insert_edge(s, &e); + } + } + } + return s; +} + +typedef int (*ref_cmp_fn)(const cbm_cypher_result_t *r, int a, int b); + +/* The reference order: an insertion sort that moves a row before an earlier + * one only when the earlier one compares greater — stable, like the old + * bubble sorts. */ +static void ref_stable_order(const cbm_cypher_result_t *r, int *order, ref_cmp_fn cmp) { + for (int i = 0; i < r->row_count; i++) { + order[i] = i; + } + for (int i = 1; i < r->row_count; i++) { + for (int j = i; j > 0 && cmp(r, order[j - 1], order[j]) > 0; j--) { + int swap = order[j]; + order[j] = order[j - 1]; + order[j - 1] = swap; + } + } +} + +/* Columns 0 (name) and 1 (line): line descending, then name ascending. */ +static int ref_line_desc_name(const cbm_cypher_result_t *r, int a, int b) { + long la = strtol(r->rows[a][1], NULL, 10); + long lb = strtol(r->rows[b][1], NULL, 10); + int cmp = (lb > la) - (lb < la); + return cmp != 0 ? cmp : strcmp(r->rows[a][0], r->rows[b][0]); +} + +static int ref_all_equal(const cbm_cypher_result_t *r, int a, int b) { + (void)r; + (void)a; + (void)b; + return 0; +} + +/* `sorted` must hold the rows of `unsorted` in the reference order. */ +static bool ref_rows_in_order(const cbm_cypher_result_t *sorted, + const cbm_cypher_result_t *unsorted, ref_cmp_fn cmp) { + if (sorted->row_count != unsorted->row_count || sorted->col_count != unsorted->col_count || + unsorted->row_count < REF_NODES) { + return false; + } + int order[REF_NODES * 4]; + ref_stable_order(unsorted, order, cmp); + for (int i = 0; i < sorted->row_count; i++) { + for (int c = 0; c < sorted->col_count; c++) { + if (strcmp(sorted->rows[i][c], unsorted->rows[order[i]][c]) != 0) { + return false; + } + } + } + return true; +} + +static bool ref_query(cbm_store_t *s, const char *query, cbm_cypher_result_t *out) { + memset(out, 0, sizeof(*out)); + return cbm_cypher_execute(s, query, "test", 0, out) == 0 && !out->truncated; +} + +TEST(cypher_order_by_is_the_stable_reference_order) { + cbm_store_t *s = setup_reference_store(); + cbm_cypher_result_t plain; + cbm_cypher_result_t sorted; + ASSERT_TRUE(ref_query(s, "MATCH (f:Function) RETURN f.name, f.start_line", &plain)); + + /* RETURN: numeric key descending, ties broken by the next key. */ + ASSERT_TRUE(ref_query( + s, "MATCH (f:Function) RETURN f.name, f.start_line ORDER BY f.start_line DESC, f.name", + &sorted)); + ASSERT_TRUE(ref_rows_in_order(&sorted, &plain, ref_line_desc_name)); + cbm_cypher_result_free(&sorted); + + /* WITH: the same keys through a projection. */ + ASSERT_TRUE(ref_query(s, + "MATCH (f:Function) WITH f.name AS n, f.start_line AS l " + "ORDER BY l DESC, n RETURN n, l", + &sorted)); + ASSERT_TRUE(ref_rows_in_order(&sorted, &plain, ref_line_desc_name)); + cbm_cypher_result_free(&sorted); + cbm_cypher_result_free(&plain); + + /* All keys tied: both sorts keep the unsorted order. */ + ASSERT_TRUE(ref_query(s, "MATCH (f:Function) RETURN f.name, f.label", &plain)); + ASSERT_TRUE( + ref_query(s, "MATCH (f:Function) RETURN f.name, f.label ORDER BY f.label", &sorted)); + ASSERT_TRUE(ref_rows_in_order(&sorted, &plain, ref_all_equal)); + cbm_cypher_result_free(&sorted); + ASSERT_TRUE(ref_query( + s, "MATCH (f:Function) WITH f.name AS n, f.label AS l ORDER BY l RETURN n, l", &sorted)); + ASSERT_TRUE(ref_rows_in_order(&sorted, &plain, ref_all_equal)); + cbm_cypher_result_free(&sorted); + cbm_cypher_result_free(&plain); + cbm_store_close(s); + PASS(); +} + +/* `got` must be the first occurrence of every value in column 0 of `all`. */ +static bool ref_first_occurrences(const cbm_cypher_result_t *got, const cbm_cypher_result_t *all) { + int kept = 0; + for (int i = 0; i < all->row_count; i++) { + bool seen = false; + for (int j = 0; j < i && !seen; j++) { + seen = strcmp(all->rows[i][0], all->rows[j][0]) == 0; + } + if (seen) { + continue; + } + if (kept >= got->row_count || strcmp(got->rows[kept][0], all->rows[i][0]) != 0) { + return false; + } + kept++; + } + return kept == got->row_count; +} + +TEST(cypher_distinct_keeps_first_occurrences) { + cbm_store_t *s = setup_reference_store(); + cbm_cypher_result_t all; + cbm_cypher_result_t got; + ASSERT_TRUE(ref_query(s, "MATCH (f:Function) RETURN f.name", &all)); + ASSERT_TRUE(ref_query(s, "MATCH (f:Function) RETURN DISTINCT f.name", &got)); + ASSERT_EQ(got.row_count, REF_NAMES); + ASSERT_TRUE(ref_first_occurrences(&got, &all)); + cbm_cypher_result_free(&got); + ASSERT_TRUE(ref_query(s, "MATCH (f:Function) WITH DISTINCT f.name AS n RETURN n", &got)); + ASSERT_EQ(got.row_count, REF_NAMES); + ASSERT_TRUE(ref_first_occurrences(&got, &all)); + cbm_cypher_result_free(&got); + cbm_cypher_result_free(&all); + cbm_store_close(s); + PASS(); +} + +/* `got` (column 0 = callee, column 1 = count) must list the callees of + * `pairs` (column 0 = callee, column 1 = caller name) in first-seen order, + * each with its caller count — of distinct names when `distinct`. */ +static bool ref_groups_match(const cbm_cypher_result_t *got, const cbm_cypher_result_t *pairs, + bool distinct) { + enum { REF_PAIRS_MAX = REF_NODES * 4 }; + if (pairs->row_count > REF_PAIRS_MAX) { + return false; + } + bool first_callee[REF_PAIRS_MAX]; + bool first_pair[REF_PAIRS_MAX]; + for (int k = 0; k < pairs->row_count; k++) { + first_callee[k] = true; + first_pair[k] = true; + for (int m = 0; m < k; m++) { + if (strcmp(pairs->rows[m][0], pairs->rows[k][0]) == 0) { + first_callee[k] = false; + first_pair[k] = first_pair[k] && strcmp(pairs->rows[m][1], pairs->rows[k][1]) != 0; + } + } + } + int groups = 0; + for (int i = 0; i < pairs->row_count; i++) { + if (!first_callee[i]) { + continue; + } + int count = 0; + for (int k = 0; k < pairs->row_count; k++) { + if (strcmp(pairs->rows[k][0], pairs->rows[i][0]) == 0) { + count += !distinct || first_pair[k] ? 1 : 0; + } + } + char want[16]; + snprintf(want, sizeof(want), "%d", count); + if (groups >= got->row_count || strcmp(got->rows[groups][0], pairs->rows[i][0]) != 0 || + strcmp(got->rows[groups][1], want) != 0) { + return false; + } + groups++; + } + return groups == got->row_count; +} + +TEST(cypher_grouping_keeps_first_seen_groups) { + cbm_store_t *s = setup_reference_store(); + cbm_cypher_result_t pairs; + cbm_cypher_result_t got; + ASSERT_TRUE( + ref_query(s, "MATCH (a)-[:CALLS]->(b:Function) RETURN b.qualified_name, a.name", &pairs)); + ASSERT_GT(pairs.row_count, REF_NODES * 2); + + ASSERT_TRUE(ref_query( + s, "MATCH (a)-[:CALLS]->(b:Function) RETURN b.qualified_name, count(a) AS c", &got)); + ASSERT_TRUE(ref_groups_match(&got, &pairs, false)); + cbm_cypher_result_free(&got); + + ASSERT_TRUE(ref_query(s, + "MATCH (a)-[:CALLS]->(b:Function) " + "RETURN b.qualified_name, count(DISTINCT a.name) AS c", + &got)); + ASSERT_TRUE(ref_groups_match(&got, &pairs, true)); + cbm_cypher_result_free(&got); + + ASSERT_TRUE(ref_query(s, + "MATCH (a)-[:CALLS]->(b:Function) WITH b.qualified_name AS q, " + "count(DISTINCT a.name) AS c RETURN q, c", + &got)); + ASSERT_TRUE(ref_groups_match(&got, &pairs, true)); + cbm_cypher_result_free(&got); + cbm_cypher_result_free(&pairs); + cbm_store_close(s); + PASS(); +} + +/* ── Streamed execution: chunks of start nodes give the single pass ── + * + * execute_single runs the start nodes through the pipeline in chunks rather + * than materialising every binding first (kernel count(*): 18 GB RSS). The + * chunked run must be indistinguishable from one pass over the whole start + * set: same rows in the same order, first-seen group order, ORDER BY ties, + * DISTINCT, LIMIT / max_rows truncation and OPTIONAL fallback rows. Each query + * runs as one chunk (the reference) and in chunks of 1, 7, 64 and the + * production default, so chunk boundaries fall inside groups and inside + * every start node's expansion order. */ + +static int stream_run(cbm_store_t *s, const char *query, int max_rows, int chunk, + cbm_cypher_result_t *out) { + memset(out, 0, sizeof(*out)); + cbm_cypher_test_set_stream_chunk(chunk); + int rc = cbm_cypher_execute(s, query, "test", max_rows, out); + cbm_cypher_test_set_stream_chunk(0); + return rc; +} + +static bool stream_same_str(const char *a, const char *b) { + return (!a && !b) || (a && b && strcmp(a, b) == 0); +} + +static bool stream_results_equal(const cbm_cypher_result_t *a, const cbm_cypher_result_t *b) { + if (a->col_count != b->col_count || a->row_count != b->row_count || + a->truncated != b->truncated || !stream_same_str(a->warning, b->warning) || + !stream_same_str(a->error, b->error)) { + return false; + } + for (int c = 0; c < a->col_count; c++) { + if (strcmp(a->columns[c], b->columns[c]) != 0) { + return false; + } + } + for (int r = 0; r < a->row_count; r++) { + for (int c = 0; c < a->col_count; c++) { + if (strcmp(a->rows[r][c], b->rows[r][c]) != 0) { + return false; + } + } + } + return true; +} + +/* True when every chunk size reproduces the single-pass result, which is + * left in *ref for the caller's own checks (caller frees). */ +static bool stream_matches_single_pass(cbm_store_t *s, const char *query, int max_rows, + cbm_cypher_result_t *ref) { + static const int chunks[] = {1, 7, 64, 0 /* production default */}; + int ref_rc = stream_run(s, query, max_rows, INT_MAX, ref); + bool same = ref_rc == 0; + for (size_t i = 0; same && i < sizeof(chunks) / sizeof(chunks[0]); i++) { + cbm_cypher_result_t got; + int rc = stream_run(s, query, max_rows, chunks[i], &got); + same = rc == ref_rc && stream_results_equal(ref, &got); + if (!same) { + printf(" chunk %d differs from the single pass: %s\n", chunks[i], query); + } + cbm_cypher_result_free(&got); + } + return same; +} + +/* Every pipeline consumer across chunk boundaries: unsorted rows, global and + * grouped aggregates (groups fed from many chunks), DISTINCT, late WHERE, + * collect, WITH aggregation and WITH sorting, RETURN * with and without a + * WITH, the default projection, edge variables and variable-length hops. */ +TEST(cypher_stream_chunks_match_single_pass) { + static const char *const queries[] = { + "MATCH (a)-[:CALLS]->(b) RETURN a.name, b.qualified_name", + "MATCH (a)-[:CALLS]->(b) RETURN count(*)", + "MATCH (a)-[:CALLS]->(b:Function) RETURN b.qualified_name, count(a) AS c ORDER BY c DESC", + "MATCH (a:Function)-[:CALLS]->(b) RETURN b.name, count(DISTINCT a.name) AS c", + "MATCH (a)-[:CALLS]->(b) RETURN DISTINCT b.name", + "MATCH (a)-[:CALLS]->(b) WHERE b.start_line > 20 RETURN a.name, b.name, b.start_line", + "MATCH (a)-[:CALLS]->(b) RETURN b.name, collect(a.name) AS cs", + ("MATCH (a:Function)-[:CALLS]->(b) WITH b, count(a) AS c WHERE c > 1 " + "RETURN b.name, c ORDER BY c DESC LIMIT 20"), + "MATCH (a)-[:CALLS]->(b) WITH a.name AS n, b.start_line AS l ORDER BY l DESC RETURN n, l", + "MATCH (a)-[:CALLS]->(b) WITH DISTINCT b.name AS n RETURN n", + "MATCH (a)-[:CALLS]->(b) RETURN *", + "MATCH (a)-[:CALLS]->(b) WITH a.name AS n RETURN *", + "MATCH (a)-[:CALLS]->(b)", + "MATCH (a)-[r:CALLS]->(b) RETURN type(r), a.name ORDER BY a.name", + "MATCH (a)-[:CALLS*1..2]->(b) RETURN a.name, b.name", + }; + cbm_store_t *s = setup_reference_store(); + for (size_t i = 0; i < sizeof(queries) / sizeof(queries[0]); i++) { + cbm_cypher_result_t ref; + ASSERT_TRUE(stream_matches_single_pass(s, queries[i], 0, &ref)); + ASSERT_GT(ref.row_count, 0); + ASSERT_FALSE(ref.truncated); + cbm_cypher_result_free(&ref); + } + cbm_store_close(s); + PASS(); +} + +/* OPTIONAL fallback rows (a start node with no match keeps its row, target + * unbound) sit between the expanded rows of their neighbours in every chunk + * layout; the multi-pattern form keeps its single pass. */ +TEST(cypher_stream_optional_across_chunks) { + cbm_store_t *s = setup_reference_store(); + cbm_cypher_result_t ref; + ASSERT_TRUE(stream_matches_single_pass( + s, "OPTIONAL MATCH (a)<-[:CALLS]-(b) RETURN a.name, b.name", 0, &ref)); + int fallback = 0; + for (int r = 0; r < ref.row_count; r++) { + fallback += ref.rows[r][1][0] == '\0' ? 1 : 0; + } + ASSERT_GT(fallback, 0); /* uncalled nodes keep a row */ + ASSERT_GT(ref.row_count, fallback); /* and called nodes expand */ + cbm_cypher_result_free(&ref); + + ASSERT_TRUE(stream_matches_single_pass( + s, "OPTIONAL MATCH (a)<-[:CALLS]-(b) RETURN a.qualified_name, count(b) AS c", 0, &ref)); + ASSERT_EQ(ref.row_count, REF_NODES); + cbm_cypher_result_free(&ref); + + ASSERT_TRUE(stream_matches_single_pass( + s, "MATCH (a:Function) OPTIONAL MATCH (a)<-[:CALLS]-(b) RETURN a.name, count(b)", 0, &ref)); + ASSERT_GT(ref.row_count, 0); + cbm_cypher_result_free(&ref); + cbm_store_close(s); + PASS(); +} + +/* A LIMIT or the max_rows budget fills up in one chunk while later chunks + * still arrive: they must add nothing, and truncation is reported exactly + * when the single pass reports it (engine budget yes, explicit LIMIT no). */ +TEST(cypher_stream_limit_and_max_rows_across_chunks) { + cbm_store_t *s = setup_reference_store(); + cbm_cypher_result_t ref; + ASSERT_TRUE(stream_matches_single_pass( + s, "MATCH (a)-[:CALLS]->(b) RETURN a.name, b.name LIMIT 25", 0, &ref)); + ASSERT_EQ(ref.row_count, 25); + ASSERT_FALSE(ref.truncated); + cbm_cypher_result_free(&ref); + + ASSERT_TRUE(stream_matches_single_pass( + s, "MATCH (a)-[:CALLS]->(b) RETURN DISTINCT b.name LIMIT 30", 0, &ref)); + ASSERT_EQ(ref.row_count, 30); + cbm_cypher_result_free(&ref); + + ASSERT_TRUE(stream_matches_single_pass( + s, "MATCH (a)-[:CALLS]->(b) RETURN a.name, b.name SKIP 5 LIMIT 10", 0, &ref)); + ASSERT_EQ(ref.row_count, 10); + cbm_cypher_result_free(&ref); + + static const char *const budgeted[] = { + "MATCH (a)-[:CALLS]->(b) RETURN a.name, b.name", + "MATCH (a)-[:CALLS]->(b) RETURN a.name, b.name ORDER BY b.name", + "MATCH (a)-[:CALLS]->(b) RETURN *", + "MATCH (a)-[:CALLS]->(b)", + }; + for (size_t i = 0; i < sizeof(budgeted) / sizeof(budgeted[0]); i++) { + ASSERT_TRUE(stream_matches_single_pass(s, budgeted[i], 30, &ref)); + ASSERT_EQ(ref.row_count, 30); + ASSERT_TRUE(ref.truncated); + cbm_cypher_result_free(&ref); + } + cbm_store_close(s); + PASS(); +} + +/* A start-node scan that fails part-way fails the whole query: the rows read + * before the failure must not answer it as if they were all of them. Pages + * hold 7 rows at chunk 7; the store seam fails a page after earlier pages + * delivered rows (the all-nodes scan's first page is its empty NULL + * file_path pass, hence one more page before the fault). */ +TEST(cypher_stream_scan_failure_fails_query) { + static const struct { + const char *query; + int pages_before_fault; + } cases[] = { + {"MATCH (a:Function)-[:CALLS]->(b) RETURN a.name, b.name", 1}, + {"MATCH (a:Function)-[:CALLS]->(b) RETURN count(*)", 1}, + {"MATCH (n) RETURN n.name", 2}, + }; + cbm_store_t *s = setup_reference_store(); + cbm_cypher_result_t r; + for (size_t i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) { + ASSERT_EQ(stream_run(s, cases[i].query, 0, 7, &r), 0); + ASSERT_GT(r.row_count, 0); + cbm_cypher_result_free(&r); + + cbm_store_node_scan_test_fail_page_after(cases[i].pages_before_fault); + int rc = stream_run(s, cases[i].query, 0, 7, &r); + cbm_store_node_scan_test_fail_page_after(-1); + ASSERT_TRUE(rc != 0); + ASSERT_NOT_NULL(r.error); + ASSERT_TRUE(strstr(r.error, "reading the graph failed") != NULL); + ASSERT_TRUE(strstr(r.error, "injected page failure") != NULL); + ASSERT_EQ(r.row_count, 0); + cbm_cypher_result_free(&r); + } + /* The failure does not outlive its query. */ + ASSERT_EQ(stream_run(s, cases[0].query, 0, 7, &r), 0); + ASSERT_GT(r.row_count, 0); + ASSERT_NULL(r.error); + cbm_cypher_result_free(&r); + cbm_store_close(s); + PASS(); +} + +/* The production chunk boundary itself: a start set of 2.5 chunks, a ring of + * calls plus five hubs whose caller groups straddle both boundaries. */ +TEST(cypher_stream_default_chunk_boundary) { + enum { STREAM_NODES = 2600, STREAM_HUBS = 5 }; + cbm_store_t *s = cbm_store_open_memory(); + cbm_store_upsert_project(s, "test", "/tmp/test"); + static int64_t ids[STREAM_NODES]; + for (int i = 0; i < STREAM_NODES; i++) { + char name[32]; + char qn[32]; + snprintf(name, sizeof(name), "fn_%d", i % 211); + snprintf(qn, sizeof(qn), "test.s%d", i); + cbm_node_t n = {.project = "test", + .label = "Function", + .name = name, + .qualified_name = qn, + .file_path = "ring.c", + .start_line = i % 37}; + ids[i] = cbm_store_upsert_node(s, &n); + } + for (int i = 0; i < STREAM_NODES; i++) { + cbm_edge_t ring = {.project = "test", + .source_id = ids[i], + .target_id = ids[(i + 1) % STREAM_NODES], + .type = "CALLS"}; + cbm_edge_t hub = {.project = "test", + .source_id = ids[i], + .target_id = ids[i % STREAM_HUBS], + .type = "CALLS"}; + cbm_store_insert_edge(s, &ring); + if (i >= STREAM_HUBS) { /* no hub self-loops */ + cbm_store_insert_edge(s, &hub); + } + } + static const char *const queries[] = { + "MATCH (a)-[:CALLS]->(b) RETURN count(*)", + "MATCH (a)-[:CALLS]->(b) RETURN a.qualified_name, b.qualified_name", + "MATCH (a:Function)-[:CALLS]->(b) RETURN b.qualified_name, count(a) AS c ORDER BY c DESC", + "MATCH (a)-[:CALLS]->(b) WITH b.name AS n, count(DISTINCT a.name) AS c RETURN n, c", + "MATCH (a)-[:CALLS]->(b) RETURN DISTINCT b.name LIMIT 300", + }; + for (size_t i = 0; i < sizeof(queries) / sizeof(queries[0]); i++) { + cbm_cypher_result_t ref; + ASSERT_TRUE(stream_matches_single_pass(s, queries[i], 0, &ref)); + ASSERT_GT(ref.row_count, 0); + cbm_cypher_result_free(&ref); + } + cbm_cypher_result_t ref; + ASSERT_TRUE(stream_matches_single_pass(s, queries[0], 0, &ref)); + ASSERT_STR_EQ(ref.rows[0][0], "5195"); /* 2600 ring + 2595 hub edges */ + cbm_cypher_result_free(&ref); + cbm_store_close(s); + PASS(); +} + /* ══════════════════════════════════════════════════════════════════ */ SUITE(cypher) { @@ -5892,4 +6405,13 @@ SUITE(cypher) { RUN_TEST(cypher_chain_with_unbound_alias_keeps_seed); RUN_TEST(cypher_many_union_branches); RUN_TEST(cypher_fifty_union_branches); + RUN_TEST(cypher_order_by_is_the_stable_reference_order); + RUN_TEST(cypher_distinct_keeps_first_occurrences); + RUN_TEST(cypher_grouping_keeps_first_seen_groups); + /* Streamed execution: chunked start set == single pass */ + RUN_TEST(cypher_stream_chunks_match_single_pass); + RUN_TEST(cypher_stream_optional_across_chunks); + RUN_TEST(cypher_stream_limit_and_max_rows_across_chunks); + RUN_TEST(cypher_stream_default_chunk_boundary); + RUN_TEST(cypher_stream_scan_failure_fails_query); } diff --git a/tests/test_daemon_application.c b/tests/test_daemon_application.c index ec35b4c4dc..9b39490072 100644 --- a/tests/test_daemon_application.c +++ b/tests/test_daemon_application.c @@ -21,6 +21,7 @@ #include "operations/json_args.h" #include "operations/result_wire.h" #include "pipeline/pipeline.h" +#include "store/graph_mem.h" #include "store/store.h" #include "ui/config.h" #include "watcher/watcher.h" @@ -5408,6 +5409,32 @@ TEST(daemon_application_worker_slice_is_aggregate_over_active_jobs) { PASS(); } +/* The daemon's resident adjacency layers are its own memory: a worker's slice + * is the aggregate minus them, and never "no cap" (0) when they exceed it. The + * held bytes come through the graph_mem seam, so no layer has to be built. */ +TEST(daemon_application_worker_slice_leaves_room_for_graph_cache) { + const size_t mib = (size_t)1024U * 1024U; + const size_t aggregate = 1000U * mib; + cbm_daemon_application_config_t config = {.aggregate_memory_budget_bytes = aggregate}; + cbm_graph_mem_drop_idle(); + cbm_daemon_application_t *application = cbm_daemon_application_new(&config); + size_t without_layers = cbm_daemon_application_worker_memory_budget_bytes(application); + cbm_graph_mem_set_cached_bytes_for_testing(200U * mib); + size_t with_layers = cbm_daemon_application_worker_memory_budget_bytes(application); + cbm_graph_mem_set_cached_bytes_for_testing(aggregate + 5U); + size_t layers_over_aggregate = cbm_daemon_application_worker_memory_budget_bytes(application); + cbm_graph_mem_set_cached_bytes_for_testing(0); + bool stopped = application && cbm_daemon_application_shutdown(application, APP_TEST_TIMEOUT_MS); + cbm_daemon_application_free(application); + + ASSERT_NOT_NULL(application); + ASSERT_TRUE(stopped); + ASSERT_EQ(without_layers, aggregate); + ASSERT_EQ(with_layers, aggregate - 200U * mib); + ASSERT_EQ(layers_over_aggregate, 1); + PASS(); +} + /* Decision A (#1997 #832): the over-budget verdict travels as a CLEAN worker * exit carrying an error response. The daemon passes that response through * verbatim on the first attempt — no crash/hang recovery loop, no quarantine @@ -6410,6 +6437,7 @@ SUITE(daemon_application) { RUN_TEST(daemon_application_queues_explicit_index_behind_physical_job_limit); RUN_TEST(daemon_application_default_limit_admits_four_and_rejects_fifth); RUN_TEST(daemon_application_worker_slice_is_aggregate_over_active_jobs); + RUN_TEST(daemon_application_worker_slice_leaves_room_for_graph_cache); RUN_TEST(daemon_application_over_budget_response_passes_through_without_recovery); RUN_TEST(daemon_application_free_reports_retained_live_ownership); RUN_TEST(daemon_application_tool_error_is_failed_job_even_after_clean_worker_exit); diff --git a/tests/test_git_context.c b/tests/test_git_context.c index 6dbb17f168..a2ad34cf34 100644 --- a/tests/test_git_context.c +++ b/tests/test_git_context.c @@ -450,9 +450,67 @@ TEST(githistory_ignores_inherited_repo_env) { #endif /* _WIN32 */ } +/* ── Branch-only resolve (list_projects) ─────────────────────────── + * The listing prints only the branch; cbm_git_branch_resolve gets it from the + * two git calls cbm_git_context_resolve uses for it, so both must agree. */ + +#ifndef _WIN32 +/* The branch resolver against the full context at `path`; NULL `want` = not git. */ +static bool branch_resolve_agrees(const char *path, const char *want) { + cbm_git_context_t ctx = {0}; + (void)cbm_git_context_resolve(path, &ctx); + bool is_git = false; + char branch[CBM_GIT_BRANCH_MAX]; + (void)cbm_git_branch_resolve(path, &is_git, branch, sizeof(branch)); + bool agrees = is_git == ctx.is_git && is_git == (want != NULL) && + (want ? (ctx.branch && strcmp(branch, want) == 0 && strcmp(ctx.branch, want) == 0) + : branch[0] == '\0'); + cbm_git_context_free(&ctx); + return agrees; +} +#endif /* _WIN32 */ + +TEST(git_branch_resolve_matches_full_context) { +#ifdef _WIN32 + SKIP_PLATFORM("git-based branch test not supported on Windows CI"); +#else + char plain[512]; + char *tmp = th_mktempdir("cbm_gitbranch_plain"); + if (!tmp) + FAIL("th_mktempdir returned NULL"); + snprintf(plain, sizeof(plain), "%s", tmp); + tmp = th_mktempdir("cbm_gitbranch"); + if (!tmp) + FAIL("th_mktempdir returned NULL"); + if (make_git_repo(tmp) != 0) { + th_rmtree(tmp); + th_rmtree(plain); + SKIP_PLATFORM("git not available to init a repo"); + } + char sub[600]; + snprintf(sub, sizeof(sub), "%s/sub", tmp); + ASSERT_EQ(th_mkdir_p(sub), 0); + char missing[600]; + snprintf(missing, sizeof(missing), "%s/missing", plain); + + ASSERT_TRUE(branch_resolve_agrees(plain, NULL)); + ASSERT_TRUE(branch_resolve_agrees(missing, NULL)); + ASSERT_EQ(git_run(tmp, "checkout -q -b feat/x"), 0); + ASSERT_TRUE(branch_resolve_agrees(tmp, "feat/x")); + ASSERT_TRUE(branch_resolve_agrees(sub, "feat/x")); + ASSERT_EQ(git_run(tmp, "checkout -q --detach"), 0); + ASSERT_TRUE(branch_resolve_agrees(tmp, "DETACHED")); + + th_rmtree(tmp); + th_rmtree(plain); + PASS(); +#endif /* _WIN32 */ +} + /* ── Suite ──────────────────────────────────────────────────────── */ SUITE(git_context) { + RUN_TEST(git_branch_resolve_matches_full_context); RUN_TEST(root_exists_non_ascii_directory); RUN_TEST(canonical_root_repo_root); RUN_TEST(canonical_root_subdir); diff --git a/tests/test_grammar_probe_f.c b/tests/test_grammar_probe_f.c index 98b6ce4366..280a0e404a 100644 --- a/tests/test_grammar_probe_f.c +++ b/tests/test_grammar_probe_f.c @@ -1777,7 +1777,7 @@ static char *gpf_deploys(const GpfFile *files, int nfiles) { "SELECT coalesce(group_concat(x, char(10)), '') FROM (SELECT " "CASE WHEN s.label IN ('Module','File') THEN '@' || s.file_path ELSE s.name END " "|| ' -> ' || CASE WHEN t.label='Chart' THEN 'chart:' || t.name || " - "CASE WHEN instr(t.qualified_name,'__helm_dep__') THEN '(ext)' ELSE '' END " + "CASE WHEN json_extract(t.properties,'$.external') THEN '(ext)' ELSE '' END " "ELSE t.label || ':' || t.file_path END " "|| ' (' || json_extract(e.properties,'$.via') || ')' AS x " "FROM edges e JOIN nodes s ON s.id=e.source_id JOIN nodes t ON t.id=e.target_id " @@ -2424,6 +2424,1309 @@ TEST(probe_container_env_sets_scoped_envvars) { PASS(); } +/* Every env join edge (CONFIGURES {strategy:env_read}) as + * " / -> ", then the containers + * the image references name: "#image_containers=...". */ +static char *gpf_env_joins(const GpfFile *files, int nfiles) { + GpfProj lp; + cbm_store_t *store = gpf_index_files(&lp, files, nfiles); + char *out = NULL; + if (store) { + sqlite3_stmt *st = NULL; + const char *sql = + "SELECT coalesce((SELECT group_concat(x, char(10)) FROM (SELECT DISTINCT s.file_path " + "|| ' ' || substr(s.qualified_name, instr(s.qualified_name, '#env/') + 5) || ' -> ' " + "|| t.file_path AS x FROM edges e JOIN nodes s ON s.id=e.source_id JOIN nodes t ON " + "t.id=e.target_id WHERE e.project=?1 AND e.type='CONFIGURES' AND " + "json_extract(e.properties,'$.strategy')='env_read' ORDER BY x)), '') || char(10) || " + "'#image_containers=' || coalesce((SELECT group_concat(c, ',') FROM (SELECT " + "json_extract(e.properties,'$.container') AS c FROM edges e JOIN nodes s ON " + "s.id=e.source_id WHERE e.project=?1 AND e.type='USAGE' AND " + "s.file_path='k8s/shop.yaml' " + "AND json_extract(e.properties,'$.ref')='image' ORDER BY c)), '')"; + if (sqlite3_prepare_v2(cbm_store_get_db(store), sql, -1, &st, NULL) == SQLITE_OK) { + sqlite3_bind_text(st, 1, lp.project, -1, SQLITE_STATIC); + if (sqlite3_step(st) == SQLITE_ROW) { + out = strdup((const char *)sqlite3_column_text(st, 0)); + } + } + sqlite3_finalize(st); + } + gpf_cleanup(&lp, store); + return out; +} + +/* Every CONNECTS_TO edge as " -> ()". */ +static char *gpf_connects(const GpfFile *files, int nfiles) { + GpfProj lp; + cbm_store_t *store = gpf_index_files(&lp, files, nfiles); + char *out = NULL; + if (store) { + sqlite3_stmt *st = NULL; + const char *sql = + "SELECT coalesce((SELECT group_concat(x, char(10)) FROM (SELECT s.file_path || ' ' || " + "s.name || ' -> ' || t.name || ' (' || json_extract(e.properties,'$.key') || ')' AS x " + "FROM edges e JOIN nodes s ON s.id=e.source_id JOIN nodes t ON t.id=e.target_id WHERE " + "e.project=?1 AND e.type='CONNECTS_TO' ORDER BY x)), '')"; + if (sqlite3_prepare_v2(cbm_store_get_db(store), sql, -1, &st, NULL) == SQLITE_OK) { + sqlite3_bind_text(st, 1, lp.project, -1, SQLITE_STATIC); + if (sqlite3_step(st) == SQLITE_ROW) { + out = strdup((const char *)sqlite3_column_text(st, 0)); + } + } + sqlite3_finalize(st); + } + gpf_cleanup(&lp, store); + return out; +} + +/* A service address in a container's environment CONNECTS_TO the Service it + * names: `svc:port`, a URL's host, `..svc...`, a bare name on an + * address key -- a K8s Service of the bundle in the named (else the + * object's own) namespace, a compose service of the same file, its .env + * interpolated (a name without a value takes the .env's, which may itself + * name others). Never: another domain's host, a templated value, localhost, + * a bare name on any other key, a namespace without that Service, a service + * addressing itself -- and never a value in the graph. */ +TEST(probe_env_address_connects_to_its_service) { + static const GpfFile files[] = { + {"k8s/app.yaml", + "apiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: web\n namespace: shop\n" + "spec:\n template:\n spec:\n containers:\n - name: web\n" + " image: acme/web\n env:\n" + " - name: CATALOG_ADDR\n value: \"catalog:3550\"\n" + " - name: AUTH_URL\n" + " value: \"http://auth.auth.svc.cluster.local:8080/login\"\n" + " - name: CACHE_HOST\n value: cache\n" + " - name: DB_HOST\n value: db.example.com\n" + " - name: TEMPLATED_ADDR\n value: \"{{ .Values.addr }}\"\n" + " - name: LOCAL_ADDR\n value: \"localhost:9000\"\n" + " - name: GREETING\n value: catalog\n" + " - name: PRICES_ADDR\n value: \"catalog.other:3550\"\n" + " - name: MAIL_URL\n value: " + "\"smtp://mail.shop.example.com:25\"\n"}, + {"k8s/svc.yaml", "apiVersion: v1\nkind: Service\nmetadata:\n name: catalog\n" + " namespace: shop\n---\napiVersion: v1\nkind: Service\nmetadata:\n" + " name: cache\n namespace: shop\n---\napiVersion: v1\nkind: Service\n" + "metadata:\n name: auth\n namespace: auth\n---\napiVersion: v1\n" + "kind: Service\nmetadata:\n name: db\n namespace: shop\n---\n" + "apiVersion: v1\nkind: Service\nmetadata:\n name: mail\n" + " namespace: shop\n"}, + {"compose.yaml", + "services:\n web:\n image: acme/web\n environment:\n DB_HOST: db\n" + " CACHE_URL: redis://cache:6379\n API_KEY: s3cr3t-value\n" + " WEB_ADDR: web\n worker:\n image: acme/worker\n environment:\n" + " - QUEUE_ADDR\n - OTHER_URL=http://${QUEUE_HOST}:80\n" + " db:\n image: postgres\n cache:\n image: redis\n queue:\n" + " image: rabbitmq\n"}, + {".env", "QUEUE_PORT=5672\nQUEUE_HOST=queue\nQUEUE_ADDR=queue:${QUEUE_PORT}\n"}}; + int n = (int)(sizeof(files) / sizeof(files[0])); + char *got = gpf_connects(files, n); + static const char want[] = "compose.yaml Service/web -> Service/cache (CACHE_URL)\n" + "compose.yaml Service/web -> Service/db (DB_HOST)\n" + "compose.yaml Service/worker -> Service/queue (QUEUE_ADDR)\n" + "k8s/app.yaml Deployment/web -> Service/auth (AUTH_URL)\n" + "k8s/app.yaml Deployment/web -> Service/cache (CACHE_HOST)\n" + "k8s/app.yaml Deployment/web -> Service/catalog (CATALOG_ADDR)"; + if (!got || strcmp(got, want) != 0) { + fprintf(stderr, "connects got:\n%s\nwant:\n%s\n", got ? got : "(null)", want); + free(got); + FAIL("CONNECTS_TO edge set"); + } + free(got); + static const char *const values[] = {"catalog:3550", "s3cr3t-value", "redis://cache:6379", + "queue:5672", "/login"}; + for (size_t i = 0; i < sizeof(values) / sizeof(values[0]); i++) { + int hits = gpf_graph_mentions(files, n, values[i]); + if (hits != 0) { + fprintf(stderr, "value %s: %d graph mentions\n", values[i], hits); + FAIL("an env value reached the graph"); + } + } + PASS(); +} + +/* Every kustomize patch edge as " -> + * @", then "patch imports " (IMPORTS of patch files), "maps + * into patches " (INFRA_MAPS into a patch file's documents) and "version + * " (base/deploy.yaml's Resource), "odd version " (other/odd.yaml's: + * none without one, missing without the Resource), "labels " + * (base/deploy.yaml's). */ +static char *gpf_patches(const GpfFile *files, int nfiles) { + GpfProj lp; + cbm_store_t *store = gpf_index_files(&lp, files, nfiles); + char *out = NULL; + if (store) { + sqlite3_stmt *st = NULL; + const char *sql = + "SELECT coalesce((SELECT group_concat(x, char(10)) FROM (SELECT s.file_path || ' ' || " + "s.label || ' -> ' || t.name || ' @' || t.file_path AS x FROM edges e JOIN nodes s ON " + "s.id=e.source_id JOIN nodes t ON t.id=e.target_id WHERE e.project=?1 AND " + "e.type='OVERRIDE' AND json_extract(e.properties,'$.via')='patch' ORDER BY x)), '') " + "|| char(10) || 'patch imports ' || (SELECT COUNT(*) FROM edges e WHERE e.project=?1 " + "AND e.type='IMPORTS' AND json_extract(e.properties,'$.local_name')='patches') || " + "char(10) || 'maps into patches ' || (SELECT COUNT(*) FROM edges e JOIN nodes t ON " + "t.id=e.target_id WHERE e.project=?1 AND e.type='INFRA_MAPS' AND t.file_path LIKE " + "'overlays/%') || char(10) || 'version ' || coalesce((SELECT " + "json_extract(properties,'$.version') FROM nodes WHERE project=?1 AND " + "label='Resource' AND file_path='base/deploy.yaml'), '-') || char(10) || " + "'odd version ' || coalesce((SELECT coalesce(json_extract(properties,'$.version'), " + "'none') FROM nodes WHERE project=?1 AND label='Resource' AND " + "file_path='other/odd.yaml'), 'missing') || char(10) || 'labels ' || " + "coalesce((SELECT json_extract(properties,'$.labels') FROM nodes WHERE project=?1 " + "AND label='Resource' AND file_path='base/deploy.yaml'), '-')"; + if (sqlite3_prepare_v2(cbm_store_get_db(store), sql, -1, &st, NULL) == SQLITE_OK) { + sqlite3_bind_text(st, 1, lp.project, -1, SQLITE_STATIC); + if (sqlite3_step(st) == SQLITE_ROW) { + out = strdup((const char *)sqlite3_column_text(st, 0)); + } + } + sqlite3_finalize(st); + } + gpf_cleanup(&lp, store); + return out; +} + +/* A kustomization's patches OVERRIDE the objects of its build they change, + * child -> base: a patch file's documents the objects of the same kind and + * name; an inline patch (from the kustomization) the object its document + * names (a `${VAR}` name as written); a JSON6902 operation file (from the + * file) the objects its target selects -- group and version included; a + * component's the objects of the build including it; a kind alone every + * object of that kind. Never: an object outside the build, one the patch + * does not name (another name, group, version or namespace), a component + * nothing includes. A label selector meets the objects whose metadata.labels + * carry its terms (`k=v`, `k`, `k!=v`; a Resource keeps them sorted). A + * patch file is an import of + * the kustomization, never a member of its build (a Service selects no patch + * document). A Resource keeps the version of its apiVersion; a bare group + * (`apiVersion: example.io`) has none. */ +TEST(probe_kustomize_patches_override_their_base) { + static const GpfFile files[] = { + {"base/kustomization.yaml", "resources:\n- deploy.yaml\n- svc.yaml\n- cm.yaml\n"}, + {"base/cm.yaml", + "apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: ${APP}-config\n namespace: app\n"}, + {"base/deploy.yaml", + "apiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: web\n namespace: app\n" + " labels:\n tier: front\n app: web\n note: a=b\nspec:\n" + " template:\n metadata:\n labels:\n app: web\n"}, + {"base/svc.yaml", + "apiVersion: v1\nkind: Service\nmetadata:\n name: web\n namespace: app\n" + "spec:\n selector:\n app: web\n"}, + {"overlays/prod/kustomization.yaml", + "namespace: app\nresources:\n- ../../base\ncomponents:\n- ../../components/extra\n" + "patches:\n" + "- path: patch-deploy.yaml\n" + "- path: ns-patch.yaml\n" + "- path: all.yaml\n target:\n kind: Service\n" + "- patch: |-\n apiVersion: v1\n kind: Service\n metadata:\n name: web\n" + " spec:\n type: NodePort\n" + "- path: ops.yaml\n target:\n kind: Deployment\n name: web\n" + "- path: ops-v2.yaml\n target:\n group: apps\n version: v2\n kind: Deployment\n" + "- path: ops-batch.yaml\n target:\n group: batch\n kind: Deployment\n" + "- path: sel.yaml\n target:\n kind: Deployment\n labelSelector: tier=none\n" + "- path: lab.yaml\n target:\n kind: Deployment\n labelSelector: app=web,tier\n" + "- path: neq.yaml\n target:\n kind: Deployment\n labelSelector: app!=web\n" + "- path: sel-only.yaml\n target:\n labelSelector: tier=front\n" + "- path: ann.yaml\n target:\n kind: Deployment\n annotationSelector: x=y\n" + "- path: cm-patch.yaml\n" + "- patch: |-\n apiVersion: apps/v1\n kind: Deployment\n metadata:\n" + " name: missing\n"}, + {"overlays/prod/ops-batch.yaml", "- op: replace\n path: /spec/replicas\n value: 5\n"}, + {"overlays/prod/ns-patch.yaml", + "apiVersion: v1\nkind: Service\nmetadata:\n name: web\n namespace: elsewhere\n"}, + {"overlays/prod/all.yaml", "- op: add\n path: /metadata/labels/tier\n value: front\n"}, + {"overlays/prod/sel.yaml", "- op: replace\n path: /spec/replicas\n value: 6\n"}, + {"overlays/prod/lab.yaml", "- op: replace\n path: /spec/replicas\n value: 7\n"}, + {"overlays/prod/neq.yaml", "- op: replace\n path: /spec/replicas\n value: 8\n"}, + {"overlays/prod/sel-only.yaml", "- op: replace\n path: /spec/replicas\n value: 9\n"}, + {"overlays/prod/ann.yaml", "- op: replace\n path: /spec/replicas\n value: 1\n"}, + {"overlays/prod/cm-patch.yaml", + "apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: ${APP}-config\n namespace: app\n" + "data:\n k: v\n"}, + {"overlays/prod/patch-deploy.yaml", + "apiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: web\n namespace: app\nspec:\n" + " template:\n metadata:\n labels:\n app: web\n"}, + {"overlays/prod/ops.yaml", "- op: replace\n path: /spec/replicas\n value: 3\n"}, + {"overlays/prod/ops-v2.yaml", "- op: replace\n path: /spec/replicas\n value: 4\n"}, + {"components/extra/kustomization.yaml", + "apiVersion: kustomize.config.k8s.io/v1alpha1\nkind: Component\npatches:\n" + "- patch: |-\n apiVersion: apps/v1\n kind: Deployment\n metadata:\n" + " name: web\n namespace: app\n"}, + {"components/unused/kustomization.yaml", + "apiVersion: kustomize.config.k8s.io/v1alpha1\nkind: Component\npatches:\n" + "- patch: |-\n apiVersion: v1\n kind: Service\n metadata:\n name: web\n"}, + {"other/deploy.yaml", + "apiVersion: apps/v1\nkind: Deployment\nmetadata:\n name: web\n namespace: app\n"}, + {"other/odd.yaml", "apiVersion: example.io\nkind: Thing\nmetadata:\n name: odd\n"}}; + int n = (int)(sizeof(files) / sizeof(files[0])); + char *got = gpf_patches(files, n); + static const char want[] = + "components/extra/kustomization.yaml File -> Deployment/web @base/deploy.yaml\n" + "overlays/prod/all.yaml File -> Service/web @base/svc.yaml\n" + "overlays/prod/cm-patch.yaml Resource -> ConfigMap/${APP}-config @base/cm.yaml\n" + "overlays/prod/kustomization.yaml File -> Service/web @base/svc.yaml\n" + "overlays/prod/lab.yaml File -> Deployment/web @base/deploy.yaml\n" + "overlays/prod/ops.yaml File -> Deployment/web @base/deploy.yaml\n" + "overlays/prod/patch-deploy.yaml Resource -> Deployment/web @base/deploy.yaml\n" + "overlays/prod/sel-only.yaml File -> Deployment/web @base/deploy.yaml\n" + "patch imports 12\n" + "maps into patches 0\n" + "version v1\n" + "odd version none\n" + "labels app=web,tier=front"; + if (!got || strcmp(got, want) != 0) { + fprintf(stderr, "patches got:\n%s\nwant:\n%s\n", got ? got : "(null)", want); + free(got); + FAIL("kustomize patch edges"); + } + free(got); + PASS(); +} + +/* Every kustomize chart deploy as " -> / ", then "values imports ". */ +static char *gpf_kcharts(const GpfFile *files, int nfiles) { + GpfProj lp; + cbm_store_t *store = gpf_index_files(&lp, files, nfiles); + char *out = NULL; + if (store) { + sqlite3_stmt *st = NULL; + const char *sql = + "SELECT coalesce((SELECT group_concat(x, char(10)) FROM (SELECT s.file_path || ' ' || " + "s.label || ' -> ' || t.label || '/' || t.name || ' ' || json_remove(e.properties, " + "'$.line') AS x FROM edges e JOIN nodes s ON s.id=e.source_id JOIN nodes t ON " + "t.id=e.target_id WHERE e.project=?1 AND e.type='DEPLOYS' AND " + "json_extract(e.properties,'$.via')='helmcharts' ORDER BY x)), '') || char(10) || " + "'values imports ' || (SELECT COUNT(*) FROM edges e WHERE e.project=?1 AND " + "e.type='IMPORTS' AND json_extract(e.properties,'$.local_name')='valuesFile')"; + if (sqlite3_prepare_v2(cbm_store_get_db(store), sql, -1, &st, NULL) == SQLITE_OK) { + sqlite3_bind_text(st, 1, lp.project, -1, SQLITE_STATIC); + if (sqlite3_step(st) == SQLITE_ROW) { + out = strdup((const char *)sqlite3_column_text(st, 0)); + } + } + sqlite3_finalize(st); + } + gpf_cleanup(&lp, store); + return out; +} + +/* A chart a kustomization inflates (`helmCharts`) DEPLOYS from that + * kustomization -- its Kustomization Resource, else (no `kind:` line) its + * File -- to the shared external Chart of the name, keeping repository, + * version, release, namespace and values file; two kustomizations of one + * chart share it. A chart name with a path names nothing; a values file + * outside the repository is not kept; one inside is an import. */ +TEST(probe_kustomize_helm_charts_deploy_their_chart) { + static const GpfFile files[] = { + {"obs/kustomization.yaml", + "apiVersion: kustomize.config.k8s.io/v1beta1\nkind: Kustomization\nhelmCharts:\n" + "- name: loki\n repo: https://grafana.github.io/helm-charts\n version: 6.46.0\n" + " releaseName: logs\n namespace: obs\n valuesFile: values.yaml\n" + "- name: org/thing\n repo: https://example.com/charts\n"}, + {"obs/values.yaml", "loki:\n auth_enabled: false\n"}, + {"plain/kustomization.yaml", + "helmCharts:\n- name: loki\n repo: oci://registry.example.com/charts\n" + " valuesFile: ../../../../outside.yaml\n"}}; + int n = (int)(sizeof(files) / sizeof(files[0])); + char *got = gpf_kcharts(files, n); + static const char want[] = + "obs/kustomization.yaml Resource -> Chart/loki {\"via\":\"helmcharts\",\"external\":true," + "\"chart\":\"loki\",\"url\":\"https://grafana.github.io/" + "helm-charts\",\"version\":\"6.46.0\"," + "\"release\":\"logs\",\"namespace\":\"obs\",\"values\":\"obs/values.yaml\"}\n" + "plain/kustomization.yaml File -> Chart/loki {\"via\":\"helmcharts\",\"external\":true," + "\"chart\":\"loki\",\"url\":\"oci://registry.example.com/charts\"}\n" + "values imports 1"; + if (!got || strcmp(got, want) != 0) { + fprintf(stderr, "kustomize charts got:\n%s\nwant:\n%s\n", got ? got : "(null)", want); + free(got); + FAIL("kustomize chart deploys"); + } + free(got); + PASS(); +} + +/* Every Terraform reference edge as + * " -> :", then a line + * "other tf edges ": USAGE / CALLS edges leaving a .tf file that the + * module resolver did not make. */ +static char *gpf_tf_refs(const GpfFile *files, int nfiles) { + GpfProj lp; + cbm_store_t *store = gpf_index_files(&lp, files, nfiles); + char *out = NULL; + if (store) { + sqlite3_stmt *st = NULL; + const char *sql = + "SELECT coalesce((SELECT group_concat(x, char(10)) FROM (SELECT s.file_path || ' ' || " + "s.name || ' -> ' || t.file_path || ' ' || t.name || ' ' || e.type || ' ' || " + "json_extract(e.properties,'$.ref') || ':' || json_extract(e.properties,'$.line') AS x " + "FROM edges e JOIN nodes s ON s.id=e.source_id JOIN nodes t ON t.id=e.target_id " + "WHERE e.project=?1 AND e.type IN ('USAGE','DEPENDS_ON') AND " + "json_extract(e.properties,'$.via')='terraform' ORDER BY x)), '') || char(10) || " + "'other tf edges ' || (SELECT COUNT(*) FROM edges e JOIN nodes s ON s.id=e.source_id " + "WHERE e.project=?1 AND e.type IN ('USAGE','CALLS') AND s.file_path LIKE '%.tf' AND " + "coalesce(json_extract(e.properties,'$.via'),'') != 'terraform')"; + if (sqlite3_prepare_v2(cbm_store_get_db(store), sql, -1, &st, NULL) == SQLITE_OK) { + sqlite3_bind_text(st, 1, lp.project, -1, SQLITE_STATIC); + if (sqlite3_step(st) == SQLITE_ROW) { + out = strdup((const char *)sqlite3_column_text(st, 0)); + } + } + sqlite3_finalize(st); + } + gpf_cleanup(&lp, store); + return out; +} + +/* A Terraform reference (var.X, local.X, module.X, data.T.N, T.N) USAGEs, + * from the top-level block holding it, the block declaring it in the same + * directory -- the module, all its .tf files -- with the reference kind and + * line; an entry of the block's depends_on list is a DEPENDS_ON (beside a + * USAGE where the block also uses the object). A bare label (`variable prefix {}`) names its block + * like a quoted one. A name mentioned in string text, a `/ *` glob inside an interpolation, a + * reference to the block holding it, and a directory declaring nothing of the name make no edge; + * the generic name matcher makes none at all for a .tf file. */ +TEST(probe_terraform_refs_resolve_in_their_module) { + static const GpfFile files[] = { + {"infra/variables.tf", "variable \"region\" {}\nvariable prefix {\n default = \"x\"\n}\n" + "variable \"unused\" {\n description = \"see var.region\"\n}\n"}, + {"infra/main.tf", + "locals {\n name = \"${var.prefix}-svc\"\n full = \"${local.name}-${var.region}\"\n}\n" + "resource \"aws_s3_bucket\" \"logs\" {\n bucket = local.full\n}\n" + "resource \"aws_s3_bucket_policy\" \"logs\" {\n bucket = aws_s3_bucket.logs.id\n" + " policy = data.aws_iam_policy_document.p.json\n" + " depends_on = [aws_s3_bucket.logs]\n}\n" + "data \"aws_iam_policy_document\" \"p\" {\n statement {\n" + " resources = [\"${aws_s3_bucket.logs.arn}/*\"]\n }\n}\n" + "module \"net\" {\n source = \"./net\"\n region = var.region\n" + " depends_on = [\n aws_s3_bucket_policy.logs,\n ]\n}\n" + "output \"net\" {\n value = module.net.id\n}\n"}, + {"infra/net/main.tf", + "variable \"region\" {}\nresource \"aws_vpc\" \"v\" {\n tags = { r = var.region }\n}\n"}, + {"other/x.tf", "resource \"null_resource\" \"n\" {\n triggers = { r = var.region }\n}\n"}}; + int n = (int)(sizeof(files) / sizeof(files[0])); + char *got = gpf_tf_refs(files, n); + static const char want[] = + "infra/main.tf data.aws_iam_policy_document.p -> infra/main.tf resource.aws_s3_bucket.logs " + "USAGE resource:15\n" + "infra/main.tf locals -> infra/variables.tf variable.prefix USAGE var:2\n" + "infra/main.tf locals -> infra/variables.tf variable.region USAGE var:3\n" + "infra/main.tf module.net -> infra/main.tf resource.aws_s3_bucket_policy.logs " + "DEPENDS_ON resource:22\n" + "infra/main.tf module.net -> infra/variables.tf variable.region USAGE var:20\n" + "infra/main.tf output.net -> infra/main.tf module.net USAGE module:26\n" + "infra/main.tf resource.aws_s3_bucket.logs -> infra/main.tf locals USAGE local:6\n" + "infra/main.tf resource.aws_s3_bucket_policy.logs -> infra/main.tf " + "data.aws_iam_policy_document.p USAGE data:10\n" + "infra/main.tf resource.aws_s3_bucket_policy.logs -> infra/main.tf " + "resource.aws_s3_bucket.logs DEPENDS_ON resource:11\n" + "infra/main.tf resource.aws_s3_bucket_policy.logs -> infra/main.tf " + "resource.aws_s3_bucket.logs USAGE resource:9\n" + "infra/net/main.tf resource.aws_vpc.v -> infra/net/main.tf variable.region USAGE var:3\n" + "other tf edges 0"; + if (!got || strcmp(got, want) != 0) { + fprintf(stderr, "terraform refs got:\n%s\nwant:\n%s\n", got ? got : "(null)", want); + free(got); + FAIL("terraform refs"); + } + free(got); + PASS(); +} + +/* A Terraform / OpenTofu module file (.tf, .tofu) extracts its blocks but no + * generic calls or usages -- its references are the module resolver's -- while + * any other HCL file (Packer, Nomad, Terragrunt .hcl) keeps them. */ +TEST(probe_terraform_module_files_extract_no_generic_refs) { + static const char src[] = "locals {\n watched = [1]\n result = length(watched)\n}\n"; + static const char *const names[] = {"main.tf", "main.tofu", "main.hcl"}; + int calls[3]; + int usages[3]; + int defs[3]; + for (int i = 0; i < 3; i++) { + CBMFileResult *r = + cbm_extract_file(src, (int)strlen(src), CBM_LANG_HCL, "probe", names[i], 0, NULL, NULL); + ASSERT_NOT_NULL(r); + calls[i] = r->calls.count; + usages[i] = r->usages.count; + defs[i] = r->defs.count; + cbm_free_result(r); + } + for (int i = 0; i < 2; i++) { + ASSERT_EQ(calls[i], 0); + ASSERT_EQ(usages[i], 0); + ASSERT_GT(defs[i], 0); + } + ASSERT_GT(calls[2], 0); + ASSERT_GT(usages[2], 0); + PASS(); +} + +/* Every Terraform module IMPORTS and output-read edge as + * " :", the target a + * Folder's path, the Project, or " ". */ +static char *gpf_tf_modules(const GpfFile *files, int nfiles) { + GpfProj lp; + cbm_store_t *store = gpf_index_files(&lp, files, nfiles); + char *out = NULL; + if (store) { + sqlite3_stmt *st = NULL; + const char *sql = + "SELECT coalesce((SELECT group_concat(x, char(10)) FROM (SELECT s.file_path || ' ' || " + "s.name || ' ' || e.type || ' ' || CASE t.label WHEN 'Folder' THEN 'Folder:' || " + "t.file_path WHEN 'Project' THEN 'Project' ELSE t.file_path || ' ' || t.name END || " + "' ' || coalesce(json_extract(e.properties,'$.local_name'), " + "json_extract(e.properties,'$.ref')) || ':' || json_extract(e.properties,'$.line') AS " + "x " + "FROM edges e JOIN nodes s ON s.id=e.source_id JOIN nodes t ON t.id=e.target_id " + "WHERE e.project=?1 AND json_extract(e.properties,'$.via')='terraform' AND " + "(e.type='IMPORTS' OR json_extract(e.properties,'$.ref')='output') ORDER BY x)), '')"; + if (sqlite3_prepare_v2(cbm_store_get_db(store), sql, -1, &st, NULL) == SQLITE_OK) { + sqlite3_bind_text(st, 1, lp.project, -1, SQLITE_STATIC); + if (sqlite3_step(st) == SQLITE_ROW) { + out = strdup((const char *)sqlite3_column_text(st, 0)); + } + } + sqlite3_finalize(st); + } + gpf_cleanup(&lp, store); + return out; +} + +/* A module call with a local source IMPORTS {local_name} the Folder of that + * directory (the root directory: the Project), and module.X.Y -- also through + * an index or a splat -- USAGEs {ref: output} the `output "Y"` there. A + * registry source (also `modules/net`: a local path starts with ./ or ../), a + * directory that does not exist, holds no .tf file or lies past the + * repository root, and an output the module does not declare make none. */ +TEST(probe_terraform_module_calls_import_their_source) { + static const GpfFile files[] = { + {"main.tf", "module \"net\" {\n source = \"./modules/net\"\n}\n" + "module \"cnt\" {\n source = \"./modules/net\"\n count = 2\n}\n" + "module \"reg\" {\n source = \"terraform-aws-modules/vpc/aws\"\n" + " version = \"5.0.0\"\n}\n" + "module \"gone\" {\n source = \"./modules/missing\"\n}\n" + "module \"out\" {\n source = \"../outside\"\n}\n" + "output \"a\" {\n value = module.net.id\n}\n" + "output \"b\" {\n value = module.cnt[0].id\n}\n" + "output \"c\" {\n value = module.cnt[*].name\n}\n" + "output \"d\" {\n value = module.net.nope\n}\n" + "output \"e\" {\n value = module.reg.vpc_id\n}\n" + "module \"bare\" {\n source = \"modules/net\"\n}\n" + "module \"docs\" {\n source = \"./docs\"\n}\n"}, + {"docs/README.md", "# docs\n"}, + {"modules/net/main.tf", + "output \"id\" {\n value = \"x\"\n}\noutput \"name\" {\n value = \"n\"\n}\n"}, + {"examples/basic/main.tf", "module \"root\" {\n source = \"../..\"\n}\noutput \"r\" {\n " + "value = module.root.a\n}\n"}}; + int n = (int)(sizeof(files) / sizeof(files[0])); + char *got = gpf_tf_modules(files, n); + static const char want[] = "examples/basic/main.tf module.root IMPORTS Project root:1\n" + "examples/basic/main.tf output.r USAGE main.tf output.a output:5\n" + "main.tf module.cnt IMPORTS Folder:modules/net cnt:4\n" + "main.tf module.net IMPORTS Folder:modules/net net:1\n" + "main.tf output.a USAGE modules/net/main.tf output.id output:19\n" + "main.tf output.b USAGE modules/net/main.tf output.id output:22\n" + "main.tf output.c USAGE modules/net/main.tf output.name output:25"; + if (!got || strcmp(got, want) != 0) { + fprintf(stderr, "terraform modules got:\n%s\nwant:\n%s\n", got ? got : "(null)", want); + free(got); + FAIL("terraform modules"); + } + free(got); + PASS(); +} + +/* Every tfvars OVERRIDE edge as " -> ", + * " inferred" appended when the edge says so. */ +static char *gpf_tf_tfvars(const GpfFile *files, int nfiles) { + GpfProj lp; + cbm_store_t *store = gpf_index_files(&lp, files, nfiles); + char *out = NULL; + if (store) { + sqlite3_stmt *st = NULL; + const char *sql = + "SELECT coalesce((SELECT group_concat(x, char(10)) FROM (SELECT s.file_path || ' ' || " + "s.name || ' -> ' || t.file_path || ' ' || t.name || ' ' || " + "json_extract(e.properties,'$.line') || CASE WHEN " + "json_extract(e.properties,'$.inferred') THEN ' inferred' ELSE '' END AS x FROM edges " + "e " + "JOIN nodes s ON s.id=e.source_id JOIN nodes t ON t.id=e.target_id WHERE e.project=?1 " + "AND e.type='OVERRIDE' AND json_extract(e.properties,'$.via')='tfvars' ORDER BY x)), " + "'')"; + if (sqlite3_prepare_v2(cbm_store_get_db(store), sql, -1, &st, NULL) == SQLITE_OK) { + sqlite3_bind_text(st, 1, lp.project, -1, SQLITE_STATIC); + if (sqlite3_step(st) == SQLITE_ROW) { + out = strdup((const char *)sqlite3_column_text(st, 0)); + } + } + sqlite3_finalize(st); + } + gpf_cleanup(&lp, store); + return out; +} + +/* A .tfvars file's assignment to a variable the module of its own directory + * declares OVERRIDEs {via: tfvars, line} that `variable` block, from the + * file's Variable of the name. A file whose directory declares none of its + * names -- no Terraform there, or a module that does not declare them -- + * sets the nearest ancestor module's that does (applied by a -var-file the + * repository does not state: `inferred`). An undeclared name, and a file + * with no declaring module above it, make none. */ +TEST(probe_terraform_tfvars_override_their_variables) { + static const GpfFile files[] = { + {"stack/variables.tf", "variable \"region\" {}\nvariable \"size\" {\n default = 1\n}\n"}, + {"stack/terraform.tfvars", "region = \"eu\"\nsize = 3\nunknown = \"x\"\n"}, + {"stack/prod/prod.tfvars", "region = \"us\"\n"}, + {"stack/sub/main.tf", "resource \"null_resource\" \"x\" {}\n"}, + {"stack/sub/sub.tfvars", "size = 9\n"}, + {"other.tfvars", "region = \"ap\"\n"}}; + int n = (int)(sizeof(files) / sizeof(files[0])); + char *got = gpf_tf_tfvars(files, n); + static const char want[] = + "stack/prod/prod.tfvars region -> stack/variables.tf variable.region 1 inferred\n" + "stack/sub/sub.tfvars size -> stack/variables.tf variable.size 1 inferred\n" + "stack/terraform.tfvars region -> stack/variables.tf variable.region 1\n" + "stack/terraform.tfvars size -> stack/variables.tf variable.size 2"; + if (!got || strcmp(got, want) != 0) { + fprintf(stderr, "terraform tfvars got:\n%s\nwant:\n%s\n", got ? got : "(null)", want); + free(got); + FAIL("terraform tfvars"); + } + free(got); + PASS(); +} + +/* Every Terragrunt edge as " | ->:", a Folder target as "Folder:". */ +static char *gpf_tg_edges(const GpfFile *files, int nfiles) { + GpfProj lp; + cbm_store_t *store = gpf_index_files(&lp, files, nfiles); + char *out = NULL; + if (store) { + sqlite3_stmt *st = NULL; + const char *sql = + "SELECT coalesce((SELECT group_concat(x, char(10)) FROM (SELECT s.file_path || ' ' || " + "s.name || ' ' || e.type || ' ' || CASE t.label WHEN 'Folder' THEN 'Folder:' || " + "t.file_path ELSE t.file_path END || ' ' || coalesce(json_extract(e.properties," + "'$.local_name'), 'read=' || json_extract(e.properties,'$.read'), '-') || ':' || " + "json_extract(e.properties,'$.line') AS x FROM edges e JOIN nodes s ON " + "s.id=e.source_id JOIN nodes t ON t.id=e.target_id WHERE e.project=?1 AND " + "json_extract(e.properties,'$.via')='terragrunt' ORDER BY x)), '')"; + if (sqlite3_prepare_v2(cbm_store_get_db(store), sql, -1, &st, NULL) == SQLITE_OK) { + sqlite3_bind_text(st, 1, lp.project, -1, SQLITE_STATIC); + if (sqlite3_step(st) == SQLITE_ROW) { + out = strdup((const char *)sqlite3_column_text(st, 0)); + } + } + sqlite3_finalize(st); + } + gpf_cleanup(&lp, store); + return out; +} + +/* A Terragrunt unit (terragrunt.hcl), evaluated in its own context: an + * include IMPORTS {local_name} the file its path names; a + * read_terragrunt_config IMPORTS {read} the file it names -- from the block + * holding the call, or, for a call in an included file (evaluated for the + * including unit: find_in_parent_folders searches from the UNIT), from the + * include block bringing that file in; a dependency DEPENDS_ON the other + * unit; a local terraform source IMPORTS its Folder. A path built from an + * unknown function (get_env) makes none. */ +TEST(probe_terragrunt_units_compose_their_files) { + static const GpfFile files[] = { + {"root.hcl", + "locals {\n acct = read_terragrunt_config(find_in_parent_folders(\"account.hcl\"))\n}\n"}, + {"live/account.hcl", "locals {\n id = \"1\"\n}\n"}, + {"live/app/terragrunt.hcl", + "include \"root\" {\n path = find_in_parent_folders(\"root.hcl\")\n}\n" + "include \"common\" {\n" + " path = \"${dirname(find_in_parent_folders(\"root.hcl\"))}/_common/app.hcl\"\n}\n" + "dependency \"db\" {\n config_path = \"../db\"\n}\n" + "terraform {\n source = \"../../modules/app\"\n}\n" + "locals {\n env = read_terragrunt_config(\"env.hcl\")\n" + " bad = read_terragrunt_config(get_env(\"X\"))\n}\n"}, + {"live/app/env.hcl", "locals {\n name = \"app\"\n}\n"}, + {"_common/app.hcl", + "locals {\n acct = read_terragrunt_config(find_in_parent_folders(\"account.hcl\"))\n}\n"}, + {"live/db/terragrunt.hcl", + "include \"root\" {\n path = find_in_parent_folders(\"root.hcl\")\n}\n"}, + {"modules/app/main.tf", "variable \"x\" {}\n"}}; + int n = (int)(sizeof(files) / sizeof(files[0])); + char *got = gpf_tg_edges(files, n); + static const char want[] = + "live/app/terragrunt.hcl dependency.db DEPENDS_ON live/db/terragrunt.hcl db:7\n" + "live/app/terragrunt.hcl include.common IMPORTS _common/app.hcl common:4\n" + "live/app/terragrunt.hcl include.common IMPORTS live/account.hcl read=_common/app.hcl:2\n" + "live/app/terragrunt.hcl include.root IMPORTS live/account.hcl read=root.hcl:2\n" + "live/app/terragrunt.hcl include.root IMPORTS root.hcl root:1\n" + "live/app/terragrunt.hcl locals IMPORTS live/app/env.hcl read=live/app/terragrunt.hcl:14\n" + "live/app/terragrunt.hcl terraform IMPORTS Folder:modules/app -:10\n" + "live/db/terragrunt.hcl include.root IMPORTS live/account.hcl read=root.hcl:2\n" + "live/db/terragrunt.hcl include.root IMPORTS root.hcl root:1"; + if (!got || strcmp(got, want) != 0) { + fprintf(stderr, "terragrunt got:\n%s\nwant:\n%s\n", got ? got : "(null)", want); + free(got); + FAIL("terragrunt"); + } + free(got); + PASS(); +} + +/* Every Ansible IMPORTS edge as "