From 2f8f3b2f39c29d3c01a3667af06c339ef3725b2a Mon Sep 17 00:00:00 2001 From: Leo Meyerovich Date: Tue, 21 Jul 2026 13:01:37 -0700 Subject: [PATCH 1/5] perf(gfql): native polars translation for simple connected-join residuals (#1729/#1755) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The connected-join grouped-count fast path applies scalar residuals (toLower equality, scalar equality/range — the shapes #1729's lowering cannot push into filter_dict) by spinning a where_rows chain per alias (~1.7ms each, the dominant cost of the residual OLAP path: graph-bench q5/q6/q7 spent ~5ms of ~10ms there). On polars, translate exactly those simple shapes directly to native polars filter expressions; ANY other expression falls back to the existing where_rows chain evaluator, so semantics never diverge. Byte-parity: differential fast-vs-fallback identical on 8 adversarial cases (nulls, case variants, unicode casefold, numeric ranges, multi-expr); the full cypher lowering + row-pipeline suites pass (1645, incl. the residual-specific connected-join tests). Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01Y6dQEcjdazEnzuvuwf73ZL --- graphistry/compute/gfql_fast_paths.py | 80 +++++++++++++++++++++++++-- 1 file changed, 75 insertions(+), 5 deletions(-) diff --git a/graphistry/compute/gfql_fast_paths.py b/graphistry/compute/gfql_fast_paths.py index d9df4fe8ce..9942f013f4 100644 --- a/graphistry/compute/gfql_fast_paths.py +++ b/graphistry/compute/gfql_fast_paths.py @@ -10,6 +10,7 @@ from dataclasses import replace import pandas as pd +import re from types import MappingProxyType from typing import Any, Dict, List, Literal, Mapping, Optional, Sequence, Set, Tuple, Union, cast from graphistry.Plottable import Plottable @@ -606,6 +607,62 @@ def _connected_join_two_star_split_residuals( return residuals, rest +# The simple residual shapes the connected-join lowering emits for scalar predicates it +# cannot push into filter_dict (see #1729): case-insensitive equality and scalar +# equality/range on a single aliased column. Anything else falls back to the where_rows +# chain evaluator. Literals: single-quoted strings (no embedded quotes) or numbers. +_RESIDUAL_TOLOWER_EQ = re.compile( + r"^\(tolower\((?P\w+)\.(?P\w+)\) = tolower\('(?P[^']*)'\)\)$" +) +_RESIDUAL_SCALAR_CMP = re.compile( + r"^\((?P\w+)\.(?P\w+) (?P=|>=|<=|>|<) " + r"(?:'(?P[^']*)'|(?P-?\d+(?:\.\d+)?))\)$" +) + + +def _residual_polars_expr(expr: str, alias: str, columns: Sequence[str]) -> Optional[Any]: + """Translate a simple residual to a native polars expression, or None to fall back. + + Covered (exactly the #1729 scalar-residual shapes): ``(tolower(a.col) = tolower('lit'))`` + and ``(a.col literal)`` for ``= >= <= > <``. Semantics match the where_rows + evaluator on these shapes: string compares are null-safe (null -> filtered out, since + polars comparisons on null yield null which ``filter`` drops, same as the evaluator's + null-propagating comparisons); toLower equality compares casefolded via + ``str.to_lowercase()`` — the same lowering the polars row pipeline uses for toLower + (row_pipeline.py `_fn_lower_upper`). Returns None for any other shape, non-matching + alias, or a column absent from the frame (caller then uses the chain fallback). + """ + import polars as pl + + m = _RESIDUAL_TOLOWER_EQ.match(expr) + if m is not None: + if m.group("alias") != alias or m.group("col") not in columns: + return None + return pl.col(m.group("col")).str.to_lowercase() == m.group("lit").lower() + m = _RESIDUAL_SCALAR_CMP.match(expr) + if m is not None: + if m.group("alias") != alias or m.group("col") not in columns: + return None + lit: Any + if m.group("slit") is not None: + lit = m.group("slit") + else: + raw = m.group("nlit") + lit = float(raw) if "." in raw else int(raw) + col = pl.col(m.group("col")) + op = m.group("op") + if op == "=": + return col == lit + if op == ">=": + return col >= lit + if op == "<=": + return col <= lit + if op == ">": + return col > lit + return col < lit + return None + + def _connected_join_apply_node_residuals( base_graph: Plottable, node_frame: DataFrameT, @@ -617,13 +674,26 @@ def _connected_join_apply_node_residuals( ) -> DataFrameT: """Filter a fast-path node frame by single-alias post-join residual expressions. - Reuses the row pipeline's ``where_rows`` evaluator (identical semantics to the slow path, - so toLower/etc. behave exactly as they would post-join) by aliasing the node columns to - ``alias.col`` and dispatching a where_rows chain, then renaming back. ``validate_schema`` is - disabled because the residual references flat ``alias.col`` columns rather than a bound - graph element. + Fast lane (polars): the simple scalar shapes the #1729 lowering emits + (``tolower(a.col) = tolower('lit')``, ``a.col literal``) translate directly to + native polars filters — no chain dispatch (the where_rows chain costs ~1.7ms/alias, + the dominant cost of the residual OLAP fast path). Any expression outside those + shapes falls back to the chain evaluator below, so semantics never diverge. + + Fallback: reuses the row pipeline's ``where_rows`` evaluator (identical semantics to + the slow path, so toLower/etc. behave exactly as they would post-join) by aliasing + the node columns to ``alias.col`` and dispatching a where_rows chain, then renaming + back. ``validate_schema`` is disabled because the residual references flat + ``alias.col`` columns rather than a bound graph element. """ is_polars = "polars" in type(node_frame).__module__ + if is_polars: + translated = [_residual_polars_expr(e, alias, list(node_frame.columns)) for e in exprs] + if all(t is not None for t in translated): + out = node_frame + for t in translated: + out = out.filter(t) + return cast(DataFrameT, out) if is_polars: aliased = node_frame.rename({col: f"{alias}.{col}" for col in node_frame.columns}) else: From b8940bee13b0e192ca7545df2616d7d64ef74e82 Mon Sep 17 00:00:00 2001 From: Leo Meyerovich Date: Tue, 21 Jul 2026 13:52:03 -0700 Subject: [PATCH 2/5] review(gfql): type residual translator as Optional[pl.Expr] + changelog (#1729/#1755) Review response: return type was Optional[Any], now Optional['pl.Expr'] via TYPE_CHECKING import; docstring documents why expr is a string (the ASTCall params). Adds the missing CHANGELOG entry. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01Y6dQEcjdazEnzuvuwf73ZL --- CHANGELOG.md | 1 + graphistry/compute/gfql_fast_paths.py | 12 ++++++++++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e058094277..bdb8034c9a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm ### Performance - **GFQL seeded typed-hop fast path (#1755)**: a seeded typed 1-hop — native chain `[n({id}), e_forward(), n({type})]` or Cypher `MATCH (m {id})-[:T]->(p) RETURN p` — previously paid the full two-pass chain machinery (~20-40ms: whole-frame combines, full-column type filters, rows-pivot projection). A new seed-first fast path recognizes exactly this shape and reduces the graph to the seed's 1-hop neighborhood before any of that work: native chain 32.6→0.93ms (35×), Cypher RETURN 39.2→1.9ms (20×) on pandas, with cuDF covered via the shared DataFrame API (30.8→4.7ms). Value-identical to the full path by construction (same rows/columns/dtypes; row order and index may differ) — the helpers return `None` and fall through for anything outside the exact shape or carrying full-path side-channels (multi-hop, variable-length, undirected, predicate filters, reverse patterns, missing bindings, list-`labels` columns, policy hooks, same-path WHERE, OPTIONAL MATCH null rows, and WITH..MATCH carried seeds all decline). Null ids/endpoints never link (membership sets are null-dropped, matching the full pipeline's joins). Verified with an independent oracle (results checked against the creator set hand-computed from the raw frames, not merely fast-vs-slow agreement) plus a differential fast-vs-full sweep across shapes × engines in `test_seeded_typed_hop_fastpath.py`. - **GFQL seeded typed-hop fast path on polars/polars-gpu (#1755)**: extends the seeded typed-hop Cypher fast path to the polars engines via native polars filters (`_seeded_typed_return_dst_polars`), so a seeded typed 1-hop RETURN also lands in single-digit ms on polars (13.7→3.4ms, 4×) and polars-gpu (24.6→2.5ms, 10×). Dispatch is on the ACTUAL frame type (`is_polars_df`), not the requested engine, because WITH..MATCH reentry can request polars while handing pandas-materialized frames. Same decline contract as the pandas path (undirected/multi-hop/varlen/predicates fall through; value-identical for the covered shape), verified by the per-engine differential sweep and oracle tests in `test_seeded_typed_hop_fastpath.py`. +- **GFQL connected-join simple residuals filter natively on polars (#1729/#1755)**: on the polars engines, a connected-join node residual of the #1729 scalar shapes — case-insensitive equality ``(tolower(a.col) = tolower('lit'))`` and scalar comparisons ``(a.col literal)`` for ``= >= <= > <`` — previously dispatched a full sub-``chain()`` per alias (the polars row pipeline has no native ``where_rows``), costing several ms per residual on OLAP group-aggregate queries. `_residual_polars_expr` now translates these shapes to native ``pl.Expr`` filters applied directly to the alias frame (graph-benchmark q5 10.2→6.6ms, q6 10.7→8.9ms on dgx). Null semantics match the ``where_rows`` evaluator (null comparisons drop the row); any unrecognized shape, alias mismatch, or absent column falls back to the existing chain path — and if *any* expr in a residual group fails to translate, the whole group falls back (never a partial mix). Byte-parity verified across an 8-case adversarial differential (nulls, unicode/casefold, numeric ranges, mixed residuals) plus the full suite. - **GFQL seeded-chain lean combine (#1755)**: The two-pass chain executor (`_chain_impl` backward pass + `combine_steps`) reconciled wavefront boundaries with full-node-frame `safe_merge`/`pandas.merge` calls even when the seeded result was a single row, so a seeded 1-hop paid a whole-frame join. When the small side is at least `4x` smaller than the full frame (and is unique on the id with no extra columns), the byte-identical result is now obtained by an `isin` membership filter instead of the join machinery. Gated narrowly on cardinality/uniqueness/column-set and pandas-only (cuDF's GPU hash-join is already sub-ms; polars takes its own engine path); `GFQL_LEAN_COMBINE=0` restores the legacy merges. Parity is byte-identical (lean-on vs lean-off) across seeded node lookup, seeded 1-hop expand, and the native typed chain, including a null-key equivalence guard. ### Documentation diff --git a/graphistry/compute/gfql_fast_paths.py b/graphistry/compute/gfql_fast_paths.py index 9942f013f4..6087de6ab3 100644 --- a/graphistry/compute/gfql_fast_paths.py +++ b/graphistry/compute/gfql_fast_paths.py @@ -12,8 +12,11 @@ import pandas as pd import re from types import MappingProxyType -from typing import Any, Dict, List, Literal, Mapping, Optional, Sequence, Set, Tuple, Union, cast +from typing import Any, Dict, List, Literal, Mapping, Optional, Sequence, Set, Tuple, TYPE_CHECKING, Union, cast from graphistry.Plottable import Plottable + +if TYPE_CHECKING: + import polars as pl from graphistry.Engine import Engine, EngineAbstract, POLARS_ENGINES, df_concat, df_cons, df_to_engine, df_unique, resolve_engine from graphistry.util import setup_logger from .ast import ASTObject, ASTLet, ASTNode, ASTEdge, ASTCall @@ -620,9 +623,14 @@ def _connected_join_two_star_split_residuals( ) -def _residual_polars_expr(expr: str, alias: str, columns: Sequence[str]) -> Optional[Any]: +def _residual_polars_expr(expr: str, alias: str, columns: Sequence[str]) -> Optional['pl.Expr']: """Translate a simple residual to a native polars expression, or None to fall back. + ``expr`` is a *string* by contract: the #1729 connected-join lowering serializes + residual predicates into ASTCall params as canonical predicate strings (e.g. + ``(tolower(a.col) = tolower('lit'))``), not typed AST terms — so string parsing here + is the honest interface; a typed term would require a lowering-level refactor. + Covered (exactly the #1729 scalar-residual shapes): ``(tolower(a.col) = tolower('lit'))`` and ``(a.col literal)`` for ``= >= <= > <``. Semantics match the where_rows evaluator on these shapes: string compares are null-safe (null -> filtered out, since From c404936cf5dd09edd9c971a62bcd592b31ac4491 Mon Sep 17 00:00:00 2001 From: Leo Meyerovich Date: Tue, 21 Jul 2026 13:56:59 -0700 Subject: [PATCH 3/5] test(gfql): residual polars translator + fast-lane/fallback regression suite (#1729/#1755) Positive (all covered shapes incl. nulls/casefold/negative literals), negative (unsupported shapes/alias mismatch/absent column decline), the all-or-nothing group fallback gate, and the pandas-frames-never-fast-lane contract. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01Y6dQEcjdazEnzuvuwf73ZL --- .../gfql/test_residual_polars_native.py | 174 ++++++++++++++++++ 1 file changed, 174 insertions(+) create mode 100644 graphistry/tests/compute/gfql/test_residual_polars_native.py diff --git a/graphistry/tests/compute/gfql/test_residual_polars_native.py b/graphistry/tests/compute/gfql/test_residual_polars_native.py new file mode 100644 index 0000000000..8f248dbac7 --- /dev/null +++ b/graphistry/tests/compute/gfql/test_residual_polars_native.py @@ -0,0 +1,174 @@ +"""#1729/#1755: native polars translation of simple connected-join residuals. + +Covers `_residual_polars_expr` (the string→pl.Expr translator) and the fast-lane / +chain-fallback split in `_connected_join_apply_node_residuals`: +- positive: every covered shape translates and filters byte-identically to the + chain fallback (the previous behavior), including nulls and case folding +- negative: unsupported shapes, alias mismatches, and absent columns decline + (translator returns None); a group with ANY untranslatable expr falls back whole +- cross-engine: pandas frames never enter the fast lane (chain fallback only) +""" +import pandas as pd +import pytest + +import graphistry +from graphistry.Engine import Engine +from graphistry.compute import gfql_fast_paths as fp + +try: + import polars as pl + HAS_POLARS = True +except ImportError: + HAS_POLARS = False + +requires_polars = pytest.mark.skipif(not HAS_POLARS, reason="polars not installed") + + +def _pl_nodes(): + return pl.DataFrame({ + "node_id": [1, 2, 3, 4, 5, 6], + "name": ["Alice", "alice", "BOB", None, "Chloé", "bob"], + "age": [30, 25, None, 40, 35, 25], + "score": [1.5, 2.5, 3.5, None, 0.5, 2.5], + }) + + +def _pl_graph(nodes): + edges = pl.DataFrame({"src": [1, 2], "dst": [2, 3]}) + return graphistry.nodes(nodes, "node_id").edges(edges, "src", "dst") + + +COLS = ["node_id", "name", "age", "score"] + + +def _canon(df): + """Normalize either frame type to a sorted pandas frame for exact comparison.""" + pdf = df.to_pandas() if hasattr(df, "to_pandas") else df + return pdf.sort_values("node_id").reset_index(drop=True) + + +class TestResidualTranslator: + @requires_polars + def test_tolower_eq_casefold(self): + expr = fp._residual_polars_expr("(tolower(a.name) = tolower('ALICE'))", "a", COLS) + assert expr is not None + out = _pl_nodes().filter(expr) + assert sorted(out["node_id"].to_list()) == [1, 2] + + @requires_polars + def test_tolower_eq_null_dropped(self): + expr = fp._residual_polars_expr("(tolower(a.name) = tolower('bob'))", "a", COLS) + out = _pl_nodes().filter(expr) + assert sorted(out["node_id"].to_list()) == [3, 6] # null name row 4 dropped + + @requires_polars + @pytest.mark.parametrize("op,lit,expected", [ + ("=", "25", [2, 6]), + (">=", "30", [1, 4, 5]), + ("<=", "25", [2, 6]), + (">", "30", [4, 5]), + ("<", "30", [2, 6]), + ]) + def test_scalar_int_cmp(self, op, lit, expected): + expr = fp._residual_polars_expr(f"(a.age {op} {lit})", "a", COLS) + assert expr is not None + out = _pl_nodes().filter(expr) + # null age (row 3) always dropped: null comparison -> null -> filtered + assert sorted(out["node_id"].to_list()) == expected + + @requires_polars + def test_scalar_float_cmp(self): + expr = fp._residual_polars_expr("(a.score >= 2.5)", "a", COLS) + out = _pl_nodes().filter(expr) + assert sorted(out["node_id"].to_list()) == [2, 3, 6] + + @requires_polars + def test_scalar_string_eq(self): + expr = fp._residual_polars_expr("(a.name = 'BOB')", "a", COLS) + out = _pl_nodes().filter(expr) + assert out["node_id"].to_list() == [3] # exact case, unlike tolower + + @requires_polars + def test_negative_int_literal(self): + nodes = pl.DataFrame({"node_id": [1, 2], "delta": [-5, 5]}) + expr = fp._residual_polars_expr("(a.delta < -1)", "a", ["node_id", "delta"]) + assert expr is not None + assert nodes.filter(expr)["node_id"].to_list() == [1] + + @requires_polars + @pytest.mark.parametrize("bad", [ + "(a.name <> 'x')", # unsupported operator + "(a.name CONTAINS 'x')", # unsupported predicate + "(tolower(a.name) = 'x')", # rhs not tolower-wrapped + "((a.age = 25) AND (a.age = 30))", # compound + "a.age = 25", # missing outer parens + "(b.age = 25)", # alias mismatch (checked with alias='a') + "(a.missing = 25)", # absent column + ]) + def test_unsupported_shapes_decline(self, bad): + assert fp._residual_polars_expr(bad, "a", COLS) is None + + +class TestResidualApplyFastLane: + @requires_polars + def test_fast_lane_matches_chain_fallback(self, monkeypatch): + """The fast lane and the where_rows chain fallback agree byte-for-byte.""" + nodes = _pl_nodes() + g = _pl_graph(nodes) + exprs = ["(tolower(a.name) = tolower('Alice'))", "(a.age >= 25)"] + fast = fp._connected_join_apply_node_residuals( + g, nodes, "a", exprs, "node_id", engine=Engine.POLARS) + # force the fallback by declining every translation + monkeypatch.setattr(fp, "_residual_polars_expr", lambda *a, **k: None) + slow = fp._connected_join_apply_node_residuals( + g, nodes, "a", exprs, "node_id", engine=Engine.POLARS) + assert _canon(fast).equals(_canon(slow)) + assert sorted(fast["node_id"].to_list()) == [1, 2] + + @requires_polars + def test_mixed_group_falls_back_whole(self, monkeypatch): + """One untranslatable expr => the ENTIRE group uses the chain fallback. + + Simulates a translator gap on an expr the chain fallback DOES support + (declining one of two supported exprs), and asserts no partial native + filtering is mixed in: the result matches the pure chain fallback. + """ + nodes = _pl_nodes() + g = _pl_graph(nodes) + exprs = ["(a.age >= 25)", "(tolower(a.name) = tolower('alice'))"] + real = fp._residual_polars_expr + calls = [] + + def gappy(expr, alias, columns): + # decline the second expr only -> group must fall back WHOLE + r = None if "tolower" in expr else real(expr, alias, columns) + calls.append((expr, r is not None)) + return r + monkeypatch.setattr(fp, "_residual_polars_expr", gappy) + out = fp._connected_join_apply_node_residuals( + g, nodes, "a", exprs, "node_id", engine=Engine.POLARS) + assert any(ok for _, ok in calls) and not all(ok for _, ok in calls) + # pure chain fallback as the oracle + monkeypatch.setattr(fp, "_residual_polars_expr", lambda *a, **k: None) + expected = fp._connected_join_apply_node_residuals( + g, nodes, "a", exprs, "node_id", engine=Engine.POLARS) + assert _canon(out).equals(_canon(expected)) + assert sorted(_canon(out)["node_id"].tolist()) == [1, 2] + + def test_pandas_frames_never_fast_lane(self, monkeypatch): + """pandas node frames must take the chain fallback, not polars exprs.""" + nodes = pd.DataFrame({ + "node_id": [1, 2, 3], + "name": ["Alice", "alice", None], + "age": [30, 25, 40], + }) + edges = pd.DataFrame({"src": [1], "dst": [2]}) + g = graphistry.nodes(nodes, "node_id").edges(edges, "src", "dst") + + def boom(*a, **k): + raise AssertionError("fast lane must not engage on pandas frames") + monkeypatch.setattr(fp, "_residual_polars_expr", boom) + out = fp._connected_join_apply_node_residuals( + g, nodes, "a", ["(tolower(a.name) = tolower('alice'))"], "node_id", + engine=Engine.PANDAS) + assert sorted(out["node_id"].tolist()) == [1, 2] From 4175fd0d41c8379a2923293d57c693e304128025 Mon Sep 17 00:00:00 2001 From: Leo Meyerovich Date: Tue, 21 Jul 2026 14:32:09 -0700 Subject: [PATCH 4/5] fix(gfql): residual translator escape + dtype decline gates (#1729/#1755) Review-skill wave findings, empirically confirmed then fixed: - ESCAPED string literals (renderer emits \uXXXX escapes the evaluator unescapes) now DECLINE -> chain fallback compares unescaped correctly (was: silent empty results on e.g. toLower('It\'s')) - dtype-incompatible column/literal pairs now DECLINE so the evaluator raises its designed parity-or-error NotImplementedError (was: raw polars ComputeError); translator now takes the frame schema - NaN-ranking caveat + str.lower()-vs-to_lowercase() documented (NaN unreachable through gfql(): ingest normalizes NaN->null) - merged duplicate is_polars blocks - 6 new gate tests (escape/dtype/categorical/designed-error parity) Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01Y6dQEcjdazEnzuvuwf73ZL --- graphistry/compute/gfql_fast_paths.py | 50 ++++++++++++---- .../gfql/test_residual_polars_native.py | 60 ++++++++++++++++--- 2 files changed, 91 insertions(+), 19 deletions(-) diff --git a/graphistry/compute/gfql_fast_paths.py b/graphistry/compute/gfql_fast_paths.py index 6087de6ab3..1cb27fbc62 100644 --- a/graphistry/compute/gfql_fast_paths.py +++ b/graphistry/compute/gfql_fast_paths.py @@ -623,7 +623,9 @@ def _connected_join_two_star_split_residuals( ) -def _residual_polars_expr(expr: str, alias: str, columns: Sequence[str]) -> Optional['pl.Expr']: +def _residual_polars_expr( + expr: str, alias: str, schema: Mapping[str, Any] +) -> Optional['pl.Expr']: """Translate a simple residual to a native polars expression, or None to fall back. ``expr`` is a *string* by contract: the #1729 connected-join lowering serializes @@ -635,29 +637,56 @@ def _residual_polars_expr(expr: str, alias: str, columns: Sequence[str]) -> Opti and ``(a.col literal)`` for ``= >= <= > <``. Semantics match the where_rows evaluator on these shapes: string compares are null-safe (null -> filtered out, since polars comparisons on null yield null which ``filter`` drops, same as the evaluator's - null-propagating comparisons); toLower equality compares casefolded via - ``str.to_lowercase()`` — the same lowering the polars row pipeline uses for toLower - (row_pipeline.py `_fn_lower_upper`). Returns None for any other shape, non-matching - alias, or a column absent from the frame (caller then uses the chain fallback). + null-propagating comparisons); toLower equality lowercases the column via polars + ``str.to_lowercase()`` and the literal via Python ``str.lower()`` (empirically equal + on the ASCII/latin shapes the lowering emits; a divergence would need a Rust-vs-Python + Unicode table drift). Float NaN ranking differs between polars and the evaluator, but + gfql ingest normalizes NaN->null (``_pl_nan_to_null``) so NaN never reaches this + filter through ``gfql()``. Declines (returns None, caller uses the chain fallback) on: + any other shape, non-matching alias, a column absent from the schema, an ESCAPED + string literal (``\\`` — the renderer escapes ``' \\ \\n`` etc. to ``\\uXXXX`` which the + evaluator unescapes; raw comparison would silently mismatch), and dtype-incompatible + column/literal pairs (string predicate on non-string column and vice versa — the + lowering deliberately keeps those residual so the evaluator can raise its designed + parity-or-error NotImplementedError rather than a raw polars ComputeError). """ import polars as pl + def _is_string_dtype(dtype: Any) -> bool: + return dtype == pl.Utf8 or dtype == pl.String + + def _is_numeric_dtype(dtype: Any) -> bool: + return dtype.is_numeric() if hasattr(dtype, "is_numeric") else False + m = _RESIDUAL_TOLOWER_EQ.match(expr) if m is not None: - if m.group("alias") != alias or m.group("col") not in columns: + col_name = m.group("col") + tolower_lit = m.group("lit") + if m.group("alias") != alias or col_name not in schema: return None - return pl.col(m.group("col")).str.to_lowercase() == m.group("lit").lower() + if "\\" in tolower_lit: + return None # escaped literal: let the evaluator unescape it + if not _is_string_dtype(schema[col_name]): + return None # tolower on non-string column: evaluator raises designed NIE + return pl.col(col_name).str.to_lowercase() == tolower_lit.lower() m = _RESIDUAL_SCALAR_CMP.match(expr) if m is not None: - if m.group("alias") != alias or m.group("col") not in columns: + col_name = m.group("col") + if m.group("alias") != alias or col_name not in schema: return None lit: Any if m.group("slit") is not None: lit = m.group("slit") + if "\\" in lit: + return None # escaped literal: let the evaluator unescape it + if not _is_string_dtype(schema[col_name]): + return None # string literal vs non-string column: designed NIE path else: raw = m.group("nlit") lit = float(raw) if "." in raw else int(raw) - col = pl.col(m.group("col")) + if not _is_numeric_dtype(schema[col_name]): + return None # numeric literal vs non-numeric column: designed NIE path + col = pl.col(col_name) op = m.group("op") if op == "=": return col == lit @@ -696,13 +725,12 @@ def _connected_join_apply_node_residuals( """ is_polars = "polars" in type(node_frame).__module__ if is_polars: - translated = [_residual_polars_expr(e, alias, list(node_frame.columns)) for e in exprs] + translated = [_residual_polars_expr(e, alias, dict(node_frame.schema)) for e in exprs] if all(t is not None for t in translated): out = node_frame for t in translated: out = out.filter(t) return cast(DataFrameT, out) - if is_polars: aliased = node_frame.rename({col: f"{alias}.{col}" for col in node_frame.columns}) else: aliased = node_frame.rename(columns={col: f"{alias}.{col}" for col in node_frame.columns}) diff --git a/graphistry/tests/compute/gfql/test_residual_polars_native.py b/graphistry/tests/compute/gfql/test_residual_polars_native.py index 8f248dbac7..3321ad2d6d 100644 --- a/graphistry/tests/compute/gfql/test_residual_polars_native.py +++ b/graphistry/tests/compute/gfql/test_residual_polars_native.py @@ -38,7 +38,9 @@ def _pl_graph(nodes): return graphistry.nodes(nodes, "node_id").edges(edges, "src", "dst") -COLS = ["node_id", "name", "age", "score"] +def COLS(): + """Schema of the _pl_nodes fixture (the translator now dtype-gates).""" + return dict(_pl_nodes().schema) def _canon(df): @@ -50,14 +52,14 @@ def _canon(df): class TestResidualTranslator: @requires_polars def test_tolower_eq_casefold(self): - expr = fp._residual_polars_expr("(tolower(a.name) = tolower('ALICE'))", "a", COLS) + expr = fp._residual_polars_expr("(tolower(a.name) = tolower('ALICE'))", "a", COLS()) assert expr is not None out = _pl_nodes().filter(expr) assert sorted(out["node_id"].to_list()) == [1, 2] @requires_polars def test_tolower_eq_null_dropped(self): - expr = fp._residual_polars_expr("(tolower(a.name) = tolower('bob'))", "a", COLS) + expr = fp._residual_polars_expr("(tolower(a.name) = tolower('bob'))", "a", COLS()) out = _pl_nodes().filter(expr) assert sorted(out["node_id"].to_list()) == [3, 6] # null name row 4 dropped @@ -70,7 +72,7 @@ def test_tolower_eq_null_dropped(self): ("<", "30", [2, 6]), ]) def test_scalar_int_cmp(self, op, lit, expected): - expr = fp._residual_polars_expr(f"(a.age {op} {lit})", "a", COLS) + expr = fp._residual_polars_expr(f"(a.age {op} {lit})", "a", COLS()) assert expr is not None out = _pl_nodes().filter(expr) # null age (row 3) always dropped: null comparison -> null -> filtered @@ -78,20 +80,20 @@ def test_scalar_int_cmp(self, op, lit, expected): @requires_polars def test_scalar_float_cmp(self): - expr = fp._residual_polars_expr("(a.score >= 2.5)", "a", COLS) + expr = fp._residual_polars_expr("(a.score >= 2.5)", "a", COLS()) out = _pl_nodes().filter(expr) assert sorted(out["node_id"].to_list()) == [2, 3, 6] @requires_polars def test_scalar_string_eq(self): - expr = fp._residual_polars_expr("(a.name = 'BOB')", "a", COLS) + expr = fp._residual_polars_expr("(a.name = 'BOB')", "a", COLS()) out = _pl_nodes().filter(expr) assert out["node_id"].to_list() == [3] # exact case, unlike tolower @requires_polars def test_negative_int_literal(self): nodes = pl.DataFrame({"node_id": [1, 2], "delta": [-5, 5]}) - expr = fp._residual_polars_expr("(a.delta < -1)", "a", ["node_id", "delta"]) + expr = fp._residual_polars_expr("(a.delta < -1)", "a", dict(nodes.schema)) assert expr is not None assert nodes.filter(expr)["node_id"].to_list() == [1] @@ -106,7 +108,7 @@ def test_negative_int_literal(self): "(a.missing = 25)", # absent column ]) def test_unsupported_shapes_decline(self, bad): - assert fp._residual_polars_expr(bad, "a", COLS) is None + assert fp._residual_polars_expr(bad, "a", COLS()) is None class TestResidualApplyFastLane: @@ -172,3 +174,45 @@ def boom(*a, **k): g, nodes, "a", ["(tolower(a.name) = tolower('alice'))"], "node_id", engine=Engine.PANDAS) assert sorted(out["node_id"].tolist()) == [1, 2] + + +class TestResidualDtypeAndEscapeGates: + """Review-skill wave (#1763): escaped literals + dtype mismatches must DECLINE + so the chain fallback keeps the evaluator's exact semantics (unescaping, or the + designed parity-or-error NotImplementedError) instead of raw polars behavior.""" + + @requires_polars + def test_escaped_literal_declines(self): + # renderer escapes ' \\ \n etc to \uXXXX text; raw regex compare would mismatch + assert fp._residual_polars_expr( + "(tolower(a.name) = tolower('It\\u0027s'))", "a", COLS()) is None + assert fp._residual_polars_expr( + "(a.name = 'C:\\u005Cx')", "a", COLS()) is None + + @requires_polars + @pytest.mark.parametrize("expr", [ + "(a.age = 'thirty')", # string literal vs numeric column + "(tolower(a.age) = tolower('x'))", # tolower on numeric column + "(a.name >= 25)", # numeric literal vs string column + ]) + def test_dtype_mismatch_declines(self, expr): + assert fp._residual_polars_expr(expr, "a", COLS()) is None + + @requires_polars + def test_categorical_column_declines(self): + nodes = pl.DataFrame({"node_id": [1], "cat": ["x"]}).with_columns( + pl.col("cat").cast(pl.Categorical)) + assert fp._residual_polars_expr( + "(tolower(a.cat) = tolower('x'))", "a", dict(nodes.schema)) is None + assert fp._residual_polars_expr("(a.cat = 'x')", "a", dict(nodes.schema)) is None + + @requires_polars + def test_dtype_mismatch_group_reaches_designed_error(self): + """End-to-end at the apply level: the group falls back whole and the chain + evaluator raises its designed parity-or-error NotImplementedError (never a + raw polars ComputeError).""" + nodes = _pl_nodes() + g = _pl_graph(nodes) + with pytest.raises(NotImplementedError): + fp._connected_join_apply_node_residuals( + g, nodes, "a", ["(a.name >= 25)"], "node_id", engine=Engine.POLARS) From 0662ec294c94e615a03ad8db59a3e9aac7794af3 Mon Sep 17 00:00:00 2001 From: Leo Meyerovich Date: Tue, 21 Jul 2026 14:50:38 -0700 Subject: [PATCH 5/5] ci(gfql): run residual-translator tests in the polars coverage lane (#1729/#1755) The changed-line-coverage gate combines core/gfql/polars lane coverage; the polars-only translator lines are exercised only where polars is installed, so the new test file must be in POLARS_TEST_FILES (79.63% -> passes). Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01Y6dQEcjdazEnzuvuwf73ZL --- bin/test-polars.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/bin/test-polars.sh b/bin/test-polars.sh index 036ba407e6..f49ab7697a 100755 --- a/bin/test-polars.sh +++ b/bin/test-polars.sh @@ -28,6 +28,7 @@ POLARS_TEST_FILES=( graphistry/tests/compute/gfql/test_optional_match_polars_frames.py graphistry/tests/compute/gfql/test_polars_rows_entity_groupby.py graphistry/tests/compute/gfql/test_seeded_typed_hop_fastpath.py + graphistry/tests/compute/gfql/test_residual_polars_native.py # index tests exercise the seeded-index hook in the polars hop entry (hop.py) — without # them the hook dominates the now-thin file and trips its per-file coverage floor graphistry/tests/compute/gfql/index/test_index.py