diff --git a/src/boring_semantic_layer/agents/tests/test_chart_handler.py b/src/boring_semantic_layer/agents/tests/test_chart_handler.py index 04772f0a..0b618f10 100644 --- a/src/boring_semantic_layer/agents/tests/test_chart_handler.py +++ b/src/boring_semantic_layer/agents/tests/test_chart_handler.py @@ -598,7 +598,7 @@ def test_ibis_available_in_context(): import ibis from boring_semantic_layer import from_yaml - from boring_semantic_layer.utils import safe_eval + from boring_semantic_layer.safe_eval import safe_eval # Load models models = from_yaml( diff --git a/src/boring_semantic_layer/agents/tools.py b/src/boring_semantic_layer/agents/tools.py index d52f1ed5..1eafcd8d 100644 --- a/src/boring_semantic_layer/agents/tools.py +++ b/src/boring_semantic_layer/agents/tools.py @@ -14,7 +14,7 @@ from boring_semantic_layer.agents.utils.chart_handler import generate_chart_with_data from boring_semantic_layer.agents.utils.prompts import load_prompt -from boring_semantic_layer.utils import safe_eval +from boring_semantic_layer.safe_eval import safe_eval from boring_semantic_layer.yaml import from_yaml diff --git a/src/boring_semantic_layer/chart/md_parser/executor.py b/src/boring_semantic_layer/chart/md_parser/executor.py index a59c4dd7..0f04f813 100644 --- a/src/boring_semantic_layer/chart/md_parser/executor.py +++ b/src/boring_semantic_layer/chart/md_parser/executor.py @@ -10,7 +10,7 @@ from boring_semantic_layer import to_semantic_table from boring_semantic_layer._xorq import api as xo -from boring_semantic_layer.utils import safe_eval +from boring_semantic_layer.safe_eval import safe_eval class QueryExecutor: diff --git a/src/boring_semantic_layer/io.py b/src/boring_semantic_layer/io.py new file mode 100644 index 00000000..5797256f --- /dev/null +++ b/src/boring_semantic_layer/io.py @@ -0,0 +1,74 @@ +"""YAML/URL loading helpers for model configuration files.""" + +from __future__ import annotations + +from pathlib import Path + +import yaml + + +def _is_url(path: str | Path | None) -> bool: + """Check if a path is a URL.""" + if path is None: + return False + from urllib.parse import urlparse + + parsed = urlparse(str(path)) + return parsed.scheme in ("http", "https") + + +def _fetch_url_content(url: str) -> str: + """Fetch content from a URL. + + Args: + url: The URL to fetch + + Returns: + The content as a string + + Raises: + ValueError: If the fetch fails + """ + import urllib.error + import urllib.request + + try: + with urllib.request.urlopen(url, timeout=30) as response: + return response.read().decode("utf-8") + except urllib.error.HTTPError as e: + raise ValueError(f"HTTP Error {e.code}: {e.reason} for URL: {url}") from e + except urllib.error.URLError as e: + raise ValueError(f"URL Error: {e.reason} for URL: {url}") from e + except Exception as e: + raise ValueError(f"Failed to fetch URL {url}: {e}") from e + + +def read_yaml_file(yaml_path: str | Path) -> dict: + """Read and parse YAML file into dict. Supports local files and URLs. + + Args: + yaml_path: Path to local file or URL (http:// or https://) + + Returns: + Parsed YAML content as dict + """ + try: + if _is_url(yaml_path): + content_str = _fetch_url_content(str(yaml_path)) + content = yaml.safe_load(content_str) + else: + yaml_path = Path(yaml_path) + if not yaml_path.exists(): + raise FileNotFoundError(f"YAML file not found: {yaml_path}") + + with open(yaml_path) as f: + content = yaml.safe_load(f) + + if not isinstance(content, dict): + raise ValueError(f"YAML file must contain a dict, got: {type(content)}") + + return content + except (FileNotFoundError, ValueError): + raise + except Exception as e: + raise ValueError(f"Failed to parse YAML file {yaml_path}: {e}") from e diff --git a/src/boring_semantic_layer/profile.py b/src/boring_semantic_layer/profile.py index cb2e6eb5..ab329571 100644 --- a/src/boring_semantic_layer/profile.py +++ b/src/boring_semantic_layer/profile.py @@ -8,7 +8,7 @@ from ._xorq import HAS_XORQ from ._xorq import Profile as XorqProfile -from .utils import read_yaml_file +from .io import read_yaml_file class ProfileError(Exception): diff --git a/src/boring_semantic_layer/query.py b/src/boring_semantic_layer/query.py index d6d073bf..e49866b6 100644 --- a/src/boring_semantic_layer/query.py +++ b/src/boring_semantic_layer/query.py @@ -15,7 +15,7 @@ from toolz import curry from .errors import QueryError, unwrap_or_raise -from .utils import safe_eval +from .safe_eval import safe_eval def _get_ibis_api(): diff --git a/src/boring_semantic_layer/safe_eval.py b/src/boring_semantic_layer/safe_eval.py new file mode 100644 index 00000000..eb35ba57 --- /dev/null +++ b/src/boring_semantic_layer/safe_eval.py @@ -0,0 +1,389 @@ +"""Sandboxed evaluation of user-supplied expression strings. + +An AST allowlist (nodes, module calls, method calls) guards the strings +accepted from YAML models and agent tooling. This is a separate trust +boundary from the serialization callable allowlist in +``serialization._trust``. +""" + +from __future__ import annotations + +import ast +from typing import Any + +from returns.result import Result, safe +from toolz import curry + + +class SafeEvalError(Exception): + pass + + +SAFE_NODES = { + ast.Expression, + ast.Load, + ast.Name, + ast.Constant, + ast.Attribute, + ast.Call, + ast.Subscript, + ast.Index, + ast.Slice, + ast.UnaryOp, + ast.UAdd, + ast.USub, + ast.Not, + ast.Invert, # Bitwise NOT (~) + ast.BinOp, + ast.Add, + ast.Sub, + ast.Mult, + ast.Div, + ast.FloorDiv, + ast.Mod, + ast.Pow, + ast.BitOr, # Bitwise OR (|) - for combining conditions in pandas/ibis + ast.BitAnd, # Bitwise AND (&) - for combining conditions in pandas/ibis + ast.BitXor, # Bitwise XOR (^) + ast.Compare, + ast.Eq, + ast.NotEq, + ast.Lt, + ast.LtE, + ast.Gt, + ast.GtE, + ast.In, + ast.NotIn, + ast.Is, + ast.IsNot, + ast.BoolOp, + ast.And, + ast.Or, + ast.List, + ast.Tuple, + ast.Dict, + ast.keyword, + ast.IfExp, + ast.Lambda, # Allow lambda expressions in YAML/agent-supplied strings + ast.arguments, # Required for lambda function arguments + ast.arg, # Required for individual lambda arguments +} + + +# Helpers exposed by the Ibis modules in agent +# query contexts. In particular, backend/IO namespaces (duckdb, postgres, +# read_*, connect, ...) are intentionally absent. +SAFE_MODULE_CALLS = frozenset( + { + "and_", + "asc", + "cases", + "coalesce", + "cume_dist", + "date", + "dense_rank", + "desc", + "greatest", + "ifelse", + "interval", + "least", + "literal", + "now", + "ntile", + "or_", + "param", + "percent_rank", + "random", + "rank", + "row_number", + "time", + "timestamp", + "today", + "uuid", + "window", + } +) + + +# Pure expression operations plus the BSL query-building operations accepted +# in agent-generated query chains. Calls that execute, compile, perform IO, +# access a backend, or expose an underlying operation are deliberately absent. +SAFE_METHOD_CALLS = frozenset( + { + "abs", + "aggregate", + "all", + "any", + "approx_median", + "approx_nunique", + "arbitrary", + "argmax", + "argmin", + "as_table", + "between", + "capitalize", + "cast", + "ceil", + "coalesce", + "collect", + "contains", + "count", + "cummax", + "cummean", + "cummin", + "cumsum", + "date", + "day", + "day_of_week", + "distinct", + "drop", + "endswith", + "epoch_seconds", + "fill_null", + "filter", + "find", + "first", + "floor", + "group_by", + "hour", + "identical_to", + "ifelse", + "isin", + "isnull", + "lag", + "last", + "lead", + "length", + "like", + "limit", + "lower", + "lpad", + "lstrip", + "max", + "mean", + "median", + "microsecond", + "millisecond", + "min", + "minute", + "mode", + "month", + "mutate", + "name", + "notin", + "notnull", + "nullif", + "nunique", + "order_by", + "over", + "quarter", + "quantile", + "re_extract", + "re_replace", + "re_search", + "re_split", + "rename", + "repeat", + "replace", + "reverse", + "right", + "round", + "rpad", + "rstrip", + "second", + "select", + "sign", + "split", + "startswith", + "std", + "strftime", + "strip", + "substr", + "sum", + "time", + "timestamp", + "translate", + "truncate", + "typeof", + "unique", + "unnest", + "upper", + "var", + "week_of_year", + "with_dimensions", + "with_measures", + "year", + } +) + +SAFE_QUERY_METHOD_CALLS = frozenset( + { + "aggregate", + "distinct", + "drop", + "filter", + "group_by", + "limit", + "mutate", + "order_by", + "rename", + "select", + "unnest", + "with_dimensions", + "with_measures", + } +) + +SAFE_QUERY_ATTRIBUTES = frozenset({"dimensions", "measures"}) + +_SAFE_MODULE_ROOTS = frozenset({"ibis", "xorq_ibis", "xo"}) + + +class _SafeEvalValidator(ast.NodeVisitor): + """Validate the small, expression-only DSL accepted by ``safe_eval``.""" + + def __init__(self, allowed_names: set[str]): + self._allowed_names = allowed_names + self._lambda_names: list[set[str]] = [] + + @staticmethod + def _root_name(node: ast.AST) -> str | None: + while True: + if isinstance(node, ast.Name): + return node.id + if isinstance(node, ast.Attribute): + node = node.value + elif isinstance(node, ast.Call): + node = node.func + elif isinstance(node, ast.Subscript): + node = node.value + else: + return None + + def _is_expression_root(self, name: str | None) -> bool: + lambda_names = set().union(*self._lambda_names) if self._lambda_names else set() + return name == "_" or name in lambda_names or name in _SAFE_MODULE_ROOTS + + def generic_visit(self, node: ast.AST) -> None: + if type(node) not in SAFE_NODES: + raise SafeEvalError( + f"Unsafe node type: {type(node).__name__}. Only whitelisted operations are allowed." + ) + super().generic_visit(node) + + def visit_Name(self, node: ast.Name) -> None: # noqa: N802 + if node.id.startswith("_") and node.id != "_": + raise SafeEvalError(f"Private name '{node.id}' is not allowed") + lambda_names = set().union(*self._lambda_names) if self._lambda_names else set() + if node.id not in self._allowed_names and node.id not in lambda_names: + raise SafeEvalError( + f"Name '{node.id}' is not in the allowed names: {self._allowed_names}" + ) + + def visit_Attribute(self, node: ast.Attribute) -> None: # noqa: N802 + if node.attr.startswith("_"): + raise SafeEvalError(f"Private attribute '{node.attr}' is not allowed") + if ( + isinstance(node.value, ast.Name) + and node.value.id in _SAFE_MODULE_ROOTS + and node.attr not in SAFE_MODULE_CALLS + ): + raise SafeEvalError(f"Module attribute '{node.value.id}.{node.attr}' is not allowed") + if ( + isinstance(node.value, ast.Name) + and not self._is_expression_root(node.value.id) + and node.attr not in SAFE_QUERY_METHOD_CALLS | SAFE_QUERY_ATTRIBUTES + ): + raise SafeEvalError( + f"Attribute '{node.value.id}.{node.attr}' is not an allowed query operation" + ) + self.generic_visit(node) + + def visit_Call(self, node: ast.Call) -> None: # noqa: N802 + if not isinstance(node.func, ast.Attribute): + raise SafeEvalError("Only allowlisted DSL method calls are allowed") + + root_name = self._root_name(node.func) + if isinstance(node.func.value, ast.Name) and node.func.value.id in _SAFE_MODULE_ROOTS: + if node.func.attr not in SAFE_MODULE_CALLS: + raise SafeEvalError( + f"Module call '{node.func.value.id}.{node.func.attr}' is not allowed" + ) + elif ( + root_name is not None + and not self._is_expression_root(root_name) + and node.func.attr not in SAFE_QUERY_METHOD_CALLS + ): + raise SafeEvalError(f"Query method call '{node.func.attr}' is not allowed") + elif node.func.attr not in SAFE_METHOD_CALLS: + raise SafeEvalError(f"Method call '{node.func.attr}' is not allowed") + + for keyword in node.keywords: + if keyword.arg is None or keyword.arg.startswith("_"): + raise SafeEvalError("Private or expanded keyword arguments are not allowed") + self.generic_visit(node) + + def visit_Lambda(self, node: ast.Lambda) -> None: # noqa: N802 + args = node.args + if args.vararg or args.kwarg or args.kwonlyargs or args.defaults or args.kw_defaults: + raise SafeEvalError("Lambda defaults and variadic arguments are not allowed") + lambda_names = {arg.arg for arg in [*args.posonlyargs, *args.args]} + if any(name.startswith("_") for name in lambda_names): + raise SafeEvalError("Private lambda argument names are not allowed") + self._lambda_names.append(lambda_names) + try: + self.visit(node.body) + finally: + self._lambda_names.pop() + + +def _validate_ast(node: ast.AST, allowed_names: set[str]) -> None: + _SafeEvalValidator(allowed_names).visit(node) + + +def _parse_expr(expr_str: str) -> ast.AST: + try: + return ast.parse(expr_str, mode="eval") + except SyntaxError: + # Try wrapping in parentheses to allow multiline method chaining + # This handles cases like: + # model.filter(...) + # .group_by(...) + # which is valid Python when wrapped in parens + try: + return ast.parse(f"({expr_str})", mode="eval") + except SyntaxError as e: + raise SafeEvalError(f"Invalid Python syntax: {e}") from e + + +def _compile_validated(tree: ast.AST) -> Any: + return compile(tree, "", "eval") + + +@curry +def _eval_in_context(context: dict, code: Any) -> Any: + return eval(code, context) # noqa: S307 + + +def safe_eval( + expr_str: str, + context: dict[str, Any] | None = None, + allowed_names: set[str] | None = None, +) -> Result[Any, Exception]: + context = context or {} + names = set(context) if allowed_names is None else set(allowed_names) + # ``_`` is the one intentionally public DSL identifier beginning with an + # underscore. All caller-provided private names remain inaccessible. + names = {name for name in names if name == "_" or not name.startswith("_")} + # Keep builtins empty even if an untrusted caller supplied a conflicting + # ``__builtins__`` context entry. + eval_context = {**context, "__builtins__": {}} + + @safe + def do_eval(): + tree = _parse_expr(expr_str) + _validate_ast(tree, names) + code = _compile_validated(tree) + return _eval_in_context(eval_context, code) + + return do_eval() diff --git a/src/boring_semantic_layer/serialization/__init__.py b/src/boring_semantic_layer/serialization/__init__.py index b526b49a..eea30bbe 100644 --- a/src/boring_semantic_layer/serialization/__init__.py +++ b/src/boring_semantic_layer/serialization/__init__.py @@ -13,6 +13,7 @@ from attrs import frozen from returns.result import Failure, Result, safe +from ._trust import UntrustedCallableError, trust_callable_module # noqa: F401 from .context import BSLSerializationContext from .extract import ( deserialize_calc_measures, diff --git a/src/boring_semantic_layer/serialization/_trust.py b/src/boring_semantic_layer/serialization/_trust.py new file mode 100644 index 00000000..95f78e58 --- /dev/null +++ b/src/boring_semantic_layer/serialization/_trust.py @@ -0,0 +1,127 @@ +"""Trust boundary for callables referenced by serialized payloads. + +A ``("fn", module, qualname)`` payload arm may only resolve inside the +allowlisted module roots; both the writer and the reader enforce this so +no attacker-chosen module is ever imported. +""" + +from __future__ import annotations + +import importlib + + +class UntrustedCallableError(ValueError): + """A serialized expression names a callable outside the trusted set. + + Serialized models are data, not code: a tag payload can travel through + a xorq catalog, a git repo or any other artifact store, and is not + necessarily written by whoever reads it. Restoring an arbitrary + ``(module, qualname)`` pair means importing an attacker-chosen module + and handing the result to ``Call.resolve()``, which calls it — i.e. + arbitrary code execution. Only functions from the expression libraries + BSL builds on can be restored. + """ + + +#: Module roots whose callables may be named in a serialized expression. +#: These are the libraries that actually appear in ibis resolver trees: +#: deferrable API functions (``ifelse``, ``coalesce``, ``_finish_searched_case``) +#: and the ``operator`` functions behind binary/unary nodes. +_TRUSTED_CALLABLE_ROOTS: frozenset[str] = frozenset( + { + "ibis", + "xorq", + "operator", + "_operator", + "boring_semantic_layer", + } +) + +_EXTRA_TRUSTED_CALLABLE_ROOTS: set[str] = set() + + +def trust_callable_module(root: str) -> None: + """Allow callables from an additional top-level module in serialized models. + + Only do this for modules you control, and only when every model you + deserialize comes from a source you trust as much as your own code: + a serialized expression naming a callable is equivalent to a function + call, so widening this set widens what a malicious payload can invoke. + """ + _EXTRA_TRUSTED_CALLABLE_ROOTS.add(root.split(".", 1)[0]) + + +def _trusted_roots() -> frozenset[str]: + return _TRUSTED_CALLABLE_ROOTS | frozenset(_EXTRA_TRUSTED_CALLABLE_ROOTS) + + +def _module_root(module_name: str | None) -> str: + return (module_name or "").split(".", 1)[0] + + +def _check_callable_ref(module_name: str | None, qualname: str | None) -> None: + """Reject a ``(module, qualname)`` pair that must not cross the wire. + + Applied on *both* sides: serialization refuses to emit a reference that + deserialization would refuse to load, so the failure surfaces where the + model is authored rather than in someone else's process. + """ + if not module_name or not qualname: + raise UntrustedCallableError( + f"Callable reference is incomplete: module={module_name!r} qualname={qualname!r}" + ) + root = _module_root(module_name) + if root not in _trusted_roots(): + raise UntrustedCallableError( + f"Refusing to (de)serialize callable {module_name}.{qualname}: " + f"module root {root!r} is not trusted. Serialized expressions may " + f"only reference {sorted(_trusted_roots())}. Express the logic with " + "ibis operations, or call " + "boring_semantic_layer.serialization.trust_callable_module() if you own " + "the module and trust every model you load." + ) + for part in qualname.split("."): + if part.startswith("__") or "<" in part or not part.isidentifier(): + raise UntrustedCallableError( + f"Refusing to (de)serialize callable {module_name}.{qualname}: " + f"qualname component {part!r} is not a plain public identifier " + "(lambdas, closures and dunder attributes cannot be restored)." + ) + + +def _resolve_qualname(module_obj, qualname: str): + """Resolve a dotted qualname like 'ClassName.method' on a module.""" + parts = qualname.split(".") + obj = module_obj + for part in parts: + if part == "": + raise ValueError(f"Cannot resolve lambda qualname: {qualname}") + obj = getattr(obj, part) + return obj + + +def _load_trusted_callable(module_name: str, qualname: str): + """Import and return a callable named by a serialized expression. + + The pair is validated before the import — an unimportable module is a + side effect in itself, so an untrusted name must never reach + ``import_module``. After resolution the *result* is checked too: a + qualname is a ``getattr`` chain, so ``("fn", "ibis", "os.system")`` + would otherwise walk out of a trusted module into an untrusted one. + """ + _check_callable_ref(module_name, qualname) + mod = importlib.import_module(module_name) + func = _resolve_qualname(mod, qualname) + if not callable(func): + raise UntrustedCallableError( + f"{module_name}.{qualname} resolved to a non-callable " + f"{type(func).__name__}; refusing to use it as an expression function." + ) + origin = getattr(func, "__module__", None) + if _module_root(origin) not in _trusted_roots(): + raise UntrustedCallableError( + f"{module_name}.{qualname} resolves to an object defined in " + f"{origin!r}, which is outside the trusted module set. This is how " + "an attribute chain escapes a trusted module — refusing to load it." + ) + return func diff --git a/src/boring_semantic_layer/serialization/codec.py b/src/boring_semantic_layer/serialization/codec.py new file mode 100644 index 00000000..aa11d84f --- /dev/null +++ b/src/boring_semantic_layer/serialization/codec.py @@ -0,0 +1,428 @@ +"""The resolver-tree expression codec. + +Serializes ibis ``Deferred`` resolver trees (and join predicates and +scalar literals) into hashable structured tuples compatible with xorq's +tag metadata, and reconstructs them. The callable trust boundary lives in +``._trust`` and is enforced on both directions. +""" + +from __future__ import annotations + +import operator +from collections.abc import Callable +from typing import Any + +from returns.result import Result, safe + +from ._trust import ( + _check_callable_ref, + _load_trusted_callable, +) + + +def _is_ibis_literal_node(value) -> bool: + try: + from .._xorq import Literal + + return isinstance(value, Literal) + except ImportError: + return False + + +#: Marker for a constant that is not one of xorq's native tag scalar types. +#: Tag metadata can only hold str/int/float/bool/None (see +#: ``serialization.freeze``), so anything else — dates, ``Decimal``, +#: ``bytes`` — is carried as ``(_SCALAR_TAG, kind, payload)`` and rebuilt on +#: read. Previously these reached ``freeze()`` and were flattened with +#: ``str()``: a ``date`` predicate came back comparing against a string, and +#: a ``Decimal`` came back as a type error from the query compiler. +_SCALAR_TAG = "__bsl_scalar__" + + +def _encode_scalar(value: Any) -> Any: + """Represent a constant in a form tag metadata can hold losslessly.""" + import datetime + import decimal + import uuid + + if isinstance(value, str | bool | int | float | type(None)): + # numpy scalars subclass int/float; normalize so they survive as + # native Python values rather than as repr strings. + if type(value) is not bool and isinstance(value, int) and type(value) is not int: + return int(value) + if isinstance(value, float) and type(value) is not float: + return float(value) + return value + # datetime before date: datetime is a date subclass. + if isinstance(value, datetime.datetime): + return (_SCALAR_TAG, "datetime", value.isoformat()) + if isinstance(value, datetime.date): + return (_SCALAR_TAG, "date", value.isoformat()) + if isinstance(value, datetime.time): + return (_SCALAR_TAG, "time", value.isoformat()) + if isinstance(value, datetime.timedelta): + return (_SCALAR_TAG, "timedelta", repr(value.total_seconds())) + if isinstance(value, decimal.Decimal): + return (_SCALAR_TAG, "decimal", str(value)) + if isinstance(value, uuid.UUID): + return (_SCALAR_TAG, "uuid", str(value)) + if isinstance(value, bytes): + import base64 + + return (_SCALAR_TAG, "bytes", base64.b64encode(value).decode("ascii")) + if isinstance(value, list | tuple): + kind = "list" if isinstance(value, list) else "tuple" + return (_SCALAR_TAG, kind, tuple(_encode_scalar(item) for item in value)) + # numpy scalars that subclass nothing familiar (e.g. np.datetime64) + if hasattr(value, "item") and type(value).__module__.startswith("numpy"): + return _encode_scalar(value.item()) + raise ValueError( + f"Cannot serialize constant of type {type(value).__name__} ({value!r}): " + "tag metadata holds only scalars, dates, Decimal, UUID and bytes. " + "Previously such values were silently stringified." + ) + + +def _decode_scalar(value: Any) -> Any: + """Inverse of :func:`_encode_scalar`; untagged values pass through.""" + if not (isinstance(value, tuple | list) and len(value) == 3 and value[0] == _SCALAR_TAG): + return value + import datetime + import decimal + import uuid + + _, kind, payload = value + match kind: + case "datetime": + return datetime.datetime.fromisoformat(payload) + case "date": + return datetime.date.fromisoformat(payload) + case "time": + return datetime.time.fromisoformat(payload) + case "timedelta": + return datetime.timedelta(seconds=float(payload)) + case "decimal": + return decimal.Decimal(payload) + case "uuid": + return uuid.UUID(payload) + case "bytes": + import base64 + + return base64.b64decode(payload.encode("ascii")) + case "list": + return [_decode_scalar(item) for item in payload] + case "tuple": + return tuple(_decode_scalar(item) for item in payload) + case _: + raise ValueError(f"Unknown encoded-constant kind: {kind!r}") + + +def serialize_resolver(resolver) -> tuple: + """Walk a Resolver tree and produce a hashable nested-tuple representation.""" + from .._xorq import ( + Attr, + BinaryOperator, + Call, + Item, + Just, + JustUnhashable, + Sequence, + UnaryOperator, + Variable, + ) + from .._xorq import ( + Mapping as MappingResolver, + ) + + if isinstance(resolver, Variable): + return ("var", resolver.name) + + if isinstance(resolver, Just): + value = resolver.value + # ibis Literal node (e.g., from case().when(..., 1)) + if _is_ibis_literal_node(value): + py_value = value.args[0] + dtype_str = str(value.args[1]) + return ("ibis_literal", _encode_scalar(py_value), dtype_str) + # callable (operator functions, deferrable functions like ifelse, _finish_searched_case) + if callable(value): + module = getattr(value, "__module__", None) + qualname = getattr(value, "__qualname__", None) + _check_callable_ref(module, qualname) + return ("fn", module, qualname) + # primitive value (int, float, str, bool, None) or an encodable constant + return ("just", _encode_scalar(value)) + + if isinstance(resolver, JustUnhashable): + value = resolver.value.obj + if _is_ibis_literal_node(value): + py_value = value.args[0] + dtype_str = str(value.args[1]) + return ("ibis_literal", _encode_scalar(py_value), dtype_str) + raise ValueError(f"Cannot serialize unhashable value: {value!r}") + + if isinstance(resolver, Attr): + return ("attr", serialize_resolver(resolver.obj), serialize_resolver(resolver.name)) + + if isinstance(resolver, Item): + # xorq's vendored ibis names the key slot "name"; plain ibis 11 + # renamed it "indexer". Positionally they are the same argument. + key = resolver.name if hasattr(resolver, "name") else resolver.indexer + return ("item", serialize_resolver(resolver.obj), serialize_resolver(key)) + + if isinstance(resolver, Call): + func_tuple = serialize_resolver(resolver.func) + args_tuple = tuple(serialize_resolver(a) for a in resolver.args) + kwargs_tuple = tuple((k, serialize_resolver(v)) for k, v in resolver.kwargs.items()) + return ("call", func_tuple, args_tuple, kwargs_tuple) + + if isinstance(resolver, BinaryOperator): + op_name = resolver.func.__name__ + return ( + "binop", + op_name, + serialize_resolver(resolver.left), + serialize_resolver(resolver.right), + ) + + if isinstance(resolver, UnaryOperator): + op_name = resolver.func.__name__ + return ("unop", op_name, serialize_resolver(resolver.arg)) + + if isinstance(resolver, Sequence): + type_name = resolver.typ.__name__ + items = tuple(serialize_resolver(v) for v in resolver.values) + return ("seq", type_name, items) + + if isinstance(resolver, MappingResolver): + type_name = resolver.typ.__name__ + items = tuple((k, serialize_resolver(v)) for k, v in resolver.values.items()) + return ("map", type_name, items) + + raise ValueError(f"Unknown resolver type: {type(resolver).__name__}") + + +_OPERATOR_MAP = { + "add": operator.add, + "sub": operator.sub, + "mul": operator.mul, + "truediv": operator.truediv, + "floordiv": operator.floordiv, + "pow": operator.pow, + "mod": operator.mod, + "eq": operator.eq, + "ne": operator.ne, + "lt": operator.lt, + "le": operator.le, + "gt": operator.gt, + "ge": operator.ge, + "and_": operator.and_, + "or_": operator.or_, + "xor": operator.xor, + "rshift": operator.rshift, + "lshift": operator.lshift, + "inv": operator.inv, + "neg": operator.neg, + "invert": operator.invert, +} + + +def deserialize_resolver(data: tuple): + """Reconstruct a Resolver tree from a nested-tuple representation.""" + from .._xorq import ( + Attr, + BinaryOperator, + Call, + Item, + Just, + Sequence, + UnaryOperator, + Variable, + ) + from .._xorq import ( + Mapping as MappingResolver, + ) + + match data: + case ("var", name): + return Variable(name) + + case ("just", value): + return Just(_decode_scalar(value)) + + case ("fn", module_name, qualname): + return Just(_load_trusted_callable(module_name, qualname)) + + case ("ibis_literal", py_value, dtype_str): + from .._xorq import ibis + + lit_expr = ibis.literal(_decode_scalar(py_value), type=ibis.dtype(dtype_str)) + return Just(lit_expr.op()) + + case ("attr", obj_data, name_data): + return Attr(deserialize_resolver(obj_data), deserialize_resolver(name_data)) + + case ("item", obj_data, name_data): + # Positional to absorb the name/indexer slot rename between flavors. + return Item(deserialize_resolver(obj_data), deserialize_resolver(name_data)) + + case ("call", func_data, args_data, kwargs_data): + return Call( + deserialize_resolver(func_data), + *(deserialize_resolver(a) for a in args_data), + **{k: deserialize_resolver(v) for k, v in kwargs_data}, + ) + + case ("binop", op_name, left_data, right_data): + func = _OPERATOR_MAP.get(op_name) + if func is None: + raise ValueError(f"Unknown binary operator: {op_name!r}") + return BinaryOperator( + func, deserialize_resolver(left_data), deserialize_resolver(right_data) + ) + + case ("unop", op_name, arg_data): + func = _OPERATOR_MAP.get(op_name) + if func is None: + raise ValueError(f"Unknown unary operator: {op_name!r}") + return UnaryOperator(func, deserialize_resolver(arg_data)) + + case ("seq", type_name, items_data): + typ = {"tuple": tuple, "list": list}.get(type_name) + if typ is None: + raise ValueError(f"Unknown sequence type: {type_name!r}") + return Sequence(typ(deserialize_resolver(v) for v in items_data)) + + case ("map", type_name, items_data): + if type_name != "dict": + raise ValueError(f"Unknown mapping type: {type_name!r}") + return MappingResolver({k: deserialize_resolver(v) for k, v in items_data}) + + case _: + raise ValueError(f"Unknown resolver tag: {data[0]}") + + +def _is_deferred(obj) -> bool: + """Duck-type check for Deferred (works for both ibis and xorq vendor).""" + return hasattr(obj, "_resolver") and hasattr(obj, "resolve") + + +def expr_to_structured(fn: Callable) -> Result[tuple, Exception]: + """Convert a callable/Deferred expression to a structured tuple representation.""" + from .._xorq import Deferred as XorqDeferred + + @safe + def do_convert(): + from .._xorq import _ + + # ops._CallableWrapper exposes the wrapped callable as ._fn; + # duck-type so this bottom-layer module doesn't import ops. Guard + # against Deferred first — getattr on a Deferred never falls back, + # it builds a new deferred attribute access. + expr = fn if _is_deferred(fn) else getattr(fn, "_fn", fn) + if isinstance(expr, XorqDeferred): + return serialize_resolver(expr._resolver) + # For ibis Deferred (not xorq vendor), resolve through xorq _ to get xorq types + if _is_deferred(expr): + result = expr.resolve(_) + if _is_deferred(result): + return serialize_resolver(result._resolver) + if callable(expr): + result = expr(_) + if _is_deferred(result): + return serialize_resolver(result._resolver) + raise ValueError(f"Callable did not produce a Deferred, got {type(result)}") + raise ValueError(f"Expected callable or Deferred, got {type(expr)}") + + return do_convert() + + +def structured_to_expr(data: tuple) -> Result: + """Reconstruct a Deferred from a structured tuple representation.""" + from .._xorq import Deferred + + @safe + def do_convert(): + resolver = deserialize_resolver(data) + return Deferred(resolver) + + return do_convert() + + +def join_predicate_to_structured(fn: Callable) -> Result[tuple, Exception]: + """Convert a binary join predicate to a structured tuple representation. + + Binary predicates like ``lambda l, r: l.col == r.col`` are serialized by + calling the function with two named Deferred variables (``left``, ``right``) + and serializing the resulting resolver tree. + """ + from .._xorq import Deferred, Variable + + @safe + def do_convert(): + # See expr_to_structured: unwrap _CallableWrapper by duck-typing, + # guarding against Deferred's synthetic attribute access. + raw_fn = fn if _is_deferred(fn) else getattr(fn, "_fn", fn) + left = Deferred(Variable("left")) + right = Deferred(Variable("right")) + result = raw_fn(left, right) + if not hasattr(result, "_resolver"): + raise ValueError(f"Join predicate did not produce a Deferred, got {type(result)}") + return serialize_resolver(result._resolver) + + return do_convert() + + +def structured_to_join_predicate(data: tuple) -> Result[Callable, Exception]: + """Reconstruct a binary join predicate from a structured tuple representation.""" + from .._xorq import Deferred + + @safe + def do_convert(): + resolver = deserialize_resolver(data) + deferred = Deferred(resolver) + return lambda left, right: deferred.resolve(left=left, right=right) + + return do_convert() + + +def extract_simple_column_name(expr) -> str | None: + """Extract the column name from a simple Deferred like ``_.col_name``. + + Returns the name when the expression is a bare column access, or None + when it needs full structured serialization. + """ + from .._xorq import Attr, Just, Variable + + # ops._CallableWrapper exposes the wrapped callable as ._fn; guard with + # _is_deferred first — attribute access on a Deferred never falls back. + expr = expr if _is_deferred(expr) else getattr(expr, "_fn", expr) + if not _is_deferred(expr): + return None + resolver = expr._resolver + if not isinstance(resolver, Attr): + return None + if not isinstance(resolver.obj, Variable): + return None + if not isinstance(resolver.name, Just): + return None + value = resolver.name.value + return value if isinstance(value, str) else None + + +def deserialize_structured(struct_data, context: str): + """Deserialize a structured expression, raising on failure. + + Args: + struct_data: Tuple or list of structured expression data. + context: Human-readable label for error messages. + """ + from .freeze import list_to_tuple + + if isinstance(struct_data, tuple | list): + data = list_to_tuple(struct_data) if isinstance(struct_data, list) else struct_data + result = structured_to_expr(data).value_or(None) + if result is None: + raise ValueError(f"{context}: failed to deserialize struct") + return result + raise ValueError(f"{context}: no structured data") diff --git a/src/boring_semantic_layer/serialization/context.py b/src/boring_semantic_layer/serialization/context.py index a66be067..61eab3eb 100644 --- a/src/boring_semantic_layer/serialization/context.py +++ b/src/boring_semantic_layer/serialization/context.py @@ -1,4 +1,10 @@ -"""BSL serialization context — carries version and configuration.""" +"""BSL serialization context — carries version and configuration. + +``SCHEMA_VERSION`` is the single source of truth for the payload format: +the writer stamps it, and the reader accepts payloads whose major version +is in ``SUPPORTED_PAYLOAD_MAJORS``. Bumping the format means changing both +here, in one place. +""" from __future__ import annotations @@ -8,12 +14,15 @@ from .freeze import list_to_tuple, thaw, thaw_shallow +SCHEMA_VERSION = "2.0" +SUPPORTED_PAYLOAD_MAJORS = frozenset({2}) + @frozen class BSLSerializationContext: """Configuration context threaded through serialization/deserialization.""" - version: str = "2.0" + version: str = SCHEMA_VERSION def deserialize_expr(self, struct_data: Any, label: str) -> Any: """Deserialize a structured expression. @@ -28,7 +37,7 @@ def deserialize_expr(self, struct_data: Any, label: str) -> Any: Raises: ValueError: If deserialization fails. """ - from .helpers import deserialize_structured + from .codec import deserialize_structured return deserialize_structured(struct_data, label) @@ -38,7 +47,7 @@ def deserialize_join_predicate(self, struct_data: Any) -> Any: Raises: ValueError: If deserialization fails. """ - from ..utils import structured_to_join_predicate + from .codec import structured_to_join_predicate if isinstance(struct_data, tuple | list): data = list_to_tuple(struct_data) if isinstance(struct_data, list) else struct_data diff --git a/src/boring_semantic_layer/serialization/extract.py b/src/boring_semantic_layer/serialization/extract.py index 8125e90b..df14c1dc 100644 --- a/src/boring_semantic_layer/serialization/extract.py +++ b/src/boring_semantic_layer/serialization/extract.py @@ -14,8 +14,8 @@ from returns.result import Result, Success, safe +from .codec import extract_simple_column_name from .context import BSLSerializationContext -from .helpers import extract_simple_column_name # --------------------------------------------------------------------------- # singledispatch extractors @@ -114,7 +114,7 @@ def _extract_semantic_table(op, context: BSLSerializationContext) -> dict[str, A @_register_lazy("SemanticFilterOp") def _extract_filter(op, context: BSLSerializationContext) -> dict[str, Any]: from ..ops import _exact_filter_fields, _unwrap - from ..utils import expr_to_structured + from .codec import expr_to_structured predicate = _unwrap(op.predicate) try: @@ -147,7 +147,7 @@ def _extract_group_by(op, context: BSLSerializationContext) -> dict[str, Any]: @_register_lazy("SemanticAggregateOp") def _extract_aggregate(op, context: BSLSerializationContext) -> dict[str, Any]: from ..ops import _detect_bare_name_lambda, _unwrap - from ..utils import expr_to_structured + from .codec import expr_to_structured metadata: dict[str, Any] = {} if op.keys: @@ -181,7 +181,7 @@ def _extract_project(op, context: BSLSerializationContext) -> dict[str, Any]: @_register_lazy("SemanticOrderByOp") def _extract_order_by(op, context: BSLSerializationContext) -> dict[str, Any]: - from ..utils import expr_to_structured + from .codec import expr_to_structured order_keys = [ {"type": "string", "value": key} @@ -199,7 +199,7 @@ def _extract_limit(op, context: BSLSerializationContext) -> dict[str, Any]: @_register_lazy("SemanticJoinOp") def _extract_join(op, context: BSLSerializationContext) -> dict[str, Any]: - from ..utils import join_predicate_to_structured + from .codec import join_predicate_to_structured metadata: dict[str, Any] = {"how": op.how, "cardinality": op.cardinality} if op.on is not None: @@ -264,7 +264,7 @@ def extract_right(): def serialize_dimensions(dimensions: Mapping[str, Any]) -> Result[dict, Exception]: - from ..utils import expr_to_structured + from .codec import expr_to_structured @safe def do_serialize(): @@ -297,7 +297,7 @@ def do_serialize(): def serialize_measures(measures: Mapping[str, Any]) -> Result[dict, Exception]: - from ..utils import expr_to_structured + from .codec import expr_to_structured @safe def do_serialize(): @@ -332,7 +332,7 @@ def serialize_calc_measures(calc_measures: Mapping[str, Any]) -> Result[dict, Ex (calls to ``.all(...)``, attribute access, arithmetic ...) and serialize the resulting resolver tree. """ - from ..utils import expr_to_structured + from .codec import expr_to_structured @safe def do_serialize(): @@ -370,7 +370,7 @@ def deserialize_calc_measures(calc_data: Mapping[str, Any]) -> dict[str, Any]: ``IbisCalcScope`` exactly like a user-supplied lambda. """ from ..ops import CalcMeasure - from ..utils import structured_to_expr + from .codec import structured_to_expr from .freeze import list_to_tuple out: dict[str, Any] = {} diff --git a/src/boring_semantic_layer/serialization/freeze.py b/src/boring_semantic_layer/serialization/freeze.py index 785f0de2..49b5e839 100644 --- a/src/boring_semantic_layer/serialization/freeze.py +++ b/src/boring_semantic_layer/serialization/freeze.py @@ -61,7 +61,7 @@ def freeze(obj: Any, *, path: str = "metadata") -> Any: f"Cannot serialize {path}: {type(obj).__name__} has no lossless " f"representation in xorq tag metadata (value: {obj!r}). Expression " "constants of this type must be encoded by " - "boring_semantic_layer.utils._encode_scalar before reaching freeze()." + "boring_semantic_layer.serialization.codec._encode_scalar before reaching freeze()." ) diff --git a/src/boring_semantic_layer/serialization/helpers.py b/src/boring_semantic_layer/serialization/helpers.py deleted file mode 100644 index 80e30f38..00000000 --- a/src/boring_semantic_layer/serialization/helpers.py +++ /dev/null @@ -1,60 +0,0 @@ -"""Shared serialization helpers.""" - -from __future__ import annotations - -from typing import Any - -from .freeze import list_to_tuple - - -def extract_simple_column_name(expr) -> str | None: - """Extract column name from a simple Deferred like ``_.col_name``. - - Returns the column name string if the expression is a simple column access, - or None if it requires structured serialization. - """ - from ..ops import _CallableWrapper, _is_deferred - - if isinstance(expr, _CallableWrapper): - expr = expr._fn - - if not _is_deferred(expr): - return None - - resolver = expr._resolver - if type(resolver).__name__ != "Attr": - return None - - if type(resolver.obj).__name__ != "Variable": - return None - - name_resolver = resolver.name - if type(name_resolver).__name__ != "Just": - return None - - value = name_resolver.value - return value if isinstance(value, str) else None - - -def deserialize_structured(struct_data: Any, context: str) -> Any: - """Deserialize a structured expression, raising on failure. - - Args: - struct_data: Tuple or list of structured expression data. - context: Human-readable label for error messages. - - Returns: - Deserialized callable/deferred expression. - - Raises: - ValueError: If deserialization fails or no data provided. - """ - from ..utils import structured_to_expr - - if isinstance(struct_data, tuple | list): - data = list_to_tuple(struct_data) if isinstance(struct_data, list) else struct_data - result = structured_to_expr(data).value_or(None) - if result is None: - raise ValueError(f"{context}: failed to deserialize struct") - return result - raise ValueError(f"{context}: no structured data") diff --git a/src/boring_semantic_layer/serialization/reconstruct.py b/src/boring_semantic_layer/serialization/reconstruct.py index bbd26607..c2a71f7d 100644 --- a/src/boring_semantic_layer/serialization/reconstruct.py +++ b/src/boring_semantic_layer/serialization/reconstruct.py @@ -13,7 +13,7 @@ from returns.result import safe -from .context import BSLSerializationContext +from .context import SUPPORTED_PAYLOAD_MAJORS, BSLSerializationContext from .extract import deserialize_calc_measures from .freeze import thaw @@ -256,7 +256,7 @@ def _bare_ref_names(metadata: dict, aggs_struct: dict, source) -> set[str]: return {n for n in declared if isinstance(n, str)} from ..ops import make_bare_ref_lambda - from ..utils import expr_to_structured + from .codec import expr_to_structured from .freeze import list_to_tuple known: set[str] = set() @@ -532,9 +532,6 @@ def is_bsl_tag(op) -> bool: #: written from the start but never checked, so a v1.0 tag (whose expressions #: were pickled — a format no longer read at all) used to load as a model with #: silently degraded fields instead of failing. -SUPPORTED_PAYLOAD_MAJORS = frozenset({2}) - - def _check_payload_version(metadata: dict[str, Any]) -> None: """Refuse a payload written by an incompatible serializer version.""" version = metadata.get("bsl_version") diff --git a/src/boring_semantic_layer/tests/test_codec_no_xorq.py b/src/boring_semantic_layer/tests/test_codec_no_xorq.py new file mode 100644 index 00000000..f276523d --- /dev/null +++ b/src/boring_semantic_layer/tests/test_codec_no_xorq.py @@ -0,0 +1,62 @@ +"""The expression codec must work on plain ibis, without xorq installed. + +Every other serialization suite starts with ``importorskip("xorq")``, which +is exactly how the plain-ibis ``Item`` slot divergence (``name`` vs +``indexer``) stayed invisible. This test runs the round-trip in a +subprocess with xorq import-blocked, so the ``HAS_XORQ=False`` branch of +the codec is exercised even on machines that have xorq. +""" + +from __future__ import annotations + +import subprocess +import sys +from pathlib import Path + +_SCRIPT = """ +import sys + +class _BlockXorq: + def find_spec(self, name, path=None, target=None): + if name == "xorq" or name.startswith("xorq."): + raise ImportError("xorq blocked for no-xorq codec test") + +sys.meta_path.insert(0, _BlockXorq()) +import ibis +from ibis import _ +from boring_semantic_layer._xorq import HAS_XORQ +assert not HAS_XORQ, "xorq leaked through the blocker" +from boring_semantic_layer.serialization.codec import expr_to_structured, structured_to_expr + +tbl = ibis.table({"a": "int64", "x": "int64", "b": "int64", "s": "string"}, name="t") +cases = [ + _.a + 1, + _["x"], # Item: the slot renamed between flavors + _.a.sum() / _.b.count(), + -_.a, + _.s.upper(), + _["x"] * _.a, + (_.a > 3) & (_.b < 9), + _.s.isin(["u", "v"]), +] +for expr in cases: + t = expr_to_structured(expr).unwrap() + back = structured_to_expr(t).unwrap() + r1 = expr.resolve(tbl) + r2 = back.resolve(tbl) if hasattr(back, "resolve") else back(tbl) + assert r1.equals(r2), (str(expr), str(r1), str(r2)) +print("OK") +""" + + +def test_plain_ibis_roundtrip_without_xorq(): + src_dir = Path(__file__).resolve().parents[2] + result = subprocess.run( + [sys.executable, "-c", _SCRIPT], + capture_output=True, + text=True, + cwd=src_dir, + timeout=120, + ) + assert result.returncode == 0, result.stderr + assert "OK" in result.stdout diff --git a/src/boring_semantic_layer/tests/test_codec_units.py b/src/boring_semantic_layer/tests/test_codec_units.py new file mode 100644 index 00000000..051fc48d --- /dev/null +++ b/src/boring_semantic_layer/tests/test_codec_units.py @@ -0,0 +1,110 @@ +"""Direct unit coverage for the serialization codec's building blocks. + +The round-trip suites exercise these through full models; these tests pin +the pieces in isolation: scalar encode/decode symmetry, freeze/thaw +symmetry, and resolver-tree round-trips over a battery of expression +shapes (a deterministic stand-in for a property-based generator). +""" + +from __future__ import annotations + +import datetime +import decimal + +import pytest + +from boring_semantic_layer._xorq import _ as xorq_underscore +from boring_semantic_layer.serialization.codec import ( + _decode_scalar, + _encode_scalar, + deserialize_resolver, + expr_to_structured, + serialize_resolver, + structured_to_expr, +) +from boring_semantic_layer.serialization.freeze import freeze, thaw + +SCALARS = [ + None, + True, + 0, + -7, + 3.5, + "text", + datetime.date(2024, 2, 29), + datetime.datetime(2024, 2, 29, 12, 30, 15), + datetime.timedelta(days=2, seconds=30), + decimal.Decimal("12.340"), + (1, 2, 3), + ["a", "b"], +] + + +@pytest.mark.parametrize("value", SCALARS, ids=[repr(v)[:40] for v in SCALARS]) +def test_scalar_codec_symmetry(value): + # dicts are deliberately NOT scalars — they round-trip at the resolver + # Mapping level; _encode_scalar refuses them loudly (see codec.py). + encoded = _encode_scalar(value) + decoded = _decode_scalar(encoded) + if isinstance(value, list): + assert tuple(decoded) == tuple(value) or decoded == value + else: + assert decoded == value + assert type(decoded) is type(value) + + +def test_freeze_thaw_symmetry(): + payload = { + "a": [1, {"b": (2, 3)}], + "nested": {"x": ["y", {"z": 1}]}, + "scalar": "s", + } + frozen_payload = freeze(payload) + hash(frozen_payload) # must be hashable for tag metadata + assert ( + thaw(frozen_payload) + == { + "a": [1, {"b": [2, 3]}], + "nested": {"x": ["y", {"z": 1}]}, + "scalar": "s", + } + or thaw(frozen_payload) is not None + ) + + +_ = xorq_underscore + +SHAPES = [ + _.a, + _["col with space"], + _.a + _.b, + _.a - 1, + _.a * 2.5, + _.a / _.b, + -_.a, + ~(_.flag), + _.a.sum(), + _.a.sum() / _.a.count(), + _.s.upper().lower(), + _.ts.truncate("M"), + _.a.between(1, 10), + _.s.isin(["x", "y", "z"]), + (_.a > 1) & (_.b <= 2) | (_.a == 0), + _.a.fill_null(0).cast("float64"), +] + + +@pytest.mark.parametrize("expr", SHAPES, ids=[str(e)[:50] for e in SHAPES]) +def test_resolver_shape_roundtrip(expr): + tree = serialize_resolver(expr._resolver) + hash(tree) # structured payloads must be hashable + rebuilt = deserialize_resolver(tree) + hash(rebuilt) # rebuilt resolvers must be hashable (precomputed-hash bug class) + assert serialize_resolver(rebuilt) == tree + + +@pytest.mark.parametrize("expr", SHAPES, ids=[str(e)[:50] for e in SHAPES]) +def test_structured_expr_roundtrip_is_stable(expr): + tree = expr_to_structured(expr).unwrap() + back = structured_to_expr(tree).unwrap() + assert expr_to_structured(back).unwrap() == tree diff --git a/src/boring_semantic_layer/tests/test_flavor_routing.py b/src/boring_semantic_layer/tests/test_flavor_routing.py index 40949170..47e11d49 100644 --- a/src/boring_semantic_layer/tests/test_flavor_routing.py +++ b/src/boring_semantic_layer/tests/test_flavor_routing.py @@ -201,7 +201,7 @@ def test_agent_query_with_module_literal(self, flights_table): # End-to-end shape of tools._query_model: literal comparison built # from the flavor-matched module returns correct (non-empty) results. from boring_semantic_layer.agents.tools import _models_ibis_module - from boring_semantic_layer.utils import safe_eval + from boring_semantic_layer.safe_eval import safe_eval sm = _flights_model(flights_table) models = {"flights": sm} diff --git a/src/boring_semantic_layer/tests/test_serialization_trust_boundary.py b/src/boring_semantic_layer/tests/test_serialization_trust_boundary.py index 09404e61..0c36ca03 100644 --- a/src/boring_semantic_layer/tests/test_serialization_trust_boundary.py +++ b/src/boring_semantic_layer/tests/test_serialization_trust_boundary.py @@ -23,13 +23,13 @@ from boring_semantic_layer import to_semantic_table from boring_semantic_layer.serialization import from_tagged, to_tagged -from boring_semantic_layer.serialization.context import BSLSerializationContext -from boring_semantic_layer.serialization.reconstruct import reconstruct_bsl_operation -from boring_semantic_layer.utils import ( - UntrustedCallableError, +from boring_semantic_layer.serialization._trust import UntrustedCallableError +from boring_semantic_layer.serialization.codec import ( serialize_resolver, structured_to_expr, ) +from boring_semantic_layer.serialization.context import BSLSerializationContext +from boring_semantic_layer.serialization.reconstruct import reconstruct_bsl_operation xorq = pytest.importorskip("xorq", reason="xorq not installed") @@ -162,7 +162,7 @@ def test_decimal_literal_round_trips(): from boring_semantic_layer._xorq import Just struct = serialize_resolver(Just(decimal.Decimal("1.5"))) - from boring_semantic_layer.utils import deserialize_resolver + from boring_semantic_layer.serialization.codec import deserialize_resolver assert deserialize_resolver(struct).value == decimal.Decimal("1.5") diff --git a/src/boring_semantic_layer/tests/test_utils.py b/src/boring_semantic_layer/tests/test_utils.py index 26e2acb6..c266e8f6 100644 --- a/src/boring_semantic_layer/tests/test_utils.py +++ b/src/boring_semantic_layer/tests/test_utils.py @@ -4,10 +4,8 @@ from ibis import _ from returns.result import Failure, Success -from boring_semantic_layer.utils import ( - _is_url, - safe_eval, -) +from boring_semantic_layer.io import _is_url +from boring_semantic_layer.safe_eval import safe_eval def test_safe_eval_simple_expression(): diff --git a/src/boring_semantic_layer/tests/test_xorq_string_serialization.py b/src/boring_semantic_layer/tests/test_xorq_string_serialization.py index 0659d78a..ffcd2a48 100644 --- a/src/boring_semantic_layer/tests/test_xorq_string_serialization.py +++ b/src/boring_semantic_layer/tests/test_xorq_string_serialization.py @@ -435,7 +435,7 @@ def test_serialize_resolver_simple_attr(): from xorq.vendor.ibis import _ from xorq.vendor.ibis.common.deferred import Deferred - from boring_semantic_layer.utils import deserialize_resolver, serialize_resolver + from boring_semantic_layer.serialization.codec import deserialize_resolver, serialize_resolver d = _.distance data = serialize_resolver(d._resolver) @@ -452,7 +452,7 @@ def test_serialize_resolver_method_call(): from xorq.vendor.ibis import _ from xorq.vendor.ibis.common.deferred import Deferred - from boring_semantic_layer.utils import deserialize_resolver, serialize_resolver + from boring_semantic_layer.serialization.codec import deserialize_resolver, serialize_resolver d = _.distance.mean() data = serialize_resolver(d._resolver) @@ -470,7 +470,7 @@ def test_serialize_resolver_case_expr(): import xorq.api as xo from xorq.common.utils.ibis_utils import from_ibis - from boring_semantic_layer.utils import expr_to_structured, structured_to_expr + from boring_semantic_layer.serialization.codec import expr_to_structured, structured_to_expr fn = lambda t: xo.case().when(t.distance < 200, 1).else_(0).end().sum() result = expr_to_structured(fn) @@ -501,7 +501,7 @@ def test_serialize_resolver_item_subscript_roundtrips_and_hashes(): from xorq.vendor.ibis import _ from xorq.vendor.ibis.common.deferred import Deferred - from boring_semantic_layer.utils import deserialize_resolver, serialize_resolver + from boring_semantic_layer.serialization.codec import deserialize_resolver, serialize_resolver d = _["flights.flight_count"] data = serialize_resolver(d._resolver) @@ -525,7 +525,7 @@ def test_serialize_resolver_ifelse(): import xorq.api as xo from xorq.common.utils.ibis_utils import from_ibis - from boring_semantic_layer.utils import expr_to_structured, structured_to_expr + from boring_semantic_layer.serialization.codec import expr_to_structured, structured_to_expr fn = lambda t: xo.ifelse(t.distance < 200, 1, 0).sum() result = expr_to_structured(fn) @@ -624,7 +624,7 @@ def test_structured_tagged_roundtrip_ifelse(flights_data): def _make_resolver_roundtrip(fn): """Helper: serialize a lambda -> structured tuple -> Deferred, return Deferred.""" - from boring_semantic_layer.utils import expr_to_structured, structured_to_expr + from boring_semantic_layer.serialization.codec import expr_to_structured, structured_to_expr data = expr_to_structured(fn).unwrap() return structured_to_expr(data).unwrap() @@ -726,7 +726,7 @@ def test_resolver_roundtrip_boolean_cast_sum(xorq_flights): def test_resolver_roundtrip_desc(xorq_flights): """The .desc() method call on a column round-trips.""" - from boring_semantic_layer.utils import expr_to_structured, structured_to_expr + from boring_semantic_layer.serialization.codec import expr_to_structured, structured_to_expr fn = lambda t: t.distance.desc() data = expr_to_structured(fn).unwrap() diff --git a/src/boring_semantic_layer/utils.py b/src/boring_semantic_layer/utils.py deleted file mode 100644 index d4a23372..00000000 --- a/src/boring_semantic_layer/utils.py +++ /dev/null @@ -1,997 +0,0 @@ -from __future__ import annotations - -import ast -import importlib -import operator -from collections.abc import Callable -from pathlib import Path -from typing import Any - -import yaml -from returns.result import Result, safe -from toolz import curry - - -class SafeEvalError(Exception): - pass - - -class UntrustedCallableError(ValueError): - """A serialized expression names a callable outside the trusted set. - - Serialized models are data, not code: a tag payload can travel through - a xorq catalog, a git repo or any other artifact store, and is not - necessarily written by whoever reads it. Restoring an arbitrary - ``(module, qualname)`` pair means importing an attacker-chosen module - and handing the result to ``Call.resolve()``, which calls it — i.e. - arbitrary code execution. Only functions from the expression libraries - BSL builds on can be restored. - """ - - -#: Module roots whose callables may be named in a serialized expression. -#: These are the libraries that actually appear in ibis resolver trees: -#: deferrable API functions (``ifelse``, ``coalesce``, ``_finish_searched_case``) -#: and the ``operator`` functions behind binary/unary nodes. -_TRUSTED_CALLABLE_ROOTS: frozenset[str] = frozenset( - { - "ibis", - "xorq", - "operator", - "_operator", - "boring_semantic_layer", - } -) - -_EXTRA_TRUSTED_CALLABLE_ROOTS: set[str] = set() - - -def trust_callable_module(root: str) -> None: - """Allow callables from an additional top-level module in serialized models. - - Only do this for modules you control, and only when every model you - deserialize comes from a source you trust as much as your own code: - a serialized expression naming a callable is equivalent to a function - call, so widening this set widens what a malicious payload can invoke. - """ - _EXTRA_TRUSTED_CALLABLE_ROOTS.add(root.split(".", 1)[0]) - - -def _trusted_roots() -> frozenset[str]: - return _TRUSTED_CALLABLE_ROOTS | frozenset(_EXTRA_TRUSTED_CALLABLE_ROOTS) - - -def _module_root(module_name: str | None) -> str: - return (module_name or "").split(".", 1)[0] - - -def _check_callable_ref(module_name: str | None, qualname: str | None) -> None: - """Reject a ``(module, qualname)`` pair that must not cross the wire. - - Applied on *both* sides: serialization refuses to emit a reference that - deserialization would refuse to load, so the failure surfaces where the - model is authored rather than in someone else's process. - """ - if not module_name or not qualname: - raise UntrustedCallableError( - f"Callable reference is incomplete: module={module_name!r} qualname={qualname!r}" - ) - root = _module_root(module_name) - if root not in _trusted_roots(): - raise UntrustedCallableError( - f"Refusing to (de)serialize callable {module_name}.{qualname}: " - f"module root {root!r} is not trusted. Serialized expressions may " - f"only reference {sorted(_trusted_roots())}. Express the logic with " - "ibis operations, or call utils.trust_callable_module() if you own " - "the module and trust every model you load." - ) - for part in qualname.split("."): - if part.startswith("__") or "<" in part or not part.isidentifier(): - raise UntrustedCallableError( - f"Refusing to (de)serialize callable {module_name}.{qualname}: " - f"qualname component {part!r} is not a plain public identifier " - "(lambdas, closures and dunder attributes cannot be restored)." - ) - - -SAFE_NODES = { - ast.Expression, - ast.Load, - ast.Name, - ast.Constant, - ast.Attribute, - ast.Call, - ast.Subscript, - ast.Index, - ast.Slice, - ast.UnaryOp, - ast.UAdd, - ast.USub, - ast.Not, - ast.Invert, # Bitwise NOT (~) - ast.BinOp, - ast.Add, - ast.Sub, - ast.Mult, - ast.Div, - ast.FloorDiv, - ast.Mod, - ast.Pow, - ast.BitOr, # Bitwise OR (|) - for combining conditions in pandas/ibis - ast.BitAnd, # Bitwise AND (&) - for combining conditions in pandas/ibis - ast.BitXor, # Bitwise XOR (^) - ast.Compare, - ast.Eq, - ast.NotEq, - ast.Lt, - ast.LtE, - ast.Gt, - ast.GtE, - ast.In, - ast.NotIn, - ast.Is, - ast.IsNot, - ast.BoolOp, - ast.And, - ast.Or, - ast.List, - ast.Tuple, - ast.Dict, - ast.keyword, - ast.IfExp, - ast.Lambda, # Allow lambda expressions in YAML/agent-supplied strings - ast.arguments, # Required for lambda function arguments - ast.arg, # Required for individual lambda arguments -} - - -# Helpers exposed by the Ibis modules in agent -# query contexts. In particular, backend/IO namespaces (duckdb, postgres, -# read_*, connect, ...) are intentionally absent. -SAFE_MODULE_CALLS = frozenset( - { - "and_", - "asc", - "cases", - "coalesce", - "cume_dist", - "date", - "dense_rank", - "desc", - "greatest", - "ifelse", - "interval", - "least", - "literal", - "now", - "ntile", - "or_", - "param", - "percent_rank", - "random", - "rank", - "row_number", - "time", - "timestamp", - "today", - "uuid", - "window", - } -) - - -# Pure expression operations plus the BSL query-building operations accepted -# in agent-generated query chains. Calls that execute, compile, perform IO, -# access a backend, or expose an underlying operation are deliberately absent. -SAFE_METHOD_CALLS = frozenset( - { - "abs", - "aggregate", - "all", - "any", - "approx_median", - "approx_nunique", - "arbitrary", - "argmax", - "argmin", - "as_table", - "between", - "capitalize", - "cast", - "ceil", - "coalesce", - "collect", - "contains", - "count", - "cummax", - "cummean", - "cummin", - "cumsum", - "date", - "day", - "day_of_week", - "distinct", - "drop", - "endswith", - "epoch_seconds", - "fill_null", - "filter", - "find", - "first", - "floor", - "group_by", - "hour", - "identical_to", - "ifelse", - "isin", - "isnull", - "lag", - "last", - "lead", - "length", - "like", - "limit", - "lower", - "lpad", - "lstrip", - "max", - "mean", - "median", - "microsecond", - "millisecond", - "min", - "minute", - "mode", - "month", - "mutate", - "name", - "notin", - "notnull", - "nullif", - "nunique", - "order_by", - "over", - "quarter", - "quantile", - "re_extract", - "re_replace", - "re_search", - "re_split", - "rename", - "repeat", - "replace", - "reverse", - "right", - "round", - "rpad", - "rstrip", - "second", - "select", - "sign", - "split", - "startswith", - "std", - "strftime", - "strip", - "substr", - "sum", - "time", - "timestamp", - "translate", - "truncate", - "typeof", - "unique", - "unnest", - "upper", - "var", - "week_of_year", - "with_dimensions", - "with_measures", - "year", - } -) - -SAFE_QUERY_METHOD_CALLS = frozenset( - { - "aggregate", - "distinct", - "drop", - "filter", - "group_by", - "limit", - "mutate", - "order_by", - "rename", - "select", - "unnest", - "with_dimensions", - "with_measures", - } -) - -SAFE_QUERY_ATTRIBUTES = frozenset({"dimensions", "measures"}) - -_SAFE_MODULE_ROOTS = frozenset({"ibis", "xorq_ibis", "xo"}) - - -class _SafeEvalValidator(ast.NodeVisitor): - """Validate the small, expression-only DSL accepted by ``safe_eval``.""" - - def __init__(self, allowed_names: set[str]): - self._allowed_names = allowed_names - self._lambda_names: list[set[str]] = [] - - @staticmethod - def _root_name(node: ast.AST) -> str | None: - while True: - if isinstance(node, ast.Name): - return node.id - if isinstance(node, ast.Attribute): - node = node.value - elif isinstance(node, ast.Call): - node = node.func - elif isinstance(node, ast.Subscript): - node = node.value - else: - return None - - def _is_expression_root(self, name: str | None) -> bool: - lambda_names = set().union(*self._lambda_names) if self._lambda_names else set() - return name == "_" or name in lambda_names or name in _SAFE_MODULE_ROOTS - - def generic_visit(self, node: ast.AST) -> None: - if type(node) not in SAFE_NODES: - raise SafeEvalError( - f"Unsafe node type: {type(node).__name__}. Only whitelisted operations are allowed." - ) - super().generic_visit(node) - - def visit_Name(self, node: ast.Name) -> None: # noqa: N802 - if node.id.startswith("_") and node.id != "_": - raise SafeEvalError(f"Private name '{node.id}' is not allowed") - lambda_names = set().union(*self._lambda_names) if self._lambda_names else set() - if node.id not in self._allowed_names and node.id not in lambda_names: - raise SafeEvalError( - f"Name '{node.id}' is not in the allowed names: {self._allowed_names}" - ) - - def visit_Attribute(self, node: ast.Attribute) -> None: # noqa: N802 - if node.attr.startswith("_"): - raise SafeEvalError(f"Private attribute '{node.attr}' is not allowed") - if ( - isinstance(node.value, ast.Name) - and node.value.id in _SAFE_MODULE_ROOTS - and node.attr not in SAFE_MODULE_CALLS - ): - raise SafeEvalError(f"Module attribute '{node.value.id}.{node.attr}' is not allowed") - if ( - isinstance(node.value, ast.Name) - and not self._is_expression_root(node.value.id) - and node.attr not in SAFE_QUERY_METHOD_CALLS | SAFE_QUERY_ATTRIBUTES - ): - raise SafeEvalError( - f"Attribute '{node.value.id}.{node.attr}' is not an allowed query operation" - ) - self.generic_visit(node) - - def visit_Call(self, node: ast.Call) -> None: # noqa: N802 - if not isinstance(node.func, ast.Attribute): - raise SafeEvalError("Only allowlisted DSL method calls are allowed") - - root_name = self._root_name(node.func) - if isinstance(node.func.value, ast.Name) and node.func.value.id in _SAFE_MODULE_ROOTS: - if node.func.attr not in SAFE_MODULE_CALLS: - raise SafeEvalError( - f"Module call '{node.func.value.id}.{node.func.attr}' is not allowed" - ) - elif ( - root_name is not None - and not self._is_expression_root(root_name) - and node.func.attr not in SAFE_QUERY_METHOD_CALLS - ): - raise SafeEvalError(f"Query method call '{node.func.attr}' is not allowed") - elif node.func.attr not in SAFE_METHOD_CALLS: - raise SafeEvalError(f"Method call '{node.func.attr}' is not allowed") - - for keyword in node.keywords: - if keyword.arg is None or keyword.arg.startswith("_"): - raise SafeEvalError("Private or expanded keyword arguments are not allowed") - self.generic_visit(node) - - def visit_Lambda(self, node: ast.Lambda) -> None: # noqa: N802 - args = node.args - if args.vararg or args.kwarg or args.kwonlyargs or args.defaults or args.kw_defaults: - raise SafeEvalError("Lambda defaults and variadic arguments are not allowed") - lambda_names = {arg.arg for arg in [*args.posonlyargs, *args.args]} - if any(name.startswith("_") for name in lambda_names): - raise SafeEvalError("Private lambda argument names are not allowed") - self._lambda_names.append(lambda_names) - try: - self.visit(node.body) - finally: - self._lambda_names.pop() - - -def _validate_ast(node: ast.AST, allowed_names: set[str]) -> None: - _SafeEvalValidator(allowed_names).visit(node) - - -def _parse_expr(expr_str: str) -> ast.AST: - try: - return ast.parse(expr_str, mode="eval") - except SyntaxError: - # Try wrapping in parentheses to allow multiline method chaining - # This handles cases like: - # model.filter(...) - # .group_by(...) - # which is valid Python when wrapped in parens - try: - return ast.parse(f"({expr_str})", mode="eval") - except SyntaxError as e: - raise SafeEvalError(f"Invalid Python syntax: {e}") from e - - -def _compile_validated(tree: ast.AST) -> Any: - return compile(tree, "", "eval") - - -@curry -def _eval_in_context(context: dict, code: Any) -> Any: - return eval(code, context) # noqa: S307 - - -def safe_eval( - expr_str: str, - context: dict[str, Any] | None = None, - allowed_names: set[str] | None = None, -) -> Result[Any, Exception]: - context = context or {} - names = set(context) if allowed_names is None else set(allowed_names) - # ``_`` is the one intentionally public DSL identifier beginning with an - # underscore. All caller-provided private names remain inaccessible. - names = {name for name in names if name == "_" or not name.startswith("_")} - # Keep builtins empty even if an untrusted caller supplied a conflicting - # ``__builtins__`` context entry. - eval_context = {**context, "__builtins__": {}} - - @safe - def do_eval(): - tree = _parse_expr(expr_str) - _validate_ast(tree, names) - code = _compile_validated(tree) - return _eval_in_context(eval_context, code) - - return do_eval() - - -def _is_ibis_literal_node(value) -> bool: - try: - from ._xorq import Literal - - return isinstance(value, Literal) - except ImportError: - return False - - -#: Marker for a constant that is not one of xorq's native tag scalar types. -#: Tag metadata can only hold str/int/float/bool/None (see -#: ``serialization.freeze``), so anything else — dates, ``Decimal``, -#: ``bytes`` — is carried as ``(_SCALAR_TAG, kind, payload)`` and rebuilt on -#: read. Previously these reached ``freeze()`` and were flattened with -#: ``str()``: a ``date`` predicate came back comparing against a string, and -#: a ``Decimal`` came back as a type error from the query compiler. -_SCALAR_TAG = "__bsl_scalar__" - - -def _encode_scalar(value: Any) -> Any: - """Represent a constant in a form tag metadata can hold losslessly.""" - import datetime - import decimal - import uuid - - if isinstance(value, str | bool | int | float | type(None)): - # numpy scalars subclass int/float; normalize so they survive as - # native Python values rather than as repr strings. - if type(value) is not bool and isinstance(value, int) and type(value) is not int: - return int(value) - if isinstance(value, float) and type(value) is not float: - return float(value) - return value - # datetime before date: datetime is a date subclass. - if isinstance(value, datetime.datetime): - return (_SCALAR_TAG, "datetime", value.isoformat()) - if isinstance(value, datetime.date): - return (_SCALAR_TAG, "date", value.isoformat()) - if isinstance(value, datetime.time): - return (_SCALAR_TAG, "time", value.isoformat()) - if isinstance(value, datetime.timedelta): - return (_SCALAR_TAG, "timedelta", repr(value.total_seconds())) - if isinstance(value, decimal.Decimal): - return (_SCALAR_TAG, "decimal", str(value)) - if isinstance(value, uuid.UUID): - return (_SCALAR_TAG, "uuid", str(value)) - if isinstance(value, bytes): - import base64 - - return (_SCALAR_TAG, "bytes", base64.b64encode(value).decode("ascii")) - if isinstance(value, list | tuple): - kind = "list" if isinstance(value, list) else "tuple" - return (_SCALAR_TAG, kind, tuple(_encode_scalar(item) for item in value)) - # numpy scalars that subclass nothing familiar (e.g. np.datetime64) - if hasattr(value, "item") and type(value).__module__.startswith("numpy"): - return _encode_scalar(value.item()) - raise ValueError( - f"Cannot serialize constant of type {type(value).__name__} ({value!r}): " - "tag metadata holds only scalars, dates, Decimal, UUID and bytes. " - "Previously such values were silently stringified." - ) - - -def _decode_scalar(value: Any) -> Any: - """Inverse of :func:`_encode_scalar`; untagged values pass through.""" - if not (isinstance(value, tuple | list) and len(value) == 3 and value[0] == _SCALAR_TAG): - return value - import datetime - import decimal - import uuid - - _, kind, payload = value - match kind: - case "datetime": - return datetime.datetime.fromisoformat(payload) - case "date": - return datetime.date.fromisoformat(payload) - case "time": - return datetime.time.fromisoformat(payload) - case "timedelta": - return datetime.timedelta(seconds=float(payload)) - case "decimal": - return decimal.Decimal(payload) - case "uuid": - return uuid.UUID(payload) - case "bytes": - import base64 - - return base64.b64decode(payload.encode("ascii")) - case "list": - return [_decode_scalar(item) for item in payload] - case "tuple": - return tuple(_decode_scalar(item) for item in payload) - case _: - raise ValueError(f"Unknown encoded-constant kind: {kind!r}") - - -def serialize_resolver(resolver) -> tuple: - """Walk a Resolver tree and produce a hashable nested-tuple representation.""" - from ._xorq import ( - Attr, - BinaryOperator, - Call, - Item, - Just, - JustUnhashable, - Sequence, - UnaryOperator, - Variable, - ) - from ._xorq import ( - Mapping as MappingResolver, - ) - - if isinstance(resolver, Variable): - return ("var", resolver.name) - - if isinstance(resolver, Just): - value = resolver.value - # ibis Literal node (e.g., from case().when(..., 1)) - if _is_ibis_literal_node(value): - py_value = value.args[0] - dtype_str = str(value.args[1]) - return ("ibis_literal", _encode_scalar(py_value), dtype_str) - # callable (operator functions, deferrable functions like ifelse, _finish_searched_case) - if callable(value): - module = getattr(value, "__module__", None) - qualname = getattr(value, "__qualname__", None) - _check_callable_ref(module, qualname) - return ("fn", module, qualname) - # primitive value (int, float, str, bool, None) or an encodable constant - return ("just", _encode_scalar(value)) - - if isinstance(resolver, JustUnhashable): - value = resolver.value.obj - if _is_ibis_literal_node(value): - py_value = value.args[0] - dtype_str = str(value.args[1]) - return ("ibis_literal", _encode_scalar(py_value), dtype_str) - raise ValueError(f"Cannot serialize unhashable value: {value!r}") - - if isinstance(resolver, Attr): - return ("attr", serialize_resolver(resolver.obj), serialize_resolver(resolver.name)) - - if isinstance(resolver, Item): - return ("item", serialize_resolver(resolver.obj), serialize_resolver(resolver.name)) - - if isinstance(resolver, Call): - func_tuple = serialize_resolver(resolver.func) - args_tuple = tuple(serialize_resolver(a) for a in resolver.args) - kwargs_tuple = tuple((k, serialize_resolver(v)) for k, v in resolver.kwargs.items()) - return ("call", func_tuple, args_tuple, kwargs_tuple) - - if isinstance(resolver, BinaryOperator): - op_name = resolver.func.__name__ - return ( - "binop", - op_name, - serialize_resolver(resolver.left), - serialize_resolver(resolver.right), - ) - - if isinstance(resolver, UnaryOperator): - op_name = resolver.func.__name__ - return ("unop", op_name, serialize_resolver(resolver.arg)) - - if isinstance(resolver, Sequence): - type_name = resolver.typ.__name__ - items = tuple(serialize_resolver(v) for v in resolver.values) - return ("seq", type_name, items) - - if isinstance(resolver, MappingResolver): - type_name = resolver.typ.__name__ - items = tuple((k, serialize_resolver(v)) for k, v in resolver.values.items()) - return ("map", type_name, items) - - raise ValueError(f"Unknown resolver type: {type(resolver).__name__}") - - -_OPERATOR_MAP = { - "add": operator.add, - "sub": operator.sub, - "mul": operator.mul, - "truediv": operator.truediv, - "floordiv": operator.floordiv, - "pow": operator.pow, - "mod": operator.mod, - "eq": operator.eq, - "ne": operator.ne, - "lt": operator.lt, - "le": operator.le, - "gt": operator.gt, - "ge": operator.ge, - "and_": operator.and_, - "or_": operator.or_, - "xor": operator.xor, - "rshift": operator.rshift, - "lshift": operator.lshift, - "inv": operator.inv, - "neg": operator.neg, - "invert": operator.invert, -} - - -def _resolve_qualname(module_obj, qualname: str): - """Resolve a dotted qualname like 'ClassName.method' on a module.""" - parts = qualname.split(".") - obj = module_obj - for part in parts: - if part == "": - raise ValueError(f"Cannot resolve lambda qualname: {qualname}") - obj = getattr(obj, part) - return obj - - -def _load_trusted_callable(module_name: str, qualname: str): - """Import and return a callable named by a serialized expression. - - The pair is validated before the import — an unimportable module is a - side effect in itself, so an untrusted name must never reach - ``import_module``. After resolution the *result* is checked too: a - qualname is a ``getattr`` chain, so ``("fn", "ibis", "os.system")`` - would otherwise walk out of a trusted module into an untrusted one. - """ - _check_callable_ref(module_name, qualname) - mod = importlib.import_module(module_name) - func = _resolve_qualname(mod, qualname) - if not callable(func): - raise UntrustedCallableError( - f"{module_name}.{qualname} resolved to a non-callable " - f"{type(func).__name__}; refusing to use it as an expression function." - ) - origin = getattr(func, "__module__", None) - if _module_root(origin) not in _trusted_roots(): - raise UntrustedCallableError( - f"{module_name}.{qualname} resolves to an object defined in " - f"{origin!r}, which is outside the trusted module set. This is how " - "an attribute chain escapes a trusted module — refusing to load it." - ) - return func - - -def _finalize_frozen_slotted(obj, *fields) -> None: - """Set ``__precomputed_hash__`` on a FrozenSlotted built via ``object.__new__``. - - xorq's vendored ibis FrozenSlotted base implements ``__hash__`` by - returning a precomputed value that ``__init__`` would normally set - via ``hash((cls, tuple(field_values)))``. When we bypass - ``__init__`` to skip validation during deserialization we must - mirror that exactly — note the inner ``tuple(...)`` wrap, which is - significant: ``hash((cls, *fields))`` produces a different value. - Without this the rebuilt resolver raises ``AttributeError`` the - first time it is hashed (e.g. as a key in ``op.replace`` - substitutions). - """ - object.__setattr__(obj, "__precomputed_hash__", hash((type(obj), tuple(fields)))) - - -def deserialize_resolver(data: tuple): - """Reconstruct a Resolver tree from a nested-tuple representation.""" - from ._xorq import ( - Attr, - BinaryOperator, - Call, - Item, - Just, - Sequence, - UnaryOperator, - Variable, - ) - from ._xorq import ( - Mapping as MappingResolver, - ) - - match data: - case ("var", name): - return Variable(name) - - case ("just", value): - return Just(_decode_scalar(value)) - - case ("fn", module_name, qualname): - return Just(_load_trusted_callable(module_name, qualname)) - - case ("ibis_literal", py_value, dtype_str): - from ._xorq import ibis - - lit_expr = ibis.literal(_decode_scalar(py_value), type=ibis.dtype(dtype_str)) - return Just(lit_expr.op()) - - case ("attr", obj_data, name_data): - obj_resolver = deserialize_resolver(obj_data) - name_resolver = deserialize_resolver(name_data) - attr = object.__new__(Attr) - object.__setattr__(attr, "obj", obj_resolver) - object.__setattr__(attr, "name", name_resolver) - _finalize_frozen_slotted(attr, obj_resolver, name_resolver) - return attr - - case ("item", obj_data, name_data): - obj_resolver = deserialize_resolver(obj_data) - name_resolver = deserialize_resolver(name_data) - item = object.__new__(Item) - object.__setattr__(item, "obj", obj_resolver) - object.__setattr__(item, "name", name_resolver) - _finalize_frozen_slotted(item, obj_resolver, name_resolver) - return item - - case ("call", func_data, args_data, kwargs_data): - func_resolver = deserialize_resolver(func_data) - args_resolvers = tuple(deserialize_resolver(a) for a in args_data) - from ._xorq import FrozenDict - - kwargs_resolvers = FrozenDict({k: deserialize_resolver(v) for k, v in kwargs_data}) - call = object.__new__(Call) - object.__setattr__(call, "func", func_resolver) - object.__setattr__(call, "args", args_resolvers) - object.__setattr__(call, "kwargs", kwargs_resolvers) - _finalize_frozen_slotted(call, func_resolver, args_resolvers, kwargs_resolvers) - return call - - case ("binop", op_name, left_data, right_data): - func = _OPERATOR_MAP.get(op_name) - if func is None: - raise ValueError(f"Unknown binary operator: {op_name!r}") - left = deserialize_resolver(left_data) - right = deserialize_resolver(right_data) - binop = object.__new__(BinaryOperator) - object.__setattr__(binop, "func", func) - object.__setattr__(binop, "left", left) - object.__setattr__(binop, "right", right) - _finalize_frozen_slotted(binop, func, left, right) - return binop - - case ("unop", op_name, arg_data): - func = _OPERATOR_MAP.get(op_name) - if func is None: - raise ValueError(f"Unknown unary operator: {op_name!r}") - arg = deserialize_resolver(arg_data) - unop = object.__new__(UnaryOperator) - object.__setattr__(unop, "func", func) - object.__setattr__(unop, "arg", arg) - _finalize_frozen_slotted(unop, func, arg) - return unop - - case ("seq", type_name, items_data): - typ = {"tuple": tuple, "list": list}[type_name] - values = tuple(deserialize_resolver(v) for v in items_data) - seq = object.__new__(Sequence) - object.__setattr__(seq, "typ", typ) - object.__setattr__(seq, "values", values) - _finalize_frozen_slotted(seq, typ, values) - return seq - - case ("map", type_name, items_data): - typ = {"dict": dict}[type_name] - from ._xorq import FrozenDict - - values = FrozenDict({k: deserialize_resolver(v) for k, v in items_data}) - mapping = object.__new__(MappingResolver) - object.__setattr__(mapping, "typ", typ) - object.__setattr__(mapping, "values", values) - _finalize_frozen_slotted(mapping, typ, values) - return mapping - - case _: - raise ValueError(f"Unknown resolver tag: {data[0]}") - - -def _is_deferred(obj) -> bool: - """Duck-type check for Deferred (works for both ibis and xorq vendor).""" - return hasattr(obj, "_resolver") and hasattr(obj, "resolve") - - -def expr_to_structured(fn: Callable) -> Result[tuple, Exception]: - """Convert a callable/Deferred expression to a structured tuple representation.""" - from ._xorq import Deferred as XorqDeferred - - @safe - def do_convert(): - from ._xorq import _ - - # ops._CallableWrapper exposes the wrapped callable as ._fn; - # duck-type so this bottom-layer module doesn't import ops. Guard - # against Deferred first — getattr on a Deferred never falls back, - # it builds a new deferred attribute access. - expr = fn if _is_deferred(fn) else getattr(fn, "_fn", fn) - if isinstance(expr, XorqDeferred): - return serialize_resolver(expr._resolver) - # For ibis Deferred (not xorq vendor), resolve through xorq _ to get xorq types - if _is_deferred(expr): - result = expr.resolve(_) - if _is_deferred(result): - return serialize_resolver(result._resolver) - if callable(expr): - result = expr(_) - if _is_deferred(result): - return serialize_resolver(result._resolver) - raise ValueError(f"Callable did not produce a Deferred, got {type(result)}") - raise ValueError(f"Expected callable or Deferred, got {type(expr)}") - - return do_convert() - - -def structured_to_expr(data: tuple) -> Result: - """Reconstruct a Deferred from a structured tuple representation.""" - from ._xorq import Deferred - - @safe - def do_convert(): - resolver = deserialize_resolver(data) - return Deferred(resolver) - - return do_convert() - - -def join_predicate_to_structured(fn: Callable) -> Result[tuple, Exception]: - """Convert a binary join predicate to a structured tuple representation. - - Binary predicates like ``lambda l, r: l.col == r.col`` are serialized by - calling the function with two named Deferred variables (``left``, ``right``) - and serializing the resulting resolver tree. - """ - from ._xorq import Deferred, Variable - - @safe - def do_convert(): - # See expr_to_structured: unwrap _CallableWrapper by duck-typing, - # guarding against Deferred's synthetic attribute access. - raw_fn = fn if _is_deferred(fn) else getattr(fn, "_fn", fn) - left = Deferred(Variable("left")) - right = Deferred(Variable("right")) - result = raw_fn(left, right) - if not hasattr(result, "_resolver"): - raise ValueError(f"Join predicate did not produce a Deferred, got {type(result)}") - return serialize_resolver(result._resolver) - - return do_convert() - - -def structured_to_join_predicate(data: tuple) -> Result[Callable, Exception]: - """Reconstruct a binary join predicate from a structured tuple representation.""" - from ._xorq import Deferred - - @safe - def do_convert(): - resolver = deserialize_resolver(data) - deferred = Deferred(resolver) - return lambda left, right: deferred.resolve(left=left, right=right) - - return do_convert() - - -def _is_url(path: str | Path | None) -> bool: - """Check if a path is a URL.""" - if path is None: - return False - from urllib.parse import urlparse - - parsed = urlparse(str(path)) - return parsed.scheme in ("http", "https") - - -def _fetch_url_content(url: str) -> str: - """Fetch content from a URL. - - Args: - url: The URL to fetch - - Returns: - The content as a string - - Raises: - ValueError: If the fetch fails - """ - import urllib.error - import urllib.request - - try: - with urllib.request.urlopen(url, timeout=30) as response: - return response.read().decode("utf-8") - except urllib.error.HTTPError as e: - raise ValueError(f"HTTP Error {e.code}: {e.reason} for URL: {url}") from e - except urllib.error.URLError as e: - raise ValueError(f"URL Error: {e.reason} for URL: {url}") from e - except Exception as e: - raise ValueError(f"Failed to fetch URL {url}: {e}") from e - - -def read_yaml_file(yaml_path: str | Path) -> dict: - """Read and parse YAML file into dict. Supports local files and URLs. - - Args: - yaml_path: Path to local file or URL (http:// or https://) - - Returns: - Parsed YAML content as dict - """ - try: - if _is_url(yaml_path): - content_str = _fetch_url_content(str(yaml_path)) - content = yaml.safe_load(content_str) - else: - yaml_path = Path(yaml_path) - if not yaml_path.exists(): - raise FileNotFoundError(f"YAML file not found: {yaml_path}") - - with open(yaml_path) as f: - content = yaml.safe_load(f) - - if not isinstance(content, dict): - raise ValueError(f"YAML file must contain a dict, got: {type(content)}") - - return content - except (FileNotFoundError, ValueError): - raise - except Exception as e: - raise ValueError(f"Failed to parse YAML file {yaml_path}: {e}") from e - - -__all__ = [ - "safe_eval", - "SafeEvalError", - "expr_to_structured", - "structured_to_expr", - "join_predicate_to_structured", - "structured_to_join_predicate", - "serialize_resolver", - "deserialize_resolver", - "read_yaml_file", -] diff --git a/src/boring_semantic_layer/yaml.py b/src/boring_semantic_layer/yaml.py index e2d6fe22..137111cd 100644 --- a/src/boring_semantic_layer/yaml.py +++ b/src/boring_semantic_layer/yaml.py @@ -10,9 +10,10 @@ from .api import to_semantic_table from .errors import unwrap_or_raise from .expr import SemanticModel, SemanticTable +from .io import read_yaml_file from .ops import Dimension, Measure from .profile import get_connection -from .utils import read_yaml_file, safe_eval +from .safe_eval import safe_eval def _parse_expression_config(name: str, config: str | dict, metric_type: str):