From 3cb7c4db9b63dafb398a89b442ff56a171350da8 Mon Sep 17 00:00:00 2001 From: Eric Prud'hommeaux Date: Sun, 27 Sep 2026 02:56:12 -0500 Subject: [PATCH 1/2] ShExMap: materialize by iteration scopes; keys, static checks, provenance, in-place update The binding tree is now written to a grammar a reader can parse without guessing (a scope is an object, or an array whose first element is the own-bindings object -- always written -- followed by one list per repeated constraint or group, in schema order, empty when nothing matched; a list's iterations are uniform), and each iteration of a repeated shape-valued constraint carries "@node", the input node it matched. A non-repeated nested shape merges into the scope that matched it. shex.js's stored bindings still compare equal once @node is stripped, and its two older layouts are still read. The materializer no longer flattens the tree into frames read by a forward-only cursor. It evaluates the output schema over the scope tree: a constraint reads its variable from the scope it is evaluated at or the nearest ancestor binding it, and reading marks nothing, so a parent's binding can be read in every item; a repetition iterates the deepest input list its body's variables are bound at, skipping items whose body fails, honouring min and max; a body reading from two unrelated lists is refused by name; OneOf, ShapeOr, optionals and extensions yield alternatives combined by product and pruned to max_accepts, the one that read the most distinct bindings winning. A nested schema now maps to itself, sibling lists transpose, a parent's binding read at the parent and in each item no longer mixes items, and the round-trip law holds on every example; the expected graphs are unchanged. ThreadedMaterializer keeps its name and results. Node identity: a shape-valued constraint with a %Map variable names its node; %Map:{ id(arg, ...) %} keys it -- variables, @node, or an IRI template <...{v:x}...> percent-encoded as R2RML does -- so equal keys merge and id(@node) maps a graph onto itself node for node; other nodes are minted from the root, constraint, depth and scope, so runs agree and materializing the same root twice adds nothing. A value a Map code produces must satisfy the constraint's value expression; a plain literal whose lexical form fits the datatype is retyped, anything else fails the constraint like an unbound variable. analyse(input_schema, output_schema) and shexmap --check decide before any data whether a pair maps coherently: unrelated lists, "which one?" reads, unbound and unused variables, double binding sites, id() misuse. Every triple carries its provenance (constraint, input scope and node, bindings and statics read, how the object arose); Bindings.matched is the subgraph the input schema selected; ThreadedMaterializer.update, and shexmap --into, replace what the output schema holds at a root and leave the rest alone. A differential test runs every example through shex.js's extension-map when SHEXJS_EXTENSION_MAP is set. Co-Authored-By: Claude Fable 5.1 --- pyshex/shexmap/__init__.py | 21 +- pyshex/shexmap/analysis.py | 374 ++++++ pyshex/shexmap/bindings.py | 174 ++- pyshex/shexmap/cli.py | 67 +- pyshex/shexmap/functions.py | 106 +- pyshex/shexmap/materializer.py | 1130 ++++++++++------- pyshex/shexmap/scopes.py | 148 +++ tests/test_shexmap/README.md | 34 +- tests/test_shexmap/shexjs_driver.js | 51 + tests/test_shexmap/test_analysis.py | 122 ++ tests/test_shexmap/test_cardinality.py | 90 +- tests/test_shexmap/test_examples.py | 88 +- tests/test_shexmap/test_keys.py | 151 +++ tests/test_shexmap/test_roundtrip.py | 160 +++ .../test_shexmap/test_shexjs_differential.py | 110 ++ tests/test_shexmap/test_threaded.py | 14 +- tests/test_shexmap/test_update.py | 106 ++ tests/test_shexmap/test_values.py | 92 ++ 18 files changed, 2479 insertions(+), 559 deletions(-) create mode 100644 pyshex/shexmap/analysis.py create mode 100644 pyshex/shexmap/scopes.py create mode 100644 tests/test_shexmap/shexjs_driver.js create mode 100644 tests/test_shexmap/test_analysis.py create mode 100644 tests/test_shexmap/test_keys.py create mode 100644 tests/test_shexmap/test_roundtrip.py create mode 100644 tests/test_shexmap/test_shexjs_differential.py create mode 100644 tests/test_shexmap/test_update.py create mode 100644 tests/test_shexmap/test_values.py diff --git a/pyshex/shexmap/__init__.py b/pyshex/shexmap/__init__.py index fc90be1..d53dd35 100644 --- a/pyshex/shexmap/__init__.py +++ b/pyshex/shexmap/__init__.py @@ -18,22 +18,27 @@ when the input conforms in several ways that bind differently it says so (``.alternatives``), and :func:`bind_all` returns every way. EXTENDS and inverse (``^p``) constraints are followed. -* :class:`ThreadedMaterializer` builds the output by running threads over each output - shape's NFA, each with its own cursor into the bindings, and keeps every accepting - thread (``.accepts``), returning the one that uses the most bindings. +* :func:`analyse` checks a schema pair before any data: every output repetition has a + list to iterate, every variable read is bound where it can be read from, and unused + bindings are reported. +* :class:`ThreadedMaterializer` builds the output from the binding tree's structure: an + output repetition iterates the input list its variables are bound at, a parent's + binding is read in every item, and the alternatives a ``OneOf`` or an extension leaves + open are kept (``.accepts``), the one that reads the most bindings returned. """ from rdflib import Graph from rdflib.term import Node -from pyshex.shexmap.bindings import (MAP_EXTENSION, AmbiguousBindingsError, Bindings, MapValidationError, bind, - bind_all, dumps, loads, normalize, shexc_prefixes) +from pyshex.shexmap.bindings import (MAP_EXTENSION, NODE_KEY, AmbiguousBindingsError, Bindings, MapValidationError, + bind, bind_all, dumps, loads, normalize, shexc_prefixes) +from pyshex.shexmap.analysis import Report, analyse from pyshex.shexmap.functions import MapFunctionError from pyshex.shexmap.materializer import Accept, MaterializationError, ThreadedMaterializer, materialize from pyshex.shexmap.semact import register -__all__ = ["MAP_EXTENSION", "Accept", "AmbiguousBindingsError", "Bindings", "MapFunctionError", - "MapValidationError", "MaterializationError", "ThreadedMaterializer", "bind", "bind_all", "dumps", - "loads", "map_graph", "materialize", "normalize", "register", "shexc_prefixes"] +__all__ = ["MAP_EXTENSION", "NODE_KEY", "Accept", "AmbiguousBindingsError", "Bindings", "MapFunctionError", + "MapValidationError", "MaterializationError", "Report", "ThreadedMaterializer", "analyse", "bind", + "bind_all", "dumps", "loads", "map_graph", "materialize", "normalize", "register", "shexc_prefixes"] def map_graph(graph: Graph, input_schema, focus: str | Node, output_schema, root: str | Node | None = None, diff --git a/pyshex/shexmap/analysis.py b/pyshex/shexmap/analysis.py new file mode 100644 index 0000000..236b814 --- /dev/null +++ b/pyshex/shexmap/analysis.py @@ -0,0 +1,374 @@ +"""Static checks of a ShExMap schema pair, before any data. + +Given the input schema and the output schema, :func:`analyse` says whether every output +repetition has a well-formed iteration scope, whether every variable the output reads is +bound in the input where it can be read from, and which bindings go unused -- the questions +materialization would otherwise answer at run time, one graph at a time. The checks are +the schema-pair conditions XSugar and biXid impose on grammar pairs, and the +"updatability" question of the view-update literature, in ShExMap's terms; see +``docs/shexmap-structural-plan.md`` §4 phase 3. + +A variable's **site** is the chain of repeated constraints and groups of the input schema +that enclose its ``%Map`` code: it is bound once per iteration of the innermost one, or once +for the whole tree when the chain is empty. An output repetition's **iteration scope** is +the deepest site among the variables its body reads, nested repetitions counting through +their parent site; it is ill-formed when two of those sites lie in unrelated lists. A +constraint may read a variable whose site is an ancestor of (or equal to) the scope it is +evaluated at; a deeper site means "which one?", and an unrelated one means the value is +not in scope. +""" +from __future__ import annotations + +from dataclasses import dataclass, field + +from ShExJSG import ShExJ + +from pyshex.shape_expressions_language.p5_context import Context +from pyshex.shapemap_structure_and_language.p3_shapemap_structure import START +from pyshex.shexmap import functions +from pyshex.shexmap.bindings import (declaration_of, expression_of, extension_candidates, is_repeated, map_actions, + references_shape, shape_parts, shexc_prefixes) + +Site = tuple[int, ...] # ids of the enclosing repeated expressions, outermost first + + +@dataclass +class Report: + """What :func:`analyse` found. ``errors`` make the pair not mappable; ``warnings`` do not. + Messages write IRIs with the schemas' prefixes.""" + errors: list[str] = field(default_factory=list) + warnings: list[str] = field(default_factory=list) + bound: dict[str, Site] = field(default_factory=dict) # input variable -> site + read: set[str] = field(default_factory=set) # variables the output reads + labels: dict[int, str] = field(default_factory=dict) # repeated expression id -> name + prefixes: dict[str, str] = field(default_factory=dict) + + @property + def ok(self) -> bool: + return not self.errors + + def where(self, site: Site) -> str: + return "the root" if not site else "each " + " / ".join(self.labels.get(i, "?") for i in site) + + def short(self, text: str) -> str: + """``text`` with every namespace the schemas declare written as its prefix.""" + for prefix, ns in sorted(self.prefixes.items(), key=lambda kv: -len(kv[1])): + if ns: + text = text.replace(ns, prefix + ":") + return text + + def error(self, message: str) -> None: + self.errors.append(self.short(message)) + + def warn(self, message: str) -> None: + self.warnings.append(self.short(message)) + + def __str__(self) -> str: + lines = [f"error: {e}" for e in self.errors] + [f"warning: {w}" for w in self.warnings] + return "\n".join(lines) if lines else "ok: the schemas map coherently" + + +def analyse(input_schema, output_schema, prefixes=None, input_start=None, output_start=None, + static_vars=()) -> Report: + """Check that ``output_schema`` can be materialized coherently from what ``input_schema`` binds. + + :param input_schema: ShExC text or a parsed ShExJ schema + :param output_schema: ShExC text or a parsed ShExJ schema + :param prefixes: prefixes for ShExMap variable names; read from ShExC texts when omitted + :param input_start, output_start: shape labels; default the schemas' start shapes + :param static_vars: variable IRIs that will be supplied as static variables + """ + from pyshex.utils.schema_loader import SchemaLoader + prefixes = dict(prefixes or {}) + schemas = [] + for text in (input_schema, output_schema): + if isinstance(text, str): + prefixes = {**shexc_prefixes(text), **prefixes} + schema = SchemaLoader().loads(text) + if schema is None: + raise ValueError("a schema does not parse") + else: + schema = text + schemas.append(schema) + report = Report(prefixes=prefixes) + inp = _Input(Context(None, schemas[0]), prefixes, report) + inp.run(_start(schemas[0], input_start)) + out = _Output(Context(None, schemas[1]), prefixes, report, set(static_vars)) + out.run(_start(schemas[1], output_start)) + for v, site in report.bound.items(): + if v not in report.read: + report.warn(f"{v} is bound (at {report.where(site)}) but the output never reads it") + for v in sorted(set(static_vars) - report.read): + report.warn(f"static variable {v} is never read") + return report + + +def _start(schema, start): + if start is None or start is START: + if schema.start is None: + raise ValueError("the schema has no start shape; name one") + return schema.start + return ShExJ.IRIREF(str(start)) + + +def _is_repetition(expr) -> bool: + """Any cardinality but exactly one, as the materializer treats it (``?`` included).""" + return not (expr.min in (None, 1) and expr.max in (None, 1)) + + +def _label(expr) -> str: + found: list[str] = [] + stack = [expr] + while stack and len(found) < 2: + e = stack.pop() + if isinstance(e, ShExJ.TripleConstraint): + found.append(("^" if e.inverse else "") + str(e.predicate)) + elif isinstance(e, (ShExJ.EachOf, ShExJ.OneOf)): + stack.extend(reversed(list(e.expressions))) + return "(" + ", ".join(found) + ")" if len(found) > 1 else (found[0] if found else "(group)") + + +def _variables(tc, prefixes) -> tuple[list[str], list[str]]: + """(variables bound or read by a constraint's Map codes, id() arguments) -- id() is not a + variable code; its arguments are returned apart.""" + plain: list[str] = [] + keys: list[str] = [] + for act in map_actions(tc): + code = str(act.code or '') + try: + if functions.is_key_code(code): + keys.extend(a for a in functions.key_arguments(code, prefixes) if a != functions.NODE_ARGUMENT) + elif functions.is_function_call(code): + plain.extend(functions.variables_of(code, prefixes)) + else: + plain.append(functions.expand_variable(code, prefixes)) + except functions.MapFunctionError: + pass + return plain, keys + + +class _Input: + """Where the input schema binds each variable.""" + + def __init__(self, cntxt: Context, prefixes, report: Report) -> None: + self.cntxt, self.prefixes, self.report = cntxt, prefixes, report + self.seen: set[int] = set() + + def run(self, se) -> None: + self.shape_expr(se, ()) + + def shape_expr(self, se, site: Site) -> None: + if isinstance(se, str): + decl = declaration_of(self.cntxt, se) + if decl is None: + return + for o in extension_candidates(self.cntxt, decl): + self.shape_expr(expression_of(o), site) + return + if id(se) in self.seen: + return + self.seen.add(id(se)) + if isinstance(se, ShExJ.ShapeDecl): + self.shape_expr(se.shapeExpr, site) + elif isinstance(se, ShExJ.Shape): + for p in shape_parts(self.cntxt, se): + if p.expression is not None: + self.expression(p.expression, site) + elif isinstance(se, (ShExJ.ShapeAnd, ShExJ.ShapeOr)): + for p in se.shapeExprs: + self.shape_expr(p, site) + + def expression(self, expr, site: Site) -> None: + if isinstance(expr, str): + expr = self.cntxt.tripleExprFor(expr) + if expr is None: + return + if is_repeated(expr): + self.report.labels[id(expr)] = _label(expr) + site = site + (id(expr),) + if isinstance(expr, ShExJ.TripleConstraint): + plain, _ = _variables(expr, self.prefixes) + for v in plain: + at = self.report.bound.setdefault(v, site) + if at != site: + self.report.error( + f"{v} is bound at two places of the input schema, {self.report.where(at)} and " + f"{self.report.where(site)}; a variable must have one binding site") + if references_shape(expr.valueExpr): + self.shape_expr(expr.valueExpr, site) + elif isinstance(expr, (ShExJ.EachOf, ShExJ.OneOf)): + for e in expr.expressions: + self.expression(e, site) + + +class _Output: + """Where the output schema reads each variable, and what each repetition iterates.""" + + def __init__(self, cntxt: Context, prefixes, report: Report, statics: set[str]) -> None: + self.cntxt, self.prefixes, self.report, self.statics = cntxt, prefixes, report, statics + self.scopes: dict[int, Site | None] = {} + self.ill_formed: set[int] = set() + self.active: set[int] = set() + + def run(self, se) -> None: + self.shape_expr(se, ()) + + # -- walking ------------------------------------------------------------------------ + def shape_expr(self, se, scope: Site) -> None: + if isinstance(se, str): + decl = declaration_of(self.cntxt, se) + if decl is None: + self.report.error(f"the output schema references {se}, which it does not define") + return + options = extension_candidates(self.cntxt, decl) + if not options: + self.report.error(f"{se} is abstract and nothing extends it") + for o in options: + self.shape_expr(expression_of(o), scope) + return + key = id(se) + if key in self.active: + return # a recursive shape: the first pass covered it + self.active.add(key) + try: + if isinstance(se, ShExJ.ShapeDecl): + self.shape_expr(se.shapeExpr, scope) + elif isinstance(se, ShExJ.Shape): + for p in shape_parts(self.cntxt, se): + if p.expression is not None: + self.expression(p.expression, scope) + elif isinstance(se, (ShExJ.ShapeAnd, ShExJ.ShapeOr)): + for p in se.shapeExprs: + if not isinstance(p, ShExJ.NodeConstraint): + self.shape_expr(p, scope) + elif isinstance(se, ShExJ.NodeConstraint): + pass + else: + self.report.error(f"{type(se).__name__} cannot be materialized") + finally: + self.active.discard(key) + + def expression(self, expr, scope: Site) -> None: + if isinstance(expr, str): + expr = self.cntxt.tripleExprFor(expr) + if expr is None: + return + if _is_repetition(expr): + iterates = self.iteration_scope(expr) + if id(expr) in self.ill_formed: + return # its structure is the error; its reads would only repeat it + body_scope = iterates if iterates is not None and len(iterates) > len(scope) else scope + else: + body_scope = scope + if isinstance(expr, ShExJ.TripleConstraint): + self.constraint(expr, body_scope) + elif isinstance(expr, (ShExJ.EachOf, ShExJ.OneOf)): + for e in expr.expressions: + self.expression(e, body_scope) + + def constraint(self, tc, scope: Site) -> None: + plain, keys = _variables(tc, self.prefixes) + acts = map_actions(tc) + if keys or any(functions.is_key_code(str(a.code or '')) for a in acts): + if len(acts) > 1: + self.report.error(f"id() must be the only Map code on {tc.predicate}") + if not references_shape(tc.valueExpr): + self.report.error(f"id() on {tc.predicate} needs a shape-valued constraint") + if any(functions.NODE_ARGUMENT in functions.key_arguments(str(a.code or ''), self.prefixes) + for a in acts if functions.is_key_code(str(a.code or ''))) and not scope: + self.report.error(f"id(@node) on {tc.predicate} is read at the root, where no input iteration " + f"provides a node") + for v in plain + keys: + self.read_variable(v, tc, scope) + if references_shape(tc.valueExpr): + self.shape_expr(tc.valueExpr, scope) + + def read_variable(self, v: str, tc, scope: Site) -> None: + self.report.read.add(v) + if v in self.statics: + return + site = self.report.bound.get(v) + if site is None: + self.report.error(f"{tc.predicate} reads {v}, which the input schema never binds") + return + if scope[:len(site)] == site: + return # bound here or above: readable + if site[:len(scope)] == scope: + self.report.error( + f"{tc.predicate} reads {v}, bound once per {self.report.where(site)}, from " + f"{self.report.where(scope)}: which one? A repetition over it is needed") + else: + self.report.error( + f"{tc.predicate} reads {v}, bound at {self.report.where(site)}, from " + f"{self.report.where(scope)}, which is not below it") + + # -- iteration scopes --------------------------------------------------------------- + def iteration_scope(self, expr) -> Site | None: + key = id(expr) + if key in self.scopes: + return self.scopes[key] + self.scopes[key] = None # a recursive repetition counts as scope-free while computed + direct: dict[str, object] = {} + nested: list = [] + self.collect(expr, direct, nested, set(), top=True) + candidates: dict[Site, str] = {} + for v in direct: + if v in self.report.bound: + candidates.setdefault(self.report.bound[v], v) + for r in nested: + sub = self.iteration_scope(r) + if sub: + candidates.setdefault(sub[:-1], f"the repetition over {_label(r)}") + result: Site | None = None + if candidates: + result = max(candidates, key=len) + for site, why in candidates.items(): + if result[:len(site)] != site: + self.report.error( + f"the repetition over {_label(expr)} reads from unrelated lists: {why} is bound at " + f"{self.report.where(site)} and {candidates[result]} at {self.report.where(result)}") + self.ill_formed.add(key) + self.report.read.update(direct) + break + self.scopes[key] = result + return result + + def collect(self, expr, direct: dict, nested: list, seen: set, top: bool) -> None: + if isinstance(expr, str): + expr = self.cntxt.tripleExprFor(expr) + if expr is None or id(expr) in seen: + return + seen.add(id(expr)) + if not top and _is_repetition(expr): + nested.append(expr) + return + if isinstance(expr, ShExJ.TripleConstraint): + plain, keys = _variables(expr, self.prefixes) + for v in plain + keys: + direct[v] = True + if references_shape(expr.valueExpr): + self.collect_shape(expr.valueExpr, direct, nested, seen) + elif isinstance(expr, (ShExJ.EachOf, ShExJ.OneOf)): + for e in expr.expressions: + self.collect(e, direct, nested, seen, top=False) + + def collect_shape(self, se, direct: dict, nested: list, seen: set) -> None: + if isinstance(se, str): + decl = declaration_of(self.cntxt, se) + if decl is None: + return + for o in extension_candidates(self.cntxt, decl): + self.collect_shape(expression_of(o), direct, nested, seen) + return + if id(se) in seen: + return + seen.add(id(se)) + if isinstance(se, ShExJ.ShapeDecl): + self.collect_shape(se.shapeExpr, direct, nested, seen) + elif isinstance(se, ShExJ.Shape): + for p in shape_parts(self.cntxt, se): + if p.expression is not None: + self.collect(p.expression, direct, nested, seen, top=False) + elif isinstance(se, (ShExJ.ShapeAnd, ShExJ.ShapeOr)): + for p in se.shapeExprs: + self.collect_shape(p, direct, nested, seen) diff --git a/pyshex/shexmap/bindings.py b/pyshex/shexmap/bindings.py index 20dbd79..0565cc3 100644 --- a/pyshex/shexmap/bindings.py +++ b/pyshex/shexmap/bindings.py @@ -1,11 +1,22 @@ """Collect ShExMap bindings from RDF that conforms to an input schema. -**Binding trees** use the structure shex.js prints and reads, with rdflib terms as values: -an object maps variable IRIs to terms, and a list is a sequence whose elements are -objects or nested lists. A repeated match (a triple constraint with max > 1, or one -iteration of a repeated group) contributes one element to a list, so everything bound -from one reading of a blood pressure stays together. :class:`Bindings` wraps a tree, -and :func:`dumps`/:func:`loads` convert to and from shex.js's JSON. +**Binding trees** use the structure shex.js prints and reads, with rdflib terms as values. +A *scope* (what one node's match binds) is an object mapping variable IRIs to terms, or, +when the match has repeated parts, an array whose first element is that object (possibly +empty) and whose other elements are *lists*; a list has one element per iteration of a +repeated constraint or group, each element a scope, so everything bound from one reading +of a blood pressure stays together. A list's elements are uniform: all objects, or all +arrays (a lone object among arrays is written as ``[object]``), so that a reader never has +to guess whether an array is a scope or a list. Every repeated constraint or group of the +shape gets its list, in schema order, empty when nothing matched, so a list's position +says which expression it came from whatever the data. A non-repeated nested shape merges +into the scope that matched it. An element made by a repeated *shape-valued* constraint also +records the node the nested shape matched under the reserved key ``"@node"`` +(:data:`NODE_KEY`; the subject for an inverse constraint). It is not a variable: +:func:`normalize` leaves it out of the frames and no output constraint can read it. It +says which input node an iteration came from, which is what tells two groups apart when +their bindings are alike. :class:`Bindings` wraps a tree, and :func:`dumps`/:func:`loads` +convert to and from shex.js's JSON. **Collecting** happens after validation succeeds. PyShEx checks that *some* partition of each node's neighbourhood satisfies the schema but does not say which, so the @@ -28,6 +39,7 @@ import re from collections.abc import Iterable, Iterator, Mapping from dataclasses import dataclass, field +from typing import Any from ShExJSG import ShExJ from rdflib import BNode, Graph, Literal, URIRef, XSD @@ -41,6 +53,8 @@ MAP_EXTENSION = "http://shex.io/extensions/Map/#" +NODE_KEY = "@node" # in an iteration's bindings: the node the nested shape matched + MAX_ALTERNATIVES = 20 # distinct binding trees to collect before giving up counting MAX_PARTITIONS = 10_000 # partitions of one neighbourhood to try @@ -60,11 +74,13 @@ def __init__(self, message: str, alternatives: list[Bindings]) -> None: # -- binding trees ---------------------------------------------------------------------- class Bindings: - """A binding tree (see module docstring), and how many distinct ones the input allowed.""" + """A binding tree (see module docstring), how many distinct ones the input allowed, and + the triples the schema matched to make it (``matched``: the subgraph it selected).""" - def __init__(self, tree, alternatives: int = 1) -> None: + def __init__(self, tree, alternatives: int = 1, matched: frozenset = frozenset()) -> None: self.tree = tree self.alternatives = alternatives + self.matched = matched @property def ambiguous(self) -> bool: @@ -106,7 +122,8 @@ def normalize(tree) -> list[dict[str, Node]]: list of their readings) is copied into every frame the sibling lists produce.""" def walk(node): if not isinstance(node, list): - return [dict(node)], True, {k: 1 for k in node} + own = {k: v for k, v in node.items() if k != NODE_KEY} + return [own], True, {k: 1 for k in own} kids = [walk(k) for k in node] counts: dict[str, int] = {} for _, _, c in kids: @@ -278,30 +295,49 @@ def triple_constraints(se, cntxt: Context | None = None) -> Iterator[ShExJ.Tripl @dataclass class _Record: - """Bindings made while matching one node: its own variables and nested matches.""" + """One scope: the bindings made while matching one node, one list of iteration records + per repeated constraint or group that matched under it, and the triples matched here.""" vars: dict[str, Node] = field(default_factory=dict) - children: list[_Record] = field(default_factory=list) - frame: bool = False + lists: list[list[_Record]] = field(default_factory=list) + triples: set = field(default_factory=set) def merged(self, other: _Record) -> _Record: - return _Record({**self.vars, **other.vars}, self.children + other.children, self.frame) + return _Record({**self.vars, **other.vars}, self.lists + other.lists, self.triples | other.triples) + + def all_triples(self) -> frozenset: + """The triples matched here and in every nested scope.""" + out = set(self.triples) + for lst in self.lists: + for r in lst: + out |= r.all_triples() + return frozenset(out) def empty(self) -> bool: - return not self.vars and all(c.empty() for c in self.children) + return not (self.vars.keys() - {NODE_KEY}) and all(r.empty() for lst in self.lists for r in lst) def tree(self): - """Non-repeated nested matches merge into this object; repeated ones become a list.""" - merged = dict(self.vars) - frames = [] - for c in self.children: - ct = c.tree() - if not c.frame and isinstance(ct, dict) and not (merged.keys() & ct.keys()): - merged.update(ct) - else: - frames.append(ct) - if not frames: - return merged - return ([merged] if merged else []) + [frames] + """The scope as a binding tree: the object alone, or ``[object, list, ...]``, each + list's elements uniform (all objects or all arrays).""" + own = dict(self.vars) + if not self.lists: + return own + out = [own] + for lst in self.lists: + iterations = [r.tree() for r in lst] + if any(isinstance(t, list) for t in iterations): + iterations = [t if isinstance(t, list) else [t] for t in iterations] + out.append(iterations) + return out + + +def _by_first(items) -> list: + """The items' expressions, in order of first occurrence.""" + seen, out = set(), [] + for item in items: + if id(item[1]) not in seen: + seen.add(id(item[1])) + out.append(item[1]) + return out def _product(alternatives: list[list[_Record]], limit: int) -> list[_Record]: @@ -397,15 +433,17 @@ def shape(self, n: Node, S: ShExJ.Shape) -> list[_Record]: [('in', t) for t in g.triples((None, None, n)) if (t[1], True) in keys]) extras = {URIRef(str(e)) for p in parts for e in (getattr(p, 'extra', None) or [])} + exprs = [p.expression for p in parts if p.expression is not None] + repeated = self._repeated_expressions(exprs) results: list[_Record] = [] tried = 0 - for items, rest in self._parts(n, [p.expression for p in parts if p.expression is not None], matchables): + for items, rest in self._parts(n, exprs, matchables): tried += 1 if tried > MAX_PARTITIONS: break if not self._valid_remainder(n, rest, extras, tcs): continue - results.extend(self._bind_items(items)) + results.extend(self._bind_items(items, repeated)) results = _dedupe(results, self.limit) if len(results) >= self.limit: break @@ -418,6 +456,25 @@ def shape(self, n: Node, S: ShExJ.Shape) -> list[_Record]: + (f" within {MAX_PARTITIONS} partitions" if tried > MAX_PARTITIONS else "")) return results + def _repeated_expressions(self, exprs: list) -> list: + """The repeated constraints and groups directly under ``exprs`` (not inside another + repeated expression), in schema order: one list each in the scope's tree.""" + out: list = [] + + def walk(e) -> None: + if isinstance(e, str): + e = self.cntxt.tripleExprFor(e) + if e is None: + return + if is_repeated(e): + out.append(e) + elif isinstance(e, (ShExJ.EachOf, ShExJ.OneOf)): + for sub in e.expressions: + walk(sub) + for e in exprs: + walk(e) + return out + def _own_tcs(self, S: ShExJ.Shape) -> list[ShExJ.TripleConstraint]: """Triple constraints of S's own expression, not of the shapes nested in them.""" out, stack = [], [S.expression] @@ -452,7 +509,8 @@ def _parts(self, n: Node, exprs: list, available: frozenset): def _match(self, n: Node, expr, available: frozenset): """Ways ``expr`` can match some of ``available``: (items, remaining) pairs, greediest - first. Items are ('tc', tc, triple) or ('group', items) for a repeated group's iteration.""" + first. Items are ('tc', tc, triple) or ('group', expr, items) for one iteration of a + repeated group.""" if isinstance(expr, str): # an inclusion: &label expr = self.cntxt.tripleExprFor(expr) if isinstance(expr, ShExJ.TripleConstraint): @@ -476,7 +534,7 @@ def _repeat(self, n: Node, expr, available: frozenset, count: int, min_: int, ma for one, rest in self._once(n, expr, available): if rest == available and count >= min_: continue # an iteration that matches nothing adds nothing - wrapped = [('group', one)] if repeated else one + wrapped = [('group', expr, one)] if repeated else one if rest == available: # matched nothing but is required: count it once yield wrapped, rest continue @@ -498,33 +556,55 @@ def _once(self, n: Node, expr, available: frozenset): def _value(tc: ShExJ.TripleConstraint, t) -> Node: return t[0] if tc.inverse else t[2] - def _bind_items(self, items) -> list[_Record]: - """Alternative records for one partition's items.""" + def _bind_items(self, items, repeated: list) -> list[_Record]: + """Alternative records for one partition's items: each of the scope's ``repeated`` + expressions makes one list, with an iteration per item (empty when it matched + nothing); a non-repeated constraint binds into the scope itself.""" + by_expr: dict[int, list] = {} alternatives: list[list[_Record]] = [] for item in items: - if item[0] == 'group': # one iteration of a repeated group: one frame - opts = [] - for r in self._bind_items(item[1]): - iteration = _Record(r.vars, r.children, True) - opts.append(_Record(children=[iteration]) if not iteration.empty() else _Record()) - alternatives.append(opts or [_Record()]) + expr = item[1] + if item[0] == 'group' or is_repeated(expr): + by_expr.setdefault(id(expr), []).append(item) continue _, tc, t = item value = self._value(tc, t) lifted = self.lift(tc, value) - per_value = is_repeated(tc) - if references_shape(tc.valueExpr): + if references_shape(tc.valueExpr): # a nested scope: merge it into this one opts = [] for sub in self.shape_expr(value, tc.valueExpr): - child = _Record({**lifted, **sub.vars}, sub.children, per_value) - opts.append(_Record(children=[child]) if not child.empty() else _Record()) + if lifted.keys() & sub.vars.keys(): + opts.append(_Record(lifted, [[sub]], {t})) # a name clash: keep it apart + else: + opts.append(_Record({**lifted, **sub.vars}, sub.lists, {t} | sub.triples)) alternatives.append(opts) - elif per_value: - alternatives.append([_Record(children=[_Record(lifted, frame=True)]) if lifted else _Record()]) else: - alternatives.append([_Record(lifted)]) + alternatives.append([_Record(lifted, triples={t})]) + for expr in repeated + [e for e in _by_first(items) if id(e) in by_expr and e not in repeated]: + per_iteration = [self._iteration(it) for it in by_expr.pop(id(expr), [])] + opts = [] + for combo in itertools.product(*per_iteration): + kept = [r for r in combo if not r.empty()] + dropped = set().union(*(r.triples for r in combo if r.empty())) if combo else set() + opts.append(_Record(lists=[kept], triples=dropped)) # an iteration that bound nothing still matched + if len(opts) >= self.limit: + break + alternatives.append(opts or [_Record(lists=[[]])]) return _product(alternatives, self.limit) + def _iteration(self, item) -> list[_Record]: + """Alternative scopes for one iteration of a repeated constraint or group.""" + if item[0] == 'group': + expr = item[1] + return self._bind_items(item[2], self._repeated_expressions(list(expr.expressions))) or [_Record()] + _, tc, t = item + value = self._value(tc, t) + lifted = self.lift(tc, value) + if references_shape(tc.valueExpr): + return [_Record({NODE_KEY: value, **lifted, **sub.vars}, sub.lists, {t} | sub.triples) + for sub in self.shape_expr(value, tc.valueExpr)] + return [_Record(lifted, triples={t})] + def lift(self, tc, value) -> dict[str, Node]: bound: dict[str, Node] = {} for act in map_actions(tc): @@ -580,8 +660,8 @@ def bind_all(graph: Graph, schema: str | ShExJ.Schema, focus: str | Node, start= raise MapValidationError(f"{focus} does not conform: " + "\n".join(reasons)) extractor = _Extractor(Context(graph, schema), prefixes, limit) records = extractor.shape_expr(focus, label if label is not START else START) - trees = [r.tree() for r in _dedupe(records, limit)] - return [Bindings(t, alternatives=len(trees)) for t in trees] + kept = _dedupe(records, limit) + return [Bindings(r.tree(), alternatives=len(kept), matched=r.all_triples()) for r in kept] def bind(graph: Graph, schema: str | ShExJ.Schema, focus: str | Node, start=None, diff --git a/pyshex/shexmap/cli.py b/pyshex/shexmap/cli.py index 8ac96e2..00b2589 100644 --- a/pyshex/shexmap/cli.py +++ b/pyshex/shexmap/cli.py @@ -11,6 +11,14 @@ Materialize from bindings saved earlier, by this tool or by shex.js:: shexmap -j bindings.json -t out.shex -r + +Check that two schemas map coherently, without data:: + + shexmap --check -s in.shex -t out.shex + +Update a graph in place, replacing what the output schema holds at the root:: + + shexmap -i data.ttl -s in.shex -f -t out.shex -r --into existing.ttl """ import json import sys @@ -19,8 +27,8 @@ from rdflib import BNode, Graph, URIRef -from pyshex.shexmap import (MapFunctionError, MapValidationError, MaterializationError, ThreadedMaterializer, bind, - dumps, loads) +from pyshex.shexmap import (MapFunctionError, MapValidationError, MaterializationError, ThreadedMaterializer, analyse, + bind, dumps, loads) from pyshex.shexmap.bindings import term_from_json @@ -48,14 +56,47 @@ def genargs(prog: str | None = None) -> ArgumentParser: p.add_argument("--static", help="JSON object of extra variable values, as shex.js staticVars") p.add_argument("--strict", action="store_true", help="fail when the input or the output can be matched in more than one way") + p.add_argument("--into", help="an existing RDF file (in --output-format) to update in place: what the output " + "schema currently holds at --root is replaced, the rest kept; written back there " + "unless --output says otherwise") + p.add_argument("--provenance", help="write one JSON object per output triple here: the triple, the output " + "constraint's predicate, the input scope and node it came from, the bindings " + "read, and how the object arose") + p.add_argument("--check", action="store_true", + help="only analyse --input-schema against --output-schema: every output repetition has a list " + "to iterate, every variable read is bound where it can be read; exit 1 on errors") p.add_argument("-o", "--output", help="write the output RDF here (default: stdout)") p.add_argument("--output-format", default="turtle", help="output RDF format (default: turtle)") return p +def provenance_json(triple, source: dict) -> dict: + """A triple's provenance record as JSON: terms in N3, the input node likewise.""" + s, p, o = triple + kind = next((k for k in ("variable", "code", "keyed", "constant", "structural", "named") if k in source), "?") + return {"subject": s.n3(), "predicate": p.n3(), "object": o.n3(), + "constraint": source["predicate"], "scope": list(source["scope"]), + "node": source["node"].n3() if source["node"] is not None else None, + "reads": [[list(path), var] for path, var in source["reads"]], "statics": list(source["statics"]), + "kind": kind, kind: source[kind] if kind in ("variable", "code", "keyed") else True} + + def main(argv: list[str] | None = None, prog: str | None = None) -> int: parser = genargs(prog) opts = parser.parse_args(argv) + if opts.check: + if not (opts.input_schema and opts.output_schema): + parser.error("--check needs --input-schema and --output-schema") + static = list(json.loads(Path(opts.static).read_text(encoding="utf-8"))) if opts.static else [] + try: + report = analyse(Path(opts.input_schema).read_text(encoding="utf-8"), + Path(opts.output_schema).read_text(encoding="utf-8"), + input_start=opts.start, output_start=opts.output_start, static_vars=static) + except ValueError as e: + print(f"shexmap: {e}", file=sys.stderr) + return 1 + print(report) + return 0 if report.ok else 1 if opts.bindings: if opts.input or opts.input_schema or opts.focus: parser.error("--bindings replaces --input, --input-schema and --focus") @@ -84,20 +125,30 @@ def main(argv: list[str] | None = None, prog: str | None = None) -> int: static = {k: term_from_json(v) for k, v in json.loads(Path(opts.static).read_text(encoding="utf-8")).items()} \ if opts.static else None m = ThreadedMaterializer(Path(opts.output_schema).read_text(encoding="utf-8"), static_vars=static) - triples = m.materialize(bindings, parse_node(opts.root) if opts.root else BNode(), start=opts.output_start) + root = parse_node(opts.root) if opts.root else BNode() + if opts.into: + out = Graph().parse(opts.into, format=opts.output_format) + added, removed = m.update(out, bindings, root, start=opts.output_start) + triples = m.chosen.triples + print(f"shexmap: {opts.into}: {len(added)} triples added, {len(removed)} removed", file=sys.stderr) + else: + out = Graph() + triples = m.materialize(bindings, root, start=opts.output_start) + for t in triples: + out.add(t) if len(m.accepts) > 1: if opts.strict: raise MaterializationError(f"the bindings fit the output schema in {len(m.accepts)} ways") print(f"shexmap: warning: the bindings fit the output schema in {len(m.accepts)} ways;" " using the one that uses the most bindings", file=sys.stderr) - out = Graph() for prefix, ns in m.prefixes.items(): out.bind(prefix, ns, override=False) - for t in triples: - out.add(t) + if opts.provenance: + Path(opts.provenance).write_text("".join(json.dumps(provenance_json(t, src)) + "\n" + for t, src in zip(triples, m.provenance)), encoding="utf-8") text = out.serialize(format=opts.output_format) - if opts.output: - Path(opts.output).write_text(text, encoding="utf-8") + if opts.output or opts.into: + Path(opts.output or opts.into).write_text(text, encoding="utf-8") else: sys.stdout.write(text) except (MapValidationError, MaterializationError, MapFunctionError) as e: diff --git a/pyshex/shexmap/functions.py b/pyshex/shexmap/functions.py index 57f01b9..78afba8 100644 --- a/pyshex/shexmap/functions.py +++ b/pyshex/shexmap/functions.py @@ -16,10 +16,11 @@ lower: find the key whose value the variable holds. """ import json +from urllib.parse import quote import re from collections.abc import Callable, Mapping -from rdflib import Literal +from rdflib import Literal, URIRef from rdflib.term import Node VARIABLE = re.compile(r'^ *(?:<([^>]*)>|([^:<>\s]*):(\S*)) *$') @@ -156,8 +157,94 @@ def _hashmap_lower(code: str, args: str, get: Callable[[str], Node | None], pref raise MapFunctionError(f'{code.strip()} cannot invert "{val}"') -LIFTERS = {'regex': _regex_lift, 'hashmap': _hashmap_lift} -LOWERERS = {'regex': _regex_lower, 'hashmap': _hashmap_lower} +# -- id ---------------------------------------------------------------------------------- + +NODE_ARGUMENT = "@node" # in id(): the input node the enclosing iteration matched +_PLACEHOLDER = re.compile(r'\{([^{}]*)\}') + + +class Template: + """An IRI template argument of ``id()``: ````.""" + + def __init__(self, text: str, prefixes: Mapping[str, str]) -> None: + self.segments: list = [] # str, or ('var', iri) + pos = 0 + for m in _PLACEHOLDER.finditer(text): + if m.start() > pos: + self.segments.append(text[pos:m.start()]) + self.segments.append(('var', expand_variable(m.group(1), prefixes))) + pos = m.end() + if pos < len(text): + self.segments.append(text[pos:]) + self.text = text + + def variables(self) -> list[str]: + return [seg[1] for seg in self.segments if isinstance(seg, tuple)] + + def expand(self, get: Callable[[str], Node | None]) -> URIRef | None: + """The IRI, values percent-encoded as R2RML does; None when a variable is unbound.""" + out = [] + for seg in self.segments: + if isinstance(seg, tuple): + value = get(seg[1]) + if value is None: + return None + out.append(quote(str(value), safe="-._~")) + else: + out.append(seg) + return URIRef("".join(out)) + + def __repr__(self) -> str: + return f"Template(<{self.text}>)" + + +def key_terms(code: str, prefixes: Mapping[str, str]) -> list: + """The arguments of an ``id(...)`` code: variable IRIs, :data:`NODE_ARGUMENT`, and + :class:`Template` s for ``<...{var}...>`` arguments.""" + name, args = _split_call(code) + if name != 'id': + raise MapFunctionError(f"{code.strip()} is not an id() code") + parts = [a.strip() for a in args.split(',')] if args.strip() else [] + if not parts: + raise MapFunctionError(f"id() needs at least one variable, template or {NODE_ARGUMENT}: {code.strip()}") + terms = [] + for a in parts: + if a == NODE_ARGUMENT: + terms.append(a) + elif a.startswith('<') and a.endswith('>') and '{' in a: + terms.append(Template(a[1:-1], prefixes)) + else: + terms.append(expand_variable(a, prefixes)) + return terms + + +def key_arguments(code: str, prefixes: Mapping[str, str]) -> list[str]: + """The variables an ``id(...)`` code reads (templates' placeholders included), and + :data:`NODE_ARGUMENT` where it appears.""" + out: list[str] = [] + for term in key_terms(code, prefixes): + out.extend(term.variables() if isinstance(term, Template) else [term]) + return out + + +def _id_lift(code: str, args: str, value, prefixes: Mapping[str, str]) -> dict[str, Node]: + """``id(...)`` names an output node; validating input it binds nothing (so one schema + serves both directions).""" + key_arguments(code, prefixes) + return {} + + +def _id_lower(code: str, args: str, get, prefixes: Mapping[str, str]) -> Node | None: + raise MapFunctionError(f"{code.strip()} names a node; it belongs on a shape-valued constraint, not a value") + + +LIFTERS = {'regex': _regex_lift, 'hashmap': _hashmap_lift, 'id': _id_lift} +LOWERERS = {'regex': _regex_lower, 'hashmap': _hashmap_lower, 'id': _id_lower} + + +def is_key_code(code: str) -> bool: + m = FUNCTION_CALL.match(code) + return m is not None and m.group(1) == 'id' def lift(code: str, value, prefixes: Mapping[str, str]) -> dict[str, Node]: @@ -168,6 +255,19 @@ def lift(code: str, value, prefixes: Mapping[str, str]) -> dict[str, Node]: return LIFTERS[name](code, args, value, prefixes) +def variables_of(code: str, prefixes: Mapping[str, str]) -> list[str]: + """The variables a function call reads when lowering (and binds when lifting).""" + name, args = _split_call(code) + if name == 'regex': + return [expand_variable(_unescape_meta(m.group(1)), prefixes) + for m in re.finditer(_CAPTURE_NAME, _regex_body(code, args))] + if name == 'hashmap': + return [_hashmap_args(code, args, prefixes)[0]] + if name == 'id': + return [a for a in key_arguments(code, prefixes) if a != NODE_ARGUMENT] + raise MapFunctionError(f"Unknown ShExMap function {name}() in {code.strip()}") + + def lower(code: str, get: Callable[[str], Node | None], prefixes: Mapping[str, str]) -> Node | None: """Build a value from bindings with a function call; None when a variable is unbound.""" name, args = _split_call(code) diff --git a/pyshex/shexmap/materializer.py b/pyshex/shexmap/materializer.py index 25e918a..9a32881 100644 --- a/pyshex/shexmap/materializer.py +++ b/pyshex/shexmap/materializer.py @@ -1,54 +1,78 @@ -"""Threaded materialization: build RDF conforming to an output schema from bindings. - -A port of shex.js's ``ThreadedMaterializer`` (packages/extension-map/doc/threaded-materializer.md). - -* The binding tree is flattened to a sequence of **frames** (:func:`normalize`), read - through a cursor that stays on the current frame while it has an unused binding for a - variable and otherwise scans forward, never back. -* Each output shape compiles to an **NFA** with four kinds of state: ``TC`` (emit one - instance of a triple constraint), ``Split`` (OneOf / ShapeOr, in priority order), - ``Rept`` (cardinality) and ``Match``. A shape-valued constraint invents a blank node, - links it and calls the nested shape's NFA; ``Match`` returns from the call. -* A **thread** is an immutable configuration -- state, call stack, cursor, emitted - triples. One whose variable is unbound just dies, taking its emissions and cursor - marks with it, so a failed optional group or disjunct never disturbs its siblings. -* Threads run depth-first in greedy order (another repetition, then the emitting arm of - an optional, then the first disjunct), except that a lookup which has to **advance to - a later frame, abandoning unused bindings there,** is deferred until the alternatives - that can still use the current frame have been explored. An advance that abandons - nothing is not a choice, so it is not deferred. -* Every accepting thread is ranked; :meth:`ThreadedMaterializer.materialize` returns the - one that consumed the most bindings (ties: fewest bindings skipped by cursor advances, - then most triples, then discovery order), or whichever ``prefer`` ranks first. The - best ``max_accepts`` are kept in :attr:`~ThreadedMaterializer.accepts`; reaching that - many does not stop the search, only a perfect accept or the step budgets do. - -Beyond shex.js: ``EXTENDS`` in the output schema materializes the extended shapes' triple -constraints on the same node, and a reference to a shape that has extensions may -materialize any non-abstract one of them -- the threads decide which fits the bindings. +"""Materialization by iteration scopes: build RDF conforming to an output schema from bindings. + +The binding tree is read as a **scope tree** (:mod:`pyshex.shexmap.scopes`): a scope's own +bindings, and one list of iteration scopes per repeated constraint or group of the input. +Nothing is flattened and nothing is consumed: + +* A constraint with a Map variable **reads** it from the scope it is evaluated at, or from + the nearest ancestor scope that binds it. Reading marks nothing, so a parent's binding + can be read in every item, and twice in one. A variable bound nowhere on that chain + fails the constraint. +* A **repetition** in the output schema iterates one list of the input: the deepest list + its body's variables are bound at (nested repetitions count through their parent list). + Its body runs once per iteration of that list, in list order, at that iteration's + scope; an iteration whose body fails is skipped; the count of successful iterations + must reach the minimum, and the repetition stops at the maximum. A body whose + variables are all bound at or above the current scope runs once, there. A body whose + variables live in two lists neither of which contains the other is ill-formed, and + says so. +* **Choices** (``OneOf``, ``ShapeOr``, an optional constraint, a shape with extensions) + yield alternatives, per scope; the alternatives of the parts of a group or conjunction + combine by product, pruned to the best ``max_accepts`` as they combine. +* Every complete alternative is an :class:`Accept`; :meth:`ThreadedMaterializer.materialize` + returns the one that read the most distinct bindings (ties: most triples, then schema + order), or whichever ``prefer`` ranks first, and keeps the rest in + :attr:`~ThreadedMaterializer.accepts`. +* A value a Map code produces must **satisfy the constraint's value expression** when + that is a node constraint (datatype, value set, facets, node kind). A plain literal + whose lexical form is valid for the constraint's datatype is retyped to it, so what + ``regex()`` and ``hashmap()`` produce can land in a typed constraint; any other mismatch + fails the constraint like an unbound variable, so an optional or another disjunct can + take over (``check_values=False`` turns this off). +* A shape-valued constraint that also carries a Map variable **names** its node: the + bound value is the node and the nested shape is materialized on it. One that carries + ``%Map:{ id(v:a, v:b) %}`` is **keyed**: its node is a function of the shape and the + values read for the key, so two constraints, or two iterations, with the same key make + the same node and their arcs merge -- grouping by value, and a shared reference. A + key of one IRI or blank node is that node itself; ``id(@node)`` reuses the input node + the enclosing iteration matched, so a schema maps a graph onto itself node for node. + Otherwise the node is a blank node determined by the root, the constraint, and the + scope (its ``@node`` when it is an iteration, else its path), so two runs over the same + bindings agree, and materializing the same root twice adds nothing. + +This replaces the frame-cursor search ported from shex.js's ``ThreadedMaterializer`` (kept +as the class name); ``docs/shexmap-semantics-draft.md`` is the normative description and +``docs/shexmap-structural-plan.md`` the reasons. ``EXTENDS`` in the output schema +materializes the extended shapes' constraints on the same node, and a reference to a +shape that has extensions may materialize any non-abstract one of them. """ from __future__ import annotations -import uuid +import hashlib +import itertools +import math from collections.abc import Callable, Mapping -from dataclasses import dataclass, field, replace +from dataclasses import dataclass from typing import Any from ShExJSG import ShExJ from rdflib import BNode, Graph, Literal, URIRef from rdflib.term import Node +from pyshex.parse_tree.parse_node import ParseNode +from pyshex.shape_expressions_language.p5_4_node_constraints import satisfiesNodeConstraint from pyshex.shape_expressions_language.p5_context import Context from pyshex.shapemap_structure_and_language.p3_shapemap_structure import START from pyshex.shexmap import functions from pyshex.shexmap.bindings import (Bindings, declaration_of, expression_of, extension_candidates, map_actions, - normalize, references_shape, shape_parts, shexc_prefixes, term_from_json) + references_shape, shape_parts, shexc_prefixes, term_from_json) +from pyshex.shexmap.scopes import BindingTreeError, ListPath, Scope, ScopeTree UNBOUNDED = -1 class MaterializationError(ValueError): - """No thread could build the requested shape from the bindings. + """No alternative materializes the requested shape from the bindings. :ivar failures: the dead ends met on the way, deepest last :ivar report: see :attr:`ThreadedMaterializer.last_report` @@ -68,141 +92,52 @@ def _describe(f: dict) -> str: return f"{f.get('predicate', '')} {what} {f.get('error', 'unbound')}".strip() -# -- the frame cursor ------------------------------------------------------------------- +# -- results ------------------------------------------------------------------------------ @dataclass(frozen=True) -class Cursor: - idx: int = 0 # current frame - used: frozenset = frozenset() # (frame, variable) pairs consumed - n: int = 0 # frame bindings consumed - skipped: int = 0 # unused bindings abandoned by advancing +class Result: + """One way to materialize something: its triples, where each came from, and the bindings + it read.""" + quads: tuple = () + reads: frozenset = frozenset() # (scope path, variable) + sources: tuple = () # parallel to quads: see Accept.provenance + def then(self, other: Result) -> Result: + return Result(self.quads + other.quads, self.reads | other.reads, self.sources + other.sources) -def cursor_get(frames: list[dict], statics: Mapping[str, Node], cursor: Cursor, var: str): - """(value, new cursor) for ``var``, or None when no unused binding remains.""" - if var in statics: # static vars: always there, never used up - return statics[var], cursor - for i in range(cursor.idx, len(frames)): - if var in frames[i] and (i, var) not in cursor.used: - used = cursor.used | {(i, var)} - skipped = cursor.skipped + sum(1 for j in range(cursor.idx, i) for v in frames[j] - if (j, v) not in used) - return frames[i][var], Cursor(i, used, cursor.n + 1, skipped) - return None +EMPTY = Result() -# -- NFAs -------------------------------------------------------------------------------- -@dataclass -class State: - type: str # TC | Split | Rept | Match - outs: list[int] = field(default_factory=list) - tc: Any = None - min: int = 1 - max: float = 1 - skippable: bool = False - - -@dataclass -class NFA: - states: list[State] - start: int - - -def _clone_into(combined: list[State], nfa: NFA) -> int: - offset = len(combined) - for s in nfa.states: - combined.append(replace(s, outs=[o + offset for o in s.outs])) - return offset - - -def _concat(parts: list[NFA]) -> NFA: - """Run each part in turn against the same subject.""" - if not parts: - return NFA([State('Match')], 0) - states: list[State] = [] - offsets = [_clone_into(states, p) for p in parts] - for i in range(len(parts) - 1): # each part's Match (state 0) hands over to the next - states[offsets[i]] = State('Split', outs=[offsets[i + 1] + parts[i + 1].start]) - return NFA(states, offsets[0] + parts[0].start) - - -def _split(parts: list[NFA]) -> NFA: - """Fork over the parts, in priority order.""" - if not parts: - return NFA([State('Match')], 0) - states: list[State] = [] - outs = [_clone_into(states, p) + p.start for p in parts] - states.append(State('Split', outs=outs)) - return NFA(states, len(states) - 1) +def _source(tc, scope: Scope, reads=(), statics=(), **kind) -> dict: + """The provenance of one triple: the constraint, the input scope it was evaluated at (its + path and the node it matched), the bindings and static variables read for it, and how + the object arose.""" + return {'tc': tc, 'predicate': str(tc.predicate), 'scope': scope.path, 'node': scope.nearest_node(), + 'reads': tuple(reads), 'statics': tuple(statics), **kind} -# -- threads ----------------------------------------------------------------------------- - -@dataclass(frozen=True) -class CallFrame: - nfa: NFA - outs: tuple - subject: Node - repeats: tuple - parent: CallFrame | None - skippable: bool - quads_mark: Any - consumed_mark: int - - -@dataclass(frozen=True) -class Thread: - nfa: NFA - state: int - subject: Node - repeats: tuple # sorted ((state, (count, consumed_at)), ...) - call_stack: CallFrame | None - cursor: Cursor - quads: Any # persistent list: (triple, previous) or None - bnode: int - - -def _repeats_get(repeats: tuple, state: int): - for k, v in repeats: - if k == state: - return v - return None - - -def _repeats_set(repeats: tuple, state: int, value) -> tuple: - kept = tuple((k, v) for k, v in repeats if k != state) - return tuple(sorted(kept + ((state, value),))) if value is not None else kept - - -def _collect(quads) -> list[tuple[Node, Node, Node]]: - out = [] - while quads is not None: - out.append(quads[0]) - quads = quads[1] - out.reverse() - seen, kept = set(), [] - for q in out: - if q not in seen: - seen.add(q) - kept.append(q) - return kept - - -def _depth(call_stack: CallFrame | None) -> int: - d = 0 - while call_stack is not None: - d, call_stack = d + 1, call_stack.parent - return d +def _rank(r: Result) -> tuple[int, int]: + return (len(r.reads), len(r.quads)) @dataclass class Accept: - """An accepting thread's result.""" + """A complete materialization. + + :ivar provenance: one record per triple: ``tc`` and ``predicate`` (the output constraint), + ``scope`` and ``node`` (the input iteration's path, and the node it matched), ``reads`` + (the ``(scope path, variable)`` bindings read for the object), ``statics`` (the static + variables read for it), and one of + ``variable``/``code`` (a Map code's value), ``constant`` (a value the schema names), + ``structural`` (a link to a minted node), ``named`` (a node a variable names) or + ``keyed`` (an ``id()`` node) -- the correspondence between output and input + """ triples: list[tuple[Node, Node, Node]] - consumed: int - skipped: int - used: frozenset + consumed: int # distinct bindings read + skipped: int # always 0: nothing is passed over any more + used: frozenset # the (scope path, variable) addresses read + provenance: list[dict] = None class ThreadedMaterializer: @@ -210,16 +145,16 @@ class ThreadedMaterializer: :param schema: output schema, as ShExC text or a parsed ShExJ schema :param prefixes: prefixes for ShExMap variable names; read from ShExC text when omitted - :param static_vars: variable values available everywhere, never used up + :param static_vars: variable values available everywhere :param prefer: comparator over :class:`Accept` s, negative when the first is better - :param max_repeat: cap on the iterations of one repetition; ``None`` (the default) - leaves ``*`` and ``+`` bounded only by the bindings, which the progress guard on - repetitions makes safe - :param max_call_depth: cap on nested shape calls (cyclic references) - :param max_steps: cap on thread steps for the whole search - :param max_accepts: how many of the best accepts to keep in :attr:`accepts` - :param explore_steps: how many steps to keep searching after the last improvement - :param require_bindings_in_subshapes: drop optional subshapes that consume no binding + :param max_repeat: cap on the iterations of one repetition; ``None`` for none + :param max_call_depth: cap on nested shape calls (recursive schemas) + :param max_accepts: how many alternatives to keep, at every level and at the end + :param require_bindings_in_subshapes: drop optional subshapes that read no binding + :param check_values: check bound values against node constraints, retyping plain + literals whose lexical form fits the datatype + :param max_steps, explore_steps: accepted for compatibility; the search has no budget + to spend any more """ def __init__(self, schema: str | ShExJ.Schema, prefixes: Mapping[str, str] | None = None, @@ -227,7 +162,7 @@ def __init__(self, schema: str | ShExJ.Schema, prefixes: Mapping[str, str] | Non prefer: Callable[[Accept, Accept], int] | None = None, max_repeat: int | None = None, max_call_depth: int = 50, max_steps: int = 1_000_000, max_accepts: int = 20, explore_steps: int = 10_000, - require_bindings_in_subshapes: bool = False) -> None: + require_bindings_in_subshapes: bool = False, check_values: bool = True) -> None: from pyshex.utils.schema_loader import SchemaLoader if isinstance(schema, str): prefixes = {**shexc_prefixes(schema), **(prefixes or {})} @@ -240,271 +175,142 @@ def __init__(self, schema: str | ShExJ.Schema, prefixes: Mapping[str, str] | Non self.statics = {k: (v if isinstance(v, Node) else term_from_json(v)) for k, v in (static_vars or {}).items()} self.prefer = prefer self.max_repeat, self.max_call_depth = max_repeat, max_call_depth - self.max_steps, self.max_accepts, self.explore_steps = max_steps, max_accepts, explore_steps + self.max_accepts = max_accepts self.require_bindings_in_subshapes = require_bindings_in_subshapes - self._cache: dict[int, NFA] = {} - self._compiling: list[tuple[int, str]] = [] + self.check_values = check_values self.accepts: list[Accept] = [] self.chosen: Accept | None = None + self.provenance: list[dict] | None = None self.last_report: dict = {} - self.frames: list[dict] = [] - - # -- compilation -------------------------------------------------------------------- - def _compile_shape_expr(self, se, label: str | None = None) -> NFA: - if isinstance(se, str): - decl = declaration_of(self.cntxt, se) - if decl is None: - raise MaterializationError(f"Shape {se} is not defined in the output schema") - return self._compile_decl(decl, str(se)) - return self._compile_expr(se, label or f"(inline {type(se).__name__})") - - def _compile_decl(self, decl, label: str) -> NFA: - """A reference: the declaration itself or any non-abstract extension of it.""" - key = id(decl) * 2 + 1 - if key in self._cache: - return self._cache[key] - options = extension_candidates(self.cntxt, decl) - if not options: - raise MaterializationError(f"Shape {label} is abstract and nothing extends it") - options.reverse() # the declaration itself first, then extensions - nfa = _split([self._compile_expr(expression_of(o), str(getattr(o, 'id', label))) for o in options]) \ - if len(options) > 1 else self._compile_expr(expression_of(options[0]), label) - self._cache[key] = nfa - return nfa - - def _compile_expr(self, se, label: str) -> NFA: - key = id(se) * 2 - if key in self._cache: - return self._cache[key] - if any(k == key for k, _ in self._compiling): - loop = [lbl for k, lbl in self._compiling[[k for k, _ in self._compiling].index(key):]] - raise MaterializationError("cycle in shape expressions: " + " -> ".join(loop + [label])) - self._compiling.append((key, label)) - try: - if isinstance(se, ShExJ.ShapeDecl): - nfa = self._compile_expr(se.shapeExpr, label) - elif isinstance(se, ShExJ.Shape): - nfa = _concat([self._nfa_for_expression(p.expression) for p in shape_parts(self.cntxt, se)]) - elif isinstance(se, (ShExJ.ShapeAnd, ShExJ.ShapeOr)): - parts = [self._compile_shape_expr(p) for p in se.shapeExprs - if not isinstance(p, ShExJ.NodeConstraint)] - nfa = _concat(parts) if isinstance(se, ShExJ.ShapeAnd) else _split(parts) - elif isinstance(se, ShExJ.NodeConstraint): - nfa = NFA([State('Match')], 0) - else: - raise MaterializationError(f"{type(se).__name__} synthesis is not supported") - finally: - self._compiling.pop() - self._cache[key] = nfa - return nfa - - def _nfa_for_expression(self, expression) -> NFA: - states: list[State] = [State('Match')] - - def mk(s: State) -> int: - states.append(s) - return len(states) - 1 - - def patch(tail: list[int], target: int) -> None: - for t in tail: - states[t].outs.append(target) - - def walk(expr) -> tuple[int, list[int]]: - if isinstance(expr, str): # an inclusion: &label - expr = self.cntxt.tripleExprFor(expr) - if isinstance(expr, ShExJ.TripleConstraint): - s = mk(State('TC', tc=expr)) - start, tail = s, [s] - elif isinstance(expr, ShExJ.OneOf): - starts, tails = [], [] - for nested in expr.expressions: - s, t = walk(nested) - starts.append(s) - tails.extend(t) - start, tail = mk(State('Split', outs=starts)), tails - elif isinstance(expr, ShExJ.EachOf): - start = tail = None - for nested in expr.expressions: - s, t = walk(nested) - if start is None: - start = s - else: - patch(tail, s) - tail = t - else: - raise MaterializationError(f"unexpected triple expression {type(expr).__name__}") - min_ = 1 if expr.min is None else expr.min - max_ = 1 if expr.max is None else (float('inf') if expr.max == UNBOUNDED else expr.max) - if isinstance(expr, ShExJ.TripleConstraint): - if min_ == 0 and max_ == 1 and self._always_synthesizable(expr): - return start, tail # skipping a constant gains nothing: emit it - if min_ == 0: - states[start].skippable = True - if min_ == 1 and max_ == 1: - return start, tail - rept = mk(State('Rept', outs=[start], min=min_, max=max_)) - patch(tail, rept) - return rept, [rept] - - start = 0 - if expression is not None: - s, tail = walk(expression) - patch(tail, 0) - start = s - return NFA(states, start) - - def _always_synthesizable(self, tc) -> bool: - acts = map_actions(tc) - if acts: - return all(not functions.is_function_call(str(a.code or '')) and - self._var(str(a.code or '')) in self.statics for a in acts) - return _single_value(tc.valueExpr) is not None - - def _var(self, code: str) -> str: - return functions.expand_variable(code, self.prefixes) + self.tree: ScopeTree | None = None + self._lists: dict[int, tuple] = {} # id(repeated expr) -> (list path or None,) + self._tc_index: dict[int, int] = {} + self._active: list[tuple[tuple, str]] = [] + self._failures: list[dict] = [] + self._referenced: set[str] = set() + self._dropped = 0 # -- running ------------------------------------------------------------------------ def materialize(self, bindings, root: str | Node | None = None, start=None) -> list[tuple[Node, Node, Node]]: """The triples of the best materialization of ``start`` (default: the schema's start) - rooted at ``root`` (default: a new blank node). The best ``max_accepts`` distinct - results are kept in :attr:`accepts`, the returned one in :attr:`chosen`. + rooted at ``root`` (default: a new blank node). The alternatives are kept in + :attr:`accepts`, best first; the returned one is :attr:`chosen`. - :raises MaterializationError: when no thread reaches an accepting state + :raises MaterializationError: when nothing materializes, or the bindings are not a + scope tree, or a repetition reads variables from unrelated lists """ tree = bindings.tree if isinstance(bindings, Bindings) else bindings - frames = normalize(tree) - self.frames = frames + try: + self.tree = ScopeTree(tree) + except BindingTreeError as e: + raise MaterializationError(str(e)) from e root = _node(root) if root is not None else BNode() if start is None or start is START: if self.schema.start is None: raise MaterializationError("The output schema has no start shape; name one with start=") - nfa = self._compile_shape_expr(self.schema.start, "START") + se = self.schema.start else: - nfa = self._compile_shape_expr(ShExJ.IRIREF(str(start))) - bnode_prefix = uuid.uuid4().hex[:8] + se = ShExJ.IRIREF(str(start)) + self._root = root + self._lists, self._tc_index, self._active = {}, {}, [] + self._failures, self._referenced, self._dropped = [], set(), 0 - failures: list[dict] = [] - referenced: set[str] = set() - available = set(self.statics) | {v for f in frames for v in f} + results = self._shape_expr(se, self.tree.root, root, 0, "START" if start is None else str(start)) + seen: set[frozenset] = set() accepts: list[Accept] = [] - self.accepts, self.chosen = accepts, None - by_used: dict[frozenset, Accept] = {} - graphs_seen: set[frozenset] = set() - total = sum(len(f) for f in frames) - found = 0 # distinct accepts, kept or not - best: Accept | None = None - - stack = [Thread(nfa, nfa.start, root, (), None, Cursor(), None, 0)] - deferred: list[Thread] = [] - seen: set = set() - steps = pruned = 0 - accepted_at = None # step of the last improvement of ``best`` - truncated = False - - while stack or deferred: - steps += 1 - if steps > self.max_steps: - if accepts: - truncated = True - break - raise MaterializationError(f"exceeded max_steps={self.max_steps}", failures, - self._report(failures, referenced, available, found, True, pruned)) - if accepted_at is not None and steps - accepted_at > self.explore_steps: - truncated = True - break - th = stack.pop() if stack else deferred.pop(0) - key = (th.state, id(th.nfa), id(th.call_stack), th.cursor.idx, th.cursor.used, th.repeats) + for r in sorted(results, key=_rank, reverse=True): # stable: schema order among ties + triples, provenance = _distinct(r.quads, r.sources) + key = frozenset(triples) if key in seen: - pruned += 1 continue seen.add(key) - st = th.nfa.states[th.state] - - if st.type == 'Match': - if th.call_stack is None: - triples = _collect(th.quads) - graph_key = frozenset(triples) - if graph_key in graphs_seen: - continue - graphs_seen.add(graph_key) - existing = by_used.get(th.cursor.used) - if existing is not None: - if len(triples) > len(existing.triples): - existing.triples, existing.skipped = triples, th.cursor.skipped - continue - accept = Accept(triples, th.cursor.n, th.cursor.skipped, th.cursor.used) - by_used[th.cursor.used] = accept - accepts.append(accept) - found += 1 - if best is None or self._better(accept, best): - best, accepted_at = accept, steps - if len(accepts) > self.max_accepts: - # keep the best max_accepts: a full list bounds memory and the - # alternatives offered, not the search -- the greedy leader's - # prefixes accept before it does, and would otherwise crowd it out - others = [a for a in accepts if a is not best] - worst = others[0] - for a in others[1:]: - if not self._better(a, worst): # ties: drop the latest - worst = a - accepts.remove(worst) - del by_used[worst.used] - if accept.consumed >= total: # perfect: nothing can beat it - break - continue - frame = th.call_stack - # vacuous descent: an optional subshape that emitted and consumed nothing - # would leave a dangling link; the skip arm already covers it - if frame.skippable and th.cursor.n == frame.consumed_mark and \ - (th.quads is frame.quads_mark or self.require_bindings_in_subshapes): - continue - for out in frame.outs: - stack.append(replace(th, nfa=frame.nfa, state=out, subject=frame.subject, - repeats=frame.repeats, call_stack=frame.parent)) - - elif st.type == 'Split': - for out in reversed(st.outs): # first disjunct explored first - stack.append(replace(th, state=out)) - - elif st.type == 'Rept': - count, at = _repeats_get(th.repeats, th.state) or (0, -1) - if count >= st.min: # exit (lower priority); reset for re-entry - stack.append(replace(th, state=st.outs[1], repeats=_repeats_set(th.repeats, th.state, None))) - # another iteration -- only if the last one consumed a frame binding, or - # constant-only subexpressions would repeat to max_repeat - limit = st.max if self.max_repeat is None else min(st.max, self.max_repeat) - if count < limit and (count == 0 or th.cursor.n > at): - stack.append(replace(th, state=st.outs[0], - repeats=_repeats_set(th.repeats, th.state, (count + 1, th.cursor.n)))) - - elif st.type == 'TC': - succs: list[Thread] = [] - self._step(th, st, frames, succs, failures, referenced, bnode_prefix) - if succs and succs[0].cursor.skipped > th.cursor.skipped: - # advancing past unused bindings is a choice: explore the alternatives - # that can still use them first. (An advance that abandons nothing - # has no alternative, and deferring it would only make the exit arms - # of enclosing repetitions accept, one per iteration, ahead of it.) - deferred.extend(succs) - else: - stack.extend(succs) - else: - raise MaterializationError(f"unexpected NFA state {st.type}") - - report = self._report(failures, referenced, available, found, truncated, pruned) - if best is None: - raise MaterializationError("no thread reached an accepting state", failures, report) + accepts.append(Accept(triples, len(r.reads), 0, r.reads, provenance)) + self.accepts = accepts + report = self._report(len(accepts)) + if not accepts: + raise MaterializationError("nothing materializes the shape from these bindings", self._failures, report) + best = accepts[0] + if self.prefer is not None: + for a in accepts[1:]: + if self.prefer(a, best) < 0: + best = a self.chosen = best + self.provenance = best.provenance return best.triples - def _better(self, a: Accept, b: Accept) -> bool: - if self.prefer is not None: - return self.prefer(a, b) < 0 - return (a.consumed, -a.skipped, len(a.triples)) > (b.consumed, -b.skipped, len(b.triples)) + def update(self, graph: Graph, bindings, root: str | Node | None, start=None) -> tuple[set, set]: + """Materialize into ``graph``, replacing what the output schema currently holds at + ``root``: the schema, used as an input schema on ``graph`` at ``root``, says which + triples it governs there; those no longer produced are removed, the new ones added, + and everything else is left alone. Returns ``(added, removed)``. - def _report(self, failures, referenced, available, found: int, truncated, pruned) -> dict: + Whether the graph holds an earlier materialization is decided by the predicates the + output schema's start shape constrains: a triple at ``root`` on one of them means + there is something to replace, and it must then conform. + + :raises MaterializationError: when ``graph`` holds something at ``root`` that does not + conform to the output schema (so what to replace cannot be told), or nothing + materializes + """ + from pyshex.shexmap.bindings import MapValidationError, bind_all + root = _node(root) if root is not None else BNode() + new = set(self.materialize(bindings, root, start=start)) + current: set = set() + if self._holds_something(graph, root, start): + try: + for b in bind_all(graph, self.schema, root, start=start, prefixes=self.prefixes): + current |= set(b.matched) + except MapValidationError as e: + raise MaterializationError(f"{root.n3()} holds something that does not conform to the output " + f"schema, so what to replace cannot be told: {e}") from e + removed = current - new + added = {t for t in new if t not in graph} + for t in removed: + graph.remove(t) + for t in new: + graph.add(t) + return added, removed + + def _holds_something(self, graph: Graph, root: Node, start) -> bool: + """Whether ``graph`` has, at ``root``, a triple on a predicate the output schema's start + shape constrains: the mark of an earlier materialization (or of a conflict).""" + se = self.schema.start if start is None or start is START else ShExJ.IRIREF(str(start)) + for pred, inverse in self._own_predicates(se, set()): + if any(graph.triples((None, pred, root) if inverse else (root, pred, None))): + return True + return False + + def _own_predicates(self, se, seen: set) -> set: + """(predicate, inverse) of the constraints on the shape's own node, extensions included.""" + if isinstance(se, str): + decl = declaration_of(self.cntxt, se) + return set().union(*(self._own_predicates(expression_of(o), seen) + for o in (extension_candidates(self.cntxt, decl) if decl is not None else []))) + if se is None or id(se) in seen: + return set() + seen.add(id(se)) + if isinstance(se, ShExJ.ShapeDecl): + return self._own_predicates(se.shapeExpr, seen) + if isinstance(se, ShExJ.Shape): + out: set = set() + stack = [p.expression for p in shape_parts(self.cntxt, se) if p.expression is not None] + while stack: + e = stack.pop() + if isinstance(e, str): + e = self.cntxt.tripleExprFor(e) + if isinstance(e, ShExJ.TripleConstraint): + out.add((URIRef(str(e.predicate)), bool(e.inverse))) + elif isinstance(e, (ShExJ.EachOf, ShExJ.OneOf)): + stack.extend(e.expressions) + return out + if isinstance(se, (ShExJ.ShapeAnd, ShExJ.ShapeOr)): + return set().union(*(self._own_predicates(p, seen) for p in se.shapeExprs)) + return set() + + def _report(self, found: int) -> dict: + available = set(self.statics) | (self.tree.variables() if self.tree else set()) seen, unbound = set(), [] - for f in failures: + for f in self._failures: v = f.get('variable') k = (v, f.get('predicate')) if v and v not in available and k not in seen: @@ -512,95 +318,479 @@ def _report(self, failures, referenced, available, found: int, truncated, pruned unbound.append(f) self.last_report = { 'unbound_variables': unbound, - 'unused_statics': [s for s in self.statics if s not in referenced], + 'unused_statics': [s for s in self.statics if s not in self._referenced], 'alternatives': found, - 'exploration_truncated': truncated, - 'configs_pruned': pruned, + 'exploration_truncated': self._dropped > 0, + 'configs_pruned': self._dropped, } return self.last_report - def _step(self, th: Thread, st: State, frames, succs, failures, referenced, bnode_prefix) -> None: - """Emit one instance of a triple constraint; successors go to ``succs``.""" - tc = st.tc + # -- shape expressions -------------------------------------------------------------- + def _shape_expr(self, se, scope: Scope, subject: Node, depth: int, label: str | None = None) -> list[Result]: + if isinstance(se, str): + decl = declaration_of(self.cntxt, se) + if decl is None: + raise MaterializationError(f"Shape {se} is not defined in the output schema") + options = extension_candidates(self.cntxt, decl) + if not options: + raise MaterializationError(f"Shape {se} is abstract and nothing extends it") + options.reverse() # the declaration itself first, then its extensions + return self._guarded(('ref', id(decl), scope.path, subject), str(se), lambda: self._prune( + [r for o in options for r in self._shape_expr(expression_of(o), scope, subject, depth, str(se))])) + label = label or f"(inline {type(se).__name__})" + if isinstance(se, ShExJ.ShapeDecl): + return self._shape_expr(se.shapeExpr, scope, subject, depth, label) + if isinstance(se, ShExJ.Shape): + parts = [p.expression for p in shape_parts(self.cntxt, se)] + return self._guarded(('shape', id(se), scope.path, subject), label, lambda: self._all( + [self._expression(e, scope, subject, depth) if e is not None else [EMPTY] for e in parts])) + if isinstance(se, ShExJ.ShapeAnd): + return self._guarded(('and', id(se), scope.path, subject), label, lambda: self._all( + [self._shape_expr(p, scope, subject, depth) for p in se.shapeExprs + if not isinstance(p, ShExJ.NodeConstraint)])) + if isinstance(se, ShExJ.ShapeOr): + return self._guarded(('or', id(se), scope.path, subject), label, lambda: self._prune( + [r for p in se.shapeExprs if not isinstance(p, ShExJ.NodeConstraint) + for r in self._shape_expr(p, scope, subject, depth)])) + if isinstance(se, ShExJ.NodeConstraint): + return [EMPTY] + raise MaterializationError(f"{type(se).__name__} synthesis is not supported") + + def _guarded(self, key: tuple, label: str, run: Callable[[], list[Result]]) -> list[Result]: + """Evaluate, refusing a shape expression that reaches itself at the same scope and + subject through references alone (`` @ OR ...``, `` @ OR ...``).""" + for i, (k, _) in enumerate(self._active): + if k == key: + raise MaterializationError("cycle in shape expressions: " + + " -> ".join(lbl for _, lbl in self._active[i:]) + " -> " + label) + self._active.append((key, label)) + try: + return run() + finally: + self._active.pop() + + # -- triple expressions ------------------------------------------------------------- + def _expression(self, expr, scope: Scope, subject: Node, depth: int) -> list[Result]: + if isinstance(expr, str): # an inclusion: &label + expr = self.cntxt.tripleExprFor(expr) + min_ = 1 if expr.min is None else expr.min + max_ = 1 if expr.max is None else (math.inf if expr.max == UNBOUNDED else expr.max) + if min_ == 1 and max_ == 1: + return self._once(expr, scope, subject, depth, False) + return self._repetition(expr, min_, max_, scope, subject, depth) + + def _once(self, expr, scope: Scope, subject: Node, depth: int, skippable: bool) -> list[Result]: + """The alternatives for one occurrence of ``expr``, cardinality aside.""" + if isinstance(expr, ShExJ.TripleConstraint): + return self._tc(expr, scope, subject, depth, skippable) + if isinstance(expr, ShExJ.EachOf): + return self._all([self._expression(e, scope, subject, depth) for e in expr.expressions]) + if isinstance(expr, ShExJ.OneOf): + return self._prune([r for e in expr.expressions for r in self._expression(e, scope, subject, depth)]) + raise MaterializationError(f"unexpected triple expression {type(expr).__name__}") + + def _repetition(self, expr, min_: int, max_: float, scope: Scope, subject: Node, depth: int) -> list[Result]: + list_path = self._list_path(expr) + skippable = min_ == 0 + if list_path is None or len(list_path) <= scope.depth: + # the body reads nothing below this scope: it runs once, here + if min_ > 1: + return [] + body = self._once(expr, scope, subject, depth, skippable) + return body if body else ([EMPTY] if min_ == 0 else []) + cap = max_ if self.max_repeat is None else min(max_, self.max_repeat) + results, count = [EMPTY], 0 + for item in scope.descendants_at(list_path): + if count >= cap: + break + body = self._once(expr, item, subject, depth, skippable) + if not body: + continue # this item does not fit the body: skip it + results = self._all([results, body]) + count += 1 + return results if count >= min_ else [] + + def _tc(self, tc: ShExJ.TripleConstraint, scope: Scope, subject: Node, depth: int, skippable: bool) -> list[Result]: pred = URIRef(str(tc.predicate)) def triple(obj: Node): if tc.inverse: if isinstance(obj, Literal): - return None # a literal cannot be a subject - return (obj, pred, th.subject) - return (th.subject, pred, obj) + self._failures.append({'predicate': str(pred), 'tc': tc, 'error': 'literal subject of inverse'}) + return None + return (obj, pred, subject) + return (subject, pred, obj) acts = map_actions(tc) + if any(functions.is_key_code(str(a.code or '')) for a in acts): + return self._keyed(tc, acts, scope, subject, depth, skippable, triple) if acts: - cursor = th.cursor - quads = th.quads + reads: set = set() + statics_read: list = [] + + def get(v: str): + self._referenced.add(v) + if v in self.statics: + statics_read.append(v) + return self.statics[v] + hit = scope.lookup(v) + if hit is None: + return None + value, at = hit + reads.add((at.path, v)) + return value + + quads = [] + objects = [] + sources = [] for act in acts: code = str(act.code or '') - if functions.is_function_call(code): - def get(v: str): - nonlocal cursor - referenced.add(v) - hit = cursor_get(frames, self.statics, cursor, v) - if hit is None: - return None - value, cursor = hit - return value - try: + before = set(reads) + del statics_read[:] + try: + if functions.is_function_call(code): value = functions.lower(code, get, self.prefixes) - except functions.MapFunctionError as e: - failures.append({'predicate': str(pred), 'tc': tc, 'code': code, 'error': str(e)}) - return + if value is None: + self._failures.append({'predicate': str(pred), 'tc': tc, 'code': code, 'error': 'unbound'}) + return [] + how = {'code': code.strip()} + else: + var = functions.expand_variable(code, self.prefixes) + value = get(var) + if value is None: + self._failures.append({'predicate': str(pred), 'tc': tc, 'variable': var, 'scope': scope.path}) + return [] + how = {'variable': var} + except functions.MapFunctionError as e: + self._failures.append({'predicate': str(pred), 'tc': tc, 'code': code, 'error': str(e)}) + return [] + if self.check_values: + value = self._checked(tc, value) if value is None: - failures.append({'predicate': str(pred), 'tc': tc, 'code': code, 'error': 'unbound'}) - return - else: - try: - var = self._var(code) - except functions.MapFunctionError as e: - failures.append({'predicate': str(pred), 'tc': tc, 'code': code, 'error': str(e)}) - return - referenced.add(var) - hit = cursor_get(frames, self.statics, cursor, var) - if hit is None: - failures.append({'predicate': str(pred), 'tc': tc, 'variable': var, 'frame': cursor.idx}) - return - value, cursor = hit + self._failures.append({'predicate': str(pred), 'tc': tc, 'code': code, + 'error': 'the value does not satisfy the value expression'}) + return [] t = triple(value) if t is None: - failures.append({'predicate': str(pred), 'tc': tc, 'error': 'literal subject of inverse'}) - return - quads = (t, quads) - for out in st.outs: - succs.append(replace(th, state=out, cursor=cursor, quads=quads)) - return + return [] + quads.append(t) + objects.append(value) + sources.append(_source(tc, scope, sorted(reads - before), list(statics_read), **how)) + if references_shape(tc.valueExpr) and len(objects) == 1 and not isinstance(objects[0], Literal): + # the variable names the node: materialize the nested shape on it + if depth >= self.max_call_depth: + self._failures.append({'predicate': str(pred), 'tc': tc, 'error': 'exceeded max_call_depth'}) + return [] + sources[0]['named'] = True + result = Result(tuple(quads), frozenset(reads), tuple(sources)) + return [result.then(n) for n in self._shape_expr(tc.valueExpr, scope, objects[0], depth + 1)] + return [Result(tuple(quads), frozenset(reads), tuple(sources))] constant = _single_value(tc.valueExpr) if constant is not None: t = triple(constant) - if t is None: - failures.append({'predicate': str(pred), 'tc': tc, 'error': 'literal subject of inverse'}) - return - for out in st.outs: - succs.append(replace(th, state=out, quads=(t, th.quads))) - return + return [Result((t,), frozenset(), (_source(tc, scope, constant=True),))] if t is not None else [] if references_shape(tc.valueExpr): - if _depth(th.call_stack) >= self.max_call_depth: - failures.append({'predicate': str(pred), 'tc': tc, 'error': 'exceeded max_call_depth'}) + if depth >= self.max_call_depth: + self._failures.append({'predicate': str(pred), 'tc': tc, 'error': 'exceeded max_call_depth'}) + return [] + node = self._mint(tc, scope, depth) + link = triple(node) + if link is None: + return [] + out = [] + for n in self._shape_expr(tc.valueExpr, scope, node, depth + 1): + if skippable and not n.reads and (not n.quads or self.require_bindings_in_subshapes): + continue # an optional island nothing asked for + out.append(Result((link,) + n.quads, n.reads, (_source(tc, scope, structural=True),) + n.sources)) + return out + + self._failures.append({'predicate': str(pred), 'tc': tc, + 'error': f"cannot synthesize {type(tc.valueExpr).__name__ if tc.valueExpr else 'any value'}" + " without a Map action"}) + return [] + + def _keyed(self, tc, acts, scope: Scope, subject: Node, depth: int, skippable: bool, triple) -> list[Result]: + """A shape-valued constraint with ``id(...)``: the node is a function of the key.""" + pred = str(tc.predicate) + if len(acts) > 1: + raise MaterializationError(f"id() must be the only Map code on the constraint at {pred}") + if not references_shape(tc.valueExpr): + raise MaterializationError(f"id() at {pred} needs a shape-valued constraint: it names a node, not a value") + code = str(acts[0].code or '') + try: + terms = functions.key_terms(code, self.prefixes) + except functions.MapFunctionError as e: + raise MaterializationError(str(e)) from e + reads: set = set() + statics_read: list = [] + missing: list = [] + + def get(a: str): + self._referenced.add(a) + if a in self.statics: + statics_read.append(a) + return self.statics[a] + hit = scope.lookup(a) + if hit is None: + missing.append(a) + return None + value, at = hit + reads.add((at.path, a)) + return value + + values = [] + for term in terms: + if term == functions.NODE_ARGUMENT: + value = scope.nearest_node() + if value is None: + self._failures.append({'predicate': pred, 'tc': tc, 'code': code, 'error': 'no @node in scope'}) + return [] + elif isinstance(term, functions.Template): + value = term.expand(get) + else: + value = get(term) + if value is None: + self._failures.append({'predicate': pred, 'tc': tc, 'variable': missing[-1] if missing else code, + 'scope': scope.path}) + return [] + values.append(value) + if len(values) == 1 and not isinstance(values[0], Literal): + node = values[0] + else: + shape = str(tc.valueExpr) if isinstance(tc.valueExpr, str) else f"inline{self._tc_index.setdefault(id(tc), len(self._tc_index))}" + node = BNode("k" + _digest(shape, *(v.n3() for v in values))) + if depth >= self.max_call_depth: + self._failures.append({'predicate': pred, 'tc': tc, 'error': 'exceeded max_call_depth'}) + return [] + link = triple(node) + if link is None: + return [] + out = [] + for n in self._shape_expr(tc.valueExpr, scope, node, depth + 1): + if skippable and not (n.reads | reads) and (not n.quads or self.require_bindings_in_subshapes): + continue + out.append(Result((link,) + n.quads, frozenset(reads) | n.reads, + (_source(tc, scope, sorted(reads), statics_read, keyed=code.strip()),) + n.sources)) + return out + + # -- value expressions -------------------------------------------------------------- + def _checked(self, tc, value: Node) -> Node | None: + """``value`` if it satisfies the constraint's value expression, a retyped copy of a + plain literal whose lexical form fits the expression's datatype, or None.""" + se = self._resolve(tc.valueExpr) + if se is None or not _is_node_constraint(se): + return value # nothing to check: a shape, or no expression at all + if self._satisfies(se, value): + return value + dt = self._datatype_of(se) + if dt and isinstance(value, Literal) and value.datatype is None and value.language is None \ + and _valid_lexical(str(value), str(dt)) is not False: + retyped = Literal(str(value), datatype=URIRef(str(dt))) + if self._satisfies(se, retyped): + return retyped + return None + + def _datatype_of(self, se): + """The datatype a value expression asks for: its own, or the one conjunct that has one.""" + se = self._resolve(se) + if isinstance(se, ShExJ.NodeConstraint): + return se.datatype + if isinstance(se, ShExJ.ShapeAnd): + found = [dt for dt in (self._datatype_of(p) for p in se.shapeExprs) if dt] + return found[0] if len(set(map(str, found))) == 1 else None + return None + + def _resolve(self, se): + for _ in range(100): + if isinstance(se, str): + decl = declaration_of(self.cntxt, se) + if decl is None: + return None + se = expression_of(decl) + elif isinstance(se, ShExJ.ShapeDecl): + se = se.shapeExpr + else: + return se + return None + + def _satisfies(self, se, value: Node) -> bool: + se = self._resolve(se) + if isinstance(se, ShExJ.NodeConstraint): + saved = self.cntxt.current_node + self.cntxt.current_node = ParseNode(satisfiesNodeConstraint, se, value, self.cntxt) + try: + return bool(satisfiesNodeConstraint(self.cntxt, value, se)) + finally: + self.cntxt.current_node = saved + if isinstance(se, ShExJ.ShapeAnd): + return all(self._satisfies(p, value) for p in se.shapeExprs if _is_node_constraint(self._resolve(p))) + if isinstance(se, ShExJ.ShapeOr): + return any(self._satisfies(p, value) for p in se.shapeExprs) + if isinstance(se, ShExJ.ShapeNot): + return not self._satisfies(se.shapeExpr, value) + return True # a shape: the nested materialization is the check + + def _mint(self, tc, scope: Scope, depth: int) -> BNode: + """The node for a shape-valued constraint without a key: determined by the root, the + constraint, its call depth, and the scope -- the input node the scope matched when it + is an iteration, else its path -- so a materialization is reproducible and two runs + over the same input agree.""" + n = self._tc_index.setdefault(id(tc), len(self._tc_index)) + where = scope.node.n3() if scope.node is not None else "p" + "_".join(map(str, scope.path)) + return BNode(f"m{_digest(self._root.n3(), str(n), str(depth), where)}") + + # -- combining alternatives --------------------------------------------------------- + def _all(self, parts: list[list[Result]]) -> list[Result]: + """Every combination of one alternative per part, pruned as it grows.""" + acc = [EMPTY] + for alternatives in parts: + if not alternatives: + return [] + acc = self._prune([a.then(b) for a in acc for b in alternatives]) + return acc + + def _prune(self, results: list[Result]) -> list[Result]: + if len(results) <= self.max_accepts: + return results + ranked = sorted(range(len(results)), key=lambda i: _rank(results[i]), reverse=True)[:self.max_accepts] + self._dropped += len(results) - len(ranked) + return [results[i] for i in sorted(ranked)] + + # -- which list a repetition iterates ----------------------------------------------- + def _list_path(self, expr) -> ListPath | None: + key = id(expr) + if key not in self._lists: + self._lists[key] = (self._analyse(expr),) + return self._lists[key][0] + + def _analyse(self, expr) -> ListPath | None: + """The list the repetition ``expr`` iterates: the deepest list its body's variables are + bound at, nested repetitions counting through their parent list.""" + direct: set[str] = set() + nested: list = [] + self._collect(expr, direct, nested, set(), top=True) + bound = self.tree.bound_at + candidates: dict[ListPath, str] = {} + for v in direct: + if v in bound: + candidates.setdefault(bound[v], v) + for r in nested: + lp = self._list_path(r) + if lp: + candidates.setdefault(lp[:-1], f"the repetition at {_predicates(r)}") + if not candidates: + return None + deepest = max(candidates, key=len) + for lp, why in candidates.items(): + if deepest[:len(lp)] != lp: + raise MaterializationError( + f"the repetition at {_predicates(expr)} reads from unrelated lists: " + f"{why} is bound at {lp} and {candidates[deepest]} at {deepest}") + return deepest + + def _collect(self, expr, direct: set[str], nested: list, seen: set, top: bool) -> None: + """Variables read directly in ``expr``'s body and the repetitions directly inside it.""" + if isinstance(expr, str): + expr = self.cntxt.tripleExprFor(expr) + if expr is None or id(expr) in seen: + return + seen.add(id(expr)) + if not top and not (expr.min in (None, 1) and expr.max in (None, 1)): + nested.append(expr) + return + if isinstance(expr, ShExJ.TripleConstraint): + for act in map_actions(expr): + code = str(act.code or '') + try: + if functions.is_function_call(code): + direct.update(functions.variables_of(code, self.prefixes)) + else: + direct.add(functions.expand_variable(code, self.prefixes)) + except functions.MapFunctionError: + pass + if references_shape(expr.valueExpr): + self._collect_shape(expr.valueExpr, direct, nested, seen) + elif isinstance(expr, (ShExJ.EachOf, ShExJ.OneOf)): + for e in expr.expressions: + self._collect(e, direct, nested, seen, top=False) + + def _collect_shape(self, se, direct: set[str], nested: list, seen: set) -> None: + if isinstance(se, str): + decl = declaration_of(self.cntxt, se) + if decl is None: return - sub = self._compile_shape_expr(tc.valueExpr) - bnode = BNode(f"{bnode_prefix}t{th.bnode}") - quads = (triple(bnode), th.quads) - succs.append(Thread(sub, sub.start, bnode, (), - CallFrame(th.nfa, tuple(st.outs), th.subject, th.repeats, th.call_stack, - st.skippable, quads, th.cursor.n), - th.cursor, quads, th.bnode + 1)) + for o in extension_candidates(self.cntxt, decl): + self._collect_shape(expression_of(o), direct, nested, seen) return - - failures.append({'predicate': str(pred), 'tc': tc, - 'error': f"cannot synthesize {type(tc.valueExpr).__name__ if tc.valueExpr else 'any value'}" - " without a Map action"}) + if id(se) in seen: + return + seen.add(id(se)) + if isinstance(se, ShExJ.ShapeDecl): + self._collect_shape(se.shapeExpr, direct, nested, seen) + elif isinstance(se, ShExJ.Shape): + for p in shape_parts(self.cntxt, se): + if p.expression is not None: + self._collect(p.expression, direct, nested, seen, top=False) + elif isinstance(se, (ShExJ.ShapeAnd, ShExJ.ShapeOr)): + for p in se.shapeExprs: + self._collect_shape(p, direct, nested, seen) + + +def _is_node_constraint(se) -> bool: + """A value expression that constrains the node itself, not its arcs.""" + if isinstance(se, ShExJ.NodeConstraint): + return True + if isinstance(se, (ShExJ.ShapeAnd, ShExJ.ShapeOr)): + return any(_is_node_constraint(p) for p in se.shapeExprs if not isinstance(p, str)) + if isinstance(se, ShExJ.ShapeNot): + return _is_node_constraint(se.shapeExpr) + return False + + +def _valid_lexical(lexical: str, datatype: str) -> bool | None: + """Whether rdflib can read ``lexical`` as ``datatype``; None for a datatype it does not know.""" + from rdflib.term import XSDToPython + dt = URIRef(datatype) + if dt not in XSDToPython: + return None + converter = XSDToPython[dt] + if converter is None: + return True + try: + converter(lexical) + except (ValueError, TypeError, ArithmeticError): + return False + return True + + +def _digest(*parts: str) -> str: + return hashlib.sha1("\x00".join(parts).encode("utf-8")).hexdigest()[:16] + + +def _predicates(expr) -> str: + """A short name for a triple expression: its first predicate(s).""" + found = [] + stack = [expr] + while stack and len(found) < 2: + e = stack.pop() + if isinstance(e, ShExJ.TripleConstraint): + found.append(str(e.predicate)) + elif isinstance(e, (ShExJ.EachOf, ShExJ.OneOf)): + stack.extend(reversed(list(e.expressions))) + return ", ".join(found) or "(no predicate)" + + +def _distinct(quads, sources) -> tuple[list, list]: + """The quads without repeats, and each one's first source.""" + seen, out, prov = set(), [], [] + for q, s in zip(quads, sources): + if q not in seen: + seen.add(q) + out.append(q) + prov.append(s) + return out, prov def _single_value(value_expr) -> Node | None: @@ -622,7 +812,7 @@ def _node(term) -> Node: def materialize(schema: str | ShExJ.Schema, bindings, root: str | Node | None = None, start=None, static_vars: Mapping[str, Node] | None = None, prefixes: Mapping[str, str] | None = None, - graph: Graph | None = None, **options) -> Graph: + graph: Graph | None = None, replace: bool = False, **options) -> Graph: """Materialize ``root`` as an instance of the output schema from ``bindings``. A convenience over :class:`ThreadedMaterializer` (which also exposes the alternative @@ -635,13 +825,17 @@ def materialize(schema: str | ShExJ.Schema, bindings, root: str | Node | None = :param static_vars: extra variable values, shared everywhere (shex.js ``staticVars``) :param prefixes: prefixes for ShExMap variable names; read from ShExC text when omitted :param graph: graph to add to; a new one by default + :param replace: with ``graph``, replace what the output schema currently holds at + ``root`` instead of adding to it (see :meth:`ThreadedMaterializer.update`) :raises MaterializationError: when the shape cannot be built """ m = ThreadedMaterializer(schema, prefixes=prefixes, static_vars=static_vars, **options) - triples = m.materialize(bindings, root, start=start) graph = graph if graph is not None else Graph() for prefix, ns in m.prefixes.items(): graph.bind(prefix, ns, override=False) - for t in triples: + if replace: + m.update(graph, bindings, root, start=start) + return graph + for t in m.materialize(bindings, root, start=start): graph.add(t) return graph diff --git a/pyshex/shexmap/scopes.py b/pyshex/shexmap/scopes.py new file mode 100644 index 0000000..7e2ec82 --- /dev/null +++ b/pyshex/shexmap/scopes.py @@ -0,0 +1,148 @@ +"""The scope tree: a binding tree read structurally, for materialization by iteration scopes. + +A binding tree (see :mod:`pyshex.shexmap.bindings`) is a *scope*: an object of own +bindings, or an array whose first element is that object and whose other elements are +*lists*, each list holding one *iteration* (a scope) per match of a repeated constraint or +group. :func:`parse_scope` builds :class:`Scope` objects from a tree, accepting the two +older layouts shex.js and PyShEx wrote (``L1``, ``L2`` in the design notes), and +:class:`ScopeTree` answers the questions materialization asks: at what list a variable is +bound, and which scopes under a given one belong to a given list. + +A **list path** names a list by position: the tuple of list indices from the root, so +``()`` is the root scope itself, ``(0,)`` the root's first list, ``(0, 1)`` the second list +of an iteration of that list. Every iteration of a list shares the list's path; a variable +bound in those iterations is *bound at* that list path. +""" +from __future__ import annotations + +from dataclasses import dataclass, field + +from rdflib.term import Node + +from pyshex.shexmap.bindings import NODE_KEY + +ListPath = tuple[int, ...] + + +class BindingTreeError(ValueError): + """The binding tree is not a scope tree (nor one of the older layouts).""" + + +@dataclass +class Scope: + own: dict[str, Node] # variables bound here (never @node) + node: Node | None = None # the node this iteration matched, if recorded + lists: list[list[Scope]] = field(default_factory=list) + parent: Scope | None = None + list_path: ListPath = () # the list this scope is an iteration of + path: tuple[int, ...] = () # (list index, iteration index, ...) from the root + + @property + def depth(self) -> int: + return len(self.list_path) + + def lookup(self, var: str): + """(value, scope) for ``var`` from here or the nearest ancestor binding it, else None.""" + s: Scope | None = self + while s is not None: + if var in s.own: + return s.own[var], s + s = s.parent + return None + + def nearest_node(self) -> Node | None: + """The node of this scope or of the nearest ancestor that recorded one.""" + s: Scope | None = self + while s is not None: + if s.node is not None: + return s.node + s = s.parent + return None + + def descendants_at(self, list_path: ListPath) -> list[Scope]: + """The scopes below this one that are iterations of ``list_path``, in document order.""" + if not _prefix(self.list_path, list_path) or list_path == self.list_path: + return [] + out: list[Scope] = [] + for lst in self.lists: + for it in lst: + if it.list_path == list_path: + out.append(it) + elif _prefix(it.list_path, list_path): + out.extend(it.descendants_at(list_path)) + return out + + def walk(self): + yield self + for lst in self.lists: + for it in lst: + yield from it.walk() + + +def _prefix(a: tuple, b: tuple) -> bool: + return b[:len(a)] == a + + +def _is_scope_array(a) -> bool: + return isinstance(a, list) and len(a) >= 1 and isinstance(a[0], dict) and all(isinstance(e, list) for e in a[1:]) + + +def parse_scope(tree) -> Scope: + """The scope tree of a binding tree (rdflib terms or JSON terms as values are both fine).""" + if isinstance(tree, dict) or _is_scope_array(tree): + return _scope(tree, None, (), ()) + if isinstance(tree, list): + # L1: a root without own bindings, written without its object + if all(isinstance(e, dict) for e in tree): # shex.js: the root's one list + return _scope([{}, tree], None, (), ()) + if all(isinstance(e, list) for e in tree): + if all(_is_scope_array(e) for e in tree): # shex.js: one list of scope iterations + return _scope([{}, tree], None, (), ()) + if all(all(isinstance(x, dict) for x in e) for e in tree): # PyShEx before 2026-09-27: the lists + return _scope([{}, *tree], None, (), ()) + raise BindingTreeError(f"not a scope: {_short(tree)}") + + +def _scope(node, parent: Scope | None, list_path: ListPath, path: tuple[int, ...]) -> Scope: + own = node if isinstance(node, dict) else node[0] + lists = [] if isinstance(node, dict) else node[1:] + scope = Scope({k: v for k, v in own.items() if not k.startswith("@")}, own.get(NODE_KEY), [], parent, list_path, path) + for i, lst in enumerate(lists): + if not isinstance(lst, list): + raise BindingTreeError(f"a scope's lists must be arrays: {_short(lst)}") + scope.lists.append([_iteration(e, scope, list_path + (i,), path + (i, j)) for j, e in enumerate(lst)]) + return scope + + +def _iteration(elt, parent: Scope, list_path: ListPath, path: tuple[int, ...]) -> Scope: + if isinstance(elt, dict): + return _scope(elt, parent, list_path, path) + if _is_scope_array(elt): # a scope array; also L2 (shex.js writes a nested scope as a sibling) + return _scope(elt, parent, list_path, path) + if isinstance(elt, list) and all(isinstance(x, dict) for x in elt): + # shex.js, a nested scope with no own bindings whose one list was unwrapped + return _scope([{}, elt], parent, list_path, path) + raise BindingTreeError(f"not an iteration: {_short(elt)}") + + +def _short(x) -> str: + s = repr(x) + return s if len(s) < 120 else s[:117] + "..." + + +class ScopeTree: + """A parsed tree plus the two indexes materialization needs.""" + + def __init__(self, tree) -> None: + self.root = parse_scope(tree) + self.bound_at: dict[str, ListPath] = {} + self.bindings = 0 + for s in self.root.walk(): + for v in s.own: + self.bindings += 1 + at = self.bound_at.setdefault(v, s.list_path) + if at != s.list_path: + raise BindingTreeError(f"variable {v} is bound at two lists, {at} and {s.list_path}") + + def variables(self) -> set[str]: + return set(self.bound_at) diff --git a/tests/test_shexmap/README.md b/tests/test_shexmap/README.md index f2ecb74..c39c578 100644 --- a/tests/test_shexmap/README.md +++ b/tests/test_shexmap/README.md @@ -7,5 +7,35 @@ Each `manifest.json` entry pairs an input schema and data with the bindings and output graph shex.js produces; `test_examples.py` checks PyShEx against them. `test_cardinality.py` covers cardinality above one: nothing caps how many values a -`*`/`+` binds or materializes, and three strict `xfail` tests pin the frame model's -known limits (nested regrouping, transposition, unchecked output value expressions). +`*`/`+` binds or materializes; a nested schema maps to itself, sibling lists transpose, +a parent's binding can be read in every item, and a body reading from unrelated lists +is refused. + +`test_values.py` covers the check of bound values against the output constraint's value +expression: retyping of plain literals, failure as an unbound variable, and conformance of +the output to its schema. + +`test_examples.py` also checks the round-trip law on every example: materializing the +bindings with the *input* schema gives a graph that binds to the same bindings. + +`test_shexjs_differential.py` runs every example through shex.js's `@shexjs/extension-map` +as well (`shexjs_driver.js`) and compares frames and graphs, when `SHEXJS_EXTENSION_MAP` +points at the package directory of a built shex.js checkout; skipped otherwise. + +`test_keys.py` covers node identity: `%Map:{ id(v:a, v:b) %}` on a shape-valued constraint +keys its node, so equal keys merge (grouping by value, shared references); `id(@node)` +reuses the input node, so a schema maps a graph onto itself; unkeyed nodes are +reproducible and specific to the root. + +`test_analysis.py` covers the static checks (`analyse`, `shexmap --check`): every example +pair and every schema against itself are coherent; unrelated lists, "which one?" reads, +unbound and unused variables, double binding sites and `id()` misuse are reported. + +`test_roundtrip.py` covers the round-trip laws of a schema pair (a schema binds back what +it wrote from; BP <-> BP back and inverse in <-> inverse out recover their bindings, the BP +pair its graph) and the provenance each output triple carries. + +`test_update.py` covers the triples a schema matched (`Bindings.matched`), the strong form +of the identity law, and updating a graph in place (`update`, `materialize(replace=True)`, +`shexmap --into`); `test_keys.py` also covers IRI templates in `id()`. + diff --git a/tests/test_shexmap/shexjs_driver.js b/tests/test_shexmap/shexjs_driver.js new file mode 100644 index 0000000..caad009 --- /dev/null +++ b/tests/test_shexmap/shexjs_driver.js @@ -0,0 +1,51 @@ +// Drive shex.js's extension-map for the differential tests (test_shexjs_differential.py). +// +// node shexjs_driver.js bind [] +// validates and prints the Map extension's bindings as JSON; "ROOT" means +// the data's one subject that is nobody's object (a blank node the caller cannot name) +// node shexjs_driver.js materialize [] [] +// runs ThreadedMaterializer and prints the result as N-Triples +// +// / are IRIs or _:labels; is a shape IRI (relative ones resolve against the +// schema base, as PyShEx's manifests write them) or "START". +"use strict"; +const [X, mode, schemaPath, dataPath, node, startArg, staticsPath] = process.argv.slice(2); +const req = m => require(require.resolve(m, {paths: [X]})); +const Fs = require("fs"); +const ShExParser = req("@shexjs/parser"), N3 = req("n3"), ShExUtil = req("@shexjs/util"), ShExTerm = req("@shexjs/term"); +const {ShExValidator, resultMapToShapeExprTest} = req("@shexjs/validator"); +const {ctor: RdfJsDb} = req("@shexjs/neighborhood-rdfjs"); +const Mapper = require(X)({rdfjs: N3, Validator: ShExValidator}); +const {ThreadedMaterializer} = require(X + "/lib/ThreadedMaterializer"); + +const TURTLE_BASE = "http://a.example/turtle/"; // what test_examples.py parses data with +const SCHEMA_BASE = "http://a.example/schema/"; +const schema = ShExParser.construct(SCHEMA_BASE, {}, {index: true}).parse(Fs.readFileSync(schemaPath, "utf8")); +const start = !startArg || startArg === "START" ? ShExValidator.Start : new URL(startArg, SCHEMA_BASE).href; + +if (mode === "bind") { + const store = new N3.Store(); + store.addQuads(new N3.Parser({baseIRI: TURTLE_BASE, format: "text/turtle"}).parse(Fs.readFileSync(dataPath, "utf8"))); + const validator = new ShExValidator(schema, RdfJsDb(store), {noCache: true}); + Mapper.register(validator, {ShExTerm, ShExUtil}); + let focus = node.startsWith("_:") ? N3.DataFactory.blankNode(node.slice(2)) : node; + if (node === "ROOT") { + const objects = new Set(store.getQuads(null, null, null, null).map(q => q.object.id)); + const roots = [...new Set(store.getQuads(null, null, null, null).map(q => q.subject).filter(s => !objects.has(s.id)))]; + if (roots.length !== 1) { console.error("expected one root, found " + roots.length); process.exit(2); } + focus = roots[0].termType === "BlankNode" ? "_:" + roots[0].value : roots[0].value; // the validator wants JSON-LD-style terms + } + const res = resultMapToShapeExprTest(validator.validateShapeMap([{node: focus, shape: start}])); + if (res.errors) { console.error(JSON.stringify(res.errors, null, 1)); process.exit(2); } + process.stdout.write(JSON.stringify(ShExUtil.valToExtension(res, Mapper.url)) + "\n"); +} else if (mode === "materialize") { + const bindings = JSON.parse(Fs.readFileSync(dataPath, "utf8")); + const staticVars = staticsPath ? JSON.parse(Fs.readFileSync(staticsPath, "utf8")) : {}; + const m = new ThreadedMaterializer(schema, {staticVars}); + const quads = m.materialize(bindings, node, start === ShExValidator.Start ? undefined : start); + const writer = new N3.Writer({format: "N-Triples"}); + quads.forEach(q => writer.addQuad(q)); + writer.end((err, out) => { if (err) throw err; process.stdout.write(out); }); +} else { + console.error("mode must be bind or materialize"); process.exit(1); +} diff --git a/tests/test_shexmap/test_analysis.py b/tests/test_shexmap/test_analysis.py new file mode 100644 index 0000000..0ef5352 --- /dev/null +++ b/tests/test_shexmap/test_analysis.py @@ -0,0 +1,122 @@ +"""Static checks of a schema pair (pyshex.shexmap.analyse): which pairs map coherently, and +why not, before any data.""" +import json +from pathlib import Path + +import pytest + +from pyshex.shexmap import analyse +from pyshex.shexmap.cli import main +from tests.test_shexmap.test_examples import MANIFEST, label, node_and_shape, text + +PREFIXES = """PREFIX : +PREFIX xsd: +PREFIX v: +PREFIX Map: +""" +NESTED_IN = PREFIXES + "start = @:P\n:P { :name . %Map:{ v:name %} ; :report { :no . %Map:{ v:no %} ; :result { :sys . %Map:{ v:sys %} }* }* }" + + +def out(schema: str) -> str: + return PREFIXES + "start = @:Out\n" + schema + + +@pytest.mark.parametrize("entry", MANIFEST, ids=label) +def test_every_example_pair_is_coherent(entry): + _, start = node_and_shape(entry["queryMap"]) + _, out_shape = node_and_shape(entry["outputShapeMap"]) + report = analyse(text(entry, "schema"), text(entry, "outputSchema"), input_start=start, output_start=out_shape, + static_vars=list(entry.get("staticVars", {}))) + assert report.errors == [], str(report) + + +@pytest.mark.parametrize("entry", MANIFEST, ids=label) +def test_every_schema_maps_to_itself(entry): + _, start = node_and_shape(entry["queryMap"]) + report = analyse(text(entry, "schema"), text(entry, "schema"), input_start=start, output_start=start) + assert report.errors == [], str(report) + assert report.warnings == [] # every binding is read back + + +def test_regrouping_and_flattening_are_coherent(): + for schema in [":Out { :group @* }\n { :label . %Map:{ v:no %} ; :reading . + %Map:{ v:sys %} }", + ":Out { :who . %Map:{ v:name %} ; :all . * %Map:{ v:sys %} }", + ":Out { :g { :label . %Map:{ v:no %} ; :reading . %Map:{ v:sys %} }* }", + ":Out { :bp { :sys . %Map:{ v:sys %} ; :who . %Map:{ v:name %} }* }"]: + report = analyse(NESTED_IN, out(schema)) + assert report.errors == [], str(report) + + +def test_a_deeper_variable_read_without_a_repetition_is_which_one(): + report = analyse(NESTED_IN, out(":Out { :first . %Map:{ v:sys %} }")) + assert len(report.errors) == 1 and "which one" in report.errors[0] + assert "each :report / :result" in report.errors[0] + + +def test_an_optional_over_a_deeper_variable_takes_the_first(): + # `?` is a repetition of at most one: it iterates the list and stops + report = analyse(NESTED_IN, out(":Out { :first . ? %Map:{ v:sys %} }")) + assert report.errors == [] + + +def test_unrelated_lists_are_named(): + two_lists = PREFIXES + "start = @:S\n:S { :a . * %Map:{ v:a %} ; :b { :v . %Map:{ v:b %} }* }" + report = analyse(two_lists, out(":Out { :pair { :x . %Map:{ v:a %} ; :y . %Map:{ v:b %} }* }")) + assert len(report.errors) == 1 + assert "unrelated lists" in report.errors[0] and "v:a" in report.errors[0] and "v:b" in report.errors[0] + assert "each :a" in report.errors[0] and "each :b" in report.errors[0] + + +def test_a_variable_from_a_sibling_list_is_not_in_scope(): + two_lists = PREFIXES + "start = @:S\n:S { :a . * %Map:{ v:a %} ; :b { :v . %Map:{ v:b %} }* }" + report = analyse(two_lists, out(":Out { :x { :va . %Map:{ v:a %} ; :vb . %Map:{ v:b %} }* }")) + assert any("unrelated lists" in e for e in report.errors) + + +def test_unbound_and_unused_variables(): + report = analyse(NESTED_IN, out(":Out { :who . %Map:{ v:name %} ; :oops . %Map:{ v:typo %} }")) + assert [e for e in report.errors if "never binds" in e and "typo" in e] + assert [w for w in report.warnings if "v:sys" in w and "never reads" in w] + report = analyse(NESTED_IN, out(":Out { :who . %Map:{ v:name %} ; :konst . %Map:{ v:stat %} }"), + static_vars=["http://v.example/stat", "http://v.example/unused"]) + assert not [e for e in report.errors if "stat" in e] + assert [w for w in report.warnings if "unused" in w and "static" in w] + + +def test_a_variable_bound_at_two_sites_is_an_error(): + twice = PREFIXES + "start = @:S\n:S { :n . %Map:{ v:x %} ; :item { :n . %Map:{ v:x %} }* }" + report = analyse(twice, out(":Out { :y . %Map:{ v:x %} }")) + assert [e for e in report.errors if "two places" in e] + + +def test_id_checks(): + assert [e for e in analyse(NESTED_IN, out(":Out { :n . %Map:{ id(v:no) %} }")).errors if "shape-valued" in e] + assert [e for e in analyse(NESTED_IN, out(":Out { :g @ %Map:{ id(v:name) %} %Map:{ v:name %} }\n { :k [:c] }")).errors + if "only Map code" in e] + assert [e for e in analyse(NESTED_IN, out(":Out { :g @ %Map:{ id(@node) %} }\n { :k [:c] }")).errors + if "@node" in e] + report = analyse(NESTED_IN, out(":Out { :g @* %Map:{ id(@node) %} }\n { :label . %Map:{ v:no %} ; :r . * %Map:{ v:sys %} }")) + assert report.errors == [], str(report) + + +def test_undefined_and_abstract_output_shapes(): + assert [e for e in analyse(NESTED_IN, out(":Out { :g @ }")).errors if "does not define" in e] + assert [e for e in analyse(NESTED_IN, out(":Out { :g @ }\nABSTRACT { :k [:c] }")).errors if "abstract" in e] + + +def test_cli_check(tmp_path, capsys): + (tmp_path / "in.shex").write_text(NESTED_IN, encoding="utf-8") + (tmp_path / "good.shex").write_text(out(":Out { :group @* }\n { :label . %Map:{ v:no %} ; :reading . + %Map:{ v:sys %} }"), + encoding="utf-8") + (tmp_path / "bad.shex").write_text(out(":Out { :first . %Map:{ v:sys %} }"), encoding="utf-8") + assert main(["--check", "-s", str(tmp_path / "in.shex"), "-t", str(tmp_path / "good.shex")]) == 0 + assert "warning" in capsys.readouterr().out # v:name is never read + assert main(["--check", "-s", str(tmp_path / "in.shex"), "-t", str(tmp_path / "bad.shex")]) == 1 + assert "which one" in capsys.readouterr().out + + +def test_template_variables_are_checked_like_the_others(): + report = analyse(NESTED_IN, out(":Out { :r @* %Map:{ id() %} }\n { :k [:c] }")) + assert report.errors == [], str(report) + report = analyse(NESTED_IN, out(":Out { :r @* %Map:{ id() %} }\n { :k [:c] }")) + assert [e for e in report.errors if "never binds" in e and "nope" in e] diff --git a/tests/test_shexmap/test_cardinality.py b/tests/test_shexmap/test_cardinality.py index 8b1311a..a2b653b 100644 --- a/tests/test_shexmap/test_cardinality.py +++ b/tests/test_shexmap/test_cardinality.py @@ -1,5 +1,6 @@ -"""Cardinality above one: nothing caps how many values bind or materialize, and the frame -model's known limits are pinned down as strict xfails, so they flip when a redesign lands.""" +"""Cardinality above one: nothing caps how many values bind or materialize; regrouping, +transposition and reads of inherited bindings come out as the input's structure says; a +body that reads from unrelated lists is refused.""" import pytest from rdflib import Graph, Literal, URIRef @@ -67,18 +68,16 @@ def test_a_starred_constraint_emits_every_binding(n): assert not m.last_report["exploration_truncated"] -def test_advances_that_forfeit_bindings_still_reach_the_full_materialization(): - # each advance abandons an unused v:u, so every repetition count accepts before the - # next one is tried; the full materialization is found last and must still win +def test_unread_bindings_beside_the_read_ones_change_nothing(): + # every item also binds v:u, which the output never reads: the frame model used to + # accept once per repetition count here and truncate at max_accepts n = 60 m = ThreadedMaterializer(STAR) triples = m.materialize(frames(n, "t", "u"), OUT) assert len(triples) == n assert m.chosen.consumed == n - assert m.chosen in m.accepts - assert len(m.accepts) == 20 # the best max_accepts are kept ... - assert sorted(a.consumed for a in m.accepts) == list(range(41, 61)) - assert m.last_report["alternatives"] == n + 1 # ... out of every distinct accept + assert m.chosen is m.accepts[0] + assert m.last_report["alternatives"] == 1 def test_a_starred_subshape_emits_every_frame(): @@ -108,14 +107,12 @@ def test_end_to_end_with_many_values(): assert len(out) == n -# -- limits of the frame model, as documented ---------------------------------------------- +# -- the structure of the input decides the structure of the output ------------------------ NESTED = PREFIXES + "start = @:P\n:P { :report { :no . %Map:{ v:no %} ; :result { :sys . %Map:{ v:sys %} }* }* }" REPORTS = ':p :report [ :no "one" ; :result [ :sys 100 ], [ :sys 101 ] ], [ :no "two" ; :result [ :sys 110 ], [ :sys 111 ] ] .' -@pytest.mark.xfail(strict=True, reason="frame model: a report's :no is copied into each reading's frame, " - "so one group per reading uses 'more' bindings") def test_a_nested_schema_maps_to_itself(): bound = bind(turtle(REPORTS), NESTED, URIRef("http://a.example/p")) out = materialize(NESTED, bound, OUT) @@ -124,8 +121,6 @@ def test_a_nested_schema_maps_to_itself(): assert sorted(len(list(out.objects(r, URIRef("http://a.example/result")))) for r in reports) == [2, 2] -@pytest.mark.xfail(strict=True, reason="frame model: the cursor never moves back, so a second repetition " - "cannot revisit the frames the first one passed") def test_sibling_lists_can_be_transposed(): m = ThreadedMaterializer(PREFIXES + "start = @:T\n:T { :allB . * %Map:{ v:b %} ; :allA . * %Map:{ v:a %} }") triples = m.materialize(frames(2, "a", "b"), OUT) @@ -133,9 +128,72 @@ def test_sibling_lists_can_be_transposed(): assert objects(triples, "allA") == ["a0", "a1"] -@pytest.mark.xfail(strict=True, reason="bound values are not checked against the output constraint's " - "value expression") def test_a_bound_value_must_satisfy_the_output_value_expression(): m = ThreadedMaterializer(PREFIXES + "start = @:T\n:T { :n xsd:integer %Map:{ v:t %} }") with pytest.raises(MaterializationError): m.materialize(Bindings({V + "t": Literal("not a number")}), OUT) + + +# -- inherited bindings and unrelated lists (structural plan §1.2, §3.3) ------------------- +# a parent's binding is read, never used up: it can be read at the parent and in every +# item, and twice in one item + +PATIENT = Bindings([{V + "name": Literal("Sue")}, + [{V + "sys": Literal("110"), V + "dia": Literal("70")}, + {V + "sys": Literal("111"), V + "dia": Literal("71")}]]) +A_EX = "http://a.example/" + + +def readings(graph_or_triples) -> list[tuple[str, str]]: + """(sys, dia) per :bp node, in sys order.""" + triples = list(graph_or_triples) + bps = {o for _, p, o in triples if str(p) == A_EX + "bp"} + value = lambda s, local: next(str(o) for s2, p, o in triples if s2 == s and str(p) == A_EX + local) + return sorted((value(b, "sys"), value(b, "dia")) for b in bps) + + +def test_a_parent_binding_read_once_per_item(): + m = ThreadedMaterializer(PREFIXES + "start = @:Out\n:Out { :bp { :sys . %Map:{ v:sys %} ; :who . %Map:{ v:name %} ; :dia . %Map:{ v:dia %} }* }") + assert readings(m.materialize(PATIENT, OUT)) == [("110", "70"), ("111", "71")] + + +def test_a_parent_binding_read_at_the_parent_and_in_each_item(): + m = ThreadedMaterializer(PREFIXES + "start = @:Out\n:Out { :who . %Map:{ v:name %} ; " + ":bp { :sys . %Map:{ v:sys %} ; :who . %Map:{ v:name %} ; :dia . %Map:{ v:dia %} }* }") + triples = m.materialize(PATIENT, OUT) + assert readings(triples) == [("110", "70"), ("111", "71")] + assert objects(triples, "who") == ["Sue", "Sue", "Sue"] + + +def test_a_parent_binding_read_twice_in_an_item(): + m = ThreadedMaterializer(PREFIXES + "start = @:Out\n:Out { :bp { :sys . %Map:{ v:sys %} ; " + ":who . %Map:{ v:name %} ; :who2 . %Map:{ v:name %} ; :dia . %Map:{ v:dia %} }* }") + assert readings(m.materialize(PATIENT, OUT)) == [("110", "70"), ("111", "71")] + + +def test_variables_from_unrelated_lists_are_refused(): + two_lists = Bindings([{}, [{V + "a": Literal("a1")}, {V + "a": Literal("a2")}], + [{V + "b": Literal("b1")}, {V + "b": Literal("b2")}]]) + m = ThreadedMaterializer(PREFIXES + "start = @:Out\n:Out { :pair { :x . %Map:{ v:a %} ; :y . %Map:{ v:b %} }* }") + with pytest.raises(MaterializationError, match="unrelated lists"): + m.materialize(two_lists, OUT) + + +def test_a_deeper_variable_read_without_a_repetition_fails(): + # "which one?" is not guessed: the read fails, as an unbound variable does + m = ThreadedMaterializer(PREFIXES + "start = @:Out\n:Out { :first . %Map:{ v:sys %} }") + with pytest.raises(MaterializationError, match="sys"): + m.materialize(PATIENT, OUT) + + +def test_a_repetition_over_a_deeper_list_flattens(): + reports = Bindings([{}, [[{V + "no": Literal("one")}, [{V + "sys": Literal("100")}, {V + "sys": Literal("101")}]], + [{V + "no": Literal("two")}, [{V + "sys": Literal("110")}]]]]) + m = ThreadedMaterializer(PREFIXES + "start = @:Out\n:Out { :all . * %Map:{ v:sys %} }") + assert objects(m.materialize(reports, OUT), "all") == ["100", "101", "110"] + # and a group per reading takes its label from the report above it + m = ThreadedMaterializer(PREFIXES + "start = @:Out\n:Out { :g { :label . %Map:{ v:no %} ; :reading . %Map:{ v:sys %} }* }") + triples = m.materialize(reports, OUT) + labels = {str(o) for _, p, o in triples if str(p) == A_EX + "label"} + assert len(objects(triples, "g")) == 3 and labels == {"one", "two"} + diff --git a/tests/test_shexmap/test_examples.py b/tests/test_shexmap/test_examples.py index 8847bd3..2daccd0 100644 --- a/tests/test_shexmap/test_examples.py +++ b/tests/test_shexmap/test_examples.py @@ -10,7 +10,7 @@ from rdflib import BNode, Graph, URIRef from rdflib.compare import isomorphic -from pyshex.shexmap import AmbiguousBindingsError, bind, bind_all, dumps, loads, materialize +from pyshex.shexmap import NODE_KEY, AmbiguousBindingsError, bind, bind_all, dumps, loads, materialize, normalize HERE = Path(__file__).parent EXAMPLES = HERE / "examples" @@ -74,15 +74,97 @@ def example(request): return entry, bindings +def without_nodes(obj): + """The tree without its ``@node`` keys, which shex.js does not write (yet).""" + if isinstance(obj, dict): + return {k: v for k, v in obj.items() if k != NODE_KEY} + if isinstance(obj, list): + return [without_nodes(e) for e in obj] + return obj + + +def canonical(obj): + """``without_nodes``, with blank-node values reduced to a marker (labels do not survive + a trip through materialization) and each list's iterations sorted (ShEx matches bags; + the order a neighbourhood is listed in follows the nodes' labels, which is arbitrary).""" + if isinstance(obj, dict): + return {k: ("_:" if isinstance(v, str) and v.startswith("_:") else v) + for k, v in obj.items() if k != NODE_KEY} + if isinstance(obj, list): + items = [canonical(e) for e in obj] + return sorted(items, key=lambda e: json.dumps(e, sort_keys=True)) + return obj + + +def nodes_in(obj) -> list[dict]: + """Every object of the tree that carries ``@node``.""" + if isinstance(obj, dict): + return [obj] if NODE_KEY in obj else [] + if isinstance(obj, list): + return [d for e in obj for d in nodes_in(e)] + return [] + + def test_bindings_match_shexjs(example): entry, bindings = example if "expectedBindingsURL" not in entry: pytest.skip("no shex.js bindings recorded") expected = json.loads(path(entry, entry["expectedBindingsURL"]).read_text(encoding="utf-8")) + got = without_nodes(bindings.to_json()) if label(entry) in BINDINGS_LAYOUT_DIFFERS: - assert bindings.to_json() != expected # if this starts failing, the layouts converged + assert got != expected # if this starts failing, the layouts converged else: - assert bindings.to_json() == expected + assert got == expected + + +# examples whose input schema has a repeated shape-valued constraint, so their trees have lists +# of iterations, each of which must say which node it matched +HAS_ITERATIONS = {"BPPatient multi-bindings / simple", "BPPatient 2 levels / simple", "splits / Ann-phone-mbox", + "splits / Ann-mbox-phone", "inverse in / Ann's heart rate", "inverse out / Ann's chart", + "EXTENDS / clinic"} +BP = "http://shex.io/extensions/Map/#BPDAM-" + + +def test_iterations_carry_the_node_they_matched(example): + entry, bindings = example + found = nodes_in(bindings.to_json()) + if label(entry) not in HAS_ITERATIONS: + assert found == [] + return + assert found + for d in found: + assert isinstance(d[NODE_KEY], str) # an IRI or a _:label, never a literal + if label(entry) == "BPPatient 2 levels / simple": + # the constraint that made the list also bound the node itself: the two must agree + assert {d[NODE_KEY] for d in found} == {d.get(BP + "reports") or d.get(BP + "bp") for d in found} + assert len(found) == 6 # two reports, four readings + if label(entry) == "BPPatient multi-bindings / simple": + assert all(d[NODE_KEY] == d[BP + "XXX"] for d in found) + if label(entry) == "inverse in / Ann's heart rate": + assert all(d[NODE_KEY].startswith("http://") for d in found) # ^:subject binds subjects + + +def test_nodes_survive_json(example): + entry, bindings = example + assert loads(dumps(bindings)).to_json() == bindings.to_json() + assert normalize(loads(dumps(bindings)).tree) == bindings.frames() + assert not any(NODE_KEY in f for f in bindings.frames()) + + +@pytest.mark.parametrize("entry", MANIFEST, ids=label) +def test_bindings_survive_a_round_trip_through_the_input_schema(entry): + """bind after materialize after bind is bind: materializing bindings with the *input* + schema gives a graph that binds to the same bindings, i.e. a schema maps to itself. + Compared as the set of parses, since an ambiguous input may parse in another order.""" + graph = input_graph(entry) + _, start = node_and_shape(entry["queryMap"]) + focus = focus_node(entry, graph) + schema = text(entry, "schema") + before = {json.dumps(canonical(b.to_json()), sort_keys=True) for b in bind_all(graph, schema, focus, start=start)} + root = BNode() if isinstance(focus, BNode) else focus + out = materialize(schema, bind(graph, schema, focus, start=start), root, start=start) + after = {json.dumps(canonical(b.to_json()), sort_keys=True) for b in bind_all(out, schema, root, start=start)} + assert after == before, out.serialize(format="turtle") def test_output_matches_shexjs(example): diff --git a/tests/test_shexmap/test_keys.py b/tests/test_shexmap/test_keys.py new file mode 100644 index 0000000..81aa848 --- /dev/null +++ b/tests/test_shexmap/test_keys.py @@ -0,0 +1,151 @@ +"""Node identity: keyed nodes (``id(...)``) merge, ``id(@node)`` reuses the input node, and +unkeyed nodes are reproducible.""" +import pytest +from rdflib import BNode, Graph, Literal, URIRef +from rdflib.compare import isomorphic + +from pyshex.shexmap import Bindings, MaterializationError, ThreadedMaterializer, bind, materialize + +PREFIXES = """PREFIX : +PREFIX xsd: +PREFIX v: +PREFIX Map: +""" +V = "http://v.example/" +A = "http://a.example/" +OUT = URIRef(A + "out") + + +def turtle(text: str) -> Graph: + return Graph().parse(data="PREFIX : \n" + text, format="turtle") + + +def by_predicate(triples, local: str): + return [(s, o) for s, p, o in triples if str(p) == A + local] + + +READINGS = Bindings([{}, [{V + "date": Literal("2026-09-01"), V + "sys": Literal("110")}, + {V + "date": Literal("2026-09-01"), V + "sys": Literal("112")}, + {V + "date": Literal("2026-09-02"), V + "sys": Literal("120")}]]) + + +def test_equal_keys_merge_into_one_node(): + # readings bound flat; grouped by date on the way out + m = ThreadedMaterializer(PREFIXES + "start = @:Out\n:Out { :day @* %Map:{ id(v:date) %} }\n" + " { :date . %Map:{ v:date %} ; :reading . %Map:{ v:sys %} }") + triples = m.materialize(READINGS, OUT) + days = {o for _, o in by_predicate(triples, "day")} + assert len(days) == 2 + readings = {(d, str(o)) for d, o in by_predicate(triples, "reading")} + first = next(d for d, o in by_predicate(triples, "date") if str(o) == "2026-09-01") + assert {r for d, r in readings if d == first} == {"110", "112"} + assert len(by_predicate(triples, "date")) == 2 # merged, not repeated + + +def test_a_key_of_one_iri_is_the_node(): + b = Bindings([{}, [{V + "who": URIRef(A + "alice"), V + "n": Literal("Alice")}, + {V + "who": URIRef(A + "bob"), V + "n": Literal("Bob")}]]) + m = ThreadedMaterializer(PREFIXES + "start = @:Out\n:Out { :member @

* %Map:{ id(v:who) %} }\n

{ :name . %Map:{ v:n %} }") + triples = m.materialize(b, OUT) + assert {o for _, o in by_predicate(triples, "member")} == {URIRef(A + "alice"), URIRef(A + "bob")} + assert (URIRef(A + "bob"), Literal("Bob")) in by_predicate(triples, "name") + + +def test_keyed_nodes_are_shared_across_constraints(): + b = Bindings([{V + "mrn": Literal("42"), V + "name": Literal("Ann")}, + [{V + "when": Literal("Mon")}, {V + "when": Literal("Tue")}]]) + m = ThreadedMaterializer(PREFIXES + "start = @:Out\n" + ":Out { :patient @

%Map:{ id(v:mrn) %} ; :visit @* }\n" + "

{ :name . %Map:{ v:name %} }\n" + " { :when . %Map:{ v:when %} ; :of @

%Map:{ id(v:mrn) %} }") + triples = m.materialize(b, OUT) + patient = by_predicate(triples, "patient")[0][1] + assert {o for _, o in by_predicate(triples, "of")} == {patient} + assert len(by_predicate(triples, "name")) == 1 + + +def test_different_shapes_with_equal_keys_stay_apart(): + b = Bindings({V + "k": Literal("Paris")}) + m = ThreadedMaterializer(PREFIXES + "start = @:Out\n:Out { :city @ %Map:{ id(v:k) %} ; :person @ %Map:{ id(v:k) %} }\n" + " { :kind [:city] }\n { :kind [:person] }") + triples = m.materialize(b, OUT) + assert by_predicate(triples, "city")[0][1] != by_predicate(triples, "person")[0][1] + + +NESTED = PREFIXES + ("start = @:P\n:P { :report @* %Map:{ id(@node) %} }\n" + " { :no . %Map:{ v:no %} ; :result @* %Map:{ id(@node) %} }\n" + " { :sys . %Map:{ v:sys %} }") +DATA = (':p :report :r1, :r2 . :r1 :no "one" ; :result [ :sys 100 ], [ :sys 101 ] . ' + ':r2 :no "two" ; :result [ :sys 110 ] .') + + +def test_id_node_maps_a_graph_onto_itself(): + graph = turtle(DATA) + bound = bind(graph, NESTED, URIRef(A + "p")) # id() binds nothing on the way in + out = materialize(NESTED, bound, URIRef(A + "p")) + assert isomorphic(out, graph) + assert (URIRef(A + "p"), URIRef(A + "report"), URIRef(A + "r1")) in out # the very same IRIs + + +def test_unkeyed_nodes_are_reproducible_and_root_specific(): + b = Bindings([{V + "name": Literal("Sue")}, [{V + "sys": Literal("110")}, {V + "sys": Literal("111")}]]) + schema = PREFIXES + "start = @:Out\n:Out { :who . %Map:{ v:name %} ; :bp { :sys . %Map:{ v:sys %} }* }" + once = ThreadedMaterializer(schema).materialize(b, OUT) + again = ThreadedMaterializer(schema).materialize(b, OUT) + assert once == again + other = ThreadedMaterializer(schema).materialize(b, URIRef(A + "elsewhere")) + assert {o for _, o in by_predicate(once, "bp")}.isdisjoint({o for _, o in by_predicate(other, "bp")}) + g = Graph() + materialize(schema, b, OUT, graph=g) + n = len(g) + materialize(schema, b, OUT, graph=g) + assert len(g) == n # the same root twice adds nothing + + +def test_an_unbound_key_fails_the_constraint(): + b = Bindings({V + "name": Literal("Sue")}) + m = ThreadedMaterializer(PREFIXES + "start = @:Out\n:Out { :who . %Map:{ v:name %} ; :home @? %Map:{ id(v:city) %} }\n { :kind [:home] }") + triples = m.materialize(b, OUT) + assert by_predicate(triples, "home") == [] # optional: left out + m = ThreadedMaterializer(PREFIXES + "start = @:Out\n:Out { :home @ %Map:{ id(v:city) %} }\n { :kind [:home] }") + with pytest.raises(MaterializationError, match="city"): + m.materialize(b, OUT) + + +@pytest.mark.parametrize("schema, message", [ + ("start = @:Out\n:Out { :n . %Map:{ id(v:k) %} }", "shape-valued"), + ("start = @:Out\n:Out { :h @ %Map:{ id(v:k) %} %Map:{ v:k %} }\n { :kind [:home] }", "only Map code"), +]) +def test_id_is_alone_and_on_a_shape(schema, message): + m = ThreadedMaterializer(PREFIXES + schema) + with pytest.raises(MaterializationError, match=message): + m.materialize(Bindings({V + "k": Literal("x")}), OUT) + + +# -- IRI templates ------------------------------------------------------------------------ + +def test_an_iri_template_names_the_node(): + b = Bindings([{V + "mrn": Literal("42"), V + "name": Literal("Ann")}, + [{V + "when": Literal("Mon")}, {V + "when": Literal("Tue")}]]) + m = ThreadedMaterializer(PREFIXES + "start = @:Out\n" + ":Out { :patient @

%Map:{ id() %} ; :visit @* }\n" + "

{ :name . %Map:{ v:name %} }\n" + " { :when . %Map:{ v:when %} ; :of @

%Map:{ id() %} }") + triples = m.materialize(b, OUT) + patient = URIRef(A + "person/42") + assert by_predicate(triples, "patient") == [(OUT, patient)] + assert {o for _, o in by_predicate(triples, "of")} == {patient} + assert len(by_predicate(triples, "name")) == 1 + assert m.provenance[[t for t in triples].index((OUT, URIRef(A + "patient"), patient))]["reads"] == (((), V + "mrn"),) + + +def test_template_values_are_iri_safe(): + b = Bindings({V + "k": Literal("a b/c?d")}) + m = ThreadedMaterializer(PREFIXES + "start = @:Out\n:Out { :x @ %Map:{ id() %} }\n { :kind [:c] }") + assert by_predicate(m.materialize(b, OUT), "x") == [(OUT, URIRef(A + "k/a%20b%2Fc%3Fd/tail"))] + + +def test_an_unbound_template_variable_fails_the_constraint(): + m = ThreadedMaterializer(PREFIXES + "start = @:Out\n:Out { :x @ %Map:{ id() %} }\n { :kind [:c] }") + with pytest.raises(MaterializationError, match="v.example/k"): + m.materialize(Bindings({V + "other": Literal("1")}), OUT) diff --git a/tests/test_shexmap/test_roundtrip.py b/tests/test_shexmap/test_roundtrip.py new file mode 100644 index 0000000..0d4dcef --- /dev/null +++ b/tests/test_shexmap/test_roundtrip.py @@ -0,0 +1,160 @@ +"""Bidirectionality: the round-trip laws of a schema pair, and the provenance that ties each +output triple to the input it came from.""" +import json + +import pytest +from rdflib import BNode, Graph, Literal, URIRef +from rdflib.compare import isomorphic + +from pyshex.shexmap import Bindings, ThreadedMaterializer, bind, bind_all, materialize +from pyshex.shexmap.bindings import term_from_json +from pyshex.shexmap.cli import main +from pyshex.shexmap.scopes import ScopeTree +from tests.test_shexmap.test_examples import (MANIFEST, canonical, focus_node, input_graph, label, node_and_shape, + output_root, text) + +BY_LABEL = {label(e): e for e in MANIFEST} +PAIRS = [("BP / simple", "BP back / simple"), ("inverse in / Ann's heart rate", "inverse out / Ann's chart")] + + +def statics(entry) -> dict: + return {k: term_from_json(v) for k, v in entry.get("staticVars", {}).items()} + + +def marker(v): + return "_:" if isinstance(v, BNode) else v.n3() + + +def bound_pairs(bindings_list) -> set: + return {(v, marker(x)) for b in bindings_list for s in ScopeTree(b.tree).root.walk() for v, x in s.own.items()} + + +def forward(entry): + """bind with the input schema, materialize with the output schema.""" + graph = input_graph(entry) + _, start = node_and_shape(entry["queryMap"]) + focus = focus_node(entry, graph) + bindings = bind(graph, text(entry, "schema"), focus, start=start) + root, shape = output_root(entry) + m = ThreadedMaterializer(text(entry, "outputSchema"), static_vars=statics(entry)) + triples = m.materialize(bindings, root, start=shape) + return graph, focus, bindings, m, triples, root, shape + + +# -- law 1: a schema binds back exactly what it wrote from ----------------------------- + +@pytest.mark.parametrize("entry", MANIFEST, ids=label) +def test_the_output_schema_binds_back_what_it_read(entry): + graph, focus, bindings, m, triples, root, shape = forward(entry) + scopes = {s.path: s for s in ScopeTree(bindings.tree).root.walk()} + read = {(v, marker(scopes[p].own[v])) for src in m.provenance for p, v in src["reads"]} + read |= {(v, marker(m.statics[v])) for src in m.provenance for v in src["statics"]} + out = Graph() + for t in triples: + out.add(t) + rebound = bound_pairs(bind_all(out, text(entry, "outputSchema"), root, start=shape)) + assert rebound == read + + +# -- law 2: a pair of schemas, there and back -------------------------------------------- + +@pytest.mark.parametrize("there, back", PAIRS, ids=[f"{a} <-> {b}" for a, b in PAIRS]) +def test_a_pair_recovers_the_bindings(there, back): + ea, eb = BY_LABEL[there], BY_LABEL[back] + graph, focus, bindings, _, triples, root, _ = forward(ea) + middle = Graph() + for t in triples: + middle.add(t) + _, start_back = node_and_shape(eb["queryMap"]) + b2 = bind(middle, text(eb, "schema"), root, start=start_back) + _, shape_back = output_root(eb) + _, start = node_and_shape(ea["queryMap"]) + returned = materialize(text(eb, "outputSchema"), b2, focus, start=shape_back, static_vars=statics(eb)) + original = {json.dumps(canonical(b.to_json()), sort_keys=True) for b in bind_all(graph, text(ea, "schema"), focus, start=start)} + again = {json.dumps(canonical(b.to_json()), sort_keys=True) for b in bind_all(returned, text(ea, "schema"), focus, start=start)} + assert again == original + + +def test_the_bp_pair_recovers_the_graph(): + # every node of the input is either the focus or carried by a variable, so the graph comes back as it was + ea, eb = BY_LABEL["BP / simple"], BY_LABEL["BP back / simple"] + graph, focus, _, _, triples, root, _ = forward(ea) + middle = Graph() + for t in triples: + middle.add(t) + _, start_back = node_and_shape(eb["queryMap"]) + _, shape_back = output_root(eb) + returned = materialize(text(eb, "outputSchema"), bind(middle, text(eb, "schema"), root, start=start_back), focus, + start=shape_back, static_vars=statics(eb)) + assert isomorphic(returned, graph) + + +def test_the_inverse_pair_loses_only_uncarried_identities(): + # the chart carries no variable for the observation nodes, so they come back as blank nodes + ea, eb = BY_LABEL["inverse in / Ann's heart rate"], BY_LABEL["inverse out / Ann's chart"] + graph, focus, _, _, triples, root, _ = forward(ea) + middle = Graph() + for t in triples: + middle.add(t) + _, start_back = node_and_shape(eb["queryMap"]) + _, shape_back = output_root(eb) + returned = materialize(text(eb, "outputSchema"), bind(middle, text(eb, "schema"), root, start=start_back), focus, + start=shape_back, static_vars=statics(eb)) + assert not isomorphic(returned, graph) + blanked = Graph() + observations = {s for s in graph.subjects() if s != focus} + for s, p, o in graph: + blanked.add((BNode(str(s)) if s in observations else s, p, BNode(str(o)) if o in observations else o)) + assert isomorphic(returned, blanked) + + +# -- provenance --------------------------------------------------------------------------- + +PREFIXES = """PREFIX : +PREFIX v: +PREFIX Map: +""" +V = "http://v.example/" +A = "http://a.example/" +OUT = URIRef(A + "out") + + +def test_every_triple_says_where_it_came_from(): + b = Bindings([{V + "name": Literal("Sue")}, + [{"@node": URIRef(A + "r1"), V + "sys": Literal("110")}, {"@node": URIRef(A + "r2"), V + "sys": Literal("111")}]]) + m = ThreadedMaterializer(PREFIXES + "start = @:Out\n:Out { :kind [:chart] ; :who . %Map:{ v:name %} ; " + ":bp @* %Map:{ id(@node) %} }\n { :sys . %Map:{ v:sys %} ; :owner . %Map:{ v:name %} }") + triples = m.materialize(b, OUT) + assert m.provenance is m.chosen.provenance and len(m.provenance) == len(triples) + by_pred = {} + for t, src in zip(triples, m.provenance): + assert src["predicate"] == str(t[1]) + by_pred.setdefault(str(t[1]).split("/")[-1], []).append((t, src)) + assert by_pred["kind"][0][1]["constant"] is True and by_pred["kind"][0][1]["reads"] == () + (t, src), = by_pred["who"] + assert src["variable"] == V + "name" and src["reads"] == (((), V + "name"),) and src["scope"] == () + for t, src in by_pred["bp"]: + assert src["keyed"] == "id(@node)" and src["node"] == t[2] # the keyed node is the input node + for t, src in by_pred["sys"]: + assert src["node"] == t[0] and src["reads"] == ((src["scope"], V + "sys"),) + for t, src in by_pred["owner"]: + assert src["reads"] == (((), V + "name"),) and src["scope"] != () # an inherited read, from the item + + +def test_structural_and_named_links_are_marked(): + b = Bindings({V + "n": Literal("x"), V + "who": URIRef(A + "alice")}) + m = ThreadedMaterializer(PREFIXES + "start = @:Out\n:Out { :a @ ; :p @

%Map:{ v:who %} }\n { :n . %Map:{ v:n %} }\n

{ :k [:c] }") + triples = m.materialize(b, OUT) + kinds = {str(t[1]).split("/")[-1]: src for t, src in zip(triples, m.provenance)} + assert kinds["a"].get("structural") is True + assert kinds["p"].get("named") is True and kinds["p"]["variable"] == V + "who" + + +def test_cli_writes_provenance(tmp_path): + (tmp_path / "b.json").write_text(json.dumps({V + "n": {"value": "x"}}), encoding="utf-8") + (tmp_path / "out.shex").write_text(PREFIXES + "start = @:Out\n:Out { :n . %Map:{ v:n %} ; :k [:c] }", encoding="utf-8") + assert main(["-j", str(tmp_path / "b.json"), "-t", str(tmp_path / "out.shex"), "-r", str(OUT), + "-o", str(tmp_path / "out.ttl"), "--provenance", str(tmp_path / "prov.jsonl")]) == 0 + records = [json.loads(line) for line in (tmp_path / "prov.jsonl").read_text(encoding="utf-8").splitlines()] + assert {r["kind"] for r in records} == {"variable", "constant"} + assert next(r for r in records if r["kind"] == "variable")["variable"] == V + "n" diff --git a/tests/test_shexmap/test_shexjs_differential.py b/tests/test_shexmap/test_shexjs_differential.py new file mode 100644 index 0000000..9765a74 --- /dev/null +++ b/tests/test_shexmap/test_shexjs_differential.py @@ -0,0 +1,110 @@ +"""Differential tests against shex.js's ``@shexjs/extension-map``. + +Set ``SHEXJS_EXTENSION_MAP`` to the package directory of a built shex.js checkout +(``…/shex.js/packages/extension-map``) to run them; they are skipped otherwise. Each +manifest example is bound and materialized on both sides and the results compared: + +* the frames of shex.js's bindings equal PyShEx's (the trees may differ in layout, and + shex.js does not write ``@node``); +* shex.js materializes PyShEx's bindings JSON to the expected graph (``@node`` stripped, as + shex.js does not know the key yet; set ``SHEXJS_KEEP_NODE=1`` to hand it over as written); +* PyShEx materializes shex.js's bindings to the expected graph. +""" +import json +import os +import subprocess +from pathlib import Path + +import pytest +from rdflib import BNode, Graph, Literal +from rdflib.compare import isomorphic + +from pyshex.shexmap import bind, loads, materialize, normalize +from tests.test_shexmap.test_examples import (MANIFEST, expected_output, focus_node, input_graph, label, + node_and_shape, output_root, text, without_nodes) + +SHEXJS = os.environ.get("SHEXJS_EXTENSION_MAP") +DRIVER = Path(__file__).parent / "shexjs_driver.js" +pytestmark = pytest.mark.skipif(not SHEXJS, reason="SHEXJS_EXTENSION_MAP not set") + + +def node_arg(term) -> str: + """A node for the driver: rdflib's blank-node labels mean nothing to n3, so a blank focus is + named "ROOT" (the data's one root) and a blank output root keeps its label (shex.js mints it).""" + return f"_:{term}" if isinstance(term, BNode) else str(term) + + +def focus_arg(term) -> str: + return "ROOT" if isinstance(term, BNode) else str(term) + + +def shexjs(mode: str, *args: str) -> str: + env = {**os.environ, "SHEXJS_TEST_PORTS": os.environ.get("SHEXJS_TEST_PORTS", "11111")} + proc = subprocess.run(["node", str(DRIVER), SHEXJS, mode, *args], capture_output=True, text=True, env=env) + assert proc.returncode == 0, proc.stderr + return proc.stdout + + +def canonical_frames(frames) -> list: + """Frames as comparable JSON: sorted (variable, value) pairs, blank-node values reduced + to a marker (their labels are implementation-specific).""" + def val(v): + if isinstance(v, BNode): + return "_:" + if isinstance(v, Literal): + return {"value": str(v), "type": str(v.datatype) if v.datatype else None, "language": v.language} + return str(v) + return sorted(sorted((k, json.dumps(val(v), sort_keys=True)) for k, v in f.items()) for f in frames) + + +# known divergences, as strict xfails +DIVERGES = { + "EXTENDS / clinic": "shex.js's valToExtension does not collect Map results through EXTENDS: it binds " + "nothing for this example, where PyShEx binds through the extension each member satisfies", +} + + +@pytest.fixture(scope="module", ids=label, + params=[pytest.param(e, marks=pytest.mark.xfail(strict=True, reason=DIVERGES[label(e)])) + if label(e) in DIVERGES else e for e in MANIFEST]) +def example(request, tmp_path_factory): + entry = request.param + d = tmp_path_factory.mktemp("shexjs") + (d / "in.shex").write_text(text(entry, "schema"), encoding="utf-8") + (d / "in.ttl").write_text(text(entry, "data"), encoding="utf-8") + (d / "out.shex").write_text(text(entry, "outputSchema"), encoding="utf-8") + return entry, d + + +def test_shexjs_binds_the_same_frames(example): + entry, d = example + graph = input_graph(entry) + _, start = node_and_shape(entry["queryMap"]) + focus = focus_node(entry, graph) + ours = bind(graph, text(entry, "schema"), focus, start=start) + theirs = loads(shexjs("bind", str(d / "in.shex"), str(d / "in.ttl"), focus_arg(focus), start or "START")) + assert canonical_frames(theirs.frames()) == canonical_frames(ours.frames()) + + +def test_shexjs_materializes_our_bindings(example): + entry, d = example + graph = input_graph(entry) + _, start = node_and_shape(entry["queryMap"]) + ours = bind(graph, text(entry, "schema"), focus_node(entry, graph), start=start) + root, shape = output_root(entry) + tree = ours.to_json() if os.environ.get("SHEXJS_KEEP_NODE") else without_nodes(ours.to_json()) + (d / "ours.json").write_text(json.dumps(tree), encoding="utf-8") + nt = shexjs("materialize", str(d / "out.shex"), str(d / "ours.json"), node_arg(root), shape or "START") + got = Graph().parse(data=nt, format="nt") + assert isomorphic(got, expected_output(entry)), nt + + +def test_we_materialize_shexjs_bindings(example): + entry, d = example + graph = input_graph(entry) + _, start = node_and_shape(entry["queryMap"]) + focus = focus_node(entry, graph) + theirs = loads(shexjs("bind", str(d / "in.shex"), str(d / "in.ttl"), focus_arg(focus), start or "START")) + root, shape = output_root(entry) + got = materialize(text(entry, "outputSchema"), theirs, root, start=shape) + assert isomorphic(got, expected_output(entry)), got.serialize(format="turtle") diff --git a/tests/test_shexmap/test_threaded.py b/tests/test_shexmap/test_threaded.py index a6c2db5..c9955fc 100644 --- a/tests/test_shexmap/test_threaded.py +++ b/tests/test_shexmap/test_threaded.py @@ -1,4 +1,5 @@ -"""The threaded materializer, ported from shex.js's ThreadedMaterializer-test.js.""" +"""The materializer's choices, failures and reports; ported from shex.js's +ThreadedMaterializer-test.js, with the frame-cursor cases restated for iteration scopes.""" import json from pathlib import Path @@ -61,12 +62,17 @@ def test_nested_groups_flatten_keeping_group_bindings_with_their_frames(): {A + "use": lit("home"), A + "tel": lit("+1")}]] -def test_cross_frame_pairing_does_not_beat_in_frame_consumption(): +def test_each_contact_keeps_its_own_use(): m, triples = run(CARD, CONTACTS, "tag:card") assert sorted(str(o) for _, p, o in triples if str(p) == A + "val") == ["+1", "h@x", "w@x"] assert m.chosen.consumed == 7 - mix = [a for a in m.accepts if a.skipped == 4] - assert mix and mix[0].consumed == 3 # frame 0's :use with frame 2's :tel, demoted + pairs = {(str(o), str(v)) for s, p, o in triples if str(p) == A + "use" + for s2, p2, v in triples if s2 == s and str(p2) == A + "val"} + assert pairs == {("work", "w@x"), ("home", "h@x"), ("home", "+1")} + # no alternative pairs a contact's :use with another contact's number + for a in m.accepts: + assert {(str(o), str(v)) for s, p, o in a.triples if str(p) == A + "use" + for s2, p2, v in a.triples if s2 == s and str(p2) == A + "val"} <= pairs def test_constant_only_variants_collapse_onto_one_accept(): diff --git a/tests/test_shexmap/test_update.py b/tests/test_shexmap/test_update.py new file mode 100644 index 0000000..99d4c13 --- /dev/null +++ b/tests/test_shexmap/test_update.py @@ -0,0 +1,106 @@ +"""The triples a schema matched, the strong identity law, and updating a graph in place.""" +import pytest +from rdflib import BNode, Graph, Literal, URIRef +from rdflib.compare import isomorphic + +from pyshex.shexmap import Bindings, MaterializationError, ThreadedMaterializer, bind, materialize +from pyshex.shexmap.cli import main + +PREFIXES = """PREFIX : +PREFIX v: +PREFIX Map: +""" +A = "http://a.example/" +V = "http://v.example/" +P = URIRef(A + "p") +NESTED = PREFIXES + ("start = @:P\n:P { :report @* %Map:{ id(@node) %} }\n" + " { :no . %Map:{ v:no %} ; :result @* %Map:{ id(@node) %} }\n { :sys . %Map:{ v:sys %} }") +DATA = (':p :report :r1, :r2 ; :other "not in the schema" . :r1 :no "one" ; :result [ :sys 100 ], [ :sys 101 ] ; :extra "x" . ' + ':r2 :no "two" .') + + +def turtle(text: str) -> Graph: + return Graph().parse(data="PREFIX : \n" + text, format="turtle") + + +def test_bindings_report_the_triples_the_schema_matched(): + graph = turtle(DATA) + bound = bind(graph, NESTED, P) + assert set(bound.matched) < set(graph) + left_out = {str(p).split("/")[-1] for _, p, _ in set(graph) - set(bound.matched)} + assert left_out == {"other", "extra"} # what the schema does not govern + assert (URIRef(A + "r2"), URIRef(A + "no"), Literal("two")) in bound.matched + + +def test_the_strong_identity_law(): + # materializing a schema's own bindings gives exactly the subgraph it selected + graph = turtle(DATA) + bound = bind(graph, NESTED, P) + selected = Graph() + for t in bound.matched: + selected.add(t) + assert isomorphic(materialize(NESTED, bound, P), selected) + + +OUT = PREFIXES + "start = @:Out\n:Out { :who . %Map:{ v:name %} ; :bp @* }\n { :sys . %Map:{ v:sys %} }" +ROOT = URIRef(A + "out") + + +def readings(graph: Graph) -> set: + return {str(o) for _, _, o in graph.triples((None, URIRef(A + "sys"), None))} + + +def test_update_replaces_what_the_schema_holds_and_keeps_the_rest(): + graph = turtle(':out :unrelated "kept" . :elsewhere :who "someone else" .') + first = Bindings([{V + "name": Literal("Sue")}, [{V + "sys": Literal("110")}, {V + "sys": Literal("111")}]]) + m = ThreadedMaterializer(OUT) + added, removed = m.update(graph, first, ROOT) + assert removed == set() and len(added) == 1 + 2 + 2 + assert readings(graph) == {"110", "111"} + second = Bindings([{V + "name": Literal("Susan")}, [{V + "sys": Literal("110")}, {V + "sys": Literal("112")}]]) + added, removed = m.update(graph, second, ROOT) + assert {str(o) for _, _, o in removed if isinstance(o, Literal)} == {"Sue", "111"} + assert len(removed) == 2 # the old name and the old value; the reading's node, minted by position, stays + assert readings(graph) == {"110", "112"} + assert (ROOT, URIRef(A + "who"), Literal("Susan")) in graph and (ROOT, URIRef(A + "who"), Literal("Sue")) not in graph + assert (ROOT, URIRef(A + "unrelated"), Literal("kept")) in graph # not governed: kept + assert (URIRef(A + "elsewhere"), URIRef(A + "who"), Literal("someone else")) in graph # another root: kept + fresh = materialize(OUT, second, ROOT) + fresh.add((ROOT, URIRef(A + "unrelated"), Literal("kept"))) + fresh.add((URIRef(A + "elsewhere"), URIRef(A + "who"), Literal("someone else"))) + assert isomorphic(graph, fresh) + + +def test_an_unchanged_input_changes_nothing(): + graph = Graph() + b = Bindings([{V + "name": Literal("Sue")}, [{V + "sys": Literal("110")}, {V + "sys": Literal("111")}]]) + materialize(OUT, b, ROOT, graph=graph) + before = set(graph) + added, removed = ThreadedMaterializer(OUT).update(graph, b, ROOT) + assert added == set() and removed == set() and set(graph) == before + + +def test_materialize_replace_is_update(): + graph = Graph() + materialize(OUT, Bindings([{V + "name": Literal("Sue")}, [{V + "sys": Literal("110")}]]), ROOT, graph=graph) + materialize(OUT, Bindings([{V + "name": Literal("Sue")}, [{V + "sys": Literal("120")}]]), ROOT, graph=graph, replace=True) + assert readings(graph) == {"120"} + materialize(OUT, Bindings([{V + "name": Literal("Sue")}, [{V + "sys": Literal("130")}]]), ROOT, graph=graph) + assert readings(graph) == {"120", "130"} # without replace: added to + + +def test_a_root_that_does_not_conform_is_refused(): + graph = turtle(':out :who "Sue" , "Also Sue" .') # two names where the schema wants one + with pytest.raises(MaterializationError, match="does not conform"): + ThreadedMaterializer(OUT).update(graph, Bindings([{V + "name": Literal("Sue")}, []]), ROOT) + + +def test_cli_into(tmp_path): + existing = tmp_path / "existing.ttl" + existing.write_text('@prefix : .\n:out :unrelated "kept" ; :who "Old" .\n', encoding="utf-8") + (tmp_path / "b.json").write_text('[{"http://v.example/name": {"value": "New"}}, []]', encoding="utf-8") + (tmp_path / "out.shex").write_text(OUT, encoding="utf-8") + assert main(["-j", str(tmp_path / "b.json"), "-t", str(tmp_path / "out.shex"), "-r", str(ROOT), "--into", str(existing)]) == 0 + g = Graph().parse(existing, format="turtle") + assert (ROOT, URIRef(A + "who"), Literal("New")) in g and (ROOT, URIRef(A + "who"), Literal("Old")) not in g + assert (ROOT, URIRef(A + "unrelated"), Literal("kept")) in g diff --git a/tests/test_shexmap/test_values.py b/tests/test_shexmap/test_values.py new file mode 100644 index 0000000..c3db132 --- /dev/null +++ b/tests/test_shexmap/test_values.py @@ -0,0 +1,92 @@ +"""Bound values against the output constraint's value expression: checked, retyped when a +plain literal's lexical form fits the datatype, otherwise failed like an unbound variable.""" +import pytest +from rdflib import Graph, Literal, URIRef, XSD + +from pyshex import ShExEvaluator +from pyshex.shexmap import Bindings, MaterializationError, ThreadedMaterializer + +PREFIXES = """PREFIX : +PREFIX xsd: +PREFIX v: +PREFIX Map: +""" +V = "http://v.example/" +A = "http://a.example/" +OUT = URIRef(A + "out") + + +def run(schema: str, **bindings): + m = ThreadedMaterializer(PREFIXES + "start = @:Out\n" + schema) + return m, m.materialize(Bindings({V + k: v for k, v in bindings.items()}), OUT) + + +def objects(triples, local: str): + return [o for _, p, o in triples if str(p) == A + local] + + +def test_a_plain_literal_is_retyped_to_the_constraint_datatype(): + _, triples = run(":Out { :n xsd:integer %Map:{ v:t %} }", t=Literal("42")) + assert objects(triples, "n") == [Literal("42", datatype=XSD.integer)] + + +def test_a_typed_literal_that_satisfies_passes_unchanged(): + _, triples = run(":Out { :n xsd:integer %Map:{ v:t %} }", t=Literal("42", datatype=XSD.integer)) + assert objects(triples, "n") == [Literal("42", datatype=XSD.integer)] + + +def test_a_value_that_does_not_fit_fails_the_constraint(): + m = ThreadedMaterializer(PREFIXES + "start = @:Out\n:Out { :n xsd:integer %Map:{ v:t %} }") + with pytest.raises(MaterializationError, match="does not satisfy"): + m.materialize(Bindings({V + "t": Literal("not a number")}), OUT) + m = ThreadedMaterializer(PREFIXES + "start = @:Out\n:Out { :n xsd:integer %Map:{ v:t %} }") + with pytest.raises(MaterializationError): # a typed literal is not retyped + m.materialize(Bindings({V + "t": Literal("1", datatype=XSD.decimal)}), OUT) + + +def test_an_optional_constraint_is_left_out_and_a_disjunct_falls_through(): + _, triples = run(":Out { :n xsd:integer ? %Map:{ v:t %} ; :s . %Map:{ v:t %} }", t=Literal("abc")) + assert objects(triples, "n") == [] and objects(triples, "s") == [Literal("abc")] + _, triples = run(":Out { :n xsd:integer %Map:{ v:t %} | :s xsd:string %Map:{ v:t %} }", t=Literal("abc")) + assert objects(triples, "n") == [] and objects(triples, "s") == [Literal("abc")] + _, triples = run(":Out { :n xsd:integer %Map:{ v:t %} | :s xsd:string %Map:{ v:t %} }", t=Literal("7")) + assert objects(triples, "n") == [Literal("7", datatype=XSD.integer)] + + +def test_value_sets_node_kinds_facets_and_references(): + _, triples = run(":Out { :k [:x :y] %Map:{ v:t %} }", t=URIRef(A + "x")) + assert objects(triples, "k") == [URIRef(A + "x")] + with pytest.raises(MaterializationError): + run(":Out { :k [:x :y] %Map:{ v:t %} }", t=URIRef(A + "z")) + with pytest.raises(MaterializationError): + run(":Out { :k IRI %Map:{ v:t %} }", t=Literal("not an iri")) + with pytest.raises(MaterializationError): + run(":Out { :s xsd:string MINLENGTH 3 %Map:{ v:t %} }", t=Literal("ab")) + _, triples = run(":Out { :n @ %Map:{ v:t %} }\n xsd:integer", t=Literal("5")) + assert objects(triples, "n") == [Literal("5", datatype=XSD.integer)] + _, triples = run(":Out { :n xsd:integer AND MININCLUSIVE 0 %Map:{ v:t %} }", t=Literal("5")) + assert objects(triples, "n") == [Literal("5", datatype=XSD.integer)] + with pytest.raises(MaterializationError): + run(":Out { :n xsd:integer AND MININCLUSIVE 0 %Map:{ v:t %} }", t=Literal("-5")) + + +def test_function_results_take_the_constraint_type(): + _, triples = run(':Out { :code xsd:integer %Map:{ hashmap(v:status, {"1": "Married", "2": "Single"}) %} }', + status=Literal("Single")) + assert objects(triples, "code") == [Literal("2", datatype=XSD.integer)] + + +def test_the_output_conforms_to_the_output_schema(): + schema = ":Out { :n xsd:integer %Map:{ v:t %} ; :k [:x] %Map:{ v:k %} }" + _, triples = run(schema, t=Literal("100"), k=URIRef(A + "x")) + g = Graph() + for t in triples: + g.add(t) + result = ShExEvaluator(rdf=g, schema=PREFIXES + "start = @:Out\n" + schema.replace("%Map:{ v:t %}", "").replace("%Map:{ v:k %}", ""), + focus=OUT).evaluate() + assert result[0].result, result[0].reason + + +def test_the_check_can_be_turned_off(): + m = ThreadedMaterializer(PREFIXES + "start = @:Out\n:Out { :n xsd:integer %Map:{ v:t %} }", check_values=False) + assert objects(m.materialize(Bindings({V + "t": Literal("not a number")}), OUT), "n") == [Literal("not a number")] From 5b7b91627c67cdcab4348ecc38edea9da3cc8c65 Mon Sep 17 00:00:00 2001 From: Eric Prud'hommeaux Date: Sun, 27 Sep 2026 03:36:32 -0500 Subject: [PATCH 2/2] ShExMap tests: re-sync the example bindings with shex.js's scope layout The recorded bindings are now what shex.js (branch shexmap-scopes, 4d53c6e1) writes: the scope layout with @node, one list per repeated expression. The example test compares the whole tree against them, blank-node labels aside, instead of only the frames. The differential driver reads shex.js's bindingTree() when the checkout has it, hands @node over as written, and the EXTENDS example no longer diverges. normalize() keeps a scope's own bindings as a frame when its lists are all empty, as shex.js's normalizeBindingTree does. Co-Authored-By: Claude Fable 5.1 --- pyshex/shexmap/bindings.py | 2 + tests/test_shexmap/README.md | 4 +- .../examples/BP-simple-bindings.json | 6 +- .../examples/BPPatient-2-levels-bindings.json | 150 +++++++++--------- .../BPPatient-multi-bindings-bindings.json | 6 +- .../examples/splits-bindings.json | 3 + tests/test_shexmap/shexjs_driver.js | 3 +- tests/test_shexmap/test_examples.py | 20 ++- .../test_shexmap/test_shexjs_differential.py | 17 +- 9 files changed, 113 insertions(+), 98 deletions(-) diff --git a/pyshex/shexmap/bindings.py b/pyshex/shexmap/bindings.py index 0565cc3..818f205 100644 --- a/pyshex/shexmap/bindings.py +++ b/pyshex/shexmap/bindings.py @@ -145,6 +145,8 @@ def walk(node): out = [] for frames, leaf in ordered: out.extend(f if leaf else {**shared, **f} for f in frames) + if not out and shared: # a scope whose lists are all empty still has its own bindings + out.append(shared) return out, False, counts return walk(tree)[0] diff --git a/tests/test_shexmap/README.md b/tests/test_shexmap/README.md index c39c578..39d7095 100644 --- a/tests/test_shexmap/README.md +++ b/tests/test_shexmap/README.md @@ -2,7 +2,9 @@ `examples/` is copied from shex.js's `@shexjs/extension-map` package (https://github.com/shexjs/shex.js/tree/main/packages/extension-map/examples, -commit 58406c3bdd4fa79cc9b0f664ef7a689db040c07b, MIT licence, by Eric Prud'hommeaux). +commit 4d53c6e1 on branch `shexmap-scopes`, MIT licence, by Eric Prud'hommeaux); the +`*-bindings.json` files are in the scope layout with `@node` that both implementations +write. Each `manifest.json` entry pairs an input schema and data with the bindings and output graph shex.js produces; `test_examples.py` checks PyShEx against them. diff --git a/tests/test_shexmap/examples/BP-simple-bindings.json b/tests/test_shexmap/examples/BP-simple-bindings.json index 3020d36..98d5ce5 100644 --- a/tests/test_shexmap/examples/BP-simple-bindings.json +++ b/tests/test_shexmap/examples/BP-simple-bindings.json @@ -1,10 +1,10 @@ { - "http://shex.io/extensions/Map/#BPDAM-family": { - "value": "Walker" - }, "http://shex.io/extensions/Map/#BPDAM-given": { "value": "Alice" }, + "http://shex.io/extensions/Map/#BPDAM-family": { + "value": "Walker" + }, "http://shex.io/extensions/Map/#BPDAM-sysVal": { "value": "110", "type": "http://www.w3.org/2001/XMLSchema#float" diff --git a/tests/test_shexmap/examples/BPPatient-2-levels-bindings.json b/tests/test_shexmap/examples/BPPatient-2-levels-bindings.json index 702a510..bd625d1 100644 --- a/tests/test_shexmap/examples/BPPatient-2-levels-bindings.json +++ b/tests/test_shexmap/examples/BPPatient-2-levels-bindings.json @@ -1,98 +1,102 @@ [ { - "http://shex.io/extensions/Map/#BPDAM-name": { "value": "Sue" } + "http://shex.io/extensions/Map/#BPDAM-name": { + "value": "Sue" + } }, [ [ { - "http://shex.io/extensions/Map/#BPDAM-reports": "http://my.data.example/medical/Report1" + "@node": "http://my.data.example/medical/Report1", + "http://shex.io/extensions/Map/#BPDAM-reports": "http://my.data.example/medical/Report1", + "http://shex.io/extensions/Map/#BPDAM-reportNo": { + "value": "one" + } }, [ { - "http://shex.io/extensions/Map/#BPDAM-reportNo": { "value": "one" } + "@node": "http://my.data.example/medical/Res00", + "http://shex.io/extensions/Map/#BPDAM-bp": "http://my.data.example/medical/Res00", + "http://shex.io/extensions/Map/#BPDAM-sysVal": { + "value": "100", + "type": "http://www.w3.org/2001/XMLSchema#float" + }, + "http://shex.io/extensions/Map/#BPDAM-sysUnits": { + "value": "mmHg" + }, + "http://shex.io/extensions/Map/#BPDAM-diaVal": { + "value": "60", + "type": "http://www.w3.org/2001/XMLSchema#float" + }, + "http://shex.io/extensions/Map/#BPDAM-diaUnits": { + "value": "mmHg" + } }, - [ - { - "http://shex.io/extensions/Map/#BPDAM-bp": "http://my.data.example/medical/Res00", - "http://shex.io/extensions/Map/#BPDAM-sysVal": { - "value": "100", - "type": "http://www.w3.org/2001/XMLSchema#float" - }, - "http://shex.io/extensions/Map/#BPDAM-sysUnits": { - "value": "mmHg" - }, - "http://shex.io/extensions/Map/#BPDAM-diaVal": { - "value": "60", - "type": "http://www.w3.org/2001/XMLSchema#float" - }, - "http://shex.io/extensions/Map/#BPDAM-diaUnits": { - "value": "mmHg" - } + { + "@node": "http://my.data.example/medical/Res01", + "http://shex.io/extensions/Map/#BPDAM-bp": "http://my.data.example/medical/Res01", + "http://shex.io/extensions/Map/#BPDAM-sysVal": { + "value": "101", + "type": "http://www.w3.org/2001/XMLSchema#float" + }, + "http://shex.io/extensions/Map/#BPDAM-sysUnits": { + "value": "mmHg" + }, + "http://shex.io/extensions/Map/#BPDAM-diaVal": { + "value": "61", + "type": "http://www.w3.org/2001/XMLSchema#float" }, - { - "http://shex.io/extensions/Map/#BPDAM-bp": "http://my.data.example/medical/Res01", - "http://shex.io/extensions/Map/#BPDAM-sysVal": { - "value": "101", - "type": "http://www.w3.org/2001/XMLSchema#float" - }, - "http://shex.io/extensions/Map/#BPDAM-sysUnits": { - "value": "mmHg" - }, - "http://shex.io/extensions/Map/#BPDAM-diaVal": { - "value": "61", - "type": "http://www.w3.org/2001/XMLSchema#float" - }, - "http://shex.io/extensions/Map/#BPDAM-diaUnits": { - "value": "mmHg" - } + "http://shex.io/extensions/Map/#BPDAM-diaUnits": { + "value": "mmHg" } - ] + } ] ], [ { - "http://shex.io/extensions/Map/#BPDAM-reports": "http://my.data.example/medical/Report2" + "@node": "http://my.data.example/medical/Report2", + "http://shex.io/extensions/Map/#BPDAM-reports": "http://my.data.example/medical/Report2", + "http://shex.io/extensions/Map/#BPDAM-reportNo": { + "value": "two" + } }, [ { - "http://shex.io/extensions/Map/#BPDAM-reportNo": { "value": "two" } + "@node": "http://my.data.example/medical/Res10", + "http://shex.io/extensions/Map/#BPDAM-bp": "http://my.data.example/medical/Res10", + "http://shex.io/extensions/Map/#BPDAM-sysVal": { + "value": "110", + "type": "http://www.w3.org/2001/XMLSchema#float" + }, + "http://shex.io/extensions/Map/#BPDAM-sysUnits": { + "value": "mmHg" + }, + "http://shex.io/extensions/Map/#BPDAM-diaVal": { + "value": "70", + "type": "http://www.w3.org/2001/XMLSchema#float" + }, + "http://shex.io/extensions/Map/#BPDAM-diaUnits": { + "value": "mmHg" + } }, - [ - { - "http://shex.io/extensions/Map/#BPDAM-bp": "http://my.data.example/medical/Res10", - "http://shex.io/extensions/Map/#BPDAM-sysVal": { - "value": "110", - "type": "http://www.w3.org/2001/XMLSchema#float" - }, - "http://shex.io/extensions/Map/#BPDAM-sysUnits": { - "value": "mmHg" - }, - "http://shex.io/extensions/Map/#BPDAM-diaVal": { - "value": "70", - "type": "http://www.w3.org/2001/XMLSchema#float" - }, - "http://shex.io/extensions/Map/#BPDAM-diaUnits": { - "value": "mmHg" - } + { + "@node": "http://my.data.example/medical/Res11", + "http://shex.io/extensions/Map/#BPDAM-bp": "http://my.data.example/medical/Res11", + "http://shex.io/extensions/Map/#BPDAM-sysVal": { + "value": "111", + "type": "http://www.w3.org/2001/XMLSchema#float" + }, + "http://shex.io/extensions/Map/#BPDAM-sysUnits": { + "value": "mmHg" + }, + "http://shex.io/extensions/Map/#BPDAM-diaVal": { + "value": "71", + "type": "http://www.w3.org/2001/XMLSchema#float" }, - { - "http://shex.io/extensions/Map/#BPDAM-bp": "http://my.data.example/medical/Res11", - "http://shex.io/extensions/Map/#BPDAM-sysVal": { - "value": "111", - "type": "http://www.w3.org/2001/XMLSchema#float" - }, - "http://shex.io/extensions/Map/#BPDAM-sysUnits": { - "value": "mmHg" - }, - "http://shex.io/extensions/Map/#BPDAM-diaVal": { - "value": "71", - "type": "http://www.w3.org/2001/XMLSchema#float" - }, - "http://shex.io/extensions/Map/#BPDAM-diaUnits": { - "value": "mmHg" - } + "http://shex.io/extensions/Map/#BPDAM-diaUnits": { + "value": "mmHg" } - ] + } ] ] ] diff --git a/tests/test_shexmap/examples/BPPatient-multi-bindings-bindings.json b/tests/test_shexmap/examples/BPPatient-multi-bindings-bindings.json index 57a3d20..b630502 100644 --- a/tests/test_shexmap/examples/BPPatient-multi-bindings-bindings.json +++ b/tests/test_shexmap/examples/BPPatient-multi-bindings-bindings.json @@ -1,9 +1,12 @@ [ { - "http://shex.io/extensions/Map/#BPDAM-name": { "value": "Sue" } + "http://shex.io/extensions/Map/#BPDAM-name": { + "value": "Sue" + } }, [ { + "@node": "tag:b0", "http://shex.io/extensions/Map/#BPDAM-XXX": "tag:b0", "http://shex.io/extensions/Map/#BPDAM-sysVal": { "value": "110", @@ -21,6 +24,7 @@ } }, { + "@node": "tag:b1", "http://shex.io/extensions/Map/#BPDAM-XXX": "tag:b1", "http://shex.io/extensions/Map/#BPDAM-sysVal": { "value": "111", diff --git a/tests/test_shexmap/examples/splits-bindings.json b/tests/test_shexmap/examples/splits-bindings.json index dcd660b..75511a7 100644 --- a/tests/test_shexmap/examples/splits-bindings.json +++ b/tests/test_shexmap/examples/splits-bindings.json @@ -6,6 +6,7 @@ }, [ { + "@node": "_:n3-22", "http://contact.example/use": { "value": "work" }, @@ -14,6 +15,7 @@ } }, { + "@node": "_:n3-23", "http://contact.example/use": { "value": "home" }, @@ -22,6 +24,7 @@ } }, { + "@node": "_:n3-24", "http://contact.example/use": { "value": "home" }, diff --git a/tests/test_shexmap/shexjs_driver.js b/tests/test_shexmap/shexjs_driver.js index caad009..61ba2a1 100644 --- a/tests/test_shexmap/shexjs_driver.js +++ b/tests/test_shexmap/shexjs_driver.js @@ -37,7 +37,8 @@ if (mode === "bind") { } const res = resultMapToShapeExprTest(validator.validateShapeMap([{node: focus, shape: start}])); if (res.errors) { console.error(JSON.stringify(res.errors, null, 1)); process.exit(2); } - process.stdout.write(JSON.stringify(ShExUtil.valToExtension(res, Mapper.url)) + "\n"); + // the scope layout with @node where the checkout has it (branch shexmap-scopes), else the legacy tree + process.stdout.write(JSON.stringify(Mapper.bindingTree ? Mapper.bindingTree(res) : ShExUtil.valToExtension(res, Mapper.url)) + "\n"); } else if (mode === "materialize") { const bindings = JSON.parse(Fs.readFileSync(dataPath, "utf8")); const staticVars = staticsPath ? JSON.parse(Fs.readFileSync(staticsPath, "utf8")) : {}; diff --git a/tests/test_shexmap/test_examples.py b/tests/test_shexmap/test_examples.py index 2daccd0..476c81c 100644 --- a/tests/test_shexmap/test_examples.py +++ b/tests/test_shexmap/test_examples.py @@ -18,9 +18,8 @@ for e in json.loads((d / "manifest.json").read_text(encoding="utf-8"))] TURTLE_BASE = "http://a.example/turtle/" # the base shex.js's test runner parses data with -# shex.js splits one binding record across two nesting levels in this example; PyShEx keeps -# a report's variables together. Both materialize to the expected graph. -BINDINGS_LAYOUT_DIFFERS = {"BPPatient 2 levels / simple"} +# the stored bindings are what shex.js writes (packages/extension-map/examples, branch +# shexmap-scopes): the scope layout with @node; blank-node labels are the parser's def label(entry) -> str: @@ -106,15 +105,20 @@ def nodes_in(obj) -> list[dict]: def test_bindings_match_shexjs(example): + """The same tree as shex.js writes, @node included, blank-node labels aside.""" entry, bindings = example if "expectedBindingsURL" not in entry: pytest.skip("no shex.js bindings recorded") expected = json.loads(path(entry, entry["expectedBindingsURL"]).read_text(encoding="utf-8")) - got = without_nodes(bindings.to_json()) - if label(entry) in BINDINGS_LAYOUT_DIFFERS: - assert got != expected # if this starts failing, the layouts converged - else: - assert got == expected + assert with_bnode_marker(bindings.to_json()) == with_bnode_marker(expected) + + +def with_bnode_marker(obj): + if isinstance(obj, dict): + return {k: ("_:" if isinstance(v, str) and v.startswith("_:") else v) for k, v in obj.items()} + if isinstance(obj, list): + return [with_bnode_marker(e) for e in obj] + return obj # examples whose input schema has a repeated shape-valued constraint, so their trees have lists diff --git a/tests/test_shexmap/test_shexjs_differential.py b/tests/test_shexmap/test_shexjs_differential.py index 9765a74..74d78dd 100644 --- a/tests/test_shexmap/test_shexjs_differential.py +++ b/tests/test_shexmap/test_shexjs_differential.py @@ -4,10 +4,9 @@ (``…/shex.js/packages/extension-map``) to run them; they are skipped otherwise. Each manifest example is bound and materialized on both sides and the results compared: -* the frames of shex.js's bindings equal PyShEx's (the trees may differ in layout, and - shex.js does not write ``@node``); -* shex.js materializes PyShEx's bindings JSON to the expected graph (``@node`` stripped, as - shex.js does not know the key yet; set ``SHEXJS_KEEP_NODE=1`` to hand it over as written); +* the frames of shex.js's bindings equal PyShEx's (the trees themselves are compared, blank + node labels aside, in ``test_examples.py`` against the bindings shex.js recorded); +* shex.js materializes PyShEx's bindings JSON, ``@node`` included, to the expected graph; * PyShEx materializes shex.js's bindings to the expected graph. """ import json @@ -21,7 +20,7 @@ from pyshex.shexmap import bind, loads, materialize, normalize from tests.test_shexmap.test_examples import (MANIFEST, expected_output, focus_node, input_graph, label, - node_and_shape, output_root, text, without_nodes) + node_and_shape, output_root, text) SHEXJS = os.environ.get("SHEXJS_EXTENSION_MAP") DRIVER = Path(__file__).parent / "shexjs_driver.js" @@ -58,10 +57,7 @@ def val(v): # known divergences, as strict xfails -DIVERGES = { - "EXTENDS / clinic": "shex.js's valToExtension does not collect Map results through EXTENDS: it binds " - "nothing for this example, where PyShEx binds through the extension each member satisfies", -} +DIVERGES: dict[str, str] = {} # label -> why the two implementations are expected to differ @pytest.fixture(scope="module", ids=label, @@ -92,8 +88,7 @@ def test_shexjs_materializes_our_bindings(example): _, start = node_and_shape(entry["queryMap"]) ours = bind(graph, text(entry, "schema"), focus_node(entry, graph), start=start) root, shape = output_root(entry) - tree = ours.to_json() if os.environ.get("SHEXJS_KEEP_NODE") else without_nodes(ours.to_json()) - (d / "ours.json").write_text(json.dumps(tree), encoding="utf-8") + (d / "ours.json").write_text(json.dumps(ours.to_json()), encoding="utf-8") nt = shexjs("materialize", str(d / "out.shex"), str(d / "ours.json"), node_arg(root), shape or "START") got = Graph().parse(data=nt, format="nt") assert isomorphic(got, expected_output(entry)), nt