diff --git a/pyshex/shexmap/__init__.py b/pyshex/shexmap/__init__.py index fc90be1..d53dd35 100644 --- a/pyshex/shexmap/__init__.py +++ b/pyshex/shexmap/__init__.py @@ -18,22 +18,27 @@ when the input conforms in several ways that bind differently it says so (``.alternatives``), and :func:`bind_all` returns every way. EXTENDS and inverse (``^p``) constraints are followed. -* :class:`ThreadedMaterializer` builds the output by running threads over each output - shape's NFA, each with its own cursor into the bindings, and keeps every accepting - thread (``.accepts``), returning the one that uses the most bindings. +* :func:`analyse` checks a schema pair before any data: every output repetition has a + list to iterate, every variable read is bound where it can be read from, and unused + bindings are reported. +* :class:`ThreadedMaterializer` builds the output from the binding tree's structure: an + output repetition iterates the input list its variables are bound at, a parent's + binding is read in every item, and the alternatives a ``OneOf`` or an extension leaves + open are kept (``.accepts``), the one that reads the most bindings returned. """ from rdflib import Graph from rdflib.term import Node -from pyshex.shexmap.bindings import (MAP_EXTENSION, AmbiguousBindingsError, Bindings, MapValidationError, bind, - bind_all, dumps, loads, normalize, shexc_prefixes) +from pyshex.shexmap.bindings import (MAP_EXTENSION, NODE_KEY, AmbiguousBindingsError, Bindings, MapValidationError, + bind, bind_all, dumps, loads, normalize, shexc_prefixes) +from pyshex.shexmap.analysis import Report, analyse from pyshex.shexmap.functions import MapFunctionError from pyshex.shexmap.materializer import Accept, MaterializationError, ThreadedMaterializer, materialize from pyshex.shexmap.semact import register -__all__ = ["MAP_EXTENSION", "Accept", "AmbiguousBindingsError", "Bindings", "MapFunctionError", - "MapValidationError", "MaterializationError", "ThreadedMaterializer", "bind", "bind_all", "dumps", - "loads", "map_graph", "materialize", "normalize", "register", "shexc_prefixes"] +__all__ = ["MAP_EXTENSION", "NODE_KEY", "Accept", "AmbiguousBindingsError", "Bindings", "MapFunctionError", + "MapValidationError", "MaterializationError", "Report", "ThreadedMaterializer", "analyse", "bind", + "bind_all", "dumps", "loads", "map_graph", "materialize", "normalize", "register", "shexc_prefixes"] def map_graph(graph: Graph, input_schema, focus: str | Node, output_schema, root: str | Node | None = None, diff --git a/pyshex/shexmap/analysis.py b/pyshex/shexmap/analysis.py new file mode 100644 index 0000000..236b814 --- /dev/null +++ b/pyshex/shexmap/analysis.py @@ -0,0 +1,374 @@ +"""Static checks of a ShExMap schema pair, before any data. + +Given the input schema and the output schema, :func:`analyse` says whether every output +repetition has a well-formed iteration scope, whether every variable the output reads is +bound in the input where it can be read from, and which bindings go unused -- the questions +materialization would otherwise answer at run time, one graph at a time. The checks are +the schema-pair conditions XSugar and biXid impose on grammar pairs, and the +"updatability" question of the view-update literature, in ShExMap's terms; see +``docs/shexmap-structural-plan.md`` §4 phase 3. + +A variable's **site** is the chain of repeated constraints and groups of the input schema +that enclose its ``%Map`` code: it is bound once per iteration of the innermost one, or once +for the whole tree when the chain is empty. An output repetition's **iteration scope** is +the deepest site among the variables its body reads, nested repetitions counting through +their parent site; it is ill-formed when two of those sites lie in unrelated lists. A +constraint may read a variable whose site is an ancestor of (or equal to) the scope it is +evaluated at; a deeper site means "which one?", and an unrelated one means the value is +not in scope. +""" +from __future__ import annotations + +from dataclasses import dataclass, field + +from ShExJSG import ShExJ + +from pyshex.shape_expressions_language.p5_context import Context +from pyshex.shapemap_structure_and_language.p3_shapemap_structure import START +from pyshex.shexmap import functions +from pyshex.shexmap.bindings import (declaration_of, expression_of, extension_candidates, is_repeated, map_actions, + references_shape, shape_parts, shexc_prefixes) + +Site = tuple[int, ...] # ids of the enclosing repeated expressions, outermost first + + +@dataclass +class Report: + """What :func:`analyse` found. ``errors`` make the pair not mappable; ``warnings`` do not. + Messages write IRIs with the schemas' prefixes.""" + errors: list[str] = field(default_factory=list) + warnings: list[str] = field(default_factory=list) + bound: dict[str, Site] = field(default_factory=dict) # input variable -> site + read: set[str] = field(default_factory=set) # variables the output reads + labels: dict[int, str] = field(default_factory=dict) # repeated expression id -> name + prefixes: dict[str, str] = field(default_factory=dict) + + @property + def ok(self) -> bool: + return not self.errors + + def where(self, site: Site) -> str: + return "the root" if not site else "each " + " / ".join(self.labels.get(i, "?") for i in site) + + def short(self, text: str) -> str: + """``text`` with every namespace the schemas declare written as its prefix.""" + for prefix, ns in sorted(self.prefixes.items(), key=lambda kv: -len(kv[1])): + if ns: + text = text.replace(ns, prefix + ":") + return text + + def error(self, message: str) -> None: + self.errors.append(self.short(message)) + + def warn(self, message: str) -> None: + self.warnings.append(self.short(message)) + + def __str__(self) -> str: + lines = [f"error: {e}" for e in self.errors] + [f"warning: {w}" for w in self.warnings] + return "\n".join(lines) if lines else "ok: the schemas map coherently" + + +def analyse(input_schema, output_schema, prefixes=None, input_start=None, output_start=None, + static_vars=()) -> Report: + """Check that ``output_schema`` can be materialized coherently from what ``input_schema`` binds. + + :param input_schema: ShExC text or a parsed ShExJ schema + :param output_schema: ShExC text or a parsed ShExJ schema + :param prefixes: prefixes for ShExMap variable names; read from ShExC texts when omitted + :param input_start, output_start: shape labels; default the schemas' start shapes + :param static_vars: variable IRIs that will be supplied as static variables + """ + from pyshex.utils.schema_loader import SchemaLoader + prefixes = dict(prefixes or {}) + schemas = [] + for text in (input_schema, output_schema): + if isinstance(text, str): + prefixes = {**shexc_prefixes(text), **prefixes} + schema = SchemaLoader().loads(text) + if schema is None: + raise ValueError("a schema does not parse") + else: + schema = text + schemas.append(schema) + report = Report(prefixes=prefixes) + inp = _Input(Context(None, schemas[0]), prefixes, report) + inp.run(_start(schemas[0], input_start)) + out = _Output(Context(None, schemas[1]), prefixes, report, set(static_vars)) + out.run(_start(schemas[1], output_start)) + for v, site in report.bound.items(): + if v not in report.read: + report.warn(f"{v} is bound (at {report.where(site)}) but the output never reads it") + for v in sorted(set(static_vars) - report.read): + report.warn(f"static variable {v} is never read") + return report + + +def _start(schema, start): + if start is None or start is START: + if schema.start is None: + raise ValueError("the schema has no start shape; name one") + return schema.start + return ShExJ.IRIREF(str(start)) + + +def _is_repetition(expr) -> bool: + """Any cardinality but exactly one, as the materializer treats it (``?`` included).""" + return not (expr.min in (None, 1) and expr.max in (None, 1)) + + +def _label(expr) -> str: + found: list[str] = [] + stack = [expr] + while stack and len(found) < 2: + e = stack.pop() + if isinstance(e, ShExJ.TripleConstraint): + found.append(("^" if e.inverse else "") + str(e.predicate)) + elif isinstance(e, (ShExJ.EachOf, ShExJ.OneOf)): + stack.extend(reversed(list(e.expressions))) + return "(" + ", ".join(found) + ")" if len(found) > 1 else (found[0] if found else "(group)") + + +def _variables(tc, prefixes) -> tuple[list[str], list[str]]: + """(variables bound or read by a constraint's Map codes, id() arguments) -- id() is not a + variable code; its arguments are returned apart.""" + plain: list[str] = [] + keys: list[str] = [] + for act in map_actions(tc): + code = str(act.code or '') + try: + if functions.is_key_code(code): + keys.extend(a for a in functions.key_arguments(code, prefixes) if a != functions.NODE_ARGUMENT) + elif functions.is_function_call(code): + plain.extend(functions.variables_of(code, prefixes)) + else: + plain.append(functions.expand_variable(code, prefixes)) + except functions.MapFunctionError: + pass + return plain, keys + + +class _Input: + """Where the input schema binds each variable.""" + + def __init__(self, cntxt: Context, prefixes, report: Report) -> None: + self.cntxt, self.prefixes, self.report = cntxt, prefixes, report + self.seen: set[int] = set() + + def run(self, se) -> None: + self.shape_expr(se, ()) + + def shape_expr(self, se, site: Site) -> None: + if isinstance(se, str): + decl = declaration_of(self.cntxt, se) + if decl is None: + return + for o in extension_candidates(self.cntxt, decl): + self.shape_expr(expression_of(o), site) + return + if id(se) in self.seen: + return + self.seen.add(id(se)) + if isinstance(se, ShExJ.ShapeDecl): + self.shape_expr(se.shapeExpr, site) + elif isinstance(se, ShExJ.Shape): + for p in shape_parts(self.cntxt, se): + if p.expression is not None: + self.expression(p.expression, site) + elif isinstance(se, (ShExJ.ShapeAnd, ShExJ.ShapeOr)): + for p in se.shapeExprs: + self.shape_expr(p, site) + + def expression(self, expr, site: Site) -> None: + if isinstance(expr, str): + expr = self.cntxt.tripleExprFor(expr) + if expr is None: + return + if is_repeated(expr): + self.report.labels[id(expr)] = _label(expr) + site = site + (id(expr),) + if isinstance(expr, ShExJ.TripleConstraint): + plain, _ = _variables(expr, self.prefixes) + for v in plain: + at = self.report.bound.setdefault(v, site) + if at != site: + self.report.error( + f"{v} is bound at two places of the input schema, {self.report.where(at)} and " + f"{self.report.where(site)}; a variable must have one binding site") + if references_shape(expr.valueExpr): + self.shape_expr(expr.valueExpr, site) + elif isinstance(expr, (ShExJ.EachOf, ShExJ.OneOf)): + for e in expr.expressions: + self.expression(e, site) + + +class _Output: + """Where the output schema reads each variable, and what each repetition iterates.""" + + def __init__(self, cntxt: Context, prefixes, report: Report, statics: set[str]) -> None: + self.cntxt, self.prefixes, self.report, self.statics = cntxt, prefixes, report, statics + self.scopes: dict[int, Site | None] = {} + self.ill_formed: set[int] = set() + self.active: set[int] = set() + + def run(self, se) -> None: + self.shape_expr(se, ()) + + # -- walking ------------------------------------------------------------------------ + def shape_expr(self, se, scope: Site) -> None: + if isinstance(se, str): + decl = declaration_of(self.cntxt, se) + if decl is None: + self.report.error(f"the output schema references {se}, which it does not define") + return + options = extension_candidates(self.cntxt, decl) + if not options: + self.report.error(f"{se} is abstract and nothing extends it") + for o in options: + self.shape_expr(expression_of(o), scope) + return + key = id(se) + if key in self.active: + return # a recursive shape: the first pass covered it + self.active.add(key) + try: + if isinstance(se, ShExJ.ShapeDecl): + self.shape_expr(se.shapeExpr, scope) + elif isinstance(se, ShExJ.Shape): + for p in shape_parts(self.cntxt, se): + if p.expression is not None: + self.expression(p.expression, scope) + elif isinstance(se, (ShExJ.ShapeAnd, ShExJ.ShapeOr)): + for p in se.shapeExprs: + if not isinstance(p, ShExJ.NodeConstraint): + self.shape_expr(p, scope) + elif isinstance(se, ShExJ.NodeConstraint): + pass + else: + self.report.error(f"{type(se).__name__} cannot be materialized") + finally: + self.active.discard(key) + + def expression(self, expr, scope: Site) -> None: + if isinstance(expr, str): + expr = self.cntxt.tripleExprFor(expr) + if expr is None: + return + if _is_repetition(expr): + iterates = self.iteration_scope(expr) + if id(expr) in self.ill_formed: + return # its structure is the error; its reads would only repeat it + body_scope = iterates if iterates is not None and len(iterates) > len(scope) else scope + else: + body_scope = scope + if isinstance(expr, ShExJ.TripleConstraint): + self.constraint(expr, body_scope) + elif isinstance(expr, (ShExJ.EachOf, ShExJ.OneOf)): + for e in expr.expressions: + self.expression(e, body_scope) + + def constraint(self, tc, scope: Site) -> None: + plain, keys = _variables(tc, self.prefixes) + acts = map_actions(tc) + if keys or any(functions.is_key_code(str(a.code or '')) for a in acts): + if len(acts) > 1: + self.report.error(f"id() must be the only Map code on {tc.predicate}") + if not references_shape(tc.valueExpr): + self.report.error(f"id() on {tc.predicate} needs a shape-valued constraint") + if any(functions.NODE_ARGUMENT in functions.key_arguments(str(a.code or ''), self.prefixes) + for a in acts if functions.is_key_code(str(a.code or ''))) and not scope: + self.report.error(f"id(@node) on {tc.predicate} is read at the root, where no input iteration " + f"provides a node") + for v in plain + keys: + self.read_variable(v, tc, scope) + if references_shape(tc.valueExpr): + self.shape_expr(tc.valueExpr, scope) + + def read_variable(self, v: str, tc, scope: Site) -> None: + self.report.read.add(v) + if v in self.statics: + return + site = self.report.bound.get(v) + if site is None: + self.report.error(f"{tc.predicate} reads {v}, which the input schema never binds") + return + if scope[:len(site)] == site: + return # bound here or above: readable + if site[:len(scope)] == scope: + self.report.error( + f"{tc.predicate} reads {v}, bound once per {self.report.where(site)}, from " + f"{self.report.where(scope)}: which one? A repetition over it is needed") + else: + self.report.error( + f"{tc.predicate} reads {v}, bound at {self.report.where(site)}, from " + f"{self.report.where(scope)}, which is not below it") + + # -- iteration scopes --------------------------------------------------------------- + def iteration_scope(self, expr) -> Site | None: + key = id(expr) + if key in self.scopes: + return self.scopes[key] + self.scopes[key] = None # a recursive repetition counts as scope-free while computed + direct: dict[str, object] = {} + nested: list = [] + self.collect(expr, direct, nested, set(), top=True) + candidates: dict[Site, str] = {} + for v in direct: + if v in self.report.bound: + candidates.setdefault(self.report.bound[v], v) + for r in nested: + sub = self.iteration_scope(r) + if sub: + candidates.setdefault(sub[:-1], f"the repetition over {_label(r)}") + result: Site | None = None + if candidates: + result = max(candidates, key=len) + for site, why in candidates.items(): + if result[:len(site)] != site: + self.report.error( + f"the repetition over {_label(expr)} reads from unrelated lists: {why} is bound at " + f"{self.report.where(site)} and {candidates[result]} at {self.report.where(result)}") + self.ill_formed.add(key) + self.report.read.update(direct) + break + self.scopes[key] = result + return result + + def collect(self, expr, direct: dict, nested: list, seen: set, top: bool) -> None: + if isinstance(expr, str): + expr = self.cntxt.tripleExprFor(expr) + if expr is None or id(expr) in seen: + return + seen.add(id(expr)) + if not top and _is_repetition(expr): + nested.append(expr) + return + if isinstance(expr, ShExJ.TripleConstraint): + plain, keys = _variables(expr, self.prefixes) + for v in plain + keys: + direct[v] = True + if references_shape(expr.valueExpr): + self.collect_shape(expr.valueExpr, direct, nested, seen) + elif isinstance(expr, (ShExJ.EachOf, ShExJ.OneOf)): + for e in expr.expressions: + self.collect(e, direct, nested, seen, top=False) + + def collect_shape(self, se, direct: dict, nested: list, seen: set) -> None: + if isinstance(se, str): + decl = declaration_of(self.cntxt, se) + if decl is None: + return + for o in extension_candidates(self.cntxt, decl): + self.collect_shape(expression_of(o), direct, nested, seen) + return + if id(se) in seen: + return + seen.add(id(se)) + if isinstance(se, ShExJ.ShapeDecl): + self.collect_shape(se.shapeExpr, direct, nested, seen) + elif isinstance(se, ShExJ.Shape): + for p in shape_parts(self.cntxt, se): + if p.expression is not None: + self.collect(p.expression, direct, nested, seen, top=False) + elif isinstance(se, (ShExJ.ShapeAnd, ShExJ.ShapeOr)): + for p in se.shapeExprs: + self.collect_shape(p, direct, nested, seen) diff --git a/pyshex/shexmap/bindings.py b/pyshex/shexmap/bindings.py index 20dbd79..818f205 100644 --- a/pyshex/shexmap/bindings.py +++ b/pyshex/shexmap/bindings.py @@ -1,11 +1,22 @@ """Collect ShExMap bindings from RDF that conforms to an input schema. -**Binding trees** use the structure shex.js prints and reads, with rdflib terms as values: -an object maps variable IRIs to terms, and a list is a sequence whose elements are -objects or nested lists. A repeated match (a triple constraint with max > 1, or one -iteration of a repeated group) contributes one element to a list, so everything bound -from one reading of a blood pressure stays together. :class:`Bindings` wraps a tree, -and :func:`dumps`/:func:`loads` convert to and from shex.js's JSON. +**Binding trees** use the structure shex.js prints and reads, with rdflib terms as values. +A *scope* (what one node's match binds) is an object mapping variable IRIs to terms, or, +when the match has repeated parts, an array whose first element is that object (possibly +empty) and whose other elements are *lists*; a list has one element per iteration of a +repeated constraint or group, each element a scope, so everything bound from one reading +of a blood pressure stays together. A list's elements are uniform: all objects, or all +arrays (a lone object among arrays is written as ``[object]``), so that a reader never has +to guess whether an array is a scope or a list. Every repeated constraint or group of the +shape gets its list, in schema order, empty when nothing matched, so a list's position +says which expression it came from whatever the data. A non-repeated nested shape merges +into the scope that matched it. An element made by a repeated *shape-valued* constraint also +records the node the nested shape matched under the reserved key ``"@node"`` +(:data:`NODE_KEY`; the subject for an inverse constraint). It is not a variable: +:func:`normalize` leaves it out of the frames and no output constraint can read it. It +says which input node an iteration came from, which is what tells two groups apart when +their bindings are alike. :class:`Bindings` wraps a tree, and :func:`dumps`/:func:`loads` +convert to and from shex.js's JSON. **Collecting** happens after validation succeeds. PyShEx checks that *some* partition of each node's neighbourhood satisfies the schema but does not say which, so the @@ -28,6 +39,7 @@ import re from collections.abc import Iterable, Iterator, Mapping from dataclasses import dataclass, field +from typing import Any from ShExJSG import ShExJ from rdflib import BNode, Graph, Literal, URIRef, XSD @@ -41,6 +53,8 @@ MAP_EXTENSION = "http://shex.io/extensions/Map/#" +NODE_KEY = "@node" # in an iteration's bindings: the node the nested shape matched + MAX_ALTERNATIVES = 20 # distinct binding trees to collect before giving up counting MAX_PARTITIONS = 10_000 # partitions of one neighbourhood to try @@ -60,11 +74,13 @@ def __init__(self, message: str, alternatives: list[Bindings]) -> None: # -- binding trees ---------------------------------------------------------------------- class Bindings: - """A binding tree (see module docstring), and how many distinct ones the input allowed.""" + """A binding tree (see module docstring), how many distinct ones the input allowed, and + the triples the schema matched to make it (``matched``: the subgraph it selected).""" - def __init__(self, tree, alternatives: int = 1) -> None: + def __init__(self, tree, alternatives: int = 1, matched: frozenset = frozenset()) -> None: self.tree = tree self.alternatives = alternatives + self.matched = matched @property def ambiguous(self) -> bool: @@ -106,7 +122,8 @@ def normalize(tree) -> list[dict[str, Node]]: list of their readings) is copied into every frame the sibling lists produce.""" def walk(node): if not isinstance(node, list): - return [dict(node)], True, {k: 1 for k in node} + own = {k: v for k, v in node.items() if k != NODE_KEY} + return [own], True, {k: 1 for k in own} kids = [walk(k) for k in node] counts: dict[str, int] = {} for _, _, c in kids: @@ -128,6 +145,8 @@ def walk(node): out = [] for frames, leaf in ordered: out.extend(f if leaf else {**shared, **f} for f in frames) + if not out and shared: # a scope whose lists are all empty still has its own bindings + out.append(shared) return out, False, counts return walk(tree)[0] @@ -278,30 +297,49 @@ def triple_constraints(se, cntxt: Context | None = None) -> Iterator[ShExJ.Tripl @dataclass class _Record: - """Bindings made while matching one node: its own variables and nested matches.""" + """One scope: the bindings made while matching one node, one list of iteration records + per repeated constraint or group that matched under it, and the triples matched here.""" vars: dict[str, Node] = field(default_factory=dict) - children: list[_Record] = field(default_factory=list) - frame: bool = False + lists: list[list[_Record]] = field(default_factory=list) + triples: set = field(default_factory=set) def merged(self, other: _Record) -> _Record: - return _Record({**self.vars, **other.vars}, self.children + other.children, self.frame) + return _Record({**self.vars, **other.vars}, self.lists + other.lists, self.triples | other.triples) + + def all_triples(self) -> frozenset: + """The triples matched here and in every nested scope.""" + out = set(self.triples) + for lst in self.lists: + for r in lst: + out |= r.all_triples() + return frozenset(out) def empty(self) -> bool: - return not self.vars and all(c.empty() for c in self.children) + return not (self.vars.keys() - {NODE_KEY}) and all(r.empty() for lst in self.lists for r in lst) def tree(self): - """Non-repeated nested matches merge into this object; repeated ones become a list.""" - merged = dict(self.vars) - frames = [] - for c in self.children: - ct = c.tree() - if not c.frame and isinstance(ct, dict) and not (merged.keys() & ct.keys()): - merged.update(ct) - else: - frames.append(ct) - if not frames: - return merged - return ([merged] if merged else []) + [frames] + """The scope as a binding tree: the object alone, or ``[object, list, ...]``, each + list's elements uniform (all objects or all arrays).""" + own = dict(self.vars) + if not self.lists: + return own + out = [own] + for lst in self.lists: + iterations = [r.tree() for r in lst] + if any(isinstance(t, list) for t in iterations): + iterations = [t if isinstance(t, list) else [t] for t in iterations] + out.append(iterations) + return out + + +def _by_first(items) -> list: + """The items' expressions, in order of first occurrence.""" + seen, out = set(), [] + for item in items: + if id(item[1]) not in seen: + seen.add(id(item[1])) + out.append(item[1]) + return out def _product(alternatives: list[list[_Record]], limit: int) -> list[_Record]: @@ -397,15 +435,17 @@ def shape(self, n: Node, S: ShExJ.Shape) -> list[_Record]: [('in', t) for t in g.triples((None, None, n)) if (t[1], True) in keys]) extras = {URIRef(str(e)) for p in parts for e in (getattr(p, 'extra', None) or [])} + exprs = [p.expression for p in parts if p.expression is not None] + repeated = self._repeated_expressions(exprs) results: list[_Record] = [] tried = 0 - for items, rest in self._parts(n, [p.expression for p in parts if p.expression is not None], matchables): + for items, rest in self._parts(n, exprs, matchables): tried += 1 if tried > MAX_PARTITIONS: break if not self._valid_remainder(n, rest, extras, tcs): continue - results.extend(self._bind_items(items)) + results.extend(self._bind_items(items, repeated)) results = _dedupe(results, self.limit) if len(results) >= self.limit: break @@ -418,6 +458,25 @@ def shape(self, n: Node, S: ShExJ.Shape) -> list[_Record]: + (f" within {MAX_PARTITIONS} partitions" if tried > MAX_PARTITIONS else "")) return results + def _repeated_expressions(self, exprs: list) -> list: + """The repeated constraints and groups directly under ``exprs`` (not inside another + repeated expression), in schema order: one list each in the scope's tree.""" + out: list = [] + + def walk(e) -> None: + if isinstance(e, str): + e = self.cntxt.tripleExprFor(e) + if e is None: + return + if is_repeated(e): + out.append(e) + elif isinstance(e, (ShExJ.EachOf, ShExJ.OneOf)): + for sub in e.expressions: + walk(sub) + for e in exprs: + walk(e) + return out + def _own_tcs(self, S: ShExJ.Shape) -> list[ShExJ.TripleConstraint]: """Triple constraints of S's own expression, not of the shapes nested in them.""" out, stack = [], [S.expression] @@ -452,7 +511,8 @@ def _parts(self, n: Node, exprs: list, available: frozenset): def _match(self, n: Node, expr, available: frozenset): """Ways ``expr`` can match some of ``available``: (items, remaining) pairs, greediest - first. Items are ('tc', tc, triple) or ('group', items) for a repeated group's iteration.""" + first. Items are ('tc', tc, triple) or ('group', expr, items) for one iteration of a + repeated group.""" if isinstance(expr, str): # an inclusion: &label expr = self.cntxt.tripleExprFor(expr) if isinstance(expr, ShExJ.TripleConstraint): @@ -476,7 +536,7 @@ def _repeat(self, n: Node, expr, available: frozenset, count: int, min_: int, ma for one, rest in self._once(n, expr, available): if rest == available and count >= min_: continue # an iteration that matches nothing adds nothing - wrapped = [('group', one)] if repeated else one + wrapped = [('group', expr, one)] if repeated else one if rest == available: # matched nothing but is required: count it once yield wrapped, rest continue @@ -498,33 +558,55 @@ def _once(self, n: Node, expr, available: frozenset): def _value(tc: ShExJ.TripleConstraint, t) -> Node: return t[0] if tc.inverse else t[2] - def _bind_items(self, items) -> list[_Record]: - """Alternative records for one partition's items.""" + def _bind_items(self, items, repeated: list) -> list[_Record]: + """Alternative records for one partition's items: each of the scope's ``repeated`` + expressions makes one list, with an iteration per item (empty when it matched + nothing); a non-repeated constraint binds into the scope itself.""" + by_expr: dict[int, list] = {} alternatives: list[list[_Record]] = [] for item in items: - if item[0] == 'group': # one iteration of a repeated group: one frame - opts = [] - for r in self._bind_items(item[1]): - iteration = _Record(r.vars, r.children, True) - opts.append(_Record(children=[iteration]) if not iteration.empty() else _Record()) - alternatives.append(opts or [_Record()]) + expr = item[1] + if item[0] == 'group' or is_repeated(expr): + by_expr.setdefault(id(expr), []).append(item) continue _, tc, t = item value = self._value(tc, t) lifted = self.lift(tc, value) - per_value = is_repeated(tc) - if references_shape(tc.valueExpr): + if references_shape(tc.valueExpr): # a nested scope: merge it into this one opts = [] for sub in self.shape_expr(value, tc.valueExpr): - child = _Record({**lifted, **sub.vars}, sub.children, per_value) - opts.append(_Record(children=[child]) if not child.empty() else _Record()) + if lifted.keys() & sub.vars.keys(): + opts.append(_Record(lifted, [[sub]], {t})) # a name clash: keep it apart + else: + opts.append(_Record({**lifted, **sub.vars}, sub.lists, {t} | sub.triples)) alternatives.append(opts) - elif per_value: - alternatives.append([_Record(children=[_Record(lifted, frame=True)]) if lifted else _Record()]) else: - alternatives.append([_Record(lifted)]) + alternatives.append([_Record(lifted, triples={t})]) + for expr in repeated + [e for e in _by_first(items) if id(e) in by_expr and e not in repeated]: + per_iteration = [self._iteration(it) for it in by_expr.pop(id(expr), [])] + opts = [] + for combo in itertools.product(*per_iteration): + kept = [r for r in combo if not r.empty()] + dropped = set().union(*(r.triples for r in combo if r.empty())) if combo else set() + opts.append(_Record(lists=[kept], triples=dropped)) # an iteration that bound nothing still matched + if len(opts) >= self.limit: + break + alternatives.append(opts or [_Record(lists=[[]])]) return _product(alternatives, self.limit) + def _iteration(self, item) -> list[_Record]: + """Alternative scopes for one iteration of a repeated constraint or group.""" + if item[0] == 'group': + expr = item[1] + return self._bind_items(item[2], self._repeated_expressions(list(expr.expressions))) or [_Record()] + _, tc, t = item + value = self._value(tc, t) + lifted = self.lift(tc, value) + if references_shape(tc.valueExpr): + return [_Record({NODE_KEY: value, **lifted, **sub.vars}, sub.lists, {t} | sub.triples) + for sub in self.shape_expr(value, tc.valueExpr)] + return [_Record(lifted, triples={t})] + def lift(self, tc, value) -> dict[str, Node]: bound: dict[str, Node] = {} for act in map_actions(tc): @@ -580,8 +662,8 @@ def bind_all(graph: Graph, schema: str | ShExJ.Schema, focus: str | Node, start= raise MapValidationError(f"{focus} does not conform: " + "\n".join(reasons)) extractor = _Extractor(Context(graph, schema), prefixes, limit) records = extractor.shape_expr(focus, label if label is not START else START) - trees = [r.tree() for r in _dedupe(records, limit)] - return [Bindings(t, alternatives=len(trees)) for t in trees] + kept = _dedupe(records, limit) + return [Bindings(r.tree(), alternatives=len(kept), matched=r.all_triples()) for r in kept] def bind(graph: Graph, schema: str | ShExJ.Schema, focus: str | Node, start=None, diff --git a/pyshex/shexmap/cli.py b/pyshex/shexmap/cli.py index 8ac96e2..00b2589 100644 --- a/pyshex/shexmap/cli.py +++ b/pyshex/shexmap/cli.py @@ -11,6 +11,14 @@ Materialize from bindings saved earlier, by this tool or by shex.js:: shexmap -j bindings.json -t out.shex -r + +Check that two schemas map coherently, without data:: + + shexmap --check -s in.shex -t out.shex + +Update a graph in place, replacing what the output schema holds at the root:: + + shexmap -i data.ttl -s in.shex -f -t out.shex -r --into existing.ttl """ import json import sys @@ -19,8 +27,8 @@ from rdflib import BNode, Graph, URIRef -from pyshex.shexmap import (MapFunctionError, MapValidationError, MaterializationError, ThreadedMaterializer, bind, - dumps, loads) +from pyshex.shexmap import (MapFunctionError, MapValidationError, MaterializationError, ThreadedMaterializer, analyse, + bind, dumps, loads) from pyshex.shexmap.bindings import term_from_json @@ -48,14 +56,47 @@ def genargs(prog: str | None = None) -> ArgumentParser: p.add_argument("--static", help="JSON object of extra variable values, as shex.js staticVars") p.add_argument("--strict", action="store_true", help="fail when the input or the output can be matched in more than one way") + p.add_argument("--into", help="an existing RDF file (in --output-format) to update in place: what the output " + "schema currently holds at --root is replaced, the rest kept; written back there " + "unless --output says otherwise") + p.add_argument("--provenance", help="write one JSON object per output triple here: the triple, the output " + "constraint's predicate, the input scope and node it came from, the bindings " + "read, and how the object arose") + p.add_argument("--check", action="store_true", + help="only analyse --input-schema against --output-schema: every output repetition has a list " + "to iterate, every variable read is bound where it can be read; exit 1 on errors") p.add_argument("-o", "--output", help="write the output RDF here (default: stdout)") p.add_argument("--output-format", default="turtle", help="output RDF format (default: turtle)") return p +def provenance_json(triple, source: dict) -> dict: + """A triple's provenance record as JSON: terms in N3, the input node likewise.""" + s, p, o = triple + kind = next((k for k in ("variable", "code", "keyed", "constant", "structural", "named") if k in source), "?") + return {"subject": s.n3(), "predicate": p.n3(), "object": o.n3(), + "constraint": source["predicate"], "scope": list(source["scope"]), + "node": source["node"].n3() if source["node"] is not None else None, + "reads": [[list(path), var] for path, var in source["reads"]], "statics": list(source["statics"]), + "kind": kind, kind: source[kind] if kind in ("variable", "code", "keyed") else True} + + def main(argv: list[str] | None = None, prog: str | None = None) -> int: parser = genargs(prog) opts = parser.parse_args(argv) + if opts.check: + if not (opts.input_schema and opts.output_schema): + parser.error("--check needs --input-schema and --output-schema") + static = list(json.loads(Path(opts.static).read_text(encoding="utf-8"))) if opts.static else [] + try: + report = analyse(Path(opts.input_schema).read_text(encoding="utf-8"), + Path(opts.output_schema).read_text(encoding="utf-8"), + input_start=opts.start, output_start=opts.output_start, static_vars=static) + except ValueError as e: + print(f"shexmap: {e}", file=sys.stderr) + return 1 + print(report) + return 0 if report.ok else 1 if opts.bindings: if opts.input or opts.input_schema or opts.focus: parser.error("--bindings replaces --input, --input-schema and --focus") @@ -84,20 +125,30 @@ def main(argv: list[str] | None = None, prog: str | None = None) -> int: static = {k: term_from_json(v) for k, v in json.loads(Path(opts.static).read_text(encoding="utf-8")).items()} \ if opts.static else None m = ThreadedMaterializer(Path(opts.output_schema).read_text(encoding="utf-8"), static_vars=static) - triples = m.materialize(bindings, parse_node(opts.root) if opts.root else BNode(), start=opts.output_start) + root = parse_node(opts.root) if opts.root else BNode() + if opts.into: + out = Graph().parse(opts.into, format=opts.output_format) + added, removed = m.update(out, bindings, root, start=opts.output_start) + triples = m.chosen.triples + print(f"shexmap: {opts.into}: {len(added)} triples added, {len(removed)} removed", file=sys.stderr) + else: + out = Graph() + triples = m.materialize(bindings, root, start=opts.output_start) + for t in triples: + out.add(t) if len(m.accepts) > 1: if opts.strict: raise MaterializationError(f"the bindings fit the output schema in {len(m.accepts)} ways") print(f"shexmap: warning: the bindings fit the output schema in {len(m.accepts)} ways;" " using the one that uses the most bindings", file=sys.stderr) - out = Graph() for prefix, ns in m.prefixes.items(): out.bind(prefix, ns, override=False) - for t in triples: - out.add(t) + if opts.provenance: + Path(opts.provenance).write_text("".join(json.dumps(provenance_json(t, src)) + "\n" + for t, src in zip(triples, m.provenance)), encoding="utf-8") text = out.serialize(format=opts.output_format) - if opts.output: - Path(opts.output).write_text(text, encoding="utf-8") + if opts.output or opts.into: + Path(opts.output or opts.into).write_text(text, encoding="utf-8") else: sys.stdout.write(text) except (MapValidationError, MaterializationError, MapFunctionError) as e: diff --git a/pyshex/shexmap/functions.py b/pyshex/shexmap/functions.py index 57f01b9..78afba8 100644 --- a/pyshex/shexmap/functions.py +++ b/pyshex/shexmap/functions.py @@ -16,10 +16,11 @@ lower: find the key whose value the variable holds. """ import json +from urllib.parse import quote import re from collections.abc import Callable, Mapping -from rdflib import Literal +from rdflib import Literal, URIRef from rdflib.term import Node VARIABLE = re.compile(r'^ *(?:<([^>]*)>|([^:<>\s]*):(\S*)) *$') @@ -156,8 +157,94 @@ def _hashmap_lower(code: str, args: str, get: Callable[[str], Node | None], pref raise MapFunctionError(f'{code.strip()} cannot invert "{val}"') -LIFTERS = {'regex': _regex_lift, 'hashmap': _hashmap_lift} -LOWERERS = {'regex': _regex_lower, 'hashmap': _hashmap_lower} +# -- id ---------------------------------------------------------------------------------- + +NODE_ARGUMENT = "@node" # in id(): the input node the enclosing iteration matched +_PLACEHOLDER = re.compile(r'\{([^{}]*)\}') + + +class Template: + """An IRI template argument of ``id()``: ````.""" + + def __init__(self, text: str, prefixes: Mapping[str, str]) -> None: + self.segments: list = [] # str, or ('var', iri) + pos = 0 + for m in _PLACEHOLDER.finditer(text): + if m.start() > pos: + self.segments.append(text[pos:m.start()]) + self.segments.append(('var', expand_variable(m.group(1), prefixes))) + pos = m.end() + if pos < len(text): + self.segments.append(text[pos:]) + self.text = text + + def variables(self) -> list[str]: + return [seg[1] for seg in self.segments if isinstance(seg, tuple)] + + def expand(self, get: Callable[[str], Node | None]) -> URIRef | None: + """The IRI, values percent-encoded as R2RML does; None when a variable is unbound.""" + out = [] + for seg in self.segments: + if isinstance(seg, tuple): + value = get(seg[1]) + if value is None: + return None + out.append(quote(str(value), safe="-._~")) + else: + out.append(seg) + return URIRef("".join(out)) + + def __repr__(self) -> str: + return f"Template(<{self.text}>)" + + +def key_terms(code: str, prefixes: Mapping[str, str]) -> list: + """The arguments of an ``id(...)`` code: variable IRIs, :data:`NODE_ARGUMENT`, and + :class:`Template` s for ``<...{var}...>`` arguments.""" + name, args = _split_call(code) + if name != 'id': + raise MapFunctionError(f"{code.strip()} is not an id() code") + parts = [a.strip() for a in args.split(',')] if args.strip() else [] + if not parts: + raise MapFunctionError(f"id() needs at least one variable, template or {NODE_ARGUMENT}: {code.strip()}") + terms = [] + for a in parts: + if a == NODE_ARGUMENT: + terms.append(a) + elif a.startswith('<') and a.endswith('>') and '{' in a: + terms.append(Template(a[1:-1], prefixes)) + else: + terms.append(expand_variable(a, prefixes)) + return terms + + +def key_arguments(code: str, prefixes: Mapping[str, str]) -> list[str]: + """The variables an ``id(...)`` code reads (templates' placeholders included), and + :data:`NODE_ARGUMENT` where it appears.""" + out: list[str] = [] + for term in key_terms(code, prefixes): + out.extend(term.variables() if isinstance(term, Template) else [term]) + return out + + +def _id_lift(code: str, args: str, value, prefixes: Mapping[str, str]) -> dict[str, Node]: + """``id(...)`` names an output node; validating input it binds nothing (so one schema + serves both directions).""" + key_arguments(code, prefixes) + return {} + + +def _id_lower(code: str, args: str, get, prefixes: Mapping[str, str]) -> Node | None: + raise MapFunctionError(f"{code.strip()} names a node; it belongs on a shape-valued constraint, not a value") + + +LIFTERS = {'regex': _regex_lift, 'hashmap': _hashmap_lift, 'id': _id_lift} +LOWERERS = {'regex': _regex_lower, 'hashmap': _hashmap_lower, 'id': _id_lower} + + +def is_key_code(code: str) -> bool: + m = FUNCTION_CALL.match(code) + return m is not None and m.group(1) == 'id' def lift(code: str, value, prefixes: Mapping[str, str]) -> dict[str, Node]: @@ -168,6 +255,19 @@ def lift(code: str, value, prefixes: Mapping[str, str]) -> dict[str, Node]: return LIFTERS[name](code, args, value, prefixes) +def variables_of(code: str, prefixes: Mapping[str, str]) -> list[str]: + """The variables a function call reads when lowering (and binds when lifting).""" + name, args = _split_call(code) + if name == 'regex': + return [expand_variable(_unescape_meta(m.group(1)), prefixes) + for m in re.finditer(_CAPTURE_NAME, _regex_body(code, args))] + if name == 'hashmap': + return [_hashmap_args(code, args, prefixes)[0]] + if name == 'id': + return [a for a in key_arguments(code, prefixes) if a != NODE_ARGUMENT] + raise MapFunctionError(f"Unknown ShExMap function {name}() in {code.strip()}") + + def lower(code: str, get: Callable[[str], Node | None], prefixes: Mapping[str, str]) -> Node | None: """Build a value from bindings with a function call; None when a variable is unbound.""" name, args = _split_call(code) diff --git a/pyshex/shexmap/materializer.py b/pyshex/shexmap/materializer.py index 25e918a..9a32881 100644 --- a/pyshex/shexmap/materializer.py +++ b/pyshex/shexmap/materializer.py @@ -1,54 +1,78 @@ -"""Threaded materialization: build RDF conforming to an output schema from bindings. - -A port of shex.js's ``ThreadedMaterializer`` (packages/extension-map/doc/threaded-materializer.md). - -* The binding tree is flattened to a sequence of **frames** (:func:`normalize`), read - through a cursor that stays on the current frame while it has an unused binding for a - variable and otherwise scans forward, never back. -* Each output shape compiles to an **NFA** with four kinds of state: ``TC`` (emit one - instance of a triple constraint), ``Split`` (OneOf / ShapeOr, in priority order), - ``Rept`` (cardinality) and ``Match``. A shape-valued constraint invents a blank node, - links it and calls the nested shape's NFA; ``Match`` returns from the call. -* A **thread** is an immutable configuration -- state, call stack, cursor, emitted - triples. One whose variable is unbound just dies, taking its emissions and cursor - marks with it, so a failed optional group or disjunct never disturbs its siblings. -* Threads run depth-first in greedy order (another repetition, then the emitting arm of - an optional, then the first disjunct), except that a lookup which has to **advance to - a later frame, abandoning unused bindings there,** is deferred until the alternatives - that can still use the current frame have been explored. An advance that abandons - nothing is not a choice, so it is not deferred. -* Every accepting thread is ranked; :meth:`ThreadedMaterializer.materialize` returns the - one that consumed the most bindings (ties: fewest bindings skipped by cursor advances, - then most triples, then discovery order), or whichever ``prefer`` ranks first. The - best ``max_accepts`` are kept in :attr:`~ThreadedMaterializer.accepts`; reaching that - many does not stop the search, only a perfect accept or the step budgets do. - -Beyond shex.js: ``EXTENDS`` in the output schema materializes the extended shapes' triple -constraints on the same node, and a reference to a shape that has extensions may -materialize any non-abstract one of them -- the threads decide which fits the bindings. +"""Materialization by iteration scopes: build RDF conforming to an output schema from bindings. + +The binding tree is read as a **scope tree** (:mod:`pyshex.shexmap.scopes`): a scope's own +bindings, and one list of iteration scopes per repeated constraint or group of the input. +Nothing is flattened and nothing is consumed: + +* A constraint with a Map variable **reads** it from the scope it is evaluated at, or from + the nearest ancestor scope that binds it. Reading marks nothing, so a parent's binding + can be read in every item, and twice in one. A variable bound nowhere on that chain + fails the constraint. +* A **repetition** in the output schema iterates one list of the input: the deepest list + its body's variables are bound at (nested repetitions count through their parent list). + Its body runs once per iteration of that list, in list order, at that iteration's + scope; an iteration whose body fails is skipped; the count of successful iterations + must reach the minimum, and the repetition stops at the maximum. A body whose + variables are all bound at or above the current scope runs once, there. A body whose + variables live in two lists neither of which contains the other is ill-formed, and + says so. +* **Choices** (``OneOf``, ``ShapeOr``, an optional constraint, a shape with extensions) + yield alternatives, per scope; the alternatives of the parts of a group or conjunction + combine by product, pruned to the best ``max_accepts`` as they combine. +* Every complete alternative is an :class:`Accept`; :meth:`ThreadedMaterializer.materialize` + returns the one that read the most distinct bindings (ties: most triples, then schema + order), or whichever ``prefer`` ranks first, and keeps the rest in + :attr:`~ThreadedMaterializer.accepts`. +* A value a Map code produces must **satisfy the constraint's value expression** when + that is a node constraint (datatype, value set, facets, node kind). A plain literal + whose lexical form is valid for the constraint's datatype is retyped to it, so what + ``regex()`` and ``hashmap()`` produce can land in a typed constraint; any other mismatch + fails the constraint like an unbound variable, so an optional or another disjunct can + take over (``check_values=False`` turns this off). +* A shape-valued constraint that also carries a Map variable **names** its node: the + bound value is the node and the nested shape is materialized on it. One that carries + ``%Map:{ id(v:a, v:b) %}`` is **keyed**: its node is a function of the shape and the + values read for the key, so two constraints, or two iterations, with the same key make + the same node and their arcs merge -- grouping by value, and a shared reference. A + key of one IRI or blank node is that node itself; ``id(@node)`` reuses the input node + the enclosing iteration matched, so a schema maps a graph onto itself node for node. + Otherwise the node is a blank node determined by the root, the constraint, and the + scope (its ``@node`` when it is an iteration, else its path), so two runs over the same + bindings agree, and materializing the same root twice adds nothing. + +This replaces the frame-cursor search ported from shex.js's ``ThreadedMaterializer`` (kept +as the class name); ``docs/shexmap-semantics-draft.md`` is the normative description and +``docs/shexmap-structural-plan.md`` the reasons. ``EXTENDS`` in the output schema +materializes the extended shapes' constraints on the same node, and a reference to a +shape that has extensions may materialize any non-abstract one of them. """ from __future__ import annotations -import uuid +import hashlib +import itertools +import math from collections.abc import Callable, Mapping -from dataclasses import dataclass, field, replace +from dataclasses import dataclass from typing import Any from ShExJSG import ShExJ from rdflib import BNode, Graph, Literal, URIRef from rdflib.term import Node +from pyshex.parse_tree.parse_node import ParseNode +from pyshex.shape_expressions_language.p5_4_node_constraints import satisfiesNodeConstraint from pyshex.shape_expressions_language.p5_context import Context from pyshex.shapemap_structure_and_language.p3_shapemap_structure import START from pyshex.shexmap import functions from pyshex.shexmap.bindings import (Bindings, declaration_of, expression_of, extension_candidates, map_actions, - normalize, references_shape, shape_parts, shexc_prefixes, term_from_json) + references_shape, shape_parts, shexc_prefixes, term_from_json) +from pyshex.shexmap.scopes import BindingTreeError, ListPath, Scope, ScopeTree UNBOUNDED = -1 class MaterializationError(ValueError): - """No thread could build the requested shape from the bindings. + """No alternative materializes the requested shape from the bindings. :ivar failures: the dead ends met on the way, deepest last :ivar report: see :attr:`ThreadedMaterializer.last_report` @@ -68,141 +92,52 @@ def _describe(f: dict) -> str: return f"{f.get('predicate', '')} {what} {f.get('error', 'unbound')}".strip() -# -- the frame cursor ------------------------------------------------------------------- +# -- results ------------------------------------------------------------------------------ @dataclass(frozen=True) -class Cursor: - idx: int = 0 # current frame - used: frozenset = frozenset() # (frame, variable) pairs consumed - n: int = 0 # frame bindings consumed - skipped: int = 0 # unused bindings abandoned by advancing +class Result: + """One way to materialize something: its triples, where each came from, and the bindings + it read.""" + quads: tuple = () + reads: frozenset = frozenset() # (scope path, variable) + sources: tuple = () # parallel to quads: see Accept.provenance + def then(self, other: Result) -> Result: + return Result(self.quads + other.quads, self.reads | other.reads, self.sources + other.sources) -def cursor_get(frames: list[dict], statics: Mapping[str, Node], cursor: Cursor, var: str): - """(value, new cursor) for ``var``, or None when no unused binding remains.""" - if var in statics: # static vars: always there, never used up - return statics[var], cursor - for i in range(cursor.idx, len(frames)): - if var in frames[i] and (i, var) not in cursor.used: - used = cursor.used | {(i, var)} - skipped = cursor.skipped + sum(1 for j in range(cursor.idx, i) for v in frames[j] - if (j, v) not in used) - return frames[i][var], Cursor(i, used, cursor.n + 1, skipped) - return None +EMPTY = Result() -# -- NFAs -------------------------------------------------------------------------------- -@dataclass -class State: - type: str # TC | Split | Rept | Match - outs: list[int] = field(default_factory=list) - tc: Any = None - min: int = 1 - max: float = 1 - skippable: bool = False - - -@dataclass -class NFA: - states: list[State] - start: int - - -def _clone_into(combined: list[State], nfa: NFA) -> int: - offset = len(combined) - for s in nfa.states: - combined.append(replace(s, outs=[o + offset for o in s.outs])) - return offset - - -def _concat(parts: list[NFA]) -> NFA: - """Run each part in turn against the same subject.""" - if not parts: - return NFA([State('Match')], 0) - states: list[State] = [] - offsets = [_clone_into(states, p) for p in parts] - for i in range(len(parts) - 1): # each part's Match (state 0) hands over to the next - states[offsets[i]] = State('Split', outs=[offsets[i + 1] + parts[i + 1].start]) - return NFA(states, offsets[0] + parts[0].start) - - -def _split(parts: list[NFA]) -> NFA: - """Fork over the parts, in priority order.""" - if not parts: - return NFA([State('Match')], 0) - states: list[State] = [] - outs = [_clone_into(states, p) + p.start for p in parts] - states.append(State('Split', outs=outs)) - return NFA(states, len(states) - 1) +def _source(tc, scope: Scope, reads=(), statics=(), **kind) -> dict: + """The provenance of one triple: the constraint, the input scope it was evaluated at (its + path and the node it matched), the bindings and static variables read for it, and how + the object arose.""" + return {'tc': tc, 'predicate': str(tc.predicate), 'scope': scope.path, 'node': scope.nearest_node(), + 'reads': tuple(reads), 'statics': tuple(statics), **kind} -# -- threads ----------------------------------------------------------------------------- - -@dataclass(frozen=True) -class CallFrame: - nfa: NFA - outs: tuple - subject: Node - repeats: tuple - parent: CallFrame | None - skippable: bool - quads_mark: Any - consumed_mark: int - - -@dataclass(frozen=True) -class Thread: - nfa: NFA - state: int - subject: Node - repeats: tuple # sorted ((state, (count, consumed_at)), ...) - call_stack: CallFrame | None - cursor: Cursor - quads: Any # persistent list: (triple, previous) or None - bnode: int - - -def _repeats_get(repeats: tuple, state: int): - for k, v in repeats: - if k == state: - return v - return None - - -def _repeats_set(repeats: tuple, state: int, value) -> tuple: - kept = tuple((k, v) for k, v in repeats if k != state) - return tuple(sorted(kept + ((state, value),))) if value is not None else kept - - -def _collect(quads) -> list[tuple[Node, Node, Node]]: - out = [] - while quads is not None: - out.append(quads[0]) - quads = quads[1] - out.reverse() - seen, kept = set(), [] - for q in out: - if q not in seen: - seen.add(q) - kept.append(q) - return kept - - -def _depth(call_stack: CallFrame | None) -> int: - d = 0 - while call_stack is not None: - d, call_stack = d + 1, call_stack.parent - return d +def _rank(r: Result) -> tuple[int, int]: + return (len(r.reads), len(r.quads)) @dataclass class Accept: - """An accepting thread's result.""" + """A complete materialization. + + :ivar provenance: one record per triple: ``tc`` and ``predicate`` (the output constraint), + ``scope`` and ``node`` (the input iteration's path, and the node it matched), ``reads`` + (the ``(scope path, variable)`` bindings read for the object), ``statics`` (the static + variables read for it), and one of + ``variable``/``code`` (a Map code's value), ``constant`` (a value the schema names), + ``structural`` (a link to a minted node), ``named`` (a node a variable names) or + ``keyed`` (an ``id()`` node) -- the correspondence between output and input + """ triples: list[tuple[Node, Node, Node]] - consumed: int - skipped: int - used: frozenset + consumed: int # distinct bindings read + skipped: int # always 0: nothing is passed over any more + used: frozenset # the (scope path, variable) addresses read + provenance: list[dict] = None class ThreadedMaterializer: @@ -210,16 +145,16 @@ class ThreadedMaterializer: :param schema: output schema, as ShExC text or a parsed ShExJ schema :param prefixes: prefixes for ShExMap variable names; read from ShExC text when omitted - :param static_vars: variable values available everywhere, never used up + :param static_vars: variable values available everywhere :param prefer: comparator over :class:`Accept` s, negative when the first is better - :param max_repeat: cap on the iterations of one repetition; ``None`` (the default) - leaves ``*`` and ``+`` bounded only by the bindings, which the progress guard on - repetitions makes safe - :param max_call_depth: cap on nested shape calls (cyclic references) - :param max_steps: cap on thread steps for the whole search - :param max_accepts: how many of the best accepts to keep in :attr:`accepts` - :param explore_steps: how many steps to keep searching after the last improvement - :param require_bindings_in_subshapes: drop optional subshapes that consume no binding + :param max_repeat: cap on the iterations of one repetition; ``None`` for none + :param max_call_depth: cap on nested shape calls (recursive schemas) + :param max_accepts: how many alternatives to keep, at every level and at the end + :param require_bindings_in_subshapes: drop optional subshapes that read no binding + :param check_values: check bound values against node constraints, retyping plain + literals whose lexical form fits the datatype + :param max_steps, explore_steps: accepted for compatibility; the search has no budget + to spend any more """ def __init__(self, schema: str | ShExJ.Schema, prefixes: Mapping[str, str] | None = None, @@ -227,7 +162,7 @@ def __init__(self, schema: str | ShExJ.Schema, prefixes: Mapping[str, str] | Non prefer: Callable[[Accept, Accept], int] | None = None, max_repeat: int | None = None, max_call_depth: int = 50, max_steps: int = 1_000_000, max_accepts: int = 20, explore_steps: int = 10_000, - require_bindings_in_subshapes: bool = False) -> None: + require_bindings_in_subshapes: bool = False, check_values: bool = True) -> None: from pyshex.utils.schema_loader import SchemaLoader if isinstance(schema, str): prefixes = {**shexc_prefixes(schema), **(prefixes or {})} @@ -240,271 +175,142 @@ def __init__(self, schema: str | ShExJ.Schema, prefixes: Mapping[str, str] | Non self.statics = {k: (v if isinstance(v, Node) else term_from_json(v)) for k, v in (static_vars or {}).items()} self.prefer = prefer self.max_repeat, self.max_call_depth = max_repeat, max_call_depth - self.max_steps, self.max_accepts, self.explore_steps = max_steps, max_accepts, explore_steps + self.max_accepts = max_accepts self.require_bindings_in_subshapes = require_bindings_in_subshapes - self._cache: dict[int, NFA] = {} - self._compiling: list[tuple[int, str]] = [] + self.check_values = check_values self.accepts: list[Accept] = [] self.chosen: Accept | None = None + self.provenance: list[dict] | None = None self.last_report: dict = {} - self.frames: list[dict] = [] - - # -- compilation -------------------------------------------------------------------- - def _compile_shape_expr(self, se, label: str | None = None) -> NFA: - if isinstance(se, str): - decl = declaration_of(self.cntxt, se) - if decl is None: - raise MaterializationError(f"Shape {se} is not defined in the output schema") - return self._compile_decl(decl, str(se)) - return self._compile_expr(se, label or f"(inline {type(se).__name__})") - - def _compile_decl(self, decl, label: str) -> NFA: - """A reference: the declaration itself or any non-abstract extension of it.""" - key = id(decl) * 2 + 1 - if key in self._cache: - return self._cache[key] - options = extension_candidates(self.cntxt, decl) - if not options: - raise MaterializationError(f"Shape {label} is abstract and nothing extends it") - options.reverse() # the declaration itself first, then extensions - nfa = _split([self._compile_expr(expression_of(o), str(getattr(o, 'id', label))) for o in options]) \ - if len(options) > 1 else self._compile_expr(expression_of(options[0]), label) - self._cache[key] = nfa - return nfa - - def _compile_expr(self, se, label: str) -> NFA: - key = id(se) * 2 - if key in self._cache: - return self._cache[key] - if any(k == key for k, _ in self._compiling): - loop = [lbl for k, lbl in self._compiling[[k for k, _ in self._compiling].index(key):]] - raise MaterializationError("cycle in shape expressions: " + " -> ".join(loop + [label])) - self._compiling.append((key, label)) - try: - if isinstance(se, ShExJ.ShapeDecl): - nfa = self._compile_expr(se.shapeExpr, label) - elif isinstance(se, ShExJ.Shape): - nfa = _concat([self._nfa_for_expression(p.expression) for p in shape_parts(self.cntxt, se)]) - elif isinstance(se, (ShExJ.ShapeAnd, ShExJ.ShapeOr)): - parts = [self._compile_shape_expr(p) for p in se.shapeExprs - if not isinstance(p, ShExJ.NodeConstraint)] - nfa = _concat(parts) if isinstance(se, ShExJ.ShapeAnd) else _split(parts) - elif isinstance(se, ShExJ.NodeConstraint): - nfa = NFA([State('Match')], 0) - else: - raise MaterializationError(f"{type(se).__name__} synthesis is not supported") - finally: - self._compiling.pop() - self._cache[key] = nfa - return nfa - - def _nfa_for_expression(self, expression) -> NFA: - states: list[State] = [State('Match')] - - def mk(s: State) -> int: - states.append(s) - return len(states) - 1 - - def patch(tail: list[int], target: int) -> None: - for t in tail: - states[t].outs.append(target) - - def walk(expr) -> tuple[int, list[int]]: - if isinstance(expr, str): # an inclusion: &label - expr = self.cntxt.tripleExprFor(expr) - if isinstance(expr, ShExJ.TripleConstraint): - s = mk(State('TC', tc=expr)) - start, tail = s, [s] - elif isinstance(expr, ShExJ.OneOf): - starts, tails = [], [] - for nested in expr.expressions: - s, t = walk(nested) - starts.append(s) - tails.extend(t) - start, tail = mk(State('Split', outs=starts)), tails - elif isinstance(expr, ShExJ.EachOf): - start = tail = None - for nested in expr.expressions: - s, t = walk(nested) - if start is None: - start = s - else: - patch(tail, s) - tail = t - else: - raise MaterializationError(f"unexpected triple expression {type(expr).__name__}") - min_ = 1 if expr.min is None else expr.min - max_ = 1 if expr.max is None else (float('inf') if expr.max == UNBOUNDED else expr.max) - if isinstance(expr, ShExJ.TripleConstraint): - if min_ == 0 and max_ == 1 and self._always_synthesizable(expr): - return start, tail # skipping a constant gains nothing: emit it - if min_ == 0: - states[start].skippable = True - if min_ == 1 and max_ == 1: - return start, tail - rept = mk(State('Rept', outs=[start], min=min_, max=max_)) - patch(tail, rept) - return rept, [rept] - - start = 0 - if expression is not None: - s, tail = walk(expression) - patch(tail, 0) - start = s - return NFA(states, start) - - def _always_synthesizable(self, tc) -> bool: - acts = map_actions(tc) - if acts: - return all(not functions.is_function_call(str(a.code or '')) and - self._var(str(a.code or '')) in self.statics for a in acts) - return _single_value(tc.valueExpr) is not None - - def _var(self, code: str) -> str: - return functions.expand_variable(code, self.prefixes) + self.tree: ScopeTree | None = None + self._lists: dict[int, tuple] = {} # id(repeated expr) -> (list path or None,) + self._tc_index: dict[int, int] = {} + self._active: list[tuple[tuple, str]] = [] + self._failures: list[dict] = [] + self._referenced: set[str] = set() + self._dropped = 0 # -- running ------------------------------------------------------------------------ def materialize(self, bindings, root: str | Node | None = None, start=None) -> list[tuple[Node, Node, Node]]: """The triples of the best materialization of ``start`` (default: the schema's start) - rooted at ``root`` (default: a new blank node). The best ``max_accepts`` distinct - results are kept in :attr:`accepts`, the returned one in :attr:`chosen`. + rooted at ``root`` (default: a new blank node). The alternatives are kept in + :attr:`accepts`, best first; the returned one is :attr:`chosen`. - :raises MaterializationError: when no thread reaches an accepting state + :raises MaterializationError: when nothing materializes, or the bindings are not a + scope tree, or a repetition reads variables from unrelated lists """ tree = bindings.tree if isinstance(bindings, Bindings) else bindings - frames = normalize(tree) - self.frames = frames + try: + self.tree = ScopeTree(tree) + except BindingTreeError as e: + raise MaterializationError(str(e)) from e root = _node(root) if root is not None else BNode() if start is None or start is START: if self.schema.start is None: raise MaterializationError("The output schema has no start shape; name one with start=") - nfa = self._compile_shape_expr(self.schema.start, "START") + se = self.schema.start else: - nfa = self._compile_shape_expr(ShExJ.IRIREF(str(start))) - bnode_prefix = uuid.uuid4().hex[:8] + se = ShExJ.IRIREF(str(start)) + self._root = root + self._lists, self._tc_index, self._active = {}, {}, [] + self._failures, self._referenced, self._dropped = [], set(), 0 - failures: list[dict] = [] - referenced: set[str] = set() - available = set(self.statics) | {v for f in frames for v in f} + results = self._shape_expr(se, self.tree.root, root, 0, "START" if start is None else str(start)) + seen: set[frozenset] = set() accepts: list[Accept] = [] - self.accepts, self.chosen = accepts, None - by_used: dict[frozenset, Accept] = {} - graphs_seen: set[frozenset] = set() - total = sum(len(f) for f in frames) - found = 0 # distinct accepts, kept or not - best: Accept | None = None - - stack = [Thread(nfa, nfa.start, root, (), None, Cursor(), None, 0)] - deferred: list[Thread] = [] - seen: set = set() - steps = pruned = 0 - accepted_at = None # step of the last improvement of ``best`` - truncated = False - - while stack or deferred: - steps += 1 - if steps > self.max_steps: - if accepts: - truncated = True - break - raise MaterializationError(f"exceeded max_steps={self.max_steps}", failures, - self._report(failures, referenced, available, found, True, pruned)) - if accepted_at is not None and steps - accepted_at > self.explore_steps: - truncated = True - break - th = stack.pop() if stack else deferred.pop(0) - key = (th.state, id(th.nfa), id(th.call_stack), th.cursor.idx, th.cursor.used, th.repeats) + for r in sorted(results, key=_rank, reverse=True): # stable: schema order among ties + triples, provenance = _distinct(r.quads, r.sources) + key = frozenset(triples) if key in seen: - pruned += 1 continue seen.add(key) - st = th.nfa.states[th.state] - - if st.type == 'Match': - if th.call_stack is None: - triples = _collect(th.quads) - graph_key = frozenset(triples) - if graph_key in graphs_seen: - continue - graphs_seen.add(graph_key) - existing = by_used.get(th.cursor.used) - if existing is not None: - if len(triples) > len(existing.triples): - existing.triples, existing.skipped = triples, th.cursor.skipped - continue - accept = Accept(triples, th.cursor.n, th.cursor.skipped, th.cursor.used) - by_used[th.cursor.used] = accept - accepts.append(accept) - found += 1 - if best is None or self._better(accept, best): - best, accepted_at = accept, steps - if len(accepts) > self.max_accepts: - # keep the best max_accepts: a full list bounds memory and the - # alternatives offered, not the search -- the greedy leader's - # prefixes accept before it does, and would otherwise crowd it out - others = [a for a in accepts if a is not best] - worst = others[0] - for a in others[1:]: - if not self._better(a, worst): # ties: drop the latest - worst = a - accepts.remove(worst) - del by_used[worst.used] - if accept.consumed >= total: # perfect: nothing can beat it - break - continue - frame = th.call_stack - # vacuous descent: an optional subshape that emitted and consumed nothing - # would leave a dangling link; the skip arm already covers it - if frame.skippable and th.cursor.n == frame.consumed_mark and \ - (th.quads is frame.quads_mark or self.require_bindings_in_subshapes): - continue - for out in frame.outs: - stack.append(replace(th, nfa=frame.nfa, state=out, subject=frame.subject, - repeats=frame.repeats, call_stack=frame.parent)) - - elif st.type == 'Split': - for out in reversed(st.outs): # first disjunct explored first - stack.append(replace(th, state=out)) - - elif st.type == 'Rept': - count, at = _repeats_get(th.repeats, th.state) or (0, -1) - if count >= st.min: # exit (lower priority); reset for re-entry - stack.append(replace(th, state=st.outs[1], repeats=_repeats_set(th.repeats, th.state, None))) - # another iteration -- only if the last one consumed a frame binding, or - # constant-only subexpressions would repeat to max_repeat - limit = st.max if self.max_repeat is None else min(st.max, self.max_repeat) - if count < limit and (count == 0 or th.cursor.n > at): - stack.append(replace(th, state=st.outs[0], - repeats=_repeats_set(th.repeats, th.state, (count + 1, th.cursor.n)))) - - elif st.type == 'TC': - succs: list[Thread] = [] - self._step(th, st, frames, succs, failures, referenced, bnode_prefix) - if succs and succs[0].cursor.skipped > th.cursor.skipped: - # advancing past unused bindings is a choice: explore the alternatives - # that can still use them first. (An advance that abandons nothing - # has no alternative, and deferring it would only make the exit arms - # of enclosing repetitions accept, one per iteration, ahead of it.) - deferred.extend(succs) - else: - stack.extend(succs) - else: - raise MaterializationError(f"unexpected NFA state {st.type}") - - report = self._report(failures, referenced, available, found, truncated, pruned) - if best is None: - raise MaterializationError("no thread reached an accepting state", failures, report) + accepts.append(Accept(triples, len(r.reads), 0, r.reads, provenance)) + self.accepts = accepts + report = self._report(len(accepts)) + if not accepts: + raise MaterializationError("nothing materializes the shape from these bindings", self._failures, report) + best = accepts[0] + if self.prefer is not None: + for a in accepts[1:]: + if self.prefer(a, best) < 0: + best = a self.chosen = best + self.provenance = best.provenance return best.triples - def _better(self, a: Accept, b: Accept) -> bool: - if self.prefer is not None: - return self.prefer(a, b) < 0 - return (a.consumed, -a.skipped, len(a.triples)) > (b.consumed, -b.skipped, len(b.triples)) + def update(self, graph: Graph, bindings, root: str | Node | None, start=None) -> tuple[set, set]: + """Materialize into ``graph``, replacing what the output schema currently holds at + ``root``: the schema, used as an input schema on ``graph`` at ``root``, says which + triples it governs there; those no longer produced are removed, the new ones added, + and everything else is left alone. Returns ``(added, removed)``. - def _report(self, failures, referenced, available, found: int, truncated, pruned) -> dict: + Whether the graph holds an earlier materialization is decided by the predicates the + output schema's start shape constrains: a triple at ``root`` on one of them means + there is something to replace, and it must then conform. + + :raises MaterializationError: when ``graph`` holds something at ``root`` that does not + conform to the output schema (so what to replace cannot be told), or nothing + materializes + """ + from pyshex.shexmap.bindings import MapValidationError, bind_all + root = _node(root) if root is not None else BNode() + new = set(self.materialize(bindings, root, start=start)) + current: set = set() + if self._holds_something(graph, root, start): + try: + for b in bind_all(graph, self.schema, root, start=start, prefixes=self.prefixes): + current |= set(b.matched) + except MapValidationError as e: + raise MaterializationError(f"{root.n3()} holds something that does not conform to the output " + f"schema, so what to replace cannot be told: {e}") from e + removed = current - new + added = {t for t in new if t not in graph} + for t in removed: + graph.remove(t) + for t in new: + graph.add(t) + return added, removed + + def _holds_something(self, graph: Graph, root: Node, start) -> bool: + """Whether ``graph`` has, at ``root``, a triple on a predicate the output schema's start + shape constrains: the mark of an earlier materialization (or of a conflict).""" + se = self.schema.start if start is None or start is START else ShExJ.IRIREF(str(start)) + for pred, inverse in self._own_predicates(se, set()): + if any(graph.triples((None, pred, root) if inverse else (root, pred, None))): + return True + return False + + def _own_predicates(self, se, seen: set) -> set: + """(predicate, inverse) of the constraints on the shape's own node, extensions included.""" + if isinstance(se, str): + decl = declaration_of(self.cntxt, se) + return set().union(*(self._own_predicates(expression_of(o), seen) + for o in (extension_candidates(self.cntxt, decl) if decl is not None else []))) + if se is None or id(se) in seen: + return set() + seen.add(id(se)) + if isinstance(se, ShExJ.ShapeDecl): + return self._own_predicates(se.shapeExpr, seen) + if isinstance(se, ShExJ.Shape): + out: set = set() + stack = [p.expression for p in shape_parts(self.cntxt, se) if p.expression is not None] + while stack: + e = stack.pop() + if isinstance(e, str): + e = self.cntxt.tripleExprFor(e) + if isinstance(e, ShExJ.TripleConstraint): + out.add((URIRef(str(e.predicate)), bool(e.inverse))) + elif isinstance(e, (ShExJ.EachOf, ShExJ.OneOf)): + stack.extend(e.expressions) + return out + if isinstance(se, (ShExJ.ShapeAnd, ShExJ.ShapeOr)): + return set().union(*(self._own_predicates(p, seen) for p in se.shapeExprs)) + return set() + + def _report(self, found: int) -> dict: + available = set(self.statics) | (self.tree.variables() if self.tree else set()) seen, unbound = set(), [] - for f in failures: + for f in self._failures: v = f.get('variable') k = (v, f.get('predicate')) if v and v not in available and k not in seen: @@ -512,95 +318,479 @@ def _report(self, failures, referenced, available, found: int, truncated, pruned unbound.append(f) self.last_report = { 'unbound_variables': unbound, - 'unused_statics': [s for s in self.statics if s not in referenced], + 'unused_statics': [s for s in self.statics if s not in self._referenced], 'alternatives': found, - 'exploration_truncated': truncated, - 'configs_pruned': pruned, + 'exploration_truncated': self._dropped > 0, + 'configs_pruned': self._dropped, } return self.last_report - def _step(self, th: Thread, st: State, frames, succs, failures, referenced, bnode_prefix) -> None: - """Emit one instance of a triple constraint; successors go to ``succs``.""" - tc = st.tc + # -- shape expressions -------------------------------------------------------------- + def _shape_expr(self, se, scope: Scope, subject: Node, depth: int, label: str | None = None) -> list[Result]: + if isinstance(se, str): + decl = declaration_of(self.cntxt, se) + if decl is None: + raise MaterializationError(f"Shape {se} is not defined in the output schema") + options = extension_candidates(self.cntxt, decl) + if not options: + raise MaterializationError(f"Shape {se} is abstract and nothing extends it") + options.reverse() # the declaration itself first, then its extensions + return self._guarded(('ref', id(decl), scope.path, subject), str(se), lambda: self._prune( + [r for o in options for r in self._shape_expr(expression_of(o), scope, subject, depth, str(se))])) + label = label or f"(inline {type(se).__name__})" + if isinstance(se, ShExJ.ShapeDecl): + return self._shape_expr(se.shapeExpr, scope, subject, depth, label) + if isinstance(se, ShExJ.Shape): + parts = [p.expression for p in shape_parts(self.cntxt, se)] + return self._guarded(('shape', id(se), scope.path, subject), label, lambda: self._all( + [self._expression(e, scope, subject, depth) if e is not None else [EMPTY] for e in parts])) + if isinstance(se, ShExJ.ShapeAnd): + return self._guarded(('and', id(se), scope.path, subject), label, lambda: self._all( + [self._shape_expr(p, scope, subject, depth) for p in se.shapeExprs + if not isinstance(p, ShExJ.NodeConstraint)])) + if isinstance(se, ShExJ.ShapeOr): + return self._guarded(('or', id(se), scope.path, subject), label, lambda: self._prune( + [r for p in se.shapeExprs if not isinstance(p, ShExJ.NodeConstraint) + for r in self._shape_expr(p, scope, subject, depth)])) + if isinstance(se, ShExJ.NodeConstraint): + return [EMPTY] + raise MaterializationError(f"{type(se).__name__} synthesis is not supported") + + def _guarded(self, key: tuple, label: str, run: Callable[[], list[Result]]) -> list[Result]: + """Evaluate, refusing a shape expression that reaches itself at the same scope and + subject through references alone (`` @ OR ...``, `` @ OR ...``).""" + for i, (k, _) in enumerate(self._active): + if k == key: + raise MaterializationError("cycle in shape expressions: " + + " -> ".join(lbl for _, lbl in self._active[i:]) + " -> " + label) + self._active.append((key, label)) + try: + return run() + finally: + self._active.pop() + + # -- triple expressions ------------------------------------------------------------- + def _expression(self, expr, scope: Scope, subject: Node, depth: int) -> list[Result]: + if isinstance(expr, str): # an inclusion: &label + expr = self.cntxt.tripleExprFor(expr) + min_ = 1 if expr.min is None else expr.min + max_ = 1 if expr.max is None else (math.inf if expr.max == UNBOUNDED else expr.max) + if min_ == 1 and max_ == 1: + return self._once(expr, scope, subject, depth, False) + return self._repetition(expr, min_, max_, scope, subject, depth) + + def _once(self, expr, scope: Scope, subject: Node, depth: int, skippable: bool) -> list[Result]: + """The alternatives for one occurrence of ``expr``, cardinality aside.""" + if isinstance(expr, ShExJ.TripleConstraint): + return self._tc(expr, scope, subject, depth, skippable) + if isinstance(expr, ShExJ.EachOf): + return self._all([self._expression(e, scope, subject, depth) for e in expr.expressions]) + if isinstance(expr, ShExJ.OneOf): + return self._prune([r for e in expr.expressions for r in self._expression(e, scope, subject, depth)]) + raise MaterializationError(f"unexpected triple expression {type(expr).__name__}") + + def _repetition(self, expr, min_: int, max_: float, scope: Scope, subject: Node, depth: int) -> list[Result]: + list_path = self._list_path(expr) + skippable = min_ == 0 + if list_path is None or len(list_path) <= scope.depth: + # the body reads nothing below this scope: it runs once, here + if min_ > 1: + return [] + body = self._once(expr, scope, subject, depth, skippable) + return body if body else ([EMPTY] if min_ == 0 else []) + cap = max_ if self.max_repeat is None else min(max_, self.max_repeat) + results, count = [EMPTY], 0 + for item in scope.descendants_at(list_path): + if count >= cap: + break + body = self._once(expr, item, subject, depth, skippable) + if not body: + continue # this item does not fit the body: skip it + results = self._all([results, body]) + count += 1 + return results if count >= min_ else [] + + def _tc(self, tc: ShExJ.TripleConstraint, scope: Scope, subject: Node, depth: int, skippable: bool) -> list[Result]: pred = URIRef(str(tc.predicate)) def triple(obj: Node): if tc.inverse: if isinstance(obj, Literal): - return None # a literal cannot be a subject - return (obj, pred, th.subject) - return (th.subject, pred, obj) + self._failures.append({'predicate': str(pred), 'tc': tc, 'error': 'literal subject of inverse'}) + return None + return (obj, pred, subject) + return (subject, pred, obj) acts = map_actions(tc) + if any(functions.is_key_code(str(a.code or '')) for a in acts): + return self._keyed(tc, acts, scope, subject, depth, skippable, triple) if acts: - cursor = th.cursor - quads = th.quads + reads: set = set() + statics_read: list = [] + + def get(v: str): + self._referenced.add(v) + if v in self.statics: + statics_read.append(v) + return self.statics[v] + hit = scope.lookup(v) + if hit is None: + return None + value, at = hit + reads.add((at.path, v)) + return value + + quads = [] + objects = [] + sources = [] for act in acts: code = str(act.code or '') - if functions.is_function_call(code): - def get(v: str): - nonlocal cursor - referenced.add(v) - hit = cursor_get(frames, self.statics, cursor, v) - if hit is None: - return None - value, cursor = hit - return value - try: + before = set(reads) + del statics_read[:] + try: + if functions.is_function_call(code): value = functions.lower(code, get, self.prefixes) - except functions.MapFunctionError as e: - failures.append({'predicate': str(pred), 'tc': tc, 'code': code, 'error': str(e)}) - return + if value is None: + self._failures.append({'predicate': str(pred), 'tc': tc, 'code': code, 'error': 'unbound'}) + return [] + how = {'code': code.strip()} + else: + var = functions.expand_variable(code, self.prefixes) + value = get(var) + if value is None: + self._failures.append({'predicate': str(pred), 'tc': tc, 'variable': var, 'scope': scope.path}) + return [] + how = {'variable': var} + except functions.MapFunctionError as e: + self._failures.append({'predicate': str(pred), 'tc': tc, 'code': code, 'error': str(e)}) + return [] + if self.check_values: + value = self._checked(tc, value) if value is None: - failures.append({'predicate': str(pred), 'tc': tc, 'code': code, 'error': 'unbound'}) - return - else: - try: - var = self._var(code) - except functions.MapFunctionError as e: - failures.append({'predicate': str(pred), 'tc': tc, 'code': code, 'error': str(e)}) - return - referenced.add(var) - hit = cursor_get(frames, self.statics, cursor, var) - if hit is None: - failures.append({'predicate': str(pred), 'tc': tc, 'variable': var, 'frame': cursor.idx}) - return - value, cursor = hit + self._failures.append({'predicate': str(pred), 'tc': tc, 'code': code, + 'error': 'the value does not satisfy the value expression'}) + return [] t = triple(value) if t is None: - failures.append({'predicate': str(pred), 'tc': tc, 'error': 'literal subject of inverse'}) - return - quads = (t, quads) - for out in st.outs: - succs.append(replace(th, state=out, cursor=cursor, quads=quads)) - return + return [] + quads.append(t) + objects.append(value) + sources.append(_source(tc, scope, sorted(reads - before), list(statics_read), **how)) + if references_shape(tc.valueExpr) and len(objects) == 1 and not isinstance(objects[0], Literal): + # the variable names the node: materialize the nested shape on it + if depth >= self.max_call_depth: + self._failures.append({'predicate': str(pred), 'tc': tc, 'error': 'exceeded max_call_depth'}) + return [] + sources[0]['named'] = True + result = Result(tuple(quads), frozenset(reads), tuple(sources)) + return [result.then(n) for n in self._shape_expr(tc.valueExpr, scope, objects[0], depth + 1)] + return [Result(tuple(quads), frozenset(reads), tuple(sources))] constant = _single_value(tc.valueExpr) if constant is not None: t = triple(constant) - if t is None: - failures.append({'predicate': str(pred), 'tc': tc, 'error': 'literal subject of inverse'}) - return - for out in st.outs: - succs.append(replace(th, state=out, quads=(t, th.quads))) - return + return [Result((t,), frozenset(), (_source(tc, scope, constant=True),))] if t is not None else [] if references_shape(tc.valueExpr): - if _depth(th.call_stack) >= self.max_call_depth: - failures.append({'predicate': str(pred), 'tc': tc, 'error': 'exceeded max_call_depth'}) + if depth >= self.max_call_depth: + self._failures.append({'predicate': str(pred), 'tc': tc, 'error': 'exceeded max_call_depth'}) + return [] + node = self._mint(tc, scope, depth) + link = triple(node) + if link is None: + return [] + out = [] + for n in self._shape_expr(tc.valueExpr, scope, node, depth + 1): + if skippable and not n.reads and (not n.quads or self.require_bindings_in_subshapes): + continue # an optional island nothing asked for + out.append(Result((link,) + n.quads, n.reads, (_source(tc, scope, structural=True),) + n.sources)) + return out + + self._failures.append({'predicate': str(pred), 'tc': tc, + 'error': f"cannot synthesize {type(tc.valueExpr).__name__ if tc.valueExpr else 'any value'}" + " without a Map action"}) + return [] + + def _keyed(self, tc, acts, scope: Scope, subject: Node, depth: int, skippable: bool, triple) -> list[Result]: + """A shape-valued constraint with ``id(...)``: the node is a function of the key.""" + pred = str(tc.predicate) + if len(acts) > 1: + raise MaterializationError(f"id() must be the only Map code on the constraint at {pred}") + if not references_shape(tc.valueExpr): + raise MaterializationError(f"id() at {pred} needs a shape-valued constraint: it names a node, not a value") + code = str(acts[0].code or '') + try: + terms = functions.key_terms(code, self.prefixes) + except functions.MapFunctionError as e: + raise MaterializationError(str(e)) from e + reads: set = set() + statics_read: list = [] + missing: list = [] + + def get(a: str): + self._referenced.add(a) + if a in self.statics: + statics_read.append(a) + return self.statics[a] + hit = scope.lookup(a) + if hit is None: + missing.append(a) + return None + value, at = hit + reads.add((at.path, a)) + return value + + values = [] + for term in terms: + if term == functions.NODE_ARGUMENT: + value = scope.nearest_node() + if value is None: + self._failures.append({'predicate': pred, 'tc': tc, 'code': code, 'error': 'no @node in scope'}) + return [] + elif isinstance(term, functions.Template): + value = term.expand(get) + else: + value = get(term) + if value is None: + self._failures.append({'predicate': pred, 'tc': tc, 'variable': missing[-1] if missing else code, + 'scope': scope.path}) + return [] + values.append(value) + if len(values) == 1 and not isinstance(values[0], Literal): + node = values[0] + else: + shape = str(tc.valueExpr) if isinstance(tc.valueExpr, str) else f"inline{self._tc_index.setdefault(id(tc), len(self._tc_index))}" + node = BNode("k" + _digest(shape, *(v.n3() for v in values))) + if depth >= self.max_call_depth: + self._failures.append({'predicate': pred, 'tc': tc, 'error': 'exceeded max_call_depth'}) + return [] + link = triple(node) + if link is None: + return [] + out = [] + for n in self._shape_expr(tc.valueExpr, scope, node, depth + 1): + if skippable and not (n.reads | reads) and (not n.quads or self.require_bindings_in_subshapes): + continue + out.append(Result((link,) + n.quads, frozenset(reads) | n.reads, + (_source(tc, scope, sorted(reads), statics_read, keyed=code.strip()),) + n.sources)) + return out + + # -- value expressions -------------------------------------------------------------- + def _checked(self, tc, value: Node) -> Node | None: + """``value`` if it satisfies the constraint's value expression, a retyped copy of a + plain literal whose lexical form fits the expression's datatype, or None.""" + se = self._resolve(tc.valueExpr) + if se is None or not _is_node_constraint(se): + return value # nothing to check: a shape, or no expression at all + if self._satisfies(se, value): + return value + dt = self._datatype_of(se) + if dt and isinstance(value, Literal) and value.datatype is None and value.language is None \ + and _valid_lexical(str(value), str(dt)) is not False: + retyped = Literal(str(value), datatype=URIRef(str(dt))) + if self._satisfies(se, retyped): + return retyped + return None + + def _datatype_of(self, se): + """The datatype a value expression asks for: its own, or the one conjunct that has one.""" + se = self._resolve(se) + if isinstance(se, ShExJ.NodeConstraint): + return se.datatype + if isinstance(se, ShExJ.ShapeAnd): + found = [dt for dt in (self._datatype_of(p) for p in se.shapeExprs) if dt] + return found[0] if len(set(map(str, found))) == 1 else None + return None + + def _resolve(self, se): + for _ in range(100): + if isinstance(se, str): + decl = declaration_of(self.cntxt, se) + if decl is None: + return None + se = expression_of(decl) + elif isinstance(se, ShExJ.ShapeDecl): + se = se.shapeExpr + else: + return se + return None + + def _satisfies(self, se, value: Node) -> bool: + se = self._resolve(se) + if isinstance(se, ShExJ.NodeConstraint): + saved = self.cntxt.current_node + self.cntxt.current_node = ParseNode(satisfiesNodeConstraint, se, value, self.cntxt) + try: + return bool(satisfiesNodeConstraint(self.cntxt, value, se)) + finally: + self.cntxt.current_node = saved + if isinstance(se, ShExJ.ShapeAnd): + return all(self._satisfies(p, value) for p in se.shapeExprs if _is_node_constraint(self._resolve(p))) + if isinstance(se, ShExJ.ShapeOr): + return any(self._satisfies(p, value) for p in se.shapeExprs) + if isinstance(se, ShExJ.ShapeNot): + return not self._satisfies(se.shapeExpr, value) + return True # a shape: the nested materialization is the check + + def _mint(self, tc, scope: Scope, depth: int) -> BNode: + """The node for a shape-valued constraint without a key: determined by the root, the + constraint, its call depth, and the scope -- the input node the scope matched when it + is an iteration, else its path -- so a materialization is reproducible and two runs + over the same input agree.""" + n = self._tc_index.setdefault(id(tc), len(self._tc_index)) + where = scope.node.n3() if scope.node is not None else "p" + "_".join(map(str, scope.path)) + return BNode(f"m{_digest(self._root.n3(), str(n), str(depth), where)}") + + # -- combining alternatives --------------------------------------------------------- + def _all(self, parts: list[list[Result]]) -> list[Result]: + """Every combination of one alternative per part, pruned as it grows.""" + acc = [EMPTY] + for alternatives in parts: + if not alternatives: + return [] + acc = self._prune([a.then(b) for a in acc for b in alternatives]) + return acc + + def _prune(self, results: list[Result]) -> list[Result]: + if len(results) <= self.max_accepts: + return results + ranked = sorted(range(len(results)), key=lambda i: _rank(results[i]), reverse=True)[:self.max_accepts] + self._dropped += len(results) - len(ranked) + return [results[i] for i in sorted(ranked)] + + # -- which list a repetition iterates ----------------------------------------------- + def _list_path(self, expr) -> ListPath | None: + key = id(expr) + if key not in self._lists: + self._lists[key] = (self._analyse(expr),) + return self._lists[key][0] + + def _analyse(self, expr) -> ListPath | None: + """The list the repetition ``expr`` iterates: the deepest list its body's variables are + bound at, nested repetitions counting through their parent list.""" + direct: set[str] = set() + nested: list = [] + self._collect(expr, direct, nested, set(), top=True) + bound = self.tree.bound_at + candidates: dict[ListPath, str] = {} + for v in direct: + if v in bound: + candidates.setdefault(bound[v], v) + for r in nested: + lp = self._list_path(r) + if lp: + candidates.setdefault(lp[:-1], f"the repetition at {_predicates(r)}") + if not candidates: + return None + deepest = max(candidates, key=len) + for lp, why in candidates.items(): + if deepest[:len(lp)] != lp: + raise MaterializationError( + f"the repetition at {_predicates(expr)} reads from unrelated lists: " + f"{why} is bound at {lp} and {candidates[deepest]} at {deepest}") + return deepest + + def _collect(self, expr, direct: set[str], nested: list, seen: set, top: bool) -> None: + """Variables read directly in ``expr``'s body and the repetitions directly inside it.""" + if isinstance(expr, str): + expr = self.cntxt.tripleExprFor(expr) + if expr is None or id(expr) in seen: + return + seen.add(id(expr)) + if not top and not (expr.min in (None, 1) and expr.max in (None, 1)): + nested.append(expr) + return + if isinstance(expr, ShExJ.TripleConstraint): + for act in map_actions(expr): + code = str(act.code or '') + try: + if functions.is_function_call(code): + direct.update(functions.variables_of(code, self.prefixes)) + else: + direct.add(functions.expand_variable(code, self.prefixes)) + except functions.MapFunctionError: + pass + if references_shape(expr.valueExpr): + self._collect_shape(expr.valueExpr, direct, nested, seen) + elif isinstance(expr, (ShExJ.EachOf, ShExJ.OneOf)): + for e in expr.expressions: + self._collect(e, direct, nested, seen, top=False) + + def _collect_shape(self, se, direct: set[str], nested: list, seen: set) -> None: + if isinstance(se, str): + decl = declaration_of(self.cntxt, se) + if decl is None: return - sub = self._compile_shape_expr(tc.valueExpr) - bnode = BNode(f"{bnode_prefix}t{th.bnode}") - quads = (triple(bnode), th.quads) - succs.append(Thread(sub, sub.start, bnode, (), - CallFrame(th.nfa, tuple(st.outs), th.subject, th.repeats, th.call_stack, - st.skippable, quads, th.cursor.n), - th.cursor, quads, th.bnode + 1)) + for o in extension_candidates(self.cntxt, decl): + self._collect_shape(expression_of(o), direct, nested, seen) return - - failures.append({'predicate': str(pred), 'tc': tc, - 'error': f"cannot synthesize {type(tc.valueExpr).__name__ if tc.valueExpr else 'any value'}" - " without a Map action"}) + if id(se) in seen: + return + seen.add(id(se)) + if isinstance(se, ShExJ.ShapeDecl): + self._collect_shape(se.shapeExpr, direct, nested, seen) + elif isinstance(se, ShExJ.Shape): + for p in shape_parts(self.cntxt, se): + if p.expression is not None: + self._collect(p.expression, direct, nested, seen, top=False) + elif isinstance(se, (ShExJ.ShapeAnd, ShExJ.ShapeOr)): + for p in se.shapeExprs: + self._collect_shape(p, direct, nested, seen) + + +def _is_node_constraint(se) -> bool: + """A value expression that constrains the node itself, not its arcs.""" + if isinstance(se, ShExJ.NodeConstraint): + return True + if isinstance(se, (ShExJ.ShapeAnd, ShExJ.ShapeOr)): + return any(_is_node_constraint(p) for p in se.shapeExprs if not isinstance(p, str)) + if isinstance(se, ShExJ.ShapeNot): + return _is_node_constraint(se.shapeExpr) + return False + + +def _valid_lexical(lexical: str, datatype: str) -> bool | None: + """Whether rdflib can read ``lexical`` as ``datatype``; None for a datatype it does not know.""" + from rdflib.term import XSDToPython + dt = URIRef(datatype) + if dt not in XSDToPython: + return None + converter = XSDToPython[dt] + if converter is None: + return True + try: + converter(lexical) + except (ValueError, TypeError, ArithmeticError): + return False + return True + + +def _digest(*parts: str) -> str: + return hashlib.sha1("\x00".join(parts).encode("utf-8")).hexdigest()[:16] + + +def _predicates(expr) -> str: + """A short name for a triple expression: its first predicate(s).""" + found = [] + stack = [expr] + while stack and len(found) < 2: + e = stack.pop() + if isinstance(e, ShExJ.TripleConstraint): + found.append(str(e.predicate)) + elif isinstance(e, (ShExJ.EachOf, ShExJ.OneOf)): + stack.extend(reversed(list(e.expressions))) + return ", ".join(found) or "(no predicate)" + + +def _distinct(quads, sources) -> tuple[list, list]: + """The quads without repeats, and each one's first source.""" + seen, out, prov = set(), [], [] + for q, s in zip(quads, sources): + if q not in seen: + seen.add(q) + out.append(q) + prov.append(s) + return out, prov def _single_value(value_expr) -> Node | None: @@ -622,7 +812,7 @@ def _node(term) -> Node: def materialize(schema: str | ShExJ.Schema, bindings, root: str | Node | None = None, start=None, static_vars: Mapping[str, Node] | None = None, prefixes: Mapping[str, str] | None = None, - graph: Graph | None = None, **options) -> Graph: + graph: Graph | None = None, replace: bool = False, **options) -> Graph: """Materialize ``root`` as an instance of the output schema from ``bindings``. A convenience over :class:`ThreadedMaterializer` (which also exposes the alternative @@ -635,13 +825,17 @@ def materialize(schema: str | ShExJ.Schema, bindings, root: str | Node | None = :param static_vars: extra variable values, shared everywhere (shex.js ``staticVars``) :param prefixes: prefixes for ShExMap variable names; read from ShExC text when omitted :param graph: graph to add to; a new one by default + :param replace: with ``graph``, replace what the output schema currently holds at + ``root`` instead of adding to it (see :meth:`ThreadedMaterializer.update`) :raises MaterializationError: when the shape cannot be built """ m = ThreadedMaterializer(schema, prefixes=prefixes, static_vars=static_vars, **options) - triples = m.materialize(bindings, root, start=start) graph = graph if graph is not None else Graph() for prefix, ns in m.prefixes.items(): graph.bind(prefix, ns, override=False) - for t in triples: + if replace: + m.update(graph, bindings, root, start=start) + return graph + for t in m.materialize(bindings, root, start=start): graph.add(t) return graph diff --git a/pyshex/shexmap/scopes.py b/pyshex/shexmap/scopes.py new file mode 100644 index 0000000..7e2ec82 --- /dev/null +++ b/pyshex/shexmap/scopes.py @@ -0,0 +1,148 @@ +"""The scope tree: a binding tree read structurally, for materialization by iteration scopes. + +A binding tree (see :mod:`pyshex.shexmap.bindings`) is a *scope*: an object of own +bindings, or an array whose first element is that object and whose other elements are +*lists*, each list holding one *iteration* (a scope) per match of a repeated constraint or +group. :func:`parse_scope` builds :class:`Scope` objects from a tree, accepting the two +older layouts shex.js and PyShEx wrote (``L1``, ``L2`` in the design notes), and +:class:`ScopeTree` answers the questions materialization asks: at what list a variable is +bound, and which scopes under a given one belong to a given list. + +A **list path** names a list by position: the tuple of list indices from the root, so +``()`` is the root scope itself, ``(0,)`` the root's first list, ``(0, 1)`` the second list +of an iteration of that list. Every iteration of a list shares the list's path; a variable +bound in those iterations is *bound at* that list path. +""" +from __future__ import annotations + +from dataclasses import dataclass, field + +from rdflib.term import Node + +from pyshex.shexmap.bindings import NODE_KEY + +ListPath = tuple[int, ...] + + +class BindingTreeError(ValueError): + """The binding tree is not a scope tree (nor one of the older layouts).""" + + +@dataclass +class Scope: + own: dict[str, Node] # variables bound here (never @node) + node: Node | None = None # the node this iteration matched, if recorded + lists: list[list[Scope]] = field(default_factory=list) + parent: Scope | None = None + list_path: ListPath = () # the list this scope is an iteration of + path: tuple[int, ...] = () # (list index, iteration index, ...) from the root + + @property + def depth(self) -> int: + return len(self.list_path) + + def lookup(self, var: str): + """(value, scope) for ``var`` from here or the nearest ancestor binding it, else None.""" + s: Scope | None = self + while s is not None: + if var in s.own: + return s.own[var], s + s = s.parent + return None + + def nearest_node(self) -> Node | None: + """The node of this scope or of the nearest ancestor that recorded one.""" + s: Scope | None = self + while s is not None: + if s.node is not None: + return s.node + s = s.parent + return None + + def descendants_at(self, list_path: ListPath) -> list[Scope]: + """The scopes below this one that are iterations of ``list_path``, in document order.""" + if not _prefix(self.list_path, list_path) or list_path == self.list_path: + return [] + out: list[Scope] = [] + for lst in self.lists: + for it in lst: + if it.list_path == list_path: + out.append(it) + elif _prefix(it.list_path, list_path): + out.extend(it.descendants_at(list_path)) + return out + + def walk(self): + yield self + for lst in self.lists: + for it in lst: + yield from it.walk() + + +def _prefix(a: tuple, b: tuple) -> bool: + return b[:len(a)] == a + + +def _is_scope_array(a) -> bool: + return isinstance(a, list) and len(a) >= 1 and isinstance(a[0], dict) and all(isinstance(e, list) for e in a[1:]) + + +def parse_scope(tree) -> Scope: + """The scope tree of a binding tree (rdflib terms or JSON terms as values are both fine).""" + if isinstance(tree, dict) or _is_scope_array(tree): + return _scope(tree, None, (), ()) + if isinstance(tree, list): + # L1: a root without own bindings, written without its object + if all(isinstance(e, dict) for e in tree): # shex.js: the root's one list + return _scope([{}, tree], None, (), ()) + if all(isinstance(e, list) for e in tree): + if all(_is_scope_array(e) for e in tree): # shex.js: one list of scope iterations + return _scope([{}, tree], None, (), ()) + if all(all(isinstance(x, dict) for x in e) for e in tree): # PyShEx before 2026-09-27: the lists + return _scope([{}, *tree], None, (), ()) + raise BindingTreeError(f"not a scope: {_short(tree)}") + + +def _scope(node, parent: Scope | None, list_path: ListPath, path: tuple[int, ...]) -> Scope: + own = node if isinstance(node, dict) else node[0] + lists = [] if isinstance(node, dict) else node[1:] + scope = Scope({k: v for k, v in own.items() if not k.startswith("@")}, own.get(NODE_KEY), [], parent, list_path, path) + for i, lst in enumerate(lists): + if not isinstance(lst, list): + raise BindingTreeError(f"a scope's lists must be arrays: {_short(lst)}") + scope.lists.append([_iteration(e, scope, list_path + (i,), path + (i, j)) for j, e in enumerate(lst)]) + return scope + + +def _iteration(elt, parent: Scope, list_path: ListPath, path: tuple[int, ...]) -> Scope: + if isinstance(elt, dict): + return _scope(elt, parent, list_path, path) + if _is_scope_array(elt): # a scope array; also L2 (shex.js writes a nested scope as a sibling) + return _scope(elt, parent, list_path, path) + if isinstance(elt, list) and all(isinstance(x, dict) for x in elt): + # shex.js, a nested scope with no own bindings whose one list was unwrapped + return _scope([{}, elt], parent, list_path, path) + raise BindingTreeError(f"not an iteration: {_short(elt)}") + + +def _short(x) -> str: + s = repr(x) + return s if len(s) < 120 else s[:117] + "..." + + +class ScopeTree: + """A parsed tree plus the two indexes materialization needs.""" + + def __init__(self, tree) -> None: + self.root = parse_scope(tree) + self.bound_at: dict[str, ListPath] = {} + self.bindings = 0 + for s in self.root.walk(): + for v in s.own: + self.bindings += 1 + at = self.bound_at.setdefault(v, s.list_path) + if at != s.list_path: + raise BindingTreeError(f"variable {v} is bound at two lists, {at} and {s.list_path}") + + def variables(self) -> set[str]: + return set(self.bound_at) diff --git a/tests/test_shexmap/README.md b/tests/test_shexmap/README.md index f2ecb74..39d7095 100644 --- a/tests/test_shexmap/README.md +++ b/tests/test_shexmap/README.md @@ -2,10 +2,42 @@ `examples/` is copied from shex.js's `@shexjs/extension-map` package (https://github.com/shexjs/shex.js/tree/main/packages/extension-map/examples, -commit 58406c3bdd4fa79cc9b0f664ef7a689db040c07b, MIT licence, by Eric Prud'hommeaux). +commit 4d53c6e1 on branch `shexmap-scopes`, MIT licence, by Eric Prud'hommeaux); the +`*-bindings.json` files are in the scope layout with `@node` that both implementations +write. Each `manifest.json` entry pairs an input schema and data with the bindings and output graph shex.js produces; `test_examples.py` checks PyShEx against them. `test_cardinality.py` covers cardinality above one: nothing caps how many values a -`*`/`+` binds or materializes, and three strict `xfail` tests pin the frame model's -known limits (nested regrouping, transposition, unchecked output value expressions). +`*`/`+` binds or materializes; a nested schema maps to itself, sibling lists transpose, +a parent's binding can be read in every item, and a body reading from unrelated lists +is refused. + +`test_values.py` covers the check of bound values against the output constraint's value +expression: retyping of plain literals, failure as an unbound variable, and conformance of +the output to its schema. + +`test_examples.py` also checks the round-trip law on every example: materializing the +bindings with the *input* schema gives a graph that binds to the same bindings. + +`test_shexjs_differential.py` runs every example through shex.js's `@shexjs/extension-map` +as well (`shexjs_driver.js`) and compares frames and graphs, when `SHEXJS_EXTENSION_MAP` +points at the package directory of a built shex.js checkout; skipped otherwise. + +`test_keys.py` covers node identity: `%Map:{ id(v:a, v:b) %}` on a shape-valued constraint +keys its node, so equal keys merge (grouping by value, shared references); `id(@node)` +reuses the input node, so a schema maps a graph onto itself; unkeyed nodes are +reproducible and specific to the root. + +`test_analysis.py` covers the static checks (`analyse`, `shexmap --check`): every example +pair and every schema against itself are coherent; unrelated lists, "which one?" reads, +unbound and unused variables, double binding sites and `id()` misuse are reported. + +`test_roundtrip.py` covers the round-trip laws of a schema pair (a schema binds back what +it wrote from; BP <-> BP back and inverse in <-> inverse out recover their bindings, the BP +pair its graph) and the provenance each output triple carries. + +`test_update.py` covers the triples a schema matched (`Bindings.matched`), the strong form +of the identity law, and updating a graph in place (`update`, `materialize(replace=True)`, +`shexmap --into`); `test_keys.py` also covers IRI templates in `id()`. + diff --git a/tests/test_shexmap/examples/BP-simple-bindings.json b/tests/test_shexmap/examples/BP-simple-bindings.json index 3020d36..98d5ce5 100644 --- a/tests/test_shexmap/examples/BP-simple-bindings.json +++ b/tests/test_shexmap/examples/BP-simple-bindings.json @@ -1,10 +1,10 @@ { - "http://shex.io/extensions/Map/#BPDAM-family": { - "value": "Walker" - }, "http://shex.io/extensions/Map/#BPDAM-given": { "value": "Alice" }, + "http://shex.io/extensions/Map/#BPDAM-family": { + "value": "Walker" + }, "http://shex.io/extensions/Map/#BPDAM-sysVal": { "value": "110", "type": "http://www.w3.org/2001/XMLSchema#float" diff --git a/tests/test_shexmap/examples/BPPatient-2-levels-bindings.json b/tests/test_shexmap/examples/BPPatient-2-levels-bindings.json index 702a510..bd625d1 100644 --- a/tests/test_shexmap/examples/BPPatient-2-levels-bindings.json +++ b/tests/test_shexmap/examples/BPPatient-2-levels-bindings.json @@ -1,98 +1,102 @@ [ { - "http://shex.io/extensions/Map/#BPDAM-name": { "value": "Sue" } + "http://shex.io/extensions/Map/#BPDAM-name": { + "value": "Sue" + } }, [ [ { - "http://shex.io/extensions/Map/#BPDAM-reports": "http://my.data.example/medical/Report1" + "@node": "http://my.data.example/medical/Report1", + "http://shex.io/extensions/Map/#BPDAM-reports": "http://my.data.example/medical/Report1", + "http://shex.io/extensions/Map/#BPDAM-reportNo": { + "value": "one" + } }, [ { - "http://shex.io/extensions/Map/#BPDAM-reportNo": { "value": "one" } + "@node": "http://my.data.example/medical/Res00", + "http://shex.io/extensions/Map/#BPDAM-bp": "http://my.data.example/medical/Res00", + "http://shex.io/extensions/Map/#BPDAM-sysVal": { + "value": "100", + "type": "http://www.w3.org/2001/XMLSchema#float" + }, + "http://shex.io/extensions/Map/#BPDAM-sysUnits": { + "value": "mmHg" + }, + "http://shex.io/extensions/Map/#BPDAM-diaVal": { + "value": "60", + "type": "http://www.w3.org/2001/XMLSchema#float" + }, + "http://shex.io/extensions/Map/#BPDAM-diaUnits": { + "value": "mmHg" + } }, - [ - { - "http://shex.io/extensions/Map/#BPDAM-bp": "http://my.data.example/medical/Res00", - "http://shex.io/extensions/Map/#BPDAM-sysVal": { - "value": "100", - "type": "http://www.w3.org/2001/XMLSchema#float" - }, - "http://shex.io/extensions/Map/#BPDAM-sysUnits": { - "value": "mmHg" - }, - "http://shex.io/extensions/Map/#BPDAM-diaVal": { - "value": "60", - "type": "http://www.w3.org/2001/XMLSchema#float" - }, - "http://shex.io/extensions/Map/#BPDAM-diaUnits": { - "value": "mmHg" - } + { + "@node": "http://my.data.example/medical/Res01", + "http://shex.io/extensions/Map/#BPDAM-bp": "http://my.data.example/medical/Res01", + "http://shex.io/extensions/Map/#BPDAM-sysVal": { + "value": "101", + "type": "http://www.w3.org/2001/XMLSchema#float" + }, + "http://shex.io/extensions/Map/#BPDAM-sysUnits": { + "value": "mmHg" + }, + "http://shex.io/extensions/Map/#BPDAM-diaVal": { + "value": "61", + "type": "http://www.w3.org/2001/XMLSchema#float" }, - { - "http://shex.io/extensions/Map/#BPDAM-bp": "http://my.data.example/medical/Res01", - "http://shex.io/extensions/Map/#BPDAM-sysVal": { - "value": "101", - "type": "http://www.w3.org/2001/XMLSchema#float" - }, - "http://shex.io/extensions/Map/#BPDAM-sysUnits": { - "value": "mmHg" - }, - "http://shex.io/extensions/Map/#BPDAM-diaVal": { - "value": "61", - "type": "http://www.w3.org/2001/XMLSchema#float" - }, - "http://shex.io/extensions/Map/#BPDAM-diaUnits": { - "value": "mmHg" - } + "http://shex.io/extensions/Map/#BPDAM-diaUnits": { + "value": "mmHg" } - ] + } ] ], [ { - "http://shex.io/extensions/Map/#BPDAM-reports": "http://my.data.example/medical/Report2" + "@node": "http://my.data.example/medical/Report2", + "http://shex.io/extensions/Map/#BPDAM-reports": "http://my.data.example/medical/Report2", + "http://shex.io/extensions/Map/#BPDAM-reportNo": { + "value": "two" + } }, [ { - "http://shex.io/extensions/Map/#BPDAM-reportNo": { "value": "two" } + "@node": "http://my.data.example/medical/Res10", + "http://shex.io/extensions/Map/#BPDAM-bp": "http://my.data.example/medical/Res10", + "http://shex.io/extensions/Map/#BPDAM-sysVal": { + "value": "110", + "type": "http://www.w3.org/2001/XMLSchema#float" + }, + "http://shex.io/extensions/Map/#BPDAM-sysUnits": { + "value": "mmHg" + }, + "http://shex.io/extensions/Map/#BPDAM-diaVal": { + "value": "70", + "type": "http://www.w3.org/2001/XMLSchema#float" + }, + "http://shex.io/extensions/Map/#BPDAM-diaUnits": { + "value": "mmHg" + } }, - [ - { - "http://shex.io/extensions/Map/#BPDAM-bp": "http://my.data.example/medical/Res10", - "http://shex.io/extensions/Map/#BPDAM-sysVal": { - "value": "110", - "type": "http://www.w3.org/2001/XMLSchema#float" - }, - "http://shex.io/extensions/Map/#BPDAM-sysUnits": { - "value": "mmHg" - }, - "http://shex.io/extensions/Map/#BPDAM-diaVal": { - "value": "70", - "type": "http://www.w3.org/2001/XMLSchema#float" - }, - "http://shex.io/extensions/Map/#BPDAM-diaUnits": { - "value": "mmHg" - } + { + "@node": "http://my.data.example/medical/Res11", + "http://shex.io/extensions/Map/#BPDAM-bp": "http://my.data.example/medical/Res11", + "http://shex.io/extensions/Map/#BPDAM-sysVal": { + "value": "111", + "type": "http://www.w3.org/2001/XMLSchema#float" + }, + "http://shex.io/extensions/Map/#BPDAM-sysUnits": { + "value": "mmHg" + }, + "http://shex.io/extensions/Map/#BPDAM-diaVal": { + "value": "71", + "type": "http://www.w3.org/2001/XMLSchema#float" }, - { - "http://shex.io/extensions/Map/#BPDAM-bp": "http://my.data.example/medical/Res11", - "http://shex.io/extensions/Map/#BPDAM-sysVal": { - "value": "111", - "type": "http://www.w3.org/2001/XMLSchema#float" - }, - "http://shex.io/extensions/Map/#BPDAM-sysUnits": { - "value": "mmHg" - }, - "http://shex.io/extensions/Map/#BPDAM-diaVal": { - "value": "71", - "type": "http://www.w3.org/2001/XMLSchema#float" - }, - "http://shex.io/extensions/Map/#BPDAM-diaUnits": { - "value": "mmHg" - } + "http://shex.io/extensions/Map/#BPDAM-diaUnits": { + "value": "mmHg" } - ] + } ] ] ] diff --git a/tests/test_shexmap/examples/BPPatient-multi-bindings-bindings.json b/tests/test_shexmap/examples/BPPatient-multi-bindings-bindings.json index 57a3d20..b630502 100644 --- a/tests/test_shexmap/examples/BPPatient-multi-bindings-bindings.json +++ b/tests/test_shexmap/examples/BPPatient-multi-bindings-bindings.json @@ -1,9 +1,12 @@ [ { - "http://shex.io/extensions/Map/#BPDAM-name": { "value": "Sue" } + "http://shex.io/extensions/Map/#BPDAM-name": { + "value": "Sue" + } }, [ { + "@node": "tag:b0", "http://shex.io/extensions/Map/#BPDAM-XXX": "tag:b0", "http://shex.io/extensions/Map/#BPDAM-sysVal": { "value": "110", @@ -21,6 +24,7 @@ } }, { + "@node": "tag:b1", "http://shex.io/extensions/Map/#BPDAM-XXX": "tag:b1", "http://shex.io/extensions/Map/#BPDAM-sysVal": { "value": "111", diff --git a/tests/test_shexmap/examples/splits-bindings.json b/tests/test_shexmap/examples/splits-bindings.json index dcd660b..75511a7 100644 --- a/tests/test_shexmap/examples/splits-bindings.json +++ b/tests/test_shexmap/examples/splits-bindings.json @@ -6,6 +6,7 @@ }, [ { + "@node": "_:n3-22", "http://contact.example/use": { "value": "work" }, @@ -14,6 +15,7 @@ } }, { + "@node": "_:n3-23", "http://contact.example/use": { "value": "home" }, @@ -22,6 +24,7 @@ } }, { + "@node": "_:n3-24", "http://contact.example/use": { "value": "home" }, diff --git a/tests/test_shexmap/shexjs_driver.js b/tests/test_shexmap/shexjs_driver.js new file mode 100644 index 0000000..61ba2a1 --- /dev/null +++ b/tests/test_shexmap/shexjs_driver.js @@ -0,0 +1,52 @@ +// Drive shex.js's extension-map for the differential tests (test_shexjs_differential.py). +// +// node shexjs_driver.js bind [] +// validates and prints the Map extension's bindings as JSON; "ROOT" means +// the data's one subject that is nobody's object (a blank node the caller cannot name) +// node shexjs_driver.js materialize [] [] +// runs ThreadedMaterializer and prints the result as N-Triples +// +// / are IRIs or _:labels; is a shape IRI (relative ones resolve against the +// schema base, as PyShEx's manifests write them) or "START". +"use strict"; +const [X, mode, schemaPath, dataPath, node, startArg, staticsPath] = process.argv.slice(2); +const req = m => require(require.resolve(m, {paths: [X]})); +const Fs = require("fs"); +const ShExParser = req("@shexjs/parser"), N3 = req("n3"), ShExUtil = req("@shexjs/util"), ShExTerm = req("@shexjs/term"); +const {ShExValidator, resultMapToShapeExprTest} = req("@shexjs/validator"); +const {ctor: RdfJsDb} = req("@shexjs/neighborhood-rdfjs"); +const Mapper = require(X)({rdfjs: N3, Validator: ShExValidator}); +const {ThreadedMaterializer} = require(X + "/lib/ThreadedMaterializer"); + +const TURTLE_BASE = "http://a.example/turtle/"; // what test_examples.py parses data with +const SCHEMA_BASE = "http://a.example/schema/"; +const schema = ShExParser.construct(SCHEMA_BASE, {}, {index: true}).parse(Fs.readFileSync(schemaPath, "utf8")); +const start = !startArg || startArg === "START" ? ShExValidator.Start : new URL(startArg, SCHEMA_BASE).href; + +if (mode === "bind") { + const store = new N3.Store(); + store.addQuads(new N3.Parser({baseIRI: TURTLE_BASE, format: "text/turtle"}).parse(Fs.readFileSync(dataPath, "utf8"))); + const validator = new ShExValidator(schema, RdfJsDb(store), {noCache: true}); + Mapper.register(validator, {ShExTerm, ShExUtil}); + let focus = node.startsWith("_:") ? N3.DataFactory.blankNode(node.slice(2)) : node; + if (node === "ROOT") { + const objects = new Set(store.getQuads(null, null, null, null).map(q => q.object.id)); + const roots = [...new Set(store.getQuads(null, null, null, null).map(q => q.subject).filter(s => !objects.has(s.id)))]; + if (roots.length !== 1) { console.error("expected one root, found " + roots.length); process.exit(2); } + focus = roots[0].termType === "BlankNode" ? "_:" + roots[0].value : roots[0].value; // the validator wants JSON-LD-style terms + } + const res = resultMapToShapeExprTest(validator.validateShapeMap([{node: focus, shape: start}])); + if (res.errors) { console.error(JSON.stringify(res.errors, null, 1)); process.exit(2); } + // the scope layout with @node where the checkout has it (branch shexmap-scopes), else the legacy tree + process.stdout.write(JSON.stringify(Mapper.bindingTree ? Mapper.bindingTree(res) : ShExUtil.valToExtension(res, Mapper.url)) + "\n"); +} else if (mode === "materialize") { + const bindings = JSON.parse(Fs.readFileSync(dataPath, "utf8")); + const staticVars = staticsPath ? JSON.parse(Fs.readFileSync(staticsPath, "utf8")) : {}; + const m = new ThreadedMaterializer(schema, {staticVars}); + const quads = m.materialize(bindings, node, start === ShExValidator.Start ? undefined : start); + const writer = new N3.Writer({format: "N-Triples"}); + quads.forEach(q => writer.addQuad(q)); + writer.end((err, out) => { if (err) throw err; process.stdout.write(out); }); +} else { + console.error("mode must be bind or materialize"); process.exit(1); +} diff --git a/tests/test_shexmap/test_analysis.py b/tests/test_shexmap/test_analysis.py new file mode 100644 index 0000000..0ef5352 --- /dev/null +++ b/tests/test_shexmap/test_analysis.py @@ -0,0 +1,122 @@ +"""Static checks of a schema pair (pyshex.shexmap.analyse): which pairs map coherently, and +why not, before any data.""" +import json +from pathlib import Path + +import pytest + +from pyshex.shexmap import analyse +from pyshex.shexmap.cli import main +from tests.test_shexmap.test_examples import MANIFEST, label, node_and_shape, text + +PREFIXES = """PREFIX : +PREFIX xsd: +PREFIX v: +PREFIX Map: +""" +NESTED_IN = PREFIXES + "start = @:P\n:P { :name . %Map:{ v:name %} ; :report { :no . %Map:{ v:no %} ; :result { :sys . %Map:{ v:sys %} }* }* }" + + +def out(schema: str) -> str: + return PREFIXES + "start = @:Out\n" + schema + + +@pytest.mark.parametrize("entry", MANIFEST, ids=label) +def test_every_example_pair_is_coherent(entry): + _, start = node_and_shape(entry["queryMap"]) + _, out_shape = node_and_shape(entry["outputShapeMap"]) + report = analyse(text(entry, "schema"), text(entry, "outputSchema"), input_start=start, output_start=out_shape, + static_vars=list(entry.get("staticVars", {}))) + assert report.errors == [], str(report) + + +@pytest.mark.parametrize("entry", MANIFEST, ids=label) +def test_every_schema_maps_to_itself(entry): + _, start = node_and_shape(entry["queryMap"]) + report = analyse(text(entry, "schema"), text(entry, "schema"), input_start=start, output_start=start) + assert report.errors == [], str(report) + assert report.warnings == [] # every binding is read back + + +def test_regrouping_and_flattening_are_coherent(): + for schema in [":Out { :group @* }\n { :label . %Map:{ v:no %} ; :reading . + %Map:{ v:sys %} }", + ":Out { :who . %Map:{ v:name %} ; :all . * %Map:{ v:sys %} }", + ":Out { :g { :label . %Map:{ v:no %} ; :reading . %Map:{ v:sys %} }* }", + ":Out { :bp { :sys . %Map:{ v:sys %} ; :who . %Map:{ v:name %} }* }"]: + report = analyse(NESTED_IN, out(schema)) + assert report.errors == [], str(report) + + +def test_a_deeper_variable_read_without_a_repetition_is_which_one(): + report = analyse(NESTED_IN, out(":Out { :first . %Map:{ v:sys %} }")) + assert len(report.errors) == 1 and "which one" in report.errors[0] + assert "each :report / :result" in report.errors[0] + + +def test_an_optional_over_a_deeper_variable_takes_the_first(): + # `?` is a repetition of at most one: it iterates the list and stops + report = analyse(NESTED_IN, out(":Out { :first . ? %Map:{ v:sys %} }")) + assert report.errors == [] + + +def test_unrelated_lists_are_named(): + two_lists = PREFIXES + "start = @:S\n:S { :a . * %Map:{ v:a %} ; :b { :v . %Map:{ v:b %} }* }" + report = analyse(two_lists, out(":Out { :pair { :x . %Map:{ v:a %} ; :y . %Map:{ v:b %} }* }")) + assert len(report.errors) == 1 + assert "unrelated lists" in report.errors[0] and "v:a" in report.errors[0] and "v:b" in report.errors[0] + assert "each :a" in report.errors[0] and "each :b" in report.errors[0] + + +def test_a_variable_from_a_sibling_list_is_not_in_scope(): + two_lists = PREFIXES + "start = @:S\n:S { :a . * %Map:{ v:a %} ; :b { :v . %Map:{ v:b %} }* }" + report = analyse(two_lists, out(":Out { :x { :va . %Map:{ v:a %} ; :vb . %Map:{ v:b %} }* }")) + assert any("unrelated lists" in e for e in report.errors) + + +def test_unbound_and_unused_variables(): + report = analyse(NESTED_IN, out(":Out { :who . %Map:{ v:name %} ; :oops . %Map:{ v:typo %} }")) + assert [e for e in report.errors if "never binds" in e and "typo" in e] + assert [w for w in report.warnings if "v:sys" in w and "never reads" in w] + report = analyse(NESTED_IN, out(":Out { :who . %Map:{ v:name %} ; :konst . %Map:{ v:stat %} }"), + static_vars=["http://v.example/stat", "http://v.example/unused"]) + assert not [e for e in report.errors if "stat" in e] + assert [w for w in report.warnings if "unused" in w and "static" in w] + + +def test_a_variable_bound_at_two_sites_is_an_error(): + twice = PREFIXES + "start = @:S\n:S { :n . %Map:{ v:x %} ; :item { :n . %Map:{ v:x %} }* }" + report = analyse(twice, out(":Out { :y . %Map:{ v:x %} }")) + assert [e for e in report.errors if "two places" in e] + + +def test_id_checks(): + assert [e for e in analyse(NESTED_IN, out(":Out { :n . %Map:{ id(v:no) %} }")).errors if "shape-valued" in e] + assert [e for e in analyse(NESTED_IN, out(":Out { :g @ %Map:{ id(v:name) %} %Map:{ v:name %} }\n { :k [:c] }")).errors + if "only Map code" in e] + assert [e for e in analyse(NESTED_IN, out(":Out { :g @ %Map:{ id(@node) %} }\n { :k [:c] }")).errors + if "@node" in e] + report = analyse(NESTED_IN, out(":Out { :g @* %Map:{ id(@node) %} }\n { :label . %Map:{ v:no %} ; :r . * %Map:{ v:sys %} }")) + assert report.errors == [], str(report) + + +def test_undefined_and_abstract_output_shapes(): + assert [e for e in analyse(NESTED_IN, out(":Out { :g @ }")).errors if "does not define" in e] + assert [e for e in analyse(NESTED_IN, out(":Out { :g @ }\nABSTRACT { :k [:c] }")).errors if "abstract" in e] + + +def test_cli_check(tmp_path, capsys): + (tmp_path / "in.shex").write_text(NESTED_IN, encoding="utf-8") + (tmp_path / "good.shex").write_text(out(":Out { :group @* }\n { :label . %Map:{ v:no %} ; :reading . + %Map:{ v:sys %} }"), + encoding="utf-8") + (tmp_path / "bad.shex").write_text(out(":Out { :first . %Map:{ v:sys %} }"), encoding="utf-8") + assert main(["--check", "-s", str(tmp_path / "in.shex"), "-t", str(tmp_path / "good.shex")]) == 0 + assert "warning" in capsys.readouterr().out # v:name is never read + assert main(["--check", "-s", str(tmp_path / "in.shex"), "-t", str(tmp_path / "bad.shex")]) == 1 + assert "which one" in capsys.readouterr().out + + +def test_template_variables_are_checked_like_the_others(): + report = analyse(NESTED_IN, out(":Out { :r @* %Map:{ id() %} }\n { :k [:c] }")) + assert report.errors == [], str(report) + report = analyse(NESTED_IN, out(":Out { :r @* %Map:{ id() %} }\n { :k [:c] }")) + assert [e for e in report.errors if "never binds" in e and "nope" in e] diff --git a/tests/test_shexmap/test_cardinality.py b/tests/test_shexmap/test_cardinality.py index 8b1311a..a2b653b 100644 --- a/tests/test_shexmap/test_cardinality.py +++ b/tests/test_shexmap/test_cardinality.py @@ -1,5 +1,6 @@ -"""Cardinality above one: nothing caps how many values bind or materialize, and the frame -model's known limits are pinned down as strict xfails, so they flip when a redesign lands.""" +"""Cardinality above one: nothing caps how many values bind or materialize; regrouping, +transposition and reads of inherited bindings come out as the input's structure says; a +body that reads from unrelated lists is refused.""" import pytest from rdflib import Graph, Literal, URIRef @@ -67,18 +68,16 @@ def test_a_starred_constraint_emits_every_binding(n): assert not m.last_report["exploration_truncated"] -def test_advances_that_forfeit_bindings_still_reach_the_full_materialization(): - # each advance abandons an unused v:u, so every repetition count accepts before the - # next one is tried; the full materialization is found last and must still win +def test_unread_bindings_beside_the_read_ones_change_nothing(): + # every item also binds v:u, which the output never reads: the frame model used to + # accept once per repetition count here and truncate at max_accepts n = 60 m = ThreadedMaterializer(STAR) triples = m.materialize(frames(n, "t", "u"), OUT) assert len(triples) == n assert m.chosen.consumed == n - assert m.chosen in m.accepts - assert len(m.accepts) == 20 # the best max_accepts are kept ... - assert sorted(a.consumed for a in m.accepts) == list(range(41, 61)) - assert m.last_report["alternatives"] == n + 1 # ... out of every distinct accept + assert m.chosen is m.accepts[0] + assert m.last_report["alternatives"] == 1 def test_a_starred_subshape_emits_every_frame(): @@ -108,14 +107,12 @@ def test_end_to_end_with_many_values(): assert len(out) == n -# -- limits of the frame model, as documented ---------------------------------------------- +# -- the structure of the input decides the structure of the output ------------------------ NESTED = PREFIXES + "start = @:P\n:P { :report { :no . %Map:{ v:no %} ; :result { :sys . %Map:{ v:sys %} }* }* }" REPORTS = ':p :report [ :no "one" ; :result [ :sys 100 ], [ :sys 101 ] ], [ :no "two" ; :result [ :sys 110 ], [ :sys 111 ] ] .' -@pytest.mark.xfail(strict=True, reason="frame model: a report's :no is copied into each reading's frame, " - "so one group per reading uses 'more' bindings") def test_a_nested_schema_maps_to_itself(): bound = bind(turtle(REPORTS), NESTED, URIRef("http://a.example/p")) out = materialize(NESTED, bound, OUT) @@ -124,8 +121,6 @@ def test_a_nested_schema_maps_to_itself(): assert sorted(len(list(out.objects(r, URIRef("http://a.example/result")))) for r in reports) == [2, 2] -@pytest.mark.xfail(strict=True, reason="frame model: the cursor never moves back, so a second repetition " - "cannot revisit the frames the first one passed") def test_sibling_lists_can_be_transposed(): m = ThreadedMaterializer(PREFIXES + "start = @:T\n:T { :allB . * %Map:{ v:b %} ; :allA . * %Map:{ v:a %} }") triples = m.materialize(frames(2, "a", "b"), OUT) @@ -133,9 +128,72 @@ def test_sibling_lists_can_be_transposed(): assert objects(triples, "allA") == ["a0", "a1"] -@pytest.mark.xfail(strict=True, reason="bound values are not checked against the output constraint's " - "value expression") def test_a_bound_value_must_satisfy_the_output_value_expression(): m = ThreadedMaterializer(PREFIXES + "start = @:T\n:T { :n xsd:integer %Map:{ v:t %} }") with pytest.raises(MaterializationError): m.materialize(Bindings({V + "t": Literal("not a number")}), OUT) + + +# -- inherited bindings and unrelated lists (structural plan §1.2, §3.3) ------------------- +# a parent's binding is read, never used up: it can be read at the parent and in every +# item, and twice in one item + +PATIENT = Bindings([{V + "name": Literal("Sue")}, + [{V + "sys": Literal("110"), V + "dia": Literal("70")}, + {V + "sys": Literal("111"), V + "dia": Literal("71")}]]) +A_EX = "http://a.example/" + + +def readings(graph_or_triples) -> list[tuple[str, str]]: + """(sys, dia) per :bp node, in sys order.""" + triples = list(graph_or_triples) + bps = {o for _, p, o in triples if str(p) == A_EX + "bp"} + value = lambda s, local: next(str(o) for s2, p, o in triples if s2 == s and str(p) == A_EX + local) + return sorted((value(b, "sys"), value(b, "dia")) for b in bps) + + +def test_a_parent_binding_read_once_per_item(): + m = ThreadedMaterializer(PREFIXES + "start = @:Out\n:Out { :bp { :sys . %Map:{ v:sys %} ; :who . %Map:{ v:name %} ; :dia . %Map:{ v:dia %} }* }") + assert readings(m.materialize(PATIENT, OUT)) == [("110", "70"), ("111", "71")] + + +def test_a_parent_binding_read_at_the_parent_and_in_each_item(): + m = ThreadedMaterializer(PREFIXES + "start = @:Out\n:Out { :who . %Map:{ v:name %} ; " + ":bp { :sys . %Map:{ v:sys %} ; :who . %Map:{ v:name %} ; :dia . %Map:{ v:dia %} }* }") + triples = m.materialize(PATIENT, OUT) + assert readings(triples) == [("110", "70"), ("111", "71")] + assert objects(triples, "who") == ["Sue", "Sue", "Sue"] + + +def test_a_parent_binding_read_twice_in_an_item(): + m = ThreadedMaterializer(PREFIXES + "start = @:Out\n:Out { :bp { :sys . %Map:{ v:sys %} ; " + ":who . %Map:{ v:name %} ; :who2 . %Map:{ v:name %} ; :dia . %Map:{ v:dia %} }* }") + assert readings(m.materialize(PATIENT, OUT)) == [("110", "70"), ("111", "71")] + + +def test_variables_from_unrelated_lists_are_refused(): + two_lists = Bindings([{}, [{V + "a": Literal("a1")}, {V + "a": Literal("a2")}], + [{V + "b": Literal("b1")}, {V + "b": Literal("b2")}]]) + m = ThreadedMaterializer(PREFIXES + "start = @:Out\n:Out { :pair { :x . %Map:{ v:a %} ; :y . %Map:{ v:b %} }* }") + with pytest.raises(MaterializationError, match="unrelated lists"): + m.materialize(two_lists, OUT) + + +def test_a_deeper_variable_read_without_a_repetition_fails(): + # "which one?" is not guessed: the read fails, as an unbound variable does + m = ThreadedMaterializer(PREFIXES + "start = @:Out\n:Out { :first . %Map:{ v:sys %} }") + with pytest.raises(MaterializationError, match="sys"): + m.materialize(PATIENT, OUT) + + +def test_a_repetition_over_a_deeper_list_flattens(): + reports = Bindings([{}, [[{V + "no": Literal("one")}, [{V + "sys": Literal("100")}, {V + "sys": Literal("101")}]], + [{V + "no": Literal("two")}, [{V + "sys": Literal("110")}]]]]) + m = ThreadedMaterializer(PREFIXES + "start = @:Out\n:Out { :all . * %Map:{ v:sys %} }") + assert objects(m.materialize(reports, OUT), "all") == ["100", "101", "110"] + # and a group per reading takes its label from the report above it + m = ThreadedMaterializer(PREFIXES + "start = @:Out\n:Out { :g { :label . %Map:{ v:no %} ; :reading . %Map:{ v:sys %} }* }") + triples = m.materialize(reports, OUT) + labels = {str(o) for _, p, o in triples if str(p) == A_EX + "label"} + assert len(objects(triples, "g")) == 3 and labels == {"one", "two"} + diff --git a/tests/test_shexmap/test_examples.py b/tests/test_shexmap/test_examples.py index 8847bd3..476c81c 100644 --- a/tests/test_shexmap/test_examples.py +++ b/tests/test_shexmap/test_examples.py @@ -10,7 +10,7 @@ from rdflib import BNode, Graph, URIRef from rdflib.compare import isomorphic -from pyshex.shexmap import AmbiguousBindingsError, bind, bind_all, dumps, loads, materialize +from pyshex.shexmap import NODE_KEY, AmbiguousBindingsError, bind, bind_all, dumps, loads, materialize, normalize HERE = Path(__file__).parent EXAMPLES = HERE / "examples" @@ -18,9 +18,8 @@ for e in json.loads((d / "manifest.json").read_text(encoding="utf-8"))] TURTLE_BASE = "http://a.example/turtle/" # the base shex.js's test runner parses data with -# shex.js splits one binding record across two nesting levels in this example; PyShEx keeps -# a report's variables together. Both materialize to the expected graph. -BINDINGS_LAYOUT_DIFFERS = {"BPPatient 2 levels / simple"} +# the stored bindings are what shex.js writes (packages/extension-map/examples, branch +# shexmap-scopes): the scope layout with @node; blank-node labels are the parser's def label(entry) -> str: @@ -74,15 +73,102 @@ def example(request): return entry, bindings +def without_nodes(obj): + """The tree without its ``@node`` keys, which shex.js does not write (yet).""" + if isinstance(obj, dict): + return {k: v for k, v in obj.items() if k != NODE_KEY} + if isinstance(obj, list): + return [without_nodes(e) for e in obj] + return obj + + +def canonical(obj): + """``without_nodes``, with blank-node values reduced to a marker (labels do not survive + a trip through materialization) and each list's iterations sorted (ShEx matches bags; + the order a neighbourhood is listed in follows the nodes' labels, which is arbitrary).""" + if isinstance(obj, dict): + return {k: ("_:" if isinstance(v, str) and v.startswith("_:") else v) + for k, v in obj.items() if k != NODE_KEY} + if isinstance(obj, list): + items = [canonical(e) for e in obj] + return sorted(items, key=lambda e: json.dumps(e, sort_keys=True)) + return obj + + +def nodes_in(obj) -> list[dict]: + """Every object of the tree that carries ``@node``.""" + if isinstance(obj, dict): + return [obj] if NODE_KEY in obj else [] + if isinstance(obj, list): + return [d for e in obj for d in nodes_in(e)] + return [] + + def test_bindings_match_shexjs(example): + """The same tree as shex.js writes, @node included, blank-node labels aside.""" entry, bindings = example if "expectedBindingsURL" not in entry: pytest.skip("no shex.js bindings recorded") expected = json.loads(path(entry, entry["expectedBindingsURL"]).read_text(encoding="utf-8")) - if label(entry) in BINDINGS_LAYOUT_DIFFERS: - assert bindings.to_json() != expected # if this starts failing, the layouts converged - else: - assert bindings.to_json() == expected + assert with_bnode_marker(bindings.to_json()) == with_bnode_marker(expected) + + +def with_bnode_marker(obj): + if isinstance(obj, dict): + return {k: ("_:" if isinstance(v, str) and v.startswith("_:") else v) for k, v in obj.items()} + if isinstance(obj, list): + return [with_bnode_marker(e) for e in obj] + return obj + + +# examples whose input schema has a repeated shape-valued constraint, so their trees have lists +# of iterations, each of which must say which node it matched +HAS_ITERATIONS = {"BPPatient multi-bindings / simple", "BPPatient 2 levels / simple", "splits / Ann-phone-mbox", + "splits / Ann-mbox-phone", "inverse in / Ann's heart rate", "inverse out / Ann's chart", + "EXTENDS / clinic"} +BP = "http://shex.io/extensions/Map/#BPDAM-" + + +def test_iterations_carry_the_node_they_matched(example): + entry, bindings = example + found = nodes_in(bindings.to_json()) + if label(entry) not in HAS_ITERATIONS: + assert found == [] + return + assert found + for d in found: + assert isinstance(d[NODE_KEY], str) # an IRI or a _:label, never a literal + if label(entry) == "BPPatient 2 levels / simple": + # the constraint that made the list also bound the node itself: the two must agree + assert {d[NODE_KEY] for d in found} == {d.get(BP + "reports") or d.get(BP + "bp") for d in found} + assert len(found) == 6 # two reports, four readings + if label(entry) == "BPPatient multi-bindings / simple": + assert all(d[NODE_KEY] == d[BP + "XXX"] for d in found) + if label(entry) == "inverse in / Ann's heart rate": + assert all(d[NODE_KEY].startswith("http://") for d in found) # ^:subject binds subjects + + +def test_nodes_survive_json(example): + entry, bindings = example + assert loads(dumps(bindings)).to_json() == bindings.to_json() + assert normalize(loads(dumps(bindings)).tree) == bindings.frames() + assert not any(NODE_KEY in f for f in bindings.frames()) + + +@pytest.mark.parametrize("entry", MANIFEST, ids=label) +def test_bindings_survive_a_round_trip_through_the_input_schema(entry): + """bind after materialize after bind is bind: materializing bindings with the *input* + schema gives a graph that binds to the same bindings, i.e. a schema maps to itself. + Compared as the set of parses, since an ambiguous input may parse in another order.""" + graph = input_graph(entry) + _, start = node_and_shape(entry["queryMap"]) + focus = focus_node(entry, graph) + schema = text(entry, "schema") + before = {json.dumps(canonical(b.to_json()), sort_keys=True) for b in bind_all(graph, schema, focus, start=start)} + root = BNode() if isinstance(focus, BNode) else focus + out = materialize(schema, bind(graph, schema, focus, start=start), root, start=start) + after = {json.dumps(canonical(b.to_json()), sort_keys=True) for b in bind_all(out, schema, root, start=start)} + assert after == before, out.serialize(format="turtle") def test_output_matches_shexjs(example): diff --git a/tests/test_shexmap/test_keys.py b/tests/test_shexmap/test_keys.py new file mode 100644 index 0000000..81aa848 --- /dev/null +++ b/tests/test_shexmap/test_keys.py @@ -0,0 +1,151 @@ +"""Node identity: keyed nodes (``id(...)``) merge, ``id(@node)`` reuses the input node, and +unkeyed nodes are reproducible.""" +import pytest +from rdflib import BNode, Graph, Literal, URIRef +from rdflib.compare import isomorphic + +from pyshex.shexmap import Bindings, MaterializationError, ThreadedMaterializer, bind, materialize + +PREFIXES = """PREFIX : +PREFIX xsd: +PREFIX v: +PREFIX Map: +""" +V = "http://v.example/" +A = "http://a.example/" +OUT = URIRef(A + "out") + + +def turtle(text: str) -> Graph: + return Graph().parse(data="PREFIX : \n" + text, format="turtle") + + +def by_predicate(triples, local: str): + return [(s, o) for s, p, o in triples if str(p) == A + local] + + +READINGS = Bindings([{}, [{V + "date": Literal("2026-09-01"), V + "sys": Literal("110")}, + {V + "date": Literal("2026-09-01"), V + "sys": Literal("112")}, + {V + "date": Literal("2026-09-02"), V + "sys": Literal("120")}]]) + + +def test_equal_keys_merge_into_one_node(): + # readings bound flat; grouped by date on the way out + m = ThreadedMaterializer(PREFIXES + "start = @:Out\n:Out { :day @* %Map:{ id(v:date) %} }\n" + " { :date . %Map:{ v:date %} ; :reading . %Map:{ v:sys %} }") + triples = m.materialize(READINGS, OUT) + days = {o for _, o in by_predicate(triples, "day")} + assert len(days) == 2 + readings = {(d, str(o)) for d, o in by_predicate(triples, "reading")} + first = next(d for d, o in by_predicate(triples, "date") if str(o) == "2026-09-01") + assert {r for d, r in readings if d == first} == {"110", "112"} + assert len(by_predicate(triples, "date")) == 2 # merged, not repeated + + +def test_a_key_of_one_iri_is_the_node(): + b = Bindings([{}, [{V + "who": URIRef(A + "alice"), V + "n": Literal("Alice")}, + {V + "who": URIRef(A + "bob"), V + "n": Literal("Bob")}]]) + m = ThreadedMaterializer(PREFIXES + "start = @:Out\n:Out { :member @

* %Map:{ id(v:who) %} }\n

{ :name . %Map:{ v:n %} }") + triples = m.materialize(b, OUT) + assert {o for _, o in by_predicate(triples, "member")} == {URIRef(A + "alice"), URIRef(A + "bob")} + assert (URIRef(A + "bob"), Literal("Bob")) in by_predicate(triples, "name") + + +def test_keyed_nodes_are_shared_across_constraints(): + b = Bindings([{V + "mrn": Literal("42"), V + "name": Literal("Ann")}, + [{V + "when": Literal("Mon")}, {V + "when": Literal("Tue")}]]) + m = ThreadedMaterializer(PREFIXES + "start = @:Out\n" + ":Out { :patient @

%Map:{ id(v:mrn) %} ; :visit @* }\n" + "

{ :name . %Map:{ v:name %} }\n" + " { :when . %Map:{ v:when %} ; :of @

%Map:{ id(v:mrn) %} }") + triples = m.materialize(b, OUT) + patient = by_predicate(triples, "patient")[0][1] + assert {o for _, o in by_predicate(triples, "of")} == {patient} + assert len(by_predicate(triples, "name")) == 1 + + +def test_different_shapes_with_equal_keys_stay_apart(): + b = Bindings({V + "k": Literal("Paris")}) + m = ThreadedMaterializer(PREFIXES + "start = @:Out\n:Out { :city @ %Map:{ id(v:k) %} ; :person @ %Map:{ id(v:k) %} }\n" + " { :kind [:city] }\n { :kind [:person] }") + triples = m.materialize(b, OUT) + assert by_predicate(triples, "city")[0][1] != by_predicate(triples, "person")[0][1] + + +NESTED = PREFIXES + ("start = @:P\n:P { :report @* %Map:{ id(@node) %} }\n" + " { :no . %Map:{ v:no %} ; :result @* %Map:{ id(@node) %} }\n" + " { :sys . %Map:{ v:sys %} }") +DATA = (':p :report :r1, :r2 . :r1 :no "one" ; :result [ :sys 100 ], [ :sys 101 ] . ' + ':r2 :no "two" ; :result [ :sys 110 ] .') + + +def test_id_node_maps_a_graph_onto_itself(): + graph = turtle(DATA) + bound = bind(graph, NESTED, URIRef(A + "p")) # id() binds nothing on the way in + out = materialize(NESTED, bound, URIRef(A + "p")) + assert isomorphic(out, graph) + assert (URIRef(A + "p"), URIRef(A + "report"), URIRef(A + "r1")) in out # the very same IRIs + + +def test_unkeyed_nodes_are_reproducible_and_root_specific(): + b = Bindings([{V + "name": Literal("Sue")}, [{V + "sys": Literal("110")}, {V + "sys": Literal("111")}]]) + schema = PREFIXES + "start = @:Out\n:Out { :who . %Map:{ v:name %} ; :bp { :sys . %Map:{ v:sys %} }* }" + once = ThreadedMaterializer(schema).materialize(b, OUT) + again = ThreadedMaterializer(schema).materialize(b, OUT) + assert once == again + other = ThreadedMaterializer(schema).materialize(b, URIRef(A + "elsewhere")) + assert {o for _, o in by_predicate(once, "bp")}.isdisjoint({o for _, o in by_predicate(other, "bp")}) + g = Graph() + materialize(schema, b, OUT, graph=g) + n = len(g) + materialize(schema, b, OUT, graph=g) + assert len(g) == n # the same root twice adds nothing + + +def test_an_unbound_key_fails_the_constraint(): + b = Bindings({V + "name": Literal("Sue")}) + m = ThreadedMaterializer(PREFIXES + "start = @:Out\n:Out { :who . %Map:{ v:name %} ; :home @? %Map:{ id(v:city) %} }\n { :kind [:home] }") + triples = m.materialize(b, OUT) + assert by_predicate(triples, "home") == [] # optional: left out + m = ThreadedMaterializer(PREFIXES + "start = @:Out\n:Out { :home @ %Map:{ id(v:city) %} }\n { :kind [:home] }") + with pytest.raises(MaterializationError, match="city"): + m.materialize(b, OUT) + + +@pytest.mark.parametrize("schema, message", [ + ("start = @:Out\n:Out { :n . %Map:{ id(v:k) %} }", "shape-valued"), + ("start = @:Out\n:Out { :h @ %Map:{ id(v:k) %} %Map:{ v:k %} }\n { :kind [:home] }", "only Map code"), +]) +def test_id_is_alone_and_on_a_shape(schema, message): + m = ThreadedMaterializer(PREFIXES + schema) + with pytest.raises(MaterializationError, match=message): + m.materialize(Bindings({V + "k": Literal("x")}), OUT) + + +# -- IRI templates ------------------------------------------------------------------------ + +def test_an_iri_template_names_the_node(): + b = Bindings([{V + "mrn": Literal("42"), V + "name": Literal("Ann")}, + [{V + "when": Literal("Mon")}, {V + "when": Literal("Tue")}]]) + m = ThreadedMaterializer(PREFIXES + "start = @:Out\n" + ":Out { :patient @

%Map:{ id() %} ; :visit @* }\n" + "

{ :name . %Map:{ v:name %} }\n" + " { :when . %Map:{ v:when %} ; :of @

%Map:{ id() %} }") + triples = m.materialize(b, OUT) + patient = URIRef(A + "person/42") + assert by_predicate(triples, "patient") == [(OUT, patient)] + assert {o for _, o in by_predicate(triples, "of")} == {patient} + assert len(by_predicate(triples, "name")) == 1 + assert m.provenance[[t for t in triples].index((OUT, URIRef(A + "patient"), patient))]["reads"] == (((), V + "mrn"),) + + +def test_template_values_are_iri_safe(): + b = Bindings({V + "k": Literal("a b/c?d")}) + m = ThreadedMaterializer(PREFIXES + "start = @:Out\n:Out { :x @ %Map:{ id() %} }\n { :kind [:c] }") + assert by_predicate(m.materialize(b, OUT), "x") == [(OUT, URIRef(A + "k/a%20b%2Fc%3Fd/tail"))] + + +def test_an_unbound_template_variable_fails_the_constraint(): + m = ThreadedMaterializer(PREFIXES + "start = @:Out\n:Out { :x @ %Map:{ id() %} }\n { :kind [:c] }") + with pytest.raises(MaterializationError, match="v.example/k"): + m.materialize(Bindings({V + "other": Literal("1")}), OUT) diff --git a/tests/test_shexmap/test_roundtrip.py b/tests/test_shexmap/test_roundtrip.py new file mode 100644 index 0000000..0d4dcef --- /dev/null +++ b/tests/test_shexmap/test_roundtrip.py @@ -0,0 +1,160 @@ +"""Bidirectionality: the round-trip laws of a schema pair, and the provenance that ties each +output triple to the input it came from.""" +import json + +import pytest +from rdflib import BNode, Graph, Literal, URIRef +from rdflib.compare import isomorphic + +from pyshex.shexmap import Bindings, ThreadedMaterializer, bind, bind_all, materialize +from pyshex.shexmap.bindings import term_from_json +from pyshex.shexmap.cli import main +from pyshex.shexmap.scopes import ScopeTree +from tests.test_shexmap.test_examples import (MANIFEST, canonical, focus_node, input_graph, label, node_and_shape, + output_root, text) + +BY_LABEL = {label(e): e for e in MANIFEST} +PAIRS = [("BP / simple", "BP back / simple"), ("inverse in / Ann's heart rate", "inverse out / Ann's chart")] + + +def statics(entry) -> dict: + return {k: term_from_json(v) for k, v in entry.get("staticVars", {}).items()} + + +def marker(v): + return "_:" if isinstance(v, BNode) else v.n3() + + +def bound_pairs(bindings_list) -> set: + return {(v, marker(x)) for b in bindings_list for s in ScopeTree(b.tree).root.walk() for v, x in s.own.items()} + + +def forward(entry): + """bind with the input schema, materialize with the output schema.""" + graph = input_graph(entry) + _, start = node_and_shape(entry["queryMap"]) + focus = focus_node(entry, graph) + bindings = bind(graph, text(entry, "schema"), focus, start=start) + root, shape = output_root(entry) + m = ThreadedMaterializer(text(entry, "outputSchema"), static_vars=statics(entry)) + triples = m.materialize(bindings, root, start=shape) + return graph, focus, bindings, m, triples, root, shape + + +# -- law 1: a schema binds back exactly what it wrote from ----------------------------- + +@pytest.mark.parametrize("entry", MANIFEST, ids=label) +def test_the_output_schema_binds_back_what_it_read(entry): + graph, focus, bindings, m, triples, root, shape = forward(entry) + scopes = {s.path: s for s in ScopeTree(bindings.tree).root.walk()} + read = {(v, marker(scopes[p].own[v])) for src in m.provenance for p, v in src["reads"]} + read |= {(v, marker(m.statics[v])) for src in m.provenance for v in src["statics"]} + out = Graph() + for t in triples: + out.add(t) + rebound = bound_pairs(bind_all(out, text(entry, "outputSchema"), root, start=shape)) + assert rebound == read + + +# -- law 2: a pair of schemas, there and back -------------------------------------------- + +@pytest.mark.parametrize("there, back", PAIRS, ids=[f"{a} <-> {b}" for a, b in PAIRS]) +def test_a_pair_recovers_the_bindings(there, back): + ea, eb = BY_LABEL[there], BY_LABEL[back] + graph, focus, bindings, _, triples, root, _ = forward(ea) + middle = Graph() + for t in triples: + middle.add(t) + _, start_back = node_and_shape(eb["queryMap"]) + b2 = bind(middle, text(eb, "schema"), root, start=start_back) + _, shape_back = output_root(eb) + _, start = node_and_shape(ea["queryMap"]) + returned = materialize(text(eb, "outputSchema"), b2, focus, start=shape_back, static_vars=statics(eb)) + original = {json.dumps(canonical(b.to_json()), sort_keys=True) for b in bind_all(graph, text(ea, "schema"), focus, start=start)} + again = {json.dumps(canonical(b.to_json()), sort_keys=True) for b in bind_all(returned, text(ea, "schema"), focus, start=start)} + assert again == original + + +def test_the_bp_pair_recovers_the_graph(): + # every node of the input is either the focus or carried by a variable, so the graph comes back as it was + ea, eb = BY_LABEL["BP / simple"], BY_LABEL["BP back / simple"] + graph, focus, _, _, triples, root, _ = forward(ea) + middle = Graph() + for t in triples: + middle.add(t) + _, start_back = node_and_shape(eb["queryMap"]) + _, shape_back = output_root(eb) + returned = materialize(text(eb, "outputSchema"), bind(middle, text(eb, "schema"), root, start=start_back), focus, + start=shape_back, static_vars=statics(eb)) + assert isomorphic(returned, graph) + + +def test_the_inverse_pair_loses_only_uncarried_identities(): + # the chart carries no variable for the observation nodes, so they come back as blank nodes + ea, eb = BY_LABEL["inverse in / Ann's heart rate"], BY_LABEL["inverse out / Ann's chart"] + graph, focus, _, _, triples, root, _ = forward(ea) + middle = Graph() + for t in triples: + middle.add(t) + _, start_back = node_and_shape(eb["queryMap"]) + _, shape_back = output_root(eb) + returned = materialize(text(eb, "outputSchema"), bind(middle, text(eb, "schema"), root, start=start_back), focus, + start=shape_back, static_vars=statics(eb)) + assert not isomorphic(returned, graph) + blanked = Graph() + observations = {s for s in graph.subjects() if s != focus} + for s, p, o in graph: + blanked.add((BNode(str(s)) if s in observations else s, p, BNode(str(o)) if o in observations else o)) + assert isomorphic(returned, blanked) + + +# -- provenance --------------------------------------------------------------------------- + +PREFIXES = """PREFIX : +PREFIX v: +PREFIX Map: +""" +V = "http://v.example/" +A = "http://a.example/" +OUT = URIRef(A + "out") + + +def test_every_triple_says_where_it_came_from(): + b = Bindings([{V + "name": Literal("Sue")}, + [{"@node": URIRef(A + "r1"), V + "sys": Literal("110")}, {"@node": URIRef(A + "r2"), V + "sys": Literal("111")}]]) + m = ThreadedMaterializer(PREFIXES + "start = @:Out\n:Out { :kind [:chart] ; :who . %Map:{ v:name %} ; " + ":bp @* %Map:{ id(@node) %} }\n { :sys . %Map:{ v:sys %} ; :owner . %Map:{ v:name %} }") + triples = m.materialize(b, OUT) + assert m.provenance is m.chosen.provenance and len(m.provenance) == len(triples) + by_pred = {} + for t, src in zip(triples, m.provenance): + assert src["predicate"] == str(t[1]) + by_pred.setdefault(str(t[1]).split("/")[-1], []).append((t, src)) + assert by_pred["kind"][0][1]["constant"] is True and by_pred["kind"][0][1]["reads"] == () + (t, src), = by_pred["who"] + assert src["variable"] == V + "name" and src["reads"] == (((), V + "name"),) and src["scope"] == () + for t, src in by_pred["bp"]: + assert src["keyed"] == "id(@node)" and src["node"] == t[2] # the keyed node is the input node + for t, src in by_pred["sys"]: + assert src["node"] == t[0] and src["reads"] == ((src["scope"], V + "sys"),) + for t, src in by_pred["owner"]: + assert src["reads"] == (((), V + "name"),) and src["scope"] != () # an inherited read, from the item + + +def test_structural_and_named_links_are_marked(): + b = Bindings({V + "n": Literal("x"), V + "who": URIRef(A + "alice")}) + m = ThreadedMaterializer(PREFIXES + "start = @:Out\n:Out { :a @ ; :p @

%Map:{ v:who %} }\n { :n . %Map:{ v:n %} }\n

{ :k [:c] }") + triples = m.materialize(b, OUT) + kinds = {str(t[1]).split("/")[-1]: src for t, src in zip(triples, m.provenance)} + assert kinds["a"].get("structural") is True + assert kinds["p"].get("named") is True and kinds["p"]["variable"] == V + "who" + + +def test_cli_writes_provenance(tmp_path): + (tmp_path / "b.json").write_text(json.dumps({V + "n": {"value": "x"}}), encoding="utf-8") + (tmp_path / "out.shex").write_text(PREFIXES + "start = @:Out\n:Out { :n . %Map:{ v:n %} ; :k [:c] }", encoding="utf-8") + assert main(["-j", str(tmp_path / "b.json"), "-t", str(tmp_path / "out.shex"), "-r", str(OUT), + "-o", str(tmp_path / "out.ttl"), "--provenance", str(tmp_path / "prov.jsonl")]) == 0 + records = [json.loads(line) for line in (tmp_path / "prov.jsonl").read_text(encoding="utf-8").splitlines()] + assert {r["kind"] for r in records} == {"variable", "constant"} + assert next(r for r in records if r["kind"] == "variable")["variable"] == V + "n" diff --git a/tests/test_shexmap/test_shexjs_differential.py b/tests/test_shexmap/test_shexjs_differential.py new file mode 100644 index 0000000..74d78dd --- /dev/null +++ b/tests/test_shexmap/test_shexjs_differential.py @@ -0,0 +1,105 @@ +"""Differential tests against shex.js's ``@shexjs/extension-map``. + +Set ``SHEXJS_EXTENSION_MAP`` to the package directory of a built shex.js checkout +(``…/shex.js/packages/extension-map``) to run them; they are skipped otherwise. Each +manifest example is bound and materialized on both sides and the results compared: + +* the frames of shex.js's bindings equal PyShEx's (the trees themselves are compared, blank + node labels aside, in ``test_examples.py`` against the bindings shex.js recorded); +* shex.js materializes PyShEx's bindings JSON, ``@node`` included, to the expected graph; +* PyShEx materializes shex.js's bindings to the expected graph. +""" +import json +import os +import subprocess +from pathlib import Path + +import pytest +from rdflib import BNode, Graph, Literal +from rdflib.compare import isomorphic + +from pyshex.shexmap import bind, loads, materialize, normalize +from tests.test_shexmap.test_examples import (MANIFEST, expected_output, focus_node, input_graph, label, + node_and_shape, output_root, text) + +SHEXJS = os.environ.get("SHEXJS_EXTENSION_MAP") +DRIVER = Path(__file__).parent / "shexjs_driver.js" +pytestmark = pytest.mark.skipif(not SHEXJS, reason="SHEXJS_EXTENSION_MAP not set") + + +def node_arg(term) -> str: + """A node for the driver: rdflib's blank-node labels mean nothing to n3, so a blank focus is + named "ROOT" (the data's one root) and a blank output root keeps its label (shex.js mints it).""" + return f"_:{term}" if isinstance(term, BNode) else str(term) + + +def focus_arg(term) -> str: + return "ROOT" if isinstance(term, BNode) else str(term) + + +def shexjs(mode: str, *args: str) -> str: + env = {**os.environ, "SHEXJS_TEST_PORTS": os.environ.get("SHEXJS_TEST_PORTS", "11111")} + proc = subprocess.run(["node", str(DRIVER), SHEXJS, mode, *args], capture_output=True, text=True, env=env) + assert proc.returncode == 0, proc.stderr + return proc.stdout + + +def canonical_frames(frames) -> list: + """Frames as comparable JSON: sorted (variable, value) pairs, blank-node values reduced + to a marker (their labels are implementation-specific).""" + def val(v): + if isinstance(v, BNode): + return "_:" + if isinstance(v, Literal): + return {"value": str(v), "type": str(v.datatype) if v.datatype else None, "language": v.language} + return str(v) + return sorted(sorted((k, json.dumps(val(v), sort_keys=True)) for k, v in f.items()) for f in frames) + + +# known divergences, as strict xfails +DIVERGES: dict[str, str] = {} # label -> why the two implementations are expected to differ + + +@pytest.fixture(scope="module", ids=label, + params=[pytest.param(e, marks=pytest.mark.xfail(strict=True, reason=DIVERGES[label(e)])) + if label(e) in DIVERGES else e for e in MANIFEST]) +def example(request, tmp_path_factory): + entry = request.param + d = tmp_path_factory.mktemp("shexjs") + (d / "in.shex").write_text(text(entry, "schema"), encoding="utf-8") + (d / "in.ttl").write_text(text(entry, "data"), encoding="utf-8") + (d / "out.shex").write_text(text(entry, "outputSchema"), encoding="utf-8") + return entry, d + + +def test_shexjs_binds_the_same_frames(example): + entry, d = example + graph = input_graph(entry) + _, start = node_and_shape(entry["queryMap"]) + focus = focus_node(entry, graph) + ours = bind(graph, text(entry, "schema"), focus, start=start) + theirs = loads(shexjs("bind", str(d / "in.shex"), str(d / "in.ttl"), focus_arg(focus), start or "START")) + assert canonical_frames(theirs.frames()) == canonical_frames(ours.frames()) + + +def test_shexjs_materializes_our_bindings(example): + entry, d = example + graph = input_graph(entry) + _, start = node_and_shape(entry["queryMap"]) + ours = bind(graph, text(entry, "schema"), focus_node(entry, graph), start=start) + root, shape = output_root(entry) + (d / "ours.json").write_text(json.dumps(ours.to_json()), encoding="utf-8") + nt = shexjs("materialize", str(d / "out.shex"), str(d / "ours.json"), node_arg(root), shape or "START") + got = Graph().parse(data=nt, format="nt") + assert isomorphic(got, expected_output(entry)), nt + + +def test_we_materialize_shexjs_bindings(example): + entry, d = example + graph = input_graph(entry) + _, start = node_and_shape(entry["queryMap"]) + focus = focus_node(entry, graph) + theirs = loads(shexjs("bind", str(d / "in.shex"), str(d / "in.ttl"), focus_arg(focus), start or "START")) + root, shape = output_root(entry) + got = materialize(text(entry, "outputSchema"), theirs, root, start=shape) + assert isomorphic(got, expected_output(entry)), got.serialize(format="turtle") diff --git a/tests/test_shexmap/test_threaded.py b/tests/test_shexmap/test_threaded.py index a6c2db5..c9955fc 100644 --- a/tests/test_shexmap/test_threaded.py +++ b/tests/test_shexmap/test_threaded.py @@ -1,4 +1,5 @@ -"""The threaded materializer, ported from shex.js's ThreadedMaterializer-test.js.""" +"""The materializer's choices, failures and reports; ported from shex.js's +ThreadedMaterializer-test.js, with the frame-cursor cases restated for iteration scopes.""" import json from pathlib import Path @@ -61,12 +62,17 @@ def test_nested_groups_flatten_keeping_group_bindings_with_their_frames(): {A + "use": lit("home"), A + "tel": lit("+1")}]] -def test_cross_frame_pairing_does_not_beat_in_frame_consumption(): +def test_each_contact_keeps_its_own_use(): m, triples = run(CARD, CONTACTS, "tag:card") assert sorted(str(o) for _, p, o in triples if str(p) == A + "val") == ["+1", "h@x", "w@x"] assert m.chosen.consumed == 7 - mix = [a for a in m.accepts if a.skipped == 4] - assert mix and mix[0].consumed == 3 # frame 0's :use with frame 2's :tel, demoted + pairs = {(str(o), str(v)) for s, p, o in triples if str(p) == A + "use" + for s2, p2, v in triples if s2 == s and str(p2) == A + "val"} + assert pairs == {("work", "w@x"), ("home", "h@x"), ("home", "+1")} + # no alternative pairs a contact's :use with another contact's number + for a in m.accepts: + assert {(str(o), str(v)) for s, p, o in a.triples if str(p) == A + "use" + for s2, p2, v in a.triples if s2 == s and str(p2) == A + "val"} <= pairs def test_constant_only_variants_collapse_onto_one_accept(): diff --git a/tests/test_shexmap/test_update.py b/tests/test_shexmap/test_update.py new file mode 100644 index 0000000..99d4c13 --- /dev/null +++ b/tests/test_shexmap/test_update.py @@ -0,0 +1,106 @@ +"""The triples a schema matched, the strong identity law, and updating a graph in place.""" +import pytest +from rdflib import BNode, Graph, Literal, URIRef +from rdflib.compare import isomorphic + +from pyshex.shexmap import Bindings, MaterializationError, ThreadedMaterializer, bind, materialize +from pyshex.shexmap.cli import main + +PREFIXES = """PREFIX : +PREFIX v: +PREFIX Map: +""" +A = "http://a.example/" +V = "http://v.example/" +P = URIRef(A + "p") +NESTED = PREFIXES + ("start = @:P\n:P { :report @* %Map:{ id(@node) %} }\n" + " { :no . %Map:{ v:no %} ; :result @* %Map:{ id(@node) %} }\n { :sys . %Map:{ v:sys %} }") +DATA = (':p :report :r1, :r2 ; :other "not in the schema" . :r1 :no "one" ; :result [ :sys 100 ], [ :sys 101 ] ; :extra "x" . ' + ':r2 :no "two" .') + + +def turtle(text: str) -> Graph: + return Graph().parse(data="PREFIX : \n" + text, format="turtle") + + +def test_bindings_report_the_triples_the_schema_matched(): + graph = turtle(DATA) + bound = bind(graph, NESTED, P) + assert set(bound.matched) < set(graph) + left_out = {str(p).split("/")[-1] for _, p, _ in set(graph) - set(bound.matched)} + assert left_out == {"other", "extra"} # what the schema does not govern + assert (URIRef(A + "r2"), URIRef(A + "no"), Literal("two")) in bound.matched + + +def test_the_strong_identity_law(): + # materializing a schema's own bindings gives exactly the subgraph it selected + graph = turtle(DATA) + bound = bind(graph, NESTED, P) + selected = Graph() + for t in bound.matched: + selected.add(t) + assert isomorphic(materialize(NESTED, bound, P), selected) + + +OUT = PREFIXES + "start = @:Out\n:Out { :who . %Map:{ v:name %} ; :bp @* }\n { :sys . %Map:{ v:sys %} }" +ROOT = URIRef(A + "out") + + +def readings(graph: Graph) -> set: + return {str(o) for _, _, o in graph.triples((None, URIRef(A + "sys"), None))} + + +def test_update_replaces_what_the_schema_holds_and_keeps_the_rest(): + graph = turtle(':out :unrelated "kept" . :elsewhere :who "someone else" .') + first = Bindings([{V + "name": Literal("Sue")}, [{V + "sys": Literal("110")}, {V + "sys": Literal("111")}]]) + m = ThreadedMaterializer(OUT) + added, removed = m.update(graph, first, ROOT) + assert removed == set() and len(added) == 1 + 2 + 2 + assert readings(graph) == {"110", "111"} + second = Bindings([{V + "name": Literal("Susan")}, [{V + "sys": Literal("110")}, {V + "sys": Literal("112")}]]) + added, removed = m.update(graph, second, ROOT) + assert {str(o) for _, _, o in removed if isinstance(o, Literal)} == {"Sue", "111"} + assert len(removed) == 2 # the old name and the old value; the reading's node, minted by position, stays + assert readings(graph) == {"110", "112"} + assert (ROOT, URIRef(A + "who"), Literal("Susan")) in graph and (ROOT, URIRef(A + "who"), Literal("Sue")) not in graph + assert (ROOT, URIRef(A + "unrelated"), Literal("kept")) in graph # not governed: kept + assert (URIRef(A + "elsewhere"), URIRef(A + "who"), Literal("someone else")) in graph # another root: kept + fresh = materialize(OUT, second, ROOT) + fresh.add((ROOT, URIRef(A + "unrelated"), Literal("kept"))) + fresh.add((URIRef(A + "elsewhere"), URIRef(A + "who"), Literal("someone else"))) + assert isomorphic(graph, fresh) + + +def test_an_unchanged_input_changes_nothing(): + graph = Graph() + b = Bindings([{V + "name": Literal("Sue")}, [{V + "sys": Literal("110")}, {V + "sys": Literal("111")}]]) + materialize(OUT, b, ROOT, graph=graph) + before = set(graph) + added, removed = ThreadedMaterializer(OUT).update(graph, b, ROOT) + assert added == set() and removed == set() and set(graph) == before + + +def test_materialize_replace_is_update(): + graph = Graph() + materialize(OUT, Bindings([{V + "name": Literal("Sue")}, [{V + "sys": Literal("110")}]]), ROOT, graph=graph) + materialize(OUT, Bindings([{V + "name": Literal("Sue")}, [{V + "sys": Literal("120")}]]), ROOT, graph=graph, replace=True) + assert readings(graph) == {"120"} + materialize(OUT, Bindings([{V + "name": Literal("Sue")}, [{V + "sys": Literal("130")}]]), ROOT, graph=graph) + assert readings(graph) == {"120", "130"} # without replace: added to + + +def test_a_root_that_does_not_conform_is_refused(): + graph = turtle(':out :who "Sue" , "Also Sue" .') # two names where the schema wants one + with pytest.raises(MaterializationError, match="does not conform"): + ThreadedMaterializer(OUT).update(graph, Bindings([{V + "name": Literal("Sue")}, []]), ROOT) + + +def test_cli_into(tmp_path): + existing = tmp_path / "existing.ttl" + existing.write_text('@prefix : .\n:out :unrelated "kept" ; :who "Old" .\n', encoding="utf-8") + (tmp_path / "b.json").write_text('[{"http://v.example/name": {"value": "New"}}, []]', encoding="utf-8") + (tmp_path / "out.shex").write_text(OUT, encoding="utf-8") + assert main(["-j", str(tmp_path / "b.json"), "-t", str(tmp_path / "out.shex"), "-r", str(ROOT), "--into", str(existing)]) == 0 + g = Graph().parse(existing, format="turtle") + assert (ROOT, URIRef(A + "who"), Literal("New")) in g and (ROOT, URIRef(A + "who"), Literal("Old")) not in g + assert (ROOT, URIRef(A + "unrelated"), Literal("kept")) in g diff --git a/tests/test_shexmap/test_values.py b/tests/test_shexmap/test_values.py new file mode 100644 index 0000000..c3db132 --- /dev/null +++ b/tests/test_shexmap/test_values.py @@ -0,0 +1,92 @@ +"""Bound values against the output constraint's value expression: checked, retyped when a +plain literal's lexical form fits the datatype, otherwise failed like an unbound variable.""" +import pytest +from rdflib import Graph, Literal, URIRef, XSD + +from pyshex import ShExEvaluator +from pyshex.shexmap import Bindings, MaterializationError, ThreadedMaterializer + +PREFIXES = """PREFIX : +PREFIX xsd: +PREFIX v: +PREFIX Map: +""" +V = "http://v.example/" +A = "http://a.example/" +OUT = URIRef(A + "out") + + +def run(schema: str, **bindings): + m = ThreadedMaterializer(PREFIXES + "start = @:Out\n" + schema) + return m, m.materialize(Bindings({V + k: v for k, v in bindings.items()}), OUT) + + +def objects(triples, local: str): + return [o for _, p, o in triples if str(p) == A + local] + + +def test_a_plain_literal_is_retyped_to_the_constraint_datatype(): + _, triples = run(":Out { :n xsd:integer %Map:{ v:t %} }", t=Literal("42")) + assert objects(triples, "n") == [Literal("42", datatype=XSD.integer)] + + +def test_a_typed_literal_that_satisfies_passes_unchanged(): + _, triples = run(":Out { :n xsd:integer %Map:{ v:t %} }", t=Literal("42", datatype=XSD.integer)) + assert objects(triples, "n") == [Literal("42", datatype=XSD.integer)] + + +def test_a_value_that_does_not_fit_fails_the_constraint(): + m = ThreadedMaterializer(PREFIXES + "start = @:Out\n:Out { :n xsd:integer %Map:{ v:t %} }") + with pytest.raises(MaterializationError, match="does not satisfy"): + m.materialize(Bindings({V + "t": Literal("not a number")}), OUT) + m = ThreadedMaterializer(PREFIXES + "start = @:Out\n:Out { :n xsd:integer %Map:{ v:t %} }") + with pytest.raises(MaterializationError): # a typed literal is not retyped + m.materialize(Bindings({V + "t": Literal("1", datatype=XSD.decimal)}), OUT) + + +def test_an_optional_constraint_is_left_out_and_a_disjunct_falls_through(): + _, triples = run(":Out { :n xsd:integer ? %Map:{ v:t %} ; :s . %Map:{ v:t %} }", t=Literal("abc")) + assert objects(triples, "n") == [] and objects(triples, "s") == [Literal("abc")] + _, triples = run(":Out { :n xsd:integer %Map:{ v:t %} | :s xsd:string %Map:{ v:t %} }", t=Literal("abc")) + assert objects(triples, "n") == [] and objects(triples, "s") == [Literal("abc")] + _, triples = run(":Out { :n xsd:integer %Map:{ v:t %} | :s xsd:string %Map:{ v:t %} }", t=Literal("7")) + assert objects(triples, "n") == [Literal("7", datatype=XSD.integer)] + + +def test_value_sets_node_kinds_facets_and_references(): + _, triples = run(":Out { :k [:x :y] %Map:{ v:t %} }", t=URIRef(A + "x")) + assert objects(triples, "k") == [URIRef(A + "x")] + with pytest.raises(MaterializationError): + run(":Out { :k [:x :y] %Map:{ v:t %} }", t=URIRef(A + "z")) + with pytest.raises(MaterializationError): + run(":Out { :k IRI %Map:{ v:t %} }", t=Literal("not an iri")) + with pytest.raises(MaterializationError): + run(":Out { :s xsd:string MINLENGTH 3 %Map:{ v:t %} }", t=Literal("ab")) + _, triples = run(":Out { :n @ %Map:{ v:t %} }\n xsd:integer", t=Literal("5")) + assert objects(triples, "n") == [Literal("5", datatype=XSD.integer)] + _, triples = run(":Out { :n xsd:integer AND MININCLUSIVE 0 %Map:{ v:t %} }", t=Literal("5")) + assert objects(triples, "n") == [Literal("5", datatype=XSD.integer)] + with pytest.raises(MaterializationError): + run(":Out { :n xsd:integer AND MININCLUSIVE 0 %Map:{ v:t %} }", t=Literal("-5")) + + +def test_function_results_take_the_constraint_type(): + _, triples = run(':Out { :code xsd:integer %Map:{ hashmap(v:status, {"1": "Married", "2": "Single"}) %} }', + status=Literal("Single")) + assert objects(triples, "code") == [Literal("2", datatype=XSD.integer)] + + +def test_the_output_conforms_to_the_output_schema(): + schema = ":Out { :n xsd:integer %Map:{ v:t %} ; :k [:x] %Map:{ v:k %} }" + _, triples = run(schema, t=Literal("100"), k=URIRef(A + "x")) + g = Graph() + for t in triples: + g.add(t) + result = ShExEvaluator(rdf=g, schema=PREFIXES + "start = @:Out\n" + schema.replace("%Map:{ v:t %}", "").replace("%Map:{ v:k %}", ""), + focus=OUT).evaluate() + assert result[0].result, result[0].reason + + +def test_the_check_can_be_turned_off(): + m = ThreadedMaterializer(PREFIXES + "start = @:Out\n:Out { :n xsd:integer %Map:{ v:t %} }", check_values=False) + assert objects(m.materialize(Bindings({V + "t": Literal("not a number")}), OUT), "n") == [Literal("not a number")]