diff --git a/README.md b/README.md index 79e180a..efc43d3 100644 --- a/README.md +++ b/README.md @@ -284,6 +284,8 @@ parse = CoordinateParse.model_validate_json(path.read_bytes()) **Reference properties.** A string property that holds another entity's `local_id` carries `"x-reference": ""` on the property itself, beside its `type`; for a multivalued slot that property is the array, not its `items`. The keyword is set from the LinkML induced slots, so `is_a` and `slot_usage` apply, and only where the range class has an identifier and the slot is not inlined. JSON Schema validators ignore it, and `study_schema.jsonschema.references(name)` returns the same relationships as `Reference(cls, slot, target, multivalued)` named tuples, sorted by class and slot. +**Unique keys.** A class with a LinkML `unique_keys` carries `"x-unique-keys": [[slot, ...], ...]` on its definition, its own keys first and then inherited ones with each key's slots in declared order, and `study_schema.jsonschema.unique_keys(name)` returns them as a dict from class to tuples of slot names. + Every model forbids fields its schema does not declare, so a file written against another version fails where it is read, naming the field, instead of losing what the reader does not recognise. Each model module records the schema version it was generated from as `version`, diff --git a/src/study_schema/jsonschema/__init__.py b/src/study_schema/jsonschema/__init__.py index 634099e..37c9cb5 100644 --- a/src/study_schema/jsonschema/__init__.py +++ b/src/study_schema/jsonschema/__init__.py @@ -46,4 +46,15 @@ def references(name: str) -> list[Reference]: return sorted(rows, key=lambda row: (row.cls, row.slot)) -__all__ = ["NAMES", "Reference", "load", "references"] +def unique_keys(name: str) -> dict[str, tuple[tuple[str, ...], ...]]: + """The unique keys of each class that declares one, sorted by class; keys are tuples.""" + return dict( + sorted( + (cls, tuple(tuple(key) for key in definition["x-unique-keys"])) + for cls, definition in load(name)["$defs"].items() + if "x-unique-keys" in definition + ) + ) + + +__all__ = ["NAMES", "Reference", "load", "references", "unique_keys"] diff --git a/src/study_schema/jsonschema/storage-study.schema.json b/src/study_schema/jsonschema/storage-study.schema.json index d166544..25671a2 100644 --- a/src/study_schema/jsonschema/storage-study.schema.json +++ b/src/study_schema/jsonschema/storage-study.schema.json @@ -412,7 +412,12 @@ "group" ], "title": "AnalysisGroup", - "type": "object" + "type": "object", + "x-unique-keys": [ + [ + "group" + ] + ] }, "AnalysisOutcome": { "description": "Whether the tested effect produced any finding at the threshold the source applied. Closed: the two are the only answers, and a source that does not say takes `not_reported` rather than a third value.", @@ -662,7 +667,13 @@ "direction" ], "title": "Cell", - "type": "object" + "type": "object", + "x-unique-keys": [ + [ + "term", + "level" + ] + ] }, "Condition": { "additionalProperties": false, @@ -1635,7 +1646,12 @@ "level" ], "title": "FactorLevel", - "type": "object" + "type": "object", + "x-unique-keys": [ + [ + "level" + ] + ] }, "FunctionalForm": { "description": "Shape of a term's relationship to the measure. A squared term with a negative coefficient is an inverted-U relationship, not a negative association, so non-linear and transformed terms must not be reported as linear. Open vocabulary: write the source's own wording in the field when nothing here fits.", @@ -2864,7 +2880,12 @@ "type" ], "title": "ModelTerm", - "type": "object" + "type": "object", + "x-unique-keys": [ + [ + "name" + ] + ] }, "NotStructurableDetails": { "additionalProperties": false, diff --git a/tests/conftest.py b/tests/conftest.py index b5d6bf9..2ff2673 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,6 +1,8 @@ from __future__ import annotations +import importlib.util import json +import sys from pathlib import Path import pytest @@ -22,5 +24,28 @@ def example_paths() -> dict[str, Path]: } +@pytest.fixture +def generator(): + path = ROOT / "tools" / "generate_models.py" + spec = importlib.util.spec_from_file_location("generate_models", path) + module = importlib.util.module_from_spec(spec) + sys.modules["generate_models"] = module + spec.loader.exec_module(module) + return module + + +def annotated_document(tmp_path: Path, yaml_text: str, top_class: str): + """The JSON Schema for `top_class` from a tiny LinkML schema, with its SchemaView.""" + from linkml.generators.jsonschemagen import JsonSchemaGenerator + from linkml_runtime.utils.schemaview import SchemaView + + path = tmp_path / "tiny.yaml" + path.write_text(yaml_text, "utf-8") + view = SchemaView(str(path)) + view.merge_imports() + document = json.loads(JsonSchemaGenerator(view.schema, top_class=top_class).serialize()) + return document, view + + def load(path: Path) -> dict: return json.loads(path.read_text("utf-8")) diff --git a/tests/test_annotate_references.py b/tests/test_annotate_references.py index adbfb40..16e6bbd 100644 --- a/tests/test_annotate_references.py +++ b/tests/test_annotate_references.py @@ -1,12 +1,8 @@ """The generator marks a property as a reference by the rules of its induced LinkML slot.""" -import importlib.util -import json -import sys - import pytest -from conftest import ROOT +from conftest import annotated_document pytest.importorskip("linkml", reason="regeneration needs the `generate` extra") @@ -55,25 +51,8 @@ """ -@pytest.fixture -def generator(): - path = ROOT / "tools" / "generate_models.py" - spec = importlib.util.spec_from_file_location("generate_models", path) - module = importlib.util.module_from_spec(spec) - sys.modules["generate_models"] = module - spec.loader.exec_module(module) - return module - - def test_annotate_references_follows_the_linkml_rules(tmp_path, generator): - from linkml.generators.jsonschemagen import JsonSchemaGenerator - from linkml_runtime.utils.schemaview import SchemaView - - path = tmp_path / "tiny.yaml" - path.write_text(TINY_SCHEMA, "utf-8") - view = SchemaView(str(path)) - view.merge_imports() - document = json.loads(JsonSchemaGenerator(view.schema, top_class="Study").serialize()) + document, view = annotated_document(tmp_path, TINY_SCHEMA, "Study") generator._annotate_references(document, view) diff --git a/tests/test_annotate_unique_keys.py b/tests/test_annotate_unique_keys.py new file mode 100644 index 0000000..acd5cfd --- /dev/null +++ b/tests/test_annotate_unique_keys.py @@ -0,0 +1,90 @@ +"""The generator writes a class's LinkML unique keys, its own and inherited, onto its definition.""" + +import pytest + +from conftest import annotated_document + +pytest.importorskip("linkml", reason="regeneration needs the `generate` extra") + +TINY_SCHEMA = """\ +id: https://example.org/tiny +name: tiny +default_prefix: tiny +prefixes: + tiny: https://example.org/tiny/ + linkml: https://w3id.org/linkml/ +default_range: string +classes: + Device: + unique_keys: + by_serial: + unique_key_slots: + - serial + attributes: + serial: + model: + Sub: + is_a: Device + unique_keys: + by_extra: + unique_key_slots: + - extra + attributes: + extra: + # Same key name as Device, so keys are deduplicated by name and the subclass wins. + Redeclared: + is_a: Device + unique_keys: + by_serial: + unique_key_slots: + - model + attributes: + model: + Stamp: + unique_keys: + by_stamp: + unique_key_slots: + - stamp + attributes: + stamp: + Mixed: + mixins: + - Stamp + attributes: + a: + Pair: + unique_keys: + by_b_then_a: + unique_key_slots: + - b + - a + by_a: + unique_key_slots: + - a + attributes: + a: + b: + Plain: + attributes: + a: +""" + + +def test_annotate_unique_keys_writes_own_and_inherited_keys(tmp_path, generator): + document, view = annotated_document(tmp_path, TINY_SCHEMA, "Device") + + generator._annotate_unique_keys(document, view) + + assert { + cls: definition.get("x-unique-keys") + for cls, definition in document["$defs"].items() + if cls in {"Device", "Sub", "Pair", "Plain", "Redeclared", "Stamp", "Mixed"} + } == { + "Device": [["serial"]], + "Sub": [["extra"], ["serial"]], + "Redeclared": [["model"]], + "Stamp": [["stamp"]], + "Mixed": [["stamp"]], + "Pair": [["b", "a"], ["a"]], + "Plain": None, + } diff --git a/tests/test_generated.py b/tests/test_generated.py index 2cf38ff..ee7dd5d 100644 --- a/tests/test_generated.py +++ b/tests/test_generated.py @@ -1,18 +1,9 @@ """The committed models and JSON Schemas are what the YAML generates today.""" -import importlib.util -import sys - import pytest -from conftest import ROOT - pytest.importorskip("linkml", reason="regeneration needs the `generate` extra") -def test_generated_files_are_current(capsys): - spec = importlib.util.spec_from_file_location("generate_models", ROOT / "tools" / "generate_models.py") - module = importlib.util.module_from_spec(spec) - sys.modules["generate_models"] = module - spec.loader.exec_module(module) - assert module.main(["--check"]) == 0, capsys.readouterr().out +def test_generated_files_are_current(generator, capsys): + assert generator.main(["--check"]) == 0, capsys.readouterr().out diff --git a/tests/test_unique_keys.py b/tests/test_unique_keys.py new file mode 100644 index 0000000..81fa41e --- /dev/null +++ b/tests/test_unique_keys.py @@ -0,0 +1,47 @@ +"""The JSON Schemas say which slots together identify one item of a class.""" + +import pytest + +from study_schema import jsonschema as js + +# A golden check of the YAML, not a sync with neurostore: these pins are neurostore's +# DECLARED_KEYS in neurostore.study_schema_tools on its review branch. Each key matches a +# unique_keys declaration in neuroimaging-study-storage/analysis.yaml, with its slots in +# declared order. +NEUROSTORE_DECLARED_KEYS = { + "Cell": (("term", "level"),), + "AnalysisGroup": (("group",),), + "ModelTerm": (("name",),), + "FactorLevel": (("level",),), +} + + +def test_storage_study_unique_keys_are_neurostore_declared_keys(): + assert js.unique_keys("storage-study") == NEUROSTORE_DECLARED_KEYS + + +def test_unique_keys_are_sorted_by_class_with_keys_as_tuples(monkeypatch): + document = { + "$defs": { + "Zed": {"x-unique-keys": [["b", "a"], ["c"]]}, + "Keyless": {"properties": {}}, + "Alpha": {"x-unique-keys": [["d"]]}, + } + } + monkeypatch.setattr(js, "load", lambda name: document) + result = js.unique_keys("storage-study") + assert list(result) == ["Alpha", "Zed"] + assert result == {"Alpha": (("d",),), "Zed": (("b", "a"), ("c",))} + + +def test_every_unique_key_slot_is_a_property_of_its_class(): + keyed = {name: js.unique_keys(name) for name in js.NAMES} + assert any(keyed.values()) + for name, classes in keyed.items(): + schema = js.load(name) + for cls, keys in classes.items(): + properties = schema["$defs"][cls]["properties"] + for key in keys: + assert key, f"{name}: {cls} has an empty key" + for slot in key: + assert slot in properties, f"{name}: {cls}.{slot}" diff --git a/tools/generate_models.py b/tools/generate_models.py index 40cadd0..a85a363 100644 --- a/tools/generate_models.py +++ b/tools/generate_models.py @@ -66,6 +66,15 @@ def _view(contract: Contract): return view +def _class_definitions(document: dict, view): + """Each class's `$defs` entry as `(cls, definition)`.""" + classes = view.all_classes() + for cls, definition in document["$defs"].items(): + # $defs also holds enums, which are not classes. + if cls in classes: + yield cls, definition + + def _annotate_references(document: dict, view) -> None: """Mark each property that holds another entity's identifier with its target class. @@ -74,10 +83,9 @@ def _annotate_references(document: dict, view) -> None: """ classes = view.all_classes() entities = {name for name in classes if view.get_identifier_slot(name)} - for cls, definition in document["$defs"].items(): + for cls, definition in _class_definitions(document, view): properties = definition.get("properties") - # $defs also holds enums, which are not classes. - if not properties or cls not in classes: + if not properties: continue for slot in view.class_induced_slots(cls): if ( @@ -88,6 +96,22 @@ def _annotate_references(document: dict, view) -> None: properties[slot.name]["x-reference"] = slot.range +def _annotate_unique_keys(document: dict, view) -> None: + """Write each class's unique keys, its own first and then inherited, as `x-unique-keys`. + + A key is a list of slot names in the order the YAML declares them. Keys are deduplicated + by name, and the nearest declaration of a name wins, so a subclass's key replaces an + ancestor's key of the same name. + """ + for cls, definition in _class_definitions(document, view): + keys: dict[str, list[str]] = {} + for ancestor in view.class_ancestors(cls): + for key in view.get_class(ancestor).unique_keys.values(): + keys.setdefault(key.unique_key_name, list(key.unique_key_slots)) + if keys: + definition["x-unique-keys"] = list(keys.values()) + + def render() -> dict[Path, str]: """Every generated file, path -> contents.""" from linkml.generators.jsonschemagen import JsonSchemaGenerator @@ -105,6 +129,7 @@ def render() -> dict[Path, str]: JsonSchemaGenerator(view.schema, top_class=top).serialize() ) _annotate_references(document, view) + _annotate_unique_keys(document, view) out[PACKAGE / "jsonschema" / f"{name}.schema.json"] = ( json.dumps(document, indent=2, sort_keys=True, ensure_ascii=False) + "\n" )