diff --git a/docs/conversion.md b/docs/conversion.md index 63c43c1..ea081a2 100644 --- a/docs/conversion.md +++ b/docs/conversion.md @@ -436,6 +436,15 @@ The vocabulary's `vcfc:missingValuePolicy` says a missing token should be reports a plain `"."` literal as an anomaly, and `--strict-conformance` promotes that from a report to a failure. +Not every `"."` is a missing value, and the check excludes the three places the +vocabulary requires a bare dot: `vcfc:fieldNumber` (`Number=.` is VCF's +variable-cardinality arity), `vcfc:genotypeString` (a fully missing call is +literally `.` or `./.`), and the `vcfc:attributeValue` of a header attribute +whose `vcfc:attributeKey` is `Number` — the structured-header layer carries each +declaration's attributes verbatim, so `Number=.` appears there too, and the +shapes require that value to be `xsd:string`. A bare dot anywhere else, including +on any other header attribute, is still reported. + This used to conflict with the published SHACL shapes, which constrained `vcfc:alt` to `sh:datatype xsd:string` and so rejected every REF-only and gVCF-style record. **The vocabulary fixed that**: `vcfc:VCFRecordShape` now diff --git a/src/validation/queries/common/preflight_missing_token_conformance.rq b/src/validation/queries/common/preflight_missing_token_conformance.rq index f06b38a..695248d 100644 --- a/src/validation/queries/common/preflight_missing_token_conformance.rq +++ b/src/validation/queries/common/preflight_missing_token_conformance.rq @@ -16,11 +16,22 @@ PREFIX vcfc: # header as a conformance failure. On the 100,000-record HG005 benchmark cell # it returned 20 such rows, and all of them were declarations the vocabulary # requires to look exactly that way. +# +# The same token appears once more, in the structured-header attribute layer: +# every declaration's attributes are also carried verbatim as +# vcfc:HeaderAttribute resources, so Number=. becomes an attributeValue of "." +# on the attribute whose attributeKey is "Number". The shapes require that +# value to be exactly xsd:string, so it can never be vcfc:Null. With only the +# two predicates above excluded, the HG005 cell still returned ten rows -- one +# per Number=. INFO declaration in its header. Only the Number attribute is +# excluded; a bare "." on any other attribute is still reported. SELECT ?s ?p ?o (DATATYPE(?o) AS ?datatype) WHERE { ?s ?p ?o . FILTER(ISLITERAL(?o) && STR(?o) = ".") FILTER(DATATYPE(?o) != vcfc:Null) FILTER(?p NOT IN (vcfc:fieldNumber, vcfc:genotypeString)) + OPTIONAL { ?s vcfc:attributeKey ?attributeKey } + FILTER(?p != vcfc:attributeValue || !BOUND(?attributeKey) || STR(?attributeKey) != "Number") } LIMIT 100 diff --git a/src/validation/queries/common/preflight_missing_token_conformance_count.rq b/src/validation/queries/common/preflight_missing_token_conformance_count.rq index f527d71..10d747d 100644 --- a/src/validation/queries/common/preflight_missing_token_conformance_count.rq +++ b/src/validation/queries/common/preflight_missing_token_conformance_count.rq @@ -1,12 +1,15 @@ PREFIX vcfc: # Exact number of missing tokens serialized as a plain "." literal. -# The predicate exclusions must match preflight_missing_token_conformance.rq -# exactly, or the count and its sample describe different populations. +# The exclusions -- the two predicates and the header Number attribute -- must +# match preflight_missing_token_conformance.rq exactly, or the count and its +# sample describe different populations. SELECT (COUNT(*) AS ?anomalyCount) WHERE { ?s ?p ?o . FILTER(ISLITERAL(?o) && STR(?o) = ".") FILTER(DATATYPE(?o) != vcfc:Null) FILTER(?p NOT IN (vcfc:fieldNumber, vcfc:genotypeString)) + OPTIONAL { ?s vcfc:attributeKey ?attributeKey } + FILTER(?p != vcfc:attributeValue || !BOUND(?attributeKey) || STR(?attributeKey) != "Number") } diff --git a/test/test_missing_token_scope_unit.py b/test/test_missing_token_scope_unit.py new file mode 100644 index 0000000..5344d77 --- /dev/null +++ b/test/test_missing_token_scope_unit.py @@ -0,0 +1,140 @@ +"""The missing-token check must not report a header's own Number=. declaration. + +``preflight_missing_token_conformance`` looks for bare "." literals that should +have been typed ``vcfc:Null``. It already excluded ``vcfc:fieldNumber`` and +``vcfc:genotypeString``, where a dot is conformant. The same token also reaches +the graph a third way: the structured-header attribute layer carries every +declaration's attributes verbatim, so ``Number=.`` becomes a +``vcfc:attributeValue`` of "." on the attribute whose key is "Number". The +shapes require that value to be exactly ``xsd:string``, so it can never be +``vcfc:Null``. + +On the v3.1.0 benchmark the narrowed check still returned ten rows on the +100,000-record HG005 cell -- one per ``Number=.`` INFO declaration in its +header. These tests run both forms of the query against a graph produced by the +real header emitter, so the exclusion is checked end to end rather than by +string inspection alone. +""" + +import tempfile +import unittest +from pathlib import Path + +# The behavioural tests need rdflib to evaluate SPARQL; the rest of the suite +# runs without it, so they skip rather than erroring discovery -- the same +# pattern test_linking_unit.py and test_validation_mutation_unit.py use. +try: + import rdflib +except ModuleNotFoundError: # pragma: no cover - exercised only without rdflib + rdflib = None + +import vcf_rdfizer +from test.helpers import VerboseTestCase + +VCFC = "https://w3id.org/vcf-core/vocab#" +QUERY_DIR = ( + Path(vcf_rdfizer.__file__).resolve().parent + / "src" / "validation" / "queries" / "common" +) +SAMPLE_QUERY = QUERY_DIR / "preflight_missing_token_conformance.rq" +COUNT_QUERY = QUERY_DIR / "preflight_missing_token_conformance_count.rq" + +HEADERS_TSV = ( + "SOURCE_FILE\tHEADER_INDEX\tHEADER_KEY\tHEADER_VALUE\tRAW_LINE\n" + "s.vcf\t1\tfileformat\tVCFv4.2\tx\n" + "s.vcf\t2\tINFO\t\tx\n" + "s.vcf\t3\tINFO\t\tx\n" + "s.vcf\t4\tFORMAT\t\tx\n" +) + + +def header_graph(tmp_path: Path) -> "rdflib.Graph": + """Emit the structured header layer exactly as a conversion would.""" + headers_tsv = tmp_path / "s.header_lines.tsv" + headers_tsv.write_text(HEADERS_TSV, encoding="utf-8") + rdf_path = tmp_path / "s.nt" + rdf_path.write_text("", encoding="utf-8") + vcf_rdfizer.append_header_representation_rdf(headers_tsv, rdf_path) + graph = rdflib.Graph() + graph.parse(rdf_path, format="nt") + return graph + + +def sample_rows(graph) -> list: + return list(graph.query(SAMPLE_QUERY.read_text(encoding="utf-8"))) + + +def anomaly_count(graph) -> int: + rows = list(graph.query(COUNT_QUERY.read_text(encoding="utf-8"))) + return int(rows[0][0]) + + +@unittest.skipIf(rdflib is None, "rdflib is required to evaluate the queries") +class HeaderNumberAttributeTests(VerboseTestCase): + """The header's Number=. is a declaration, not a missing value.""" + + def test_the_emitter_really_writes_the_dot_this_test_is_about(self): + """Guard the premise: without it the other tests would pass vacuously.""" + with tempfile.TemporaryDirectory() as td: + graph = header_graph(Path(td)) + number_values = { + str(value) + for attribute, _, key in graph.triples( + (None, rdflib.URIRef(VCFC + "attributeKey"), None)) + if str(key) == "Number" + for value in graph.objects(attribute, rdflib.URIRef(VCFC + "attributeValue")) + } + self.assertIn(".", number_values) + + def test_number_dot_declarations_are_not_reported(self): + with tempfile.TemporaryDirectory() as td: + graph = header_graph(Path(td)) + self.assertEqual(sample_rows(graph), []) + self.assertEqual(anomaly_count(graph), 0) + + def test_a_bare_dot_elsewhere_is_still_reported(self): + """The exclusion is the Number attribute only, not the whole layer.""" + with tempfile.TemporaryDirectory() as td: + graph = header_graph(Path(td)) + # A genuinely untyped missing value on a record field ... + graph.add(( + rdflib.URIRef("file://s.vcf#record/1"), + rdflib.URIRef(VCFC + "recordId"), + rdflib.Literal("."), + )) + # ... and a dot on a header attribute that is not Number. + attribute = rdflib.URIRef("file://s.vcf#header/line/3/attribute/9") + graph.add((attribute, rdflib.URIRef(VCFC + "attributeKey"), rdflib.Literal("Source"))) + graph.add((attribute, rdflib.URIRef(VCFC + "attributeValue"), rdflib.Literal("."))) + + reported = {(str(row[0]), str(row[1])) for row in sample_rows(graph)} + self.assertEqual(reported, { + ("file://s.vcf#record/1", VCFC + "recordId"), + (str(attribute), VCFC + "attributeValue"), + }) + self.assertEqual(anomaly_count(graph), 2) + + +class ExclusionTextTests(VerboseTestCase): + """Static guarantees that hold without rdflib.""" + + def test_both_queries_exclude_only_the_number_attribute(self): + for path in (SAMPLE_QUERY, COUNT_QUERY): + text = path.read_text(encoding="utf-8") + self.assertIn('STR(?attributeKey) != "Number"', text, path.name) + self.assertIn("OPTIONAL { ?s vcfc:attributeKey ?attributeKey }", text, path.name) + + def test_the_bundled_shapes_still_require_attribute_values_as_strings(self): + """If a shape ever allowed vcfc:Null here, the exclusion must be revisited.""" + shapes = ( + Path(vcf_rdfizer.__file__).resolve().parent + / "vcf_rdfizer_data" / "shacl" / "vcf-core-vocabulary.shacl.ttl" + ).read_text(encoding="utf-8") + self.assertIn( + "sh:path vcfc:attributeValue ; sh:minCount 1 ; sh:maxCount 1 ; sh:datatype xsd:string", + shapes, + ) + + +if __name__ == "__main__": + unittest.main()