From f43ec15c8d7fe0b915471bba560ef962a547c5bc Mon Sep 17 00:00:00 2001 From: ecrum19 Date: Thu, 24 Sep 2026 16:20:06 +0200 Subject: [PATCH] Stop the missing-token check reporting a header's own Number=. preflight_missing_token_conformance looks for bare "." literals that should have been typed vcfc:Null. It already excluded vcfc:fieldNumber and vcfc:genotypeString, where the vocabulary requires a dot. The same token reaches the graph a third way: the structured-header layer carries every declaration's attributes verbatim as vcfc:HeaderAttribute resources, so Number=. becomes an attributeValue of "." on the attribute keyed "Number", and the core shapes require that value to be exactly xsd:string. On the v3.1.0 benchmark the narrowed check still returned ten rows on the 100,000-record HG005 cell, one per Number=. INFO declaration in its header, and reported them as EXPECTED_CONFORMANCE_FAILURE on every validation of that input. Exclude only the Number attribute's value, in both the sample query and its count, so a bare dot on any other attribute or field is still reported. It is written with OPTIONAL and BOUND rather than NOT EXISTS because those are already used by queries that pass on all four validation engines. test_missing_token_scope_unit.py runs both queries with rdflib over a graph from the real header emitter: Number=. declarations are not reported, while a bare dot on a record field and on a non-Number attribute still are. The behavioural tests fail against the previous queries. Co-Authored-By: Claude Opus 5.5 --- docs/conversion.md | 9 ++ .../preflight_missing_token_conformance.rq | 11 ++ ...eflight_missing_token_conformance_count.rq | 7 +- test/test_missing_token_scope_unit.py | 140 ++++++++++++++++++ 4 files changed, 165 insertions(+), 2 deletions(-) create mode 100644 test/test_missing_token_scope_unit.py diff --git a/docs/conversion.md b/docs/conversion.md index 63c43c1..ea081a2 100644 --- a/docs/conversion.md +++ b/docs/conversion.md @@ -436,6 +436,15 @@ The vocabulary's `vcfc:missingValuePolicy` says a missing token should be reports a plain `"."` literal as an anomaly, and `--strict-conformance` promotes that from a report to a failure. +Not every `"."` is a missing value, and the check excludes the three places the +vocabulary requires a bare dot: `vcfc:fieldNumber` (`Number=.` is VCF's +variable-cardinality arity), `vcfc:genotypeString` (a fully missing call is +literally `.` or `./.`), and the `vcfc:attributeValue` of a header attribute +whose `vcfc:attributeKey` is `Number` — the structured-header layer carries each +declaration's attributes verbatim, so `Number=.` appears there too, and the +shapes require that value to be `xsd:string`. A bare dot anywhere else, including +on any other header attribute, is still reported. + This used to conflict with the published SHACL shapes, which constrained `vcfc:alt` to `sh:datatype xsd:string` and so rejected every REF-only and gVCF-style record. **The vocabulary fixed that**: `vcfc:VCFRecordShape` now diff --git a/src/validation/queries/common/preflight_missing_token_conformance.rq b/src/validation/queries/common/preflight_missing_token_conformance.rq index f06b38a..695248d 100644 --- a/src/validation/queries/common/preflight_missing_token_conformance.rq +++ b/src/validation/queries/common/preflight_missing_token_conformance.rq @@ -16,11 +16,22 @@ PREFIX vcfc: # header as a conformance failure. On the 100,000-record HG005 benchmark cell # it returned 20 such rows, and all of them were declarations the vocabulary # requires to look exactly that way. +# +# The same token appears once more, in the structured-header attribute layer: +# every declaration's attributes are also carried verbatim as +# vcfc:HeaderAttribute resources, so Number=. becomes an attributeValue of "." +# on the attribute whose attributeKey is "Number". The shapes require that +# value to be exactly xsd:string, so it can never be vcfc:Null. With only the +# two predicates above excluded, the HG005 cell still returned ten rows -- one +# per Number=. INFO declaration in its header. Only the Number attribute is +# excluded; a bare "." on any other attribute is still reported. SELECT ?s ?p ?o (DATATYPE(?o) AS ?datatype) WHERE { ?s ?p ?o . FILTER(ISLITERAL(?o) && STR(?o) = ".") FILTER(DATATYPE(?o) != vcfc:Null) FILTER(?p NOT IN (vcfc:fieldNumber, vcfc:genotypeString)) + OPTIONAL { ?s vcfc:attributeKey ?attributeKey } + FILTER(?p != vcfc:attributeValue || !BOUND(?attributeKey) || STR(?attributeKey) != "Number") } LIMIT 100 diff --git a/src/validation/queries/common/preflight_missing_token_conformance_count.rq b/src/validation/queries/common/preflight_missing_token_conformance_count.rq index f527d71..10d747d 100644 --- a/src/validation/queries/common/preflight_missing_token_conformance_count.rq +++ b/src/validation/queries/common/preflight_missing_token_conformance_count.rq @@ -1,12 +1,15 @@ PREFIX vcfc: # Exact number of missing tokens serialized as a plain "." literal. -# The predicate exclusions must match preflight_missing_token_conformance.rq -# exactly, or the count and its sample describe different populations. +# The exclusions -- the two predicates and the header Number attribute -- must +# match preflight_missing_token_conformance.rq exactly, or the count and its +# sample describe different populations. SELECT (COUNT(*) AS ?anomalyCount) WHERE { ?s ?p ?o . FILTER(ISLITERAL(?o) && STR(?o) = ".") FILTER(DATATYPE(?o) != vcfc:Null) FILTER(?p NOT IN (vcfc:fieldNumber, vcfc:genotypeString)) + OPTIONAL { ?s vcfc:attributeKey ?attributeKey } + FILTER(?p != vcfc:attributeValue || !BOUND(?attributeKey) || STR(?attributeKey) != "Number") } diff --git a/test/test_missing_token_scope_unit.py b/test/test_missing_token_scope_unit.py new file mode 100644 index 0000000..5344d77 --- /dev/null +++ b/test/test_missing_token_scope_unit.py @@ -0,0 +1,140 @@ +"""The missing-token check must not report a header's own Number=. declaration. + +``preflight_missing_token_conformance`` looks for bare "." literals that should +have been typed ``vcfc:Null``. It already excluded ``vcfc:fieldNumber`` and +``vcfc:genotypeString``, where a dot is conformant. The same token also reaches +the graph a third way: the structured-header attribute layer carries every +declaration's attributes verbatim, so ``Number=.`` becomes a +``vcfc:attributeValue`` of "." on the attribute whose key is "Number". The +shapes require that value to be exactly ``xsd:string``, so it can never be +``vcfc:Null``. + +On the v3.1.0 benchmark the narrowed check still returned ten rows on the +100,000-record HG005 cell -- one per ``Number=.`` INFO declaration in its +header. These tests run both forms of the query against a graph produced by the +real header emitter, so the exclusion is checked end to end rather than by +string inspection alone. +""" + +import tempfile +import unittest +from pathlib import Path + +# The behavioural tests need rdflib to evaluate SPARQL; the rest of the suite +# runs without it, so they skip rather than erroring discovery -- the same +# pattern test_linking_unit.py and test_validation_mutation_unit.py use. +try: + import rdflib +except ModuleNotFoundError: # pragma: no cover - exercised only without rdflib + rdflib = None + +import vcf_rdfizer +from test.helpers import VerboseTestCase + +VCFC = "https://w3id.org/vcf-core/vocab#" +QUERY_DIR = ( + Path(vcf_rdfizer.__file__).resolve().parent + / "src" / "validation" / "queries" / "common" +) +SAMPLE_QUERY = QUERY_DIR / "preflight_missing_token_conformance.rq" +COUNT_QUERY = QUERY_DIR / "preflight_missing_token_conformance_count.rq" + +HEADERS_TSV = ( + "SOURCE_FILE\tHEADER_INDEX\tHEADER_KEY\tHEADER_VALUE\tRAW_LINE\n" + "s.vcf\t1\tfileformat\tVCFv4.2\tx\n" + "s.vcf\t2\tINFO\t\tx\n" + "s.vcf\t3\tINFO\t\tx\n" + "s.vcf\t4\tFORMAT\t\tx\n" +) + + +def header_graph(tmp_path: Path) -> "rdflib.Graph": + """Emit the structured header layer exactly as a conversion would.""" + headers_tsv = tmp_path / "s.header_lines.tsv" + headers_tsv.write_text(HEADERS_TSV, encoding="utf-8") + rdf_path = tmp_path / "s.nt" + rdf_path.write_text("", encoding="utf-8") + vcf_rdfizer.append_header_representation_rdf(headers_tsv, rdf_path) + graph = rdflib.Graph() + graph.parse(rdf_path, format="nt") + return graph + + +def sample_rows(graph) -> list: + return list(graph.query(SAMPLE_QUERY.read_text(encoding="utf-8"))) + + +def anomaly_count(graph) -> int: + rows = list(graph.query(COUNT_QUERY.read_text(encoding="utf-8"))) + return int(rows[0][0]) + + +@unittest.skipIf(rdflib is None, "rdflib is required to evaluate the queries") +class HeaderNumberAttributeTests(VerboseTestCase): + """The header's Number=. is a declaration, not a missing value.""" + + def test_the_emitter_really_writes_the_dot_this_test_is_about(self): + """Guard the premise: without it the other tests would pass vacuously.""" + with tempfile.TemporaryDirectory() as td: + graph = header_graph(Path(td)) + number_values = { + str(value) + for attribute, _, key in graph.triples( + (None, rdflib.URIRef(VCFC + "attributeKey"), None)) + if str(key) == "Number" + for value in graph.objects(attribute, rdflib.URIRef(VCFC + "attributeValue")) + } + self.assertIn(".", number_values) + + def test_number_dot_declarations_are_not_reported(self): + with tempfile.TemporaryDirectory() as td: + graph = header_graph(Path(td)) + self.assertEqual(sample_rows(graph), []) + self.assertEqual(anomaly_count(graph), 0) + + def test_a_bare_dot_elsewhere_is_still_reported(self): + """The exclusion is the Number attribute only, not the whole layer.""" + with tempfile.TemporaryDirectory() as td: + graph = header_graph(Path(td)) + # A genuinely untyped missing value on a record field ... + graph.add(( + rdflib.URIRef("file://s.vcf#record/1"), + rdflib.URIRef(VCFC + "recordId"), + rdflib.Literal("."), + )) + # ... and a dot on a header attribute that is not Number. + attribute = rdflib.URIRef("file://s.vcf#header/line/3/attribute/9") + graph.add((attribute, rdflib.URIRef(VCFC + "attributeKey"), rdflib.Literal("Source"))) + graph.add((attribute, rdflib.URIRef(VCFC + "attributeValue"), rdflib.Literal("."))) + + reported = {(str(row[0]), str(row[1])) for row in sample_rows(graph)} + self.assertEqual(reported, { + ("file://s.vcf#record/1", VCFC + "recordId"), + (str(attribute), VCFC + "attributeValue"), + }) + self.assertEqual(anomaly_count(graph), 2) + + +class ExclusionTextTests(VerboseTestCase): + """Static guarantees that hold without rdflib.""" + + def test_both_queries_exclude_only_the_number_attribute(self): + for path in (SAMPLE_QUERY, COUNT_QUERY): + text = path.read_text(encoding="utf-8") + self.assertIn('STR(?attributeKey) != "Number"', text, path.name) + self.assertIn("OPTIONAL { ?s vcfc:attributeKey ?attributeKey }", text, path.name) + + def test_the_bundled_shapes_still_require_attribute_values_as_strings(self): + """If a shape ever allowed vcfc:Null here, the exclusion must be revisited.""" + shapes = ( + Path(vcf_rdfizer.__file__).resolve().parent + / "vcf_rdfizer_data" / "shacl" / "vcf-core-vocabulary.shacl.ttl" + ).read_text(encoding="utf-8") + self.assertIn( + "sh:path vcfc:attributeValue ; sh:minCount 1 ; sh:maxCount 1 ; sh:datatype xsd:string", + shapes, + ) + + +if __name__ == "__main__": + unittest.main()