diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 05c8481..47ff643 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -29,6 +29,16 @@ jobs: with: python-version: "3.11" + - name: Install the package and its declared dependencies + # rdflib is a required runtime dependency (see pyproject.toml) and the + # CLI imports it through the data-linking module while building its + # argument parser, so even `--help` needs it. Installing the project + # editable pulls exactly what pyproject declares and cannot drift from + # it, while tests still run against the checkout. + run: | + python -m pip install --upgrade pip + python -m pip install -e . + - name: Run cross-platform wrapper tests (no real external tools) run: python -m unittest -v test.test_vcf_rdfizer_cross_platform_unit @@ -51,6 +61,16 @@ jobs: with: python-version: "3.11" + - name: Install the package and its declared dependencies + # rdflib is a required runtime dependency (see pyproject.toml) and the + # CLI imports it through the data-linking module while building its + # argument parser, so even `--help` needs it. Installing the project + # editable pulls exactly what pyproject declares and cannot drift from + # it, while tests still run against the checkout. + run: | + python -m pip install --upgrade pip + python -m pip install -e . + - name: Run full unit test suite run: python -m unittest discover -s test -p "test_*_unit.py" -v @@ -72,6 +92,7 @@ jobs: - name: Run coverage run: | python -m pip install --upgrade pip coverage + python -m pip install -e . coverage run -m unittest discover -s test -p "test_*_unit.py" coverage xml -o coverage.xml diff --git a/.gitignore b/.gitignore index 0b1216f..ab99cfc 100644 --- a/.gitignore +++ b/.gitignore @@ -1,26 +1,36 @@ -!docker/configuration.properties -scripts/node_modules +# Run outputs and intermediates +run_output/ +run_metrics/ +out/ +tsv/ -!time-* +# Downloaded toolchain +RMLStreamer-*-standalone.jar -run_output/ +# Test inputs: large VCFs are generated locally; only the small fixtures are tracked test/test_vcf_files/* !test/test_vcf_files/test-100.vcf !test/test_vcf_files/test-1k.vcf !test/test_vcf_files/test-10k.vcf !test/test_vcf_files/test-larger.vcf.gz +!test/test_vcf_files/test-larger-multisample.vcf.gz -RMLStreamer-v2.5.0-standalone.jar -out/ -tsv/ -run_metrics/ +# Python build and cache artifacts __pycache__/ *.py[cod] .venv/ -.DS_Store -*.cover -test/*.cover -vcf_rdfizer.cover dist/ build/ *.egg-info/ + +# Coverage +*.cover +test/*.cover +vcf_rdfizer.cover +.coverage +.coverage.* +coverage.xml +htmlcov/ + +# OS +.DS_Store diff --git a/CITATION.cff b/CITATION.cff index 8ad11d2..cc199c6 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -2,7 +2,7 @@ cff-version: 1.2.0 message: "If you use VCF-RDFizer in your research, please cite it using the metadata below." title: "VCF-RDFizer" type: software -version: "2.1.0" +version: "3.0.0" authors: - name: "VCF-RDFizer maintainers" repository-code: "https://github.com/ecrum19/VCF-RDFizer" diff --git a/Dockerfile b/Dockerfile index 59d042d..1a5d06f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -166,9 +166,14 @@ COPY THIRD_PARTY_NOTICES.md /usr/share/licenses/vcf-rdfizer/THIRD_PARTY_NOTICES. COPY src/*.sh /opt/vcf-rdfizer/ COPY src/*.py /opt/vcf-rdfizer/ COPY src/validation/ /opt/vcf-rdfizer/validation/ -# Shared, dependency-free helper used by both the host CLI and the in-container -# conversion runner, so it lives at the repository root rather than in src/. +# Shared, dependency-free helpers used by both the host CLI and the in-container +# runners, so they live at the repository root rather than in src/. COPY vcf_rdfizer_gzip.py /opt/vcf-rdfizer/ +# The vocabulary terms, the VCF-version model and the lexical parsers. The +# validation runner imports this as a sibling module, so the oracle and the +# emitters share one description of what the graph should contain instead of +# mirroring each other. +COPY vcf_rdfizer_vocab.py /opt/vcf-rdfizer/validation/ RUN chmod +x /opt/vcf-rdfizer/*.sh \ && chmod +x /usr/local/bin/rdf2hdt \ diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..74ec6c5 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,13 @@ +# The sdist ships the installable package, not the development tree. +# +# setuptools' default sdist walk picks up `test/*.py`, but not the support files +# those modules import (`test/__init__.py`, `helpers.py`, `validation_fixtures.py`, +# `validation_mutations.py`), nor the fixtures and container scripts they read +# from `src/`, `rules/`, `examples/` and `test/test_vcf_files/`. The result was a +# published archive whose test suite could not run at all. Shipping no tests is +# honest; shipping a suite that fails on arrival is not. +# +# The full suite runs from a git checkout, which is what CONTRIBUTING and +# `.github/workflows/tests.yml` both assume: +# python3 -m unittest discover -s test -p "test_*_unit.py" +prune test diff --git a/README.md b/README.md index 744ee26..936ad48 100644 --- a/README.md +++ b/README.md @@ -14,16 +14,25 @@ VCF-RDFizer is a Docker-first CLI wrapper for: 1. VCF -> RDF (N-Triples) with RMLStreamer -2. Optional RDF compression/decompression +2. Optional RDF compression/decompression, into queryable HDT and COTTAS artifacts 3. Semantic validation of a compressed RDF graph against its source VCF - -The VCF-RDFizer vocabulary is available at [https://w3id.org/vcf-rdfizer/vocab#](https://w3id.org/vcf-rdfizer/vocab#). - -This README is the task-oriented reference. For how the tool works, why it is -built that way, and where it stops working, see the documentation set in -**[`docs/`](docs/README.md)** - starting with -[Architecture](docs/architecture.md) and, before you rely on the output, -[Limitations](docs/limitations.md). +4. Data linking, which writes a provenance-tracked side-graph of external links + +The conversion targets the **VCF Core vocabulary**, published at +[https://w3id.org/vcf-core/vocab#](https://w3id.org/vcf-core/vocab#) (prefix +`vcfc:`). It replaces the retired VCF-RDFizer vocabulary +(`https://w3id.org/vcf-rdfizer/vocab#`), which is not redirected: that namespace +serves a deprecation document linking every former term to its successor. VCF +Core is a semantic target any conversion system can adopt, and its version line +is independent of this converter's — the emitted IRIs reference the namespace, +not a pinned version. + +This README covers day-to-day use: installation, the CLI flags, and worked +commands. The documentation set in **[`docs/`](docs/README.md)** covers +everything else — how the tool works internally, why it is designed that way, +and where it stops working. Start with +[Architecture](docs/architecture.md), and read +[Limitations](docs/limitations.md) before you rely on the output. ## Requirements @@ -96,7 +105,7 @@ In `full` mode with multiple VCF inputs, failures are isolated per input: ## Main Flags (Most Used) -- `-m, --mode {full,compress,decompress,tsv,validation,index}` +- `-m, --mode {full,compress,decompress,tsv,index,validation,link}` - `-o, --out` required output root directory - `--rdf-compression` final raw RDF codecs: `gzip`, `brotli`, or `none` - `--representations` queryable RDF outputs: `hdt`, `cottas`, or `none` @@ -195,7 +204,7 @@ vocabulary's published shapes. ```bash vcf-rdfizer --mode validation -i ./cohort.vcf.gz --rdf ./results/cohort/cohort.nt.gz \ - --shacl-shapes ./vocabulary/shacl/vcf-rdfizer-vocabulary.shacl.ttl \ + --shacl-shapes ./vocabulary/shacl/vcf-core-vocabulary.shacl.ttl \ --strict-conformance -o ./validation-results ``` @@ -255,7 +264,7 @@ host filesystem. - `--filter-oracle {auto,bcftools,cyvcf2}` FILTER oracle for `--validate` - `--quiet` suppress terminal progress and validation query chatter while retaining logs/metrics - `--no-progress` disable progress sidecars and terminal progress displays -- `--rdf-storage-mode {plain,space-optimized}` required full-mode aggregate storage policy +- `--rdf-storage-mode {plain,space-optimized}` full-mode aggregate storage policy (default: `space-optimized`) - `plain`: merge RMLStreamer parts into one uncompressed `.nt` - `space-optimized`: gzip each part into one `.nt.gz` aggregate and delete the source part immediately - `--rdf-compression {gzip,brotli,none}` raw RDF artifacts to retain @@ -284,11 +293,36 @@ structure is emitted; all default to the richer form. | Option | Effect | | --- | --- | | `--sample-representation {expanded,condensed}` | Genotype shape (see below) | -| `--info-representation {structured,raw}` | `structured` adds one `vcfr:InfoFieldValue` per record and key, with typed values, alongside `vcfr:infoRaw` | -| `--header-representation {structured,basic}` | `structured` types each `##` line with its vocabulary subclass and lifts FILTER/ALT/contig attributes into their own properties | - -QUAL is always emitted, typed `xsd:decimal` or `vcfr:Null`, because the -published SHACL shape requires a datatype that depends on the value. +| `--info-representation {structured,raw}` | `structured` adds one `vcfc:InfoFieldValue` per record and key, with typed values, alongside `vcfc:infoRaw` | +| `--vcf-version {auto,4.1,4.2,4.3,4.4,4.5}` | `auto` (the default) reads each input's `##fileformat` line, so nothing has to be supplied; an explicit version overrides it for a file whose declaration is missing or wrong | +| `--header-representation {structured,basic}` | `structured` types each `##` line with its vocabulary subclass, emits the ordered `vcfc:HeaderAttribute` resources the SHACL profile requires, and lifts the INFO/FORMAT/FILTER/ALT/contig/META/SAMPLE/PEDIGREE attributes into their own properties | + +`ID`, `ALT`, `QUAL`, `FILTER` and `infoRaw` are always emitted by the wrapper +rather than by the RML mapping, because each may be the VCF missing token and +the vocabulary requires that as `"."^^vcfc:Null` — a datatype that depends on +the row, which RML cannot express. QUAL additionally takes `xsd:decimal` for a +finite value and `vcfc:VCFFloat` for the `INF`/`INFINITY`/`NAN` spellings. + +### VCF versions + +VCF Core ships a SHACL overlay per VCF version (4.1–4.5). The converter detects +each input's version from its `##fileformat` line — required to be the first +line of a conforming VCF — and follows it **per input**, so a directory of mixed +versions converts correctly without any flag. The version selects the +`vcfc:VCF4xFile` class in the mapping and the behaviour that genuinely differs +between versions: whether `CIPOS`/`CIEND` carry one pair per record (4.1–4.3) or +one per ALT allele (4.4+), whether `CILEN`/`CICN` and `EVENTTYPE` exist at all, +and whether the local-allele and base-modification FORMAT families are defined. + +VCF 4.0 has no conformance overlay, so it converts with the newest supported +rules and no version class, and the run says so rather than assuming silently. + +`--info-representation structured` also carries the **allele layer** (ordered +`vcfc:ReferenceAllele`/`vcfc:AltAllele` with their kind, symbolic type and +breakend components), the `vcfc:FieldValueItem` decomposition of +`Number=A/R/G/P` values, and the structural-variant carriers. +`--header-representation basic` is the smallest header form but does **not** +produce a SHACL-conformant graph. [`docs/vcf-coverage.md`](docs/vcf-coverage.md) maps every VCF element to its RDF terms and to the validation check that covers it. @@ -306,11 +340,17 @@ selection trade-offs. ### Expanded (default) Use `--sample-representation expanded` for single-sample and low-sample VCFs. It -preserves the original vocabulary model: - -- every record/sample pair is a `vcfr:SampleCall`; -- every represented FORMAT slot is a `vcfr:FormatFieldValue`; -- the VCF file declares `vcfr:representationProfile vcfr:ExpandedRepresentation`. +preserves the per-sample vocabulary model: + +- every record/sample pair is a `vcfc:SampleCall`, linked to a reusable + `vcfc:VCFSample` with `vcfc:forSample`; +- every represented FORMAT slot is a `vcfc:FormatFieldValue`, linked to its + declaration with `vcfc:declaredBy`; +- `GT` is parsed into a `vcfc:Genotype` with an explicit `vcfc:phasingStatus` + and one `vcfc:GenotypeAlleleCall` per position, and `FT`, `PS`/`PSL`, `LAA`, + the copy-number and haplotype keys, and the `M`/`DPM`/`ADM` base-modification + families get their own resources; +- the VCF file declares `vcfc:representationProfile vcfc:ExpandedRepresentation`. With the default rules, these triples are appended directly from `records.tsv`; the large materialized helper TSVs are not created. The final graph is still @@ -326,19 +366,25 @@ vcf-rdfizer --mode full \ ### Condensed -Use `--sample-representation condensed` for large multi-sample cohorts. It -uses the vocabulary introduced in VCF-RDFizer Vocabulary 1.1.0: +Use `--sample-representation condensed` for large multi-sample cohorts: -- sample columns are declared once as an ordered `vcfr:SampleSet` of reusable - `vcfr:VCFSample` resources; -- each genotype-bearing call has one `vcfr:CohortCallMatrix`; -- each FORMAT key has one `vcfr:FormatValueVector`, rather than one RDF value +- sample columns are declared once as an ordered `vcfc:SampleSet` of reusable + `vcfc:VCFSample` resources; +- each genotype-bearing call has one `vcfc:CohortCallMatrix`; +- each FORMAT key has one `vcfc:FormatValueVector`, rather than one RDF value resource per sample; - the VCF file declares - `vcfr:representationProfile vcfr:CondensedRepresentation`. + `vcfc:representationProfile vcfc:CondensedRepresentation`. + +Condensed mode deliberately stops at the vector: it does **not** derive the +per-sample genotype, phase-set or base-modification resources that expanded mode +does, because that is exactly the per-sample materialization the profile exists +to avoid. Every value stays recoverable by decoding a vector against its FORMAT +definition and the matrix `SampleSet`. Both profiles do emit the file-level +`vcfc:SampleSet`, which costs one resource per sample column for the whole file. -`vcfr:encodedValues` uses `vcfr:VCFTextVector`: one tab-separated lexical item -per sample in `vcfr:sampleIndex` order. Commas inside a FORMAT value remain part +`vcfc:encodedValues` uses `vcfc:VCFTextVector`: one tab-separated lexical item +per sample in `vcfc:sampleIndex` order. Commas inside a FORMAT value remain part of that item, and absent values are emitted as `.` so all vectors stay aligned. Consumers reconstruct sample `i`'s value for a FORMAT key by selecting position `i` from its vector. This changes genotype graph growth to approximately @@ -843,6 +889,23 @@ finishes. - default rules file: `rules/default_rules.ttl` - rules guide: `rules/README.md` +### Data linking plug-ins + +Three installed examples cover declarative dbSNP links (`rsid-dbsnp`), interval +joins against a synthetic GFF3 bundle (`gene-demo`), and an Ensembl API resolver +(`rsid-ensembl`). Add `--link ` in full mode, or link an existing aggregate +without Docker: + +```bash +vcf-rdfizer --mode link --rdf ./results/sample/sample.nt.gz \ + --link rsid-dbsnp --offline -o ./linked-results +``` + +Links go into `sample.links.nt`; the base graph is unchanged. The +`vcf-rdfizer-link` companion CLI lists, scaffolds, checks and previews plug-ins. +See [Data linking](docs/datalinking.md) for all three worked examples, reference +and network safeguards, provenance, and the remaining design limitations. + ### Custom RML Mappings `--rules` accepts any RML mapping, so you can change what RDF the pipeline @@ -924,6 +987,8 @@ launches Docker, and reads back the JSON/CSV reports each stage writes. | --- | --- | | `vcf_rdfizer.py` | Host CLI: argument validation, output-collision planning, Docker orchestration, metrics assembly, mode dispatch. Also emits the multi-sample genotype RDF (see below). | | `vcf_rdfizer_rules.py` | `vcf-rdfizer-rules` CLI: scaffold, document, and validate custom RML mappings. | +| `vcf_rdfizer_link.py`, `vcf_rdfizer_linking/` | Linker authoring CLI and shared token/interval/API runner. | +| `vcf_rdfizer_data/linkers/` | Packaged examples of all three plug-in tiers. | | `vcf_rdfizer_gzip.py` | Uncompressed size of a gzip/BGZF VCF without decompressing it. Used by the host preflight estimate and, inside the image, by `run_conversion.sh`. | | `src/vcf_as_tsv.sh` | VCF -> per-input `records`/`header_lines`/`file_metadata` TSV, in one `awk` pass. | | `src/run_conversion.sh` | Runs RMLStreamer, normalizes Spark part files, merges them into one `.nt`/`.nt.gz` aggregate, records conversion metrics. | @@ -959,11 +1024,12 @@ how each part of the tool works, why, and where it stops working. | [CLI reference](docs/cli-reference.md) | Every flag, with constraints and interactions | | [Limitations](docs/limitations.md) | Everything the tool cannot do, in one place | | [Roadmap](docs/roadmap.md) | Planned work, known defects, and rejected options | -| [Data linking design](docs/datalinking-design.md) | Proposal: a plug-in system for external links | +| [Data linking](docs/datalinking.md) | Runnable examples of all three plug-in tiers, authoring, safeguards, and provenance | +| [Data linking design](docs/datalinking-design.md) | Broader proposal and remaining work | | [Privacy policy design](docs/privacy-policy-design.md) | Proposal: ODRL-based granular disclosure control over the graph | -- [`changelog.md`](changelog.md) - dated change history - [`ACKNOWLEDGEMENTS.md`](ACKNOWLEDGEMENTS.md) - funding and attribution +- [Releases](https://github.com/ecrum19/VCF-RDFizer/releases) - release notes per version ## Troubleshooting @@ -1014,7 +1080,7 @@ Safe termination: If you use VCF-RDFizer in a publication, please cite: -VCF-RDFizer maintainers. (2026). *VCF-RDFizer* (Version 2.1.0) [Computer software]. GitHub. https://github.com/ecrum19/VCF-RDFizer +VCF-RDFizer maintainers. (2026). *VCF-RDFizer* (Version 3.0.0) [Computer software]. GitHub. https://github.com/ecrum19/VCF-RDFizer BibTeX: @@ -1023,7 +1089,7 @@ BibTeX: author = {{VCF-RDFizer maintainers}}, title = {VCF-RDFizer}, year = {2026}, - version = {2.1.0}, + version = {3.0.0}, url = {https://github.com/ecrum19/VCF-RDFizer}, note = {Computer software} } diff --git a/changelog.md b/changelog.md deleted file mode 100644 index cf5a184..0000000 --- a/changelog.md +++ /dev/null @@ -1,966 +0,0 @@ -# Changelog - -## 2026-09-05 — Fix a unit test that inverted under CI - -`shell+pipeline-unit` failed on all three OS matrix legs of the `dev` -> `main` -merge (ubuntu-latest, ubuntu-22.04, macos-latest) while passing on developer -machines. One test, three red checks. - -### Fixed - -- `test_validation_forwards_progress_sidecar_and_quiet_flag` asserted that - `run_validation_mode` forwards `--progress-path` to the container. It does — - but `progress_events_enabled()` returns `False` whenever `CI` is set, and - GitHub Actions sets `CI=true`, so the sidecar is deliberately not requested - there. The test pinned `_PROGRESS_ALLOWED`, `_PROGRESS_EVENTS_ALLOWED` and - `_QUIET` but not the environment, so it tested a different code path in CI - than the one it was written for. Reproducible locally with - `CI=true python -m unittest discover -s test -p "test_*_unit.py"`. - - The tool's behaviour is correct and unchanged: the sidecar exists only to - drive the terminal display and `ProgressSession.__exit__` unlinks it, so - suppressing it where nothing renders costs nothing. - -### Changed - -- `stable_progress_env()` replaces the `{"VCF_RDFIZER_NO_PROGRESS": "", "CI": ""}` - dict that two sibling progress tests already carried inline. The third test - omitted it, which is how this happened; a named helper with the reason in its - docstring makes the requirement visible to the next test author. -- The validation-command setup is factored into - `_run_validation_mode_capturing_commands()`, shared by both sides of the - contract. - -### Added - -- `test_validation_omits_progress_sidecar_under_ci` pins the CI behaviour that - was previously only implicit: no `--progress-path` is requested, and `--quiet` - still is. Verified as a real guard by removing the `CI` check from - `progress_events_enabled()` and confirming the test fails. - -## 2026-09-05 — Vocabulary alignment for the condensed representation (upstream) - -The two open items in [`docs/roadmap.md`](docs/roadmap.md) under "Blocking -publication" are closed. Both were work in the -[vocabulary repository](https://github.com/ecrum19/VCF-RDFizer-vocabulary), -released there as **v1.1.0**; **no change was needed in this repository**, and -this entry records what the conversion may now rely on. - -### Condensed graphs are ontology-backed - -The conversion emitted 17 terms the vocabulary did not define — `SampleSet`, -`VCFSample`, `CohortCallMatrix`, `FormatValueVector`, `representationProfile`, -`sampleIndex`, `encodedValues` and the rest — so a condensed graph could not be -dereferenced or meaningfully SHACL-checked. All 17 are now defined, with two -superclasses (`RepresentationProfile`, `VectorEncoding`) giving the enumerations -a range, plus SHACL shapes for the profile and a worked example that is the -expanded example's own record in the other profile. - -The condensed shapes constrain what positional decoding depends on: exactly one -`sampleIndex` per sample, a sample set on every matrix, and an encoding and -FORMAT declaration on every vector. Ten targeted corruptions of a condensed -graph are each reported as a violation, so the shapes are load-bearing. - -`--shacl-shapes` therefore now covers `--sample-representation condensed`, which -it previously could not. - -### The missing-value contradiction is resolved - -`vcfr:missingValuePolicy` required `"."^^vcfr:Null` for a missing token while -`VCFRecordShape` constrained `vcfr:alt` to `sh:datatype xsd:string`, so a record -with `ALT=.` could satisfy neither and the tool could not be conformant on any -VCF containing one. - -The fix draws the boundary the policy was missing rather than relaxing every -field: ALT and ID admit the missing token in VCF 4.5, so their shapes now accept -`xsd:string` or `vcfr:Null` as `qual` already did; CHROM, POS and REF are -required and have no missing form, so their shapes stay exact — a `.` there is -malformed input and should still be reported. The policy also now states that a -missing value inside a `FormatValueVector` stays a `.` character at its sample's -position, rather than becoming a typed literal that would break alignment. - -The conversion already followed the policy, so it needs no change. Reproduced -against the 1.0.1 shapes before the fix and confirmed gone after. - -### Caveat - -The terms only dereference for a third-party consumer once vocabulary v1.1.0 is -published to `https://w3id.org/vcf-rdfizer/vocab#`. Until then the definitions -exist but are not yet live, which is why the docs below say "pending release" -rather than "done". - -### Documentation - -- [`docs/vcf-coverage.md`](docs/vcf-coverage.md) — the "Vocabulary alignment" - section now records both gaps as closed, with what was verified. -- [`docs/roadmap.md`](docs/roadmap.md) — both publication blockers marked done. -- [`docs/limitations.md`](docs/limitations.md), - [`docs/conversion.md`](docs/conversion.md), - [`docs/validation.md`](docs/validation.md) updated to match. - -## 2026-09-04 — Multi-engine validation, native HDT/COTTAS querying, and benchmarking - -Validation was one engine against one N-Triples file. It is now up to four -engines against the artifact each of them reads best, in a single run, timed -and cross-checked. - -### Added - -- **Several SPARQL engines in one run.** `--validation-engine` accepts a - comma-separated list or `all`. Every requested engine answers the whole query - set; the first is the primary and keeps the existing single-engine report - layout, while each engine additionally gets `engines//` with its own - preflight, SPARQL, comparison and execution reports. The run's `status` is - `PASS` only if every engine passed, and `summary.json` carries - `engineStatuses`. -- **Cross-engine agreement on the real graph.** Every engine's normalized - results are compared against each other and written to - `engine-agreement.json`, naming the queries that differ. This is not - theoretical: QLever's literal canonicalisation had already been caught this - way in the test harness. -- **Native HDT and COTTAS querying** (`--validation-engine hdt` / `cottas`). - `--validation-target hdt,cottas` validates those artifacts by *decoding* them - first, which proves the decode is faithful and says nothing about querying - them — and measures the wrong thing entirely in a performance comparison. The - new engines query the artifact in place: HDT through `comunica-sparql-hdt`, - COTTAS in-process through `pycottas.COTTASStore`, an rdflib `Store` over the - Parquet artifact. When the run's own artifact is already in the engine's - format it is used directly; otherwise one is built in container scratch and - that build is timed as **setup**, kept out of the query total. Each report - records which happened, in `artifactOrigin`. -- **`@comunica/query-sparql-hdt` in the image**, installed alongside - `build-essential` (needed for its native bindings) which is purged in the same - layer. -- **Timings for everything, and a comparison against the parser.** Every run - writes `benchmark.json` and a long-format `benchmark.csv` — one row per engine - and query, ready to plot without reshaping. `benchmark.json` adds per-engine - setup and query totals, the slowest query per engine, materialization and - SHACL time, and the **oracle**: what it costs to compute the same answers - directly from the VCF with cyvcf2, split into parse and census. The suite - already computes every expected value twice, once by parsing and once by - querying, so those two costs are for identical work on identical input — the - performance comparison the reports were missing. -- **Benchmark columns in the run's `metrics.csv`**: `validation_engines`, - `validation_oracle_seconds`, `validation_engine_query_seconds` and - `validation_engine_setup_seconds`, the last two as `engine=seconds` pairs. - -### Fixed - -- **Comunica could not query an HDT file by path.** A bare path is treated as a - link to dereference and failed with "Could not dereference". The source is now - addressed as `hdt@`. Found by the first real four-engine run, not by - inspection. -- **The parser oracle was reading htslib's header, not the file's.** cyvcf2's - `raw_header` is normalised: htslib injects - `##FILTER=` into files that never - declared it. The oracle therefore expected a `FilterDefinition` resource the - graph could not contain, and every header check failed. `read_vcf_header_text` - now reads the header block from the file itself — the same text the conversion - reads — for both plain and gzipped VCFs. - -### Verified - -- A four-engine run over the fixture: all four `PASS`, all four agree, and the - benchmark records per-query timings for each. -- `test/cross_engine_agreement.py` extended from two engines to all four, - across both representations, with per-run scratch so the native engines cannot - read each other's artifacts. All queries agree and every engine agrees with - the Python oracle. -- 347 host tests pass (24 new); mutation score unchanged at **76/78**. - -### Documentation - -- [`docs/validation.md`](docs/validation.md) — rewritten engine section covering - all four engines, multi-engine runs, native artifact querying, and the - benchmark report. Its "What is *not* tested" section was stale — it predated - the census, the identity digests, and the QUAL/INFO coverage work, and - contradicted [`docs/vcf-coverage.md`](docs/vcf-coverage.md) — and now states - the gaps that actually remain. -- [`docs/validation-methodology.md`](docs/validation-methodology.md) — "Two - engines" became "Four engines", with the two new cross-engine findings and a - section on why a multi-engine run is a controlled benchmark rather than an - incidental one. -- [`README.md`](README.md), [`docs/cli-reference.md`](docs/cli-reference.md), - [`test/README.md`](test/README.md) updated; the README's stale mutation score - (36 mutations, 64/66) corrected to 42 and 76/78. - -## 2026-09-04 — Design proposal: granular privacy policies over the VCF graph - -[`docs/privacy-policy-design.md`](docs/privacy-policy-design.md) — **design -proposal, not implemented.** A system for governed, partial release of a VCF -graph: which participants, which regions, which fields, at what resolution, to -whom, for what purpose — declared in ODRL and enforced by the pipeline. - -### Added - -- **An ODRL profile whose assets are graph selectors.** ODRL's `odrl:target` - addresses an Asset IRI and has no notion of "these triples", so the profile - supplies `ClassSelector`, `PredicateSelector`, `SampleSelector`, - `RegionSelector`, `FieldSelector`, `HeaderSelector` and a SPARQL - `PatternSelector` escape hatch. It also pins the conflict semantics ODRL - leaves informative: deny-overrides, default-deny, most-restrictive-first - composition. -- **Effects beyond permit/prohibit**, because the useful answer for genomic data - is usually less resolution rather than nothing: `drop`, `pseudonymize`, - `generalize`, `threshold`, `aggregateOnly`, and `maskVectorPositions`. -- **Three enforcement tiers derived from the existing pipeline** — TSV - pre-filtering before RMLStreamer (cheapest, covers RML-produced triples), - emitter-time filtering inside `_append_rdf_atomically` (full record context, - no second pass), and a post-hoc `--mode redact` (needs two passes for region - rules, since an N-Triples stream has no ordering guarantee). The compiler - assigns each rule to the cheapest tier that can express it, and **aborts** on - any rule no tier can enforce rather than warning. -- **GA4GH DUO consent codes as `odrl:purpose` right operands**, so a policy is - reviewable by the data access committees that already speak DUO. -- **Release manifests and policy conformance verification**: prohibitions - compile to `ASK` preflights that must return zero, with mutation-catalogue - entries that break the redactor and assert the checks catch it — the same - "measured, not asserted" standard as - [`validation-methodology.md`](docs/validation-methodology.md). - -### Findings that affect work outside the proposal - -- **`ParsedSampleRecord` carries no `CHROM` or `POS`.** `_parse_row` reads - columns 0, 1, 7, 9, 10 and −1, skipping 2 and 3. Any region-scoped feature - evaluated at emission time needs both; adding them is two dataclass fields and - two index reads. -- **Condensed mode cannot express per-sample protection by triple filtering.** - One `FormatValueVector` literal holds every participant's value for a FORMAT - key, so the unit of protection is finer than the unit of storage. Redaction - must rewrite the literal in place (replacing a position with `.` to preserve - `sampleIndex` alignment), and a single masked position is itself disclosive. -- **IRIs and header lines leak independently of genotypes.** Sample names, the - source filename and a monotonic row counter are all embedded in IRIs, and - `##source` / `##SAMPLE` / `##PEDIGREE` / free-text `Description` fields are - transcribed verbatim. Recorded in - [`docs/limitations.md`](docs/limitations.md). - -### Framing kept throughout - -The document is explicit that this is **governed release, not anonymization**. -Genotypes are identifiers — a few dozen independent common variants single out -an individual — so an access-control layer governs who receives what and creates -an audit trail; it does not make released data non-identifying. Differential -privacy is discussed and explicitly **not** proposed, because without a -persistent per-recipient budget ledger it provides no protection. - -## 2026-09-04 — Documentation set, and a data-linking design proposal - -`docs/` becomes an in-depth explanation of the whole tool rather than four -validation-focused documents. Every part of the pipeline is now described, with -its limitations stated next to its capabilities rather than in a footnote. - -### Added - -- [`docs/README.md`](docs/README.md) — index, reading paths per audience, and - the conventions the documentation set follows. -- [`docs/architecture.md`](docs/architecture.md) — the host/container split, - what runs where, the **three places that split deliberately leaks** (genotype, - header and QUAL/INFO emission happen on the host because RML cannot choose a - datatype or class per row), the three-way failure policy, and the pinned - toolchain versions. -- [`docs/conversion.md`](docs/conversion.md) — VCF -> TSV -> RDF stage by stage, - including what the single-`awk`-pass parser can and cannot see, the complete - IRI template table, and the datatype and missing-value decisions. -- [`docs/representations.md`](docs/representations.md) — the compression plan's - three independent decisions, record-safe chunking, the `hdtc` and PyArrow - rationales, and what the round-trip check does *not* prove. -- [`docs/output-and-metrics.md`](docs/output-and-metrics.md) — output layout, - the `run_metrics/` tree, input size accounting, progress, interrupts, exit - codes. -- [`docs/rml-mappings.md`](docs/rml-mappings.md) — the `--rules` contract in - full, plus what a custom mapping does *not* control and what it costs in - validation. -- [`docs/cli-reference.md`](docs/cli-reference.md) — every flag, grouped, with - enforced constraints, environment variables, and exit codes. -- [`docs/limitations.md`](docs/limitations.md) — one consolidated, honest - account: operational, input handling, modelling, extension points, - compression, validation, vocabulary, and scope. -- [`docs/roadmap.md`](docs/roadmap.md) — planned work, known defects, and - options that were assessed and deliberately rejected. -- [`docs/datalinking-design.md`](docs/datalinking-design.md) — **design - proposal, not implemented.** A plug-in architecture for linking the graph to - external resources, built on the observation that rsID, gene and clinical - linking are all one of three joins (`token`, `interval`, `allele`). Three - plugin tiers (declarative template, local reference bundle, live service), - enforced network safeguards, side-graph output with provenance, and a build - order that freezes the extension contract before capability is added. - -### Fixed - -- **Documented behaviour that did not match the code.** `README.md` and - `docs/validation.md` both stated that the wrapper switches `q09`-`q13` to - report-only under a custom `--rules`. It does not: `--mapping-policy` exists - in `validation_runner.py` but `vcf_rdfizer.py` never forwards it, so the - runner always executes `strict` and a *correct* custom-mapping conversion is - reported as `MISMATCH`. The claim is corrected in both places, recorded in - [`docs/limitations.md`](docs/limitations.md), and tracked as a defect in - [`docs/roadmap.md`](docs/roadmap.md). No code change yet. - -### Changed - -- Existing documents gained consistent navigation headers and "See also" - footers, and `README.md` now points at `docs/README.md` as the documentation - entry point rather than listing two files. - -## 2026-09-04 — Graph integrity: blank nodes, empty terms, duplicate statements - -Three checks that verify the N-Triples graph itself, independently of what the -VCF contains. Mutation score **64/66 -> 76/78**, catalogue 36 -> 42 mutations. - -### Added - -- `preflight_blank_nodes` — any blank node, in subject or object position. Every - class in the vocabulary declares an `vcfr:iriTemplate`, so a blank node means - a term map produced no IRI; the record and value digests could not address - such a node, and neither could anything that later merges the graph. A - predicate cannot be blank in RDF, so it is not examined. -- `preflight_empty_values` — empty or whitespace-only literals and IRIs, each - labelled `EMPTY_LITERAL` or `EMPTY_IRI` in the sample. An empty literal means - a value was lost rather than marked missing (the pipeline writes - `"."^^vcfr:Null` for a genuine missing token); an empty IRI means a template - substitution collapsed. Whitespace-only counts as empty: it carries no more - information and is just as certainly a defect. -- `preflight_duplicate_triples` — the same statement emitted more than once. - -Both anomaly checks have exact-count companions, so severity is measured rather -than saturating at the 100-row diagnostic sample. - -### How duplicates are detected, and why it could not be a query - -A SPARQL store holds a **set**. A repeated line is collapsed on load, so it is -invisible to every other check here — `COUNT(*)` over `?s ?p ?o` returns the -distinct total on Comunica, QLever and rdflib alike. Verified before building -anything: a three-line file with one repeat reports 2. - -The detector compares the statements the parser read against the distinct -triples the store holds. Raptor supplies the first (`rapper -c` counts parsed -statements including repeats — confirmed in the image: it reports 3 where the -store reports 2), `preflight_distinct_triple_count` the second, and the -difference is exactly the number of redundant statements. Duplicating the whole -fixture graph reports `parsed=624, distinct=312, duplicates=312`. - -This is worth having because duplicated RDF parts are a known failure mode of -the conversion — `run_conversion.sh` already carries a defensive dedupe for it — -and a duplicated aggregate costs twice the storage for no added information. - -### Behaviour - -- All three are blocking: a graph that fails one is not worth comparing. -- When an input a check needs is unavailable, it reports `NOT_EVALUATED` rather - than `PASS`, and `NOT_EVALUATED` never fails a run. A check that could not run - must not be mistaken for a clean result in either direction. - -### Verified - -- All 22 queries (13 core, 9 preflight/count) return identical values under - Comunica and QLever for both representations, and both engines produce `PASS` - against the Python oracle — including the new `ISBLANK`, `ISIRI` and - `REGEX`-based checks. -- 317 tests pass. Six new mutations (`introduce_blank_node`, - `blank_node_object`, `empty_literal`, `whitespace_only_literal`, - `duplicate_triple`, `duplicate_whole_graph`) are all detected. - -### Documentation - -- `docs/validation.md` gained a "Graph integrity" section explaining why - duplicates cannot be found by a query. -- `docs/vcf-coverage.md` lists the three under graph-level properties. -- `docs/validation-methodology.md` now describes four independent layers rather - than three. - -## 2026-09-04 — Full VCF coverage: census, identity digests, INFO, headers, SHACL - -Phases 2-6 of the validation-coverage plan. Mutation score **29/45 -> 64/66** -(64% -> 97%), across a catalogue that grew from 21 to 36 distinct mutations. - -### Fixed — silent data loss - -- **QUAL was extracted from every VCF and never mapped into RDF.** It is now - emitted, typed `xsd:decimal` or `"."^^vcfr:Null` as the published SHACL shape - requires. RML cannot choose a datatype per row, so it comes from a new - record-detail emitter rather than `default_rules.ttl`. -- **`##fileDate` was never mapped.** Now emitted as `vcfr:fileDate`, typed - `xsd:date` when the value's form allows and lexically otherwise. - -### Added — completeness (Phase 2) - -- `q09_predicate_census` and `q10_class_census` compare the graph's predicate - and class inventory against counts derived from the VCF. One comparison - catches three things: a predicate missing, one with the wrong cardinality, and - one that should not be in the graph at all. Expectations are derived from the - VCF and the emitters' documented shapes, never from `default_rules.ttl` - - deriving them from the mapping is how QUAL stayed invisible. -- `--mapping-policy {strict,report-only}`: a custom `--rules` changes the - inventory and IRI templates by design, so these checks become report-only. - -### Added — record and value identity (Phases 3-4) - -- `q11_record_digest`, `q12_info_value_digest`, `q13_format_value_digest` hash - each record, INFO value and FORMAT value **together with its own IRI** and - bucket the result. Binding identity into the hash is what catches a - permutation; bucketing keeps the result at most 256 rows for any graph size - and needs no ordering guarantee, which `GROUP_CONCAT` could not have given - portably. Fields are separated by U+001F, which cannot occur in a VCF field. -- These close the two largest gaps: values permuted between records, and non-GT - FORMAT values (a mangled DP, GQ or AD previously passed). - -### Added — structured INFO (Phase 4) - -- `--info-representation {structured,raw}`, default `structured`: one - `vcfr:InfoFieldValue` per record and key at the vocabulary's declared IRI - template, linked by `vcfr:declaredBy` to an `InfoFieldDefinition` carrying - `fieldId`/`fieldNumber`/`fieldType`/`fieldDescription`. Single-valued - Integer/Float fields also get `fieldValueInteger`/`fieldValueDecimal`; a Flag - gets `fieldValueBoolean true`. `vcfr:infoRaw` is retained. -- The model was already fully defined in the vocabulary and simply unused. - -### Added — header section (Phase 5) - -- `--header-representation {structured,basic}`, default `structured`: each `##` - line is typed with its vocabulary subclass, and FILTER, ALT and contig - declarations get `filterId`, `altId`, `contigId`, `contigLength`, - `contigMd5`, `contigAssembly` and `contigCount`. An unrecognized key keeps - only the base `HeaderLine` type rather than inventing an undefined term. - -### Added — SHACL (Phase 6) - -- `--shacl-shapes PATH` validates the graph against a SHACL shapes file with - `pyshacl`, inside the container, as a layer independent of the VCF entirely. - Off by default because `pyshacl` loads the graph into memory. A violation - blocks the run; a missing `pyshacl` is reported as `EXECUTION_FAILED`, never - as a conformance failure. -- It found three violations the other layers could not see. Two are fixed above. - The third is a **contradiction inside the published vocabulary**: - `vcfr:missingValuePolicy` says a missing token SHOULD be `"."^^vcfr:Null`, - while `VCFRecordShape` constrains `vcfr:alt` to `xsd:string`, so a record with - `ALT=.` cannot satisfy both. Recorded in `docs/vcf-coverage.md` for a decision - in the vocabulary repository. - -### Changed - -- `evaluate_validation()` is the single place the PASS/MISMATCH/BLOCKED decision - is made, used by real runs and the mutation harness alike. -- `test_validation_logic_unit.py` now shares the canonical fixture instead of - maintaining a second description of the same data. - -### Verified - -- All 13 core queries plus 4 exact-count preflights return **identical values - under Comunica and QLever**, for both representations, and both engines - produce `PASS` against the Python oracle end to end. The agreement script now - checks that last part too: engines agreeing with each other is not enough when - the digests are compared against values Python computes. -- 297 tests pass; the suite still passes without `rdflib` (those tests skip). - -### Documentation - -- `docs/vcf-coverage.md` rewritten: every VCF element with separate - "represented" and "validated" columns, each validated claim backed by a named - mutation, the remaining gaps, and the vocabulary alignment problem. -- `docs/validation-methodology.md`: the three independent layers, why identity - digests are histograms, and how to reproduce the score. -- README gained a "VCF Coverage" section for the three representation options. - -## 2026-09-04 — Validation mutation harness, and the first coverage gaps closed - -### Added — measurement (Phase 0) - -- A mutation-testing harness for the semantic validation suite. A correct graph - is corrupted in ~25 named ways and the validator is asked for a verdict; the - proportion detected is a reproducible **mutation score**. Before this, the - suite's coverage was prose, and that prose was wrong: `QUAL` is extracted from - every VCF and then never mapped into RDF, and no check could have noticed. - - `test/validation_fixtures.py` derives the VCF, the RDF graph and the parser - oracle from one declarative specification, so they cannot drift apart. The - genotype triples come from the project's own emitters. - - `test/validation_mutations.py` is the catalogue: each entry names the VCF - element it targets, the check expected to catch it, and — for a recorded - gap — why it is not caught. - - `test/test_validation_mutation_unit.py` evaluates queries in-process with - rdflib (test-only dependency, tests skip without it) and routes every - verdict through the shipped `evaluate_validation()`, so the harness measures - real code rather than a reimplementation. - - **Known gaps are assertions.** A mutation marked `known_undetected` is - asserted to still be undetected, so closing a gap fails a test and forces - the catalogue and `docs/vcf-coverage.md` to be updated. -- `test/cross_engine_agreement.py` plus a CI job asserting every validation - query returns identical values under Comunica and QLever. Verified: all 34 - executions across both representations agree. -- `.github/workflows/validation-mutation.yml` publishes `mutation-score.json` - as a build artifact. - -### Added — coverage (Phase 1) - -- **Exact anomaly counts.** Each structural anomaly preflight now has a - companion aggregate returning the true count alongside the `LIMIT 100` - diagnostic sample. Reports carry `anomalyCount`, `anomalyCountReturned` and - `sampleTruncated`, so ten million anomalies is no longer indistinguishable - from a hundred. Verified against a graph with 250 anomalies: exact count 250, - sample 100, truncated true. -- **`--strict-conformance`** (runner and wrapper) promotes a missing-token - conformance failure from a report-only observation to a validation failure. -- **File metadata and header coverage.** New `q07_file_metadata` compares the - `##fileformat` / `##reference` / `##source` declarations, and - `q08_header_line_census` compares how many `##` lines carry each header key. - `parse_vcf` gained `parse_header_metadata` to compute the oracle side. These - close the two entirely uncovered triples maps. - -Mutation score: **23/45 → 29/45 (51% → 64%)**. - -### Changed - -- The PASS / MISMATCH / BLOCKED_BY_PREFLIGHT decision is extracted into - `evaluate_validation()`, the single place real runs and the mutation harness - both use. -- `normalize()` generalises its single-row handling beyond `q03_titv` via - `SINGLE_ROW_QUERIES`. -- `rdflib>=7.0.0` added to the `dev` extra. It is not a runtime dependency and - the test suite passes without it. - -### Documentation - -- **`docs/vcf-coverage.md`** — the coverage matrix: one row per VCF element, - with separate "represented" and "validated" columns, each validated claim - backed by a named mutation. Includes the vocabulary alignment gap (17 emitted - terms the vocabulary does not define, all condensed-representation) and the - open gaps in priority order. This is the table intended for publication. -- **`docs/validation-methodology.md`** — the mutation-testing method, why - self-reported coverage failed, the two-engine two-layer design, and how to - reproduce the score. -- `docs/validation.md` — Q7/Q8 documented, the exact-count behaviour and - `--strict-conformance` described, and the "What is *not* tested" section - narrowed to what is still true. - -## 2026-09-04 — QLever integration verified against a live container - -Everything below came out of actually running the integration, and each item -was wrong or missing beforehand. - -### Fixed - -- **`preflight_position_datatype` would have failed every QLever run.** QLever - canonicalises numeric literals at index time, so `"100"^^xsd:integer` is - reported by `DATATYPE()` as `xsd:int`. The query required exactly - `xsd:integer`, so on QLever it flagged every record and the run ended as - `BLOCKED_BY_PREFLIGHT`, while passing on Comunica. It now accepts the XSD - integer family, which is what the check means; a plain-string or - `xsd:decimal` POS is still reported as an anomaly on both engines (verified). -- **QLever's binaries are not called `IndexBuilderMain`/`ServerMain`, and are - not in `/usr/bin`.** They are `/qlever/qlever-index` and - `/qlever/qlever-server`. The Dockerfile COPY paths and the default argv were - both wrong and would have failed the build. -- **Copying the binaries alone does not work.** The upstream image is Ubuntu - 24.04 and this base is Ubuntu 26.04; the binaries need Boost 1.83, ICU 74, - jemalloc and io_uring sonames that do not exist here and cannot be - apt-installed (the sonames are release-pinned). Those nine libraries are now - copied into `/opt/qlever/lib`, and `LD_LIBRARY_PATH` is set for QLever's own - processes only, so they cannot shadow anything else in the image. glibc is - deliberately not copied - it is backward compatible and this base is newer. -- **`qlever-server` defaults to a 30-second query timeout**, far below what the - aggregate queries need. The server is now started with `-s` matching - `--validation-query-timeout`. -- Index/server flags corrected against the real CLI: `-m` is `--stxxl-memory` - for the indexer and `--memory-max-size` for the server (`--stxxl-memory` is - not a server flag). - -### Verified - -- Both binaries link and run in the target image (`qlever-index --version` - reports build `bfd5741`). -- A real index build, server start, readiness poll, HTTP query, SPARQL Results - JSON parse and `normalize()` round-trip, driven through the shipped - `QleverEngine` code rather than a reimplementation. -- **Engine equivalence**: all eleven validation queries run under both Comunica - and QLever against the same expanded and condensed graphs — graphs built with - the project's own sample emitters — produce identical normalized results, and - the values are independently correct. - -### Tests - -- `QleverEnvironmentTests`: the private library path is prepended for QLever - only, and the POS datatype preflight accepts the integer family while still - rejecting string/decimal/double. -- The QLever lifecycle test now asserts the verified binary names and flags, - including the explicit server-side query timeout. - -### Still unverified - -- The HDT and COTTAS decode paths, and a full `docker build .` of the real - image. Those need hdt-cpp and the pycottas venv; the QLever work above was - done against a minimal probe image that isolates the integration. - -## 2026-09-04 — Validation: QLever engine, HDT/COTTAS artifacts, coverage audit - -### Added - -- `--validation-engine {comunica,qlever}`. Comunica remains the default and - queries the graph in memory; [QLever](https://github.com/ad-freiburg/qlever) - builds an on-disk index inside the container, serves it on a container-local - port, answers the queries over HTTP, and tears both down. Both engines answer - identical queries and feed the same comparison layer, so the choice is a scale - decision, never a semantic one, and every report records which engine ran. - Tunable with `--qlever-memory-gb`, `--qlever-port`, - `--qlever-startup-timeout`, and `--validation-query-timeout`. -- QLever binaries are copied into the image from the upstream - `adfreiburg/qlever` image; pin a version with - `--build-arg QLEVER_IMAGE=adfreiburg/qlever:`. A build-time linkage - check records whether they resolve against this base, and the validator - surfaces that note if the engine cannot start. -- QLever's CLI has changed across releases, so both command lines are - overridable without code changes: repeatable `--qlever-index-arg` / - `--qlever-server-arg`, or full replacement via `QLEVER_INDEX_COMMAND` / - `QLEVER_SERVER_COMMAND` with `{index}`, `{input}`, `{memory}`, `{port}` - placeholders. The exact argv that ran is recorded in `manifest.json`. -- Validation of compressed and indexed artifacts. `--rdf` now accepts `.nt`, - `.nt.gz`, `.nt.br`, `.hdt`, `.cottas`, `.cottas.gz`, and `.cottas.br`; - `--rdf-format` overrides filename detection. Anything that is not already - plain N-Triples is decoded in container scratch (`hdt2rdf`, - `cottas_tool.py decompress`, gzip/brotli) and then put through the full - semantic suite. That is strictly stronger than the triple-count round-trip - `validate_compression.py` performs during compression: it proves the artifact - decodes to a graph that still reproduces every VCF summary. Nothing decoded - is written beneath `--out`. -- `--validate-artifacts {aggregate,hdt,cottas,all}` for full mode. Each - requested artifact is validated independently with its own report directory - (``, `__hdt`, `__cottas`), so one run can prove the aggregate and - both representations agree with the VCF. A representation that was not - selected, or whose artifact is missing after a recoverable index warning, is - skipped rather than reported as a failure. Default is `aggregate`, which - preserves the previous behaviour. -- `materialization.json` per validation run, recording how the artifact was - decoded and how many triples that yielded; `manifest.json` gained `engine` - and a format-tagged `sourceRdf`. Rapper's parsed triple count is now captured - rather than discarded. - -### Changed - -- The validator's cyvcf2 import is lazy, so the pure normalization/comparison - layer is importable and testable on the host without the container. -- `--mode validation` no longer requires `.nt.gz`; any supported artifact works. - `--rdf-nt` / `--rdf-gz` remain accepted as aliases. - -### Tests - -- `test/test_validation_logic_unit.py` (16 tests): mutation tests that drive the - comparison layer directly and record what it detects — dropped records, - misclassified shapes, flipped genotypes, corrupted FILTER lexicals, Ti/Tv - swaps, allele-count errors, spurious records, impossible AC/AN — **and** what - it does not: record permutation, and the entirely unvalidated `ID`, `QUAL`, - `INFO`, non-`GT` `FORMAT`, and header/metadata columns. The blind-spot tests - are deliberate, so closing a gap fails a test instead of passing unnoticed. -- `test/test_validation_engines_unit.py` (25 tests): artifact format detection - and host/container agreement, each decode path, decode-failure reporting, - engine registry, Comunica command construction, QLever index/serve/teardown, - crashed-server diagnostics, command-line overrides, CLI argument validation, - and the wrapper's target resolution and docker argv. - -### Documentation - -- `docs/validation.md`: new "Which artifact is validated", "Which SPARQL engine - runs the queries", and — importantly — "What is *not* tested", which states - the coverage limits plainly: aggregate-only comparison with no per-record - identity, unvalidated VCF columns, no completeness bound, saturating preflight - anomaly counts. -- README: validation artifacts/engines subsection with a short note on what a - `PASS` does and does not mean. - -## 2026-09-04 — Custom-mapping tooling, cheap gzip sizing, dead-script removal - -### Added - -- `vcf-rdfizer-rules`, a second console script (`vcf_rdfizer_rules.py`) for - authoring custom RML mappings. It replaces the `update_rules.sh` sed script - with something that serves the actual need: - - `columns` documents the five generated TSV sources and every column each - one provides, including why `records.tsv`'s final column has no fixed name. - - `init` writes an annotated copy of the default mapping to start from. - - `check` validates a mapping against the wrapper contract *before* a long - run: logical-source paths the wrapper cannot rewrite per input, referenced - columns no TSV provides, which `--sample-representation` values remain - usable, and whether the mapping forces the large sample helper tables to be - materialized. Exits 1 on a contract violation; `--json` for scripting. - The old script rewrote a rules file *in place* to point at fixed TSV paths, - which the pipeline has not needed since `render_rules_for_triplet` began - rendering a per-input copy - and which silently broke the per-input rewrite - it was supposed to help with. -- `vcf_rdfizer_gzip.py`: uncompressed size of a gzip/BGZF file without - decompressing it. Three tiers, and the method used is recorded next to the - size so the metric stays auditable: - - `bgzf` - sums the per-block `ISIZE` trailers of a `bgzip` file by walking - block headers. Exact, no inflate, and immune to the 32-bit wrap. This is - what `bcftools`/`tabix`/`htslib` produce, so it covers the usual case. - - `gzip-sample` / `gzip-trailer` - single-member gzip: measured outright when - it fits the sampling budget, otherwise the 32-bit `ISIZE` trailer resolved - against a compression ratio sampled from the file's own prefix. - - `inflate` - full pass, used whenever the structure cannot settle the - answer. Concatenated non-BGZF members are detected and routed here rather - than trusting a trailer that describes only the final member. - -### Changed - -- `run_conversion.sh` uses that helper for `input_vcf_size_bytes`, which - previously always cost a full `gzip -dc | wc -c` pass over the source VCF. - On the repository's 52 MB test fixture this is 1.115s -> 0.066s (17x); on a - 634 MB gzip whose uncompressed size exceeds 4 GiB, 13.6s -> 0.029s (470x), - where a naive trailer read would have been 93% wrong. The shell keeps its - `gzip -dc` fallback if the helper is unavailable. -- `stages/conversion/*.json` gains `input_vcf_size_method` recording how the - size was obtained. -- `--estimate-size` now uses the real uncompressed input size when the file's - structure can supply it, instead of always assuming a 5x expansion, and says - so when it had to fall back to the assumption. - -### Removed - -- `src/compression.sh` and `test/test_compression_unit.py`. The wrapper issues - its own gzip/brotli/rdf2hdt commands and delegates chunked work to - `partitioned_compression.py`; nothing had called this script for some time, - and it wrote a stale flat `metrics.csv` schema. -- `src/update_rules.sh` and `test/test_update_rules_unit.py`, superseded by - `vcf-rdfizer-rules` above. - -### Documentation - -- README: new "Custom RML Mappings" section with the three-point contract, the - logical-source table, and the `vcf-rdfizer-rules` workflow; a metrics table - explaining each `input_vcf_size_method`; repository-layout and test-suite - entries for the two new modules. -- `rules/README.md` points at the new CLI as the way to author a mapping. - -### Tests - -- `test/test_rules_helper_unit.py` (12 tests), including a drift guard that - runs `src/vcf_as_tsv.sh` and asserts the documented column lists still match - the headers it writes. -- `test/test_gzip_size_unit.py` (13 tests), each checking a measured size - against a full-inflate ground truth across BGZF, plain, concatenated, and - optional-header-field gzip layouts. - -## 2026-09-04 — Code review: cleanup, hot-path performance, and bug fixes - -Behaviour-preserving unless noted. Emitted RDF was verified byte-identical -(SHA-256) against the previous implementation for both sample representations -and for both `run_conversion.sh` storage modes. - -### Fixed - -- `run_compression_methods_for_rdf`: an existing `.hdt` that failed validation - during a full run recorded a **COTTAS** index warning (wrong format, wrong - artifact path, wrong message) via a copy-pasted block whose assignment also - never propagated, because `cottas_failure_warning` was not `nonlocal` there. - The block is removed; `validate_container_artifact` already downgrades a - recoverable HDT index problem to a warning, so reaching that point means the - artifact is genuinely unusable. -- `--mode validation` raised an uncaught `ValueError` traceback when the - results directory already existed and was non-empty, because that check sits - after the argument-validation `try/except`. It now prints `Error: ...` and - exits 2 like every other input error. Covered by a new regression test. -- `count_triples_in_nt_files` (the host-side fallback triple count) used a - slightly different rule than the container-side counters in - `validate_compression.py` / `partitioned_compression.py`, so a fallback count - could disagree with the authoritative one. The predicate is now shared as - `is_triple_line` with identical semantics. - -### Performance - -- Sample RDF emission (`append_expanded_sample_rdf`, - `append_condensed_sample_rdf`) no longer percent-encodes the same values on - every record: sample-column URI components and `sampleId` literals are - computed once per file, and FORMAT-key components are memoized. Previously - these ran `variants x samples` and `variants x samples x FORMAT keys` times. - ~1.6x faster at 50 samples, with the gain growing with cohort width. -- `_rml_uri_component` and `_ntriples_string_literal` short-circuit values that - need no encoding/escaping (~2.7x and ~1.3x on typical VCF tokens). -- `SampleRecordStream` caches the derived FORMAT-key tuple and its - duplicate-key check per distinct `FORMAT` string instead of rebuilding both - for every record. -- `count_triples_in_nt_files` scans bytes instead of decoding to `str`, and - reads gzip aggregates through a `BufferedReader` rather than `GzipFile` - line-by-line. -- `run()` discards subprocess output at the file-descriptor level instead of - buffering a whole container's output in memory only to drop it. -- `run_conversion.sh`: the `"."` -> `"."^^vcfr:Null` rewrite is now applied to - each RMLStreamer part while it is streamed into the aggregate, instead of as - a separate pass over the finished aggregate. This removes one full read and - one full write of the complete RDF output, and removes the transient - full-size temporary copy (which previously required 2x the aggregate size in - free disk at the end of a plain-storage run). -- `run_conversion.sh`: triple counting uses one `LC_ALL=C grep -c` instead of - `awk` / `grep | wc -l`. On a 228 MB gzip aggregate this is ~5x faster - (6.5s -> 1.2s); it is the last unavoidable full pass over the RDF output. -- `run_conversion.sh`: the `stat` dialect is detected once instead of probed on - every call, and the per-second progress heartbeat does one directory walk - with no `basename` subprocess per part. - -### Changed - -- Space-optimized aggregates are ~14 bytes per RMLStreamer part smaller, - because gzip members are now produced from a pipe and therefore no longer - embed the temporary part filename and mtime. Decompressed content is - unchanged and the output is more reproducible. - -### Removed - -- Dead host-side code: `resolve_input` (superseded by - `resolve_input_snapshot`), and the host chunk planners - `plan_record_safe_rdf_chunks`, `plan_partitioned_hdt_chunks`, - `split_nt_file_for_hdt`, `write_nt_chunk`, `iter_rdf_binary_lines`. Chunking - has been the container runner's job (`src/partitioned_compression.py`) since - partitioned compression moved into an ephemeral Docker volume; keeping a - second host-side implementation was exactly what - `run_partitioned_representation_methods_for_rdf_files` documents against. - Their two tests were removed with them. -- An unreachable `else:` arm in `run_full_mode`'s output summary (it iterated a - list the guard had just proved empty), the unused `input_metrics_target` - parameter of `run_full_mode`, an always-true `is not None` guard, and two - unused local assignments. - -### Documentation - -- `vcf_rdfizer.py` gained a module-level division-of-labour note and section - map, plus section banners for the previously unmarked regions (triple - counting, artifact naming, destructive filesystem operations, preflight - estimation, genotype representations, compression-plan parsing, metrics - layout). -- `README.md`: new "Repository Layout" section explaining the host/container - split and what each file does; fixed the broken `RELEASING.md` link - (`scripts/RELEASING.md`); added links to `docs/`, `changelog.md`, and - `ACKNOWLEDGEMENTS.md`. -- `src/compression.sh` and `src/update_rules.sh` headers now state that they - are standalone utilities the pipeline does not call, and what supersedes - each. `compression.sh` previously claimed the Python wrapper was its primary - caller, which has not been true since compression moved inline. -- Added `ACKNOWLEDGEMENTS.md` with funding/attribution and a paper - acknowledgement template. - -## 2026-09-04 — Validation progress and quiet mode - -- Integrated semantic validation with the existing JSONL progress sidecar and - host `ProgressSession`; validation now reports its preflight/core query - progress using the same terminal UI as conversion and compression. -- Added `--quiet` to suppress terminal progress displays and validation query - chatter while retaining progress bookkeeping, command logs, and metrics. -- Kept `--no-progress` as the stronger opt-out that disables sidecar creation - and progress rendering entirely. - -## 2026-09-04 — Full-run semantic validation - -- Added `--validate`/`--run-validation` to run the semantic VCF/RDF validator - once per input as the final stage of a `--mode full` run. -- Full-mode validation accepts the generated plain `.nt` or gzip `.nt.gz` - aggregate, and retains the validator's container-local temporary-file - guarantees. -- Validation timing, status, exit code, input RDF, and report paths are now - included in the run's `metrics.csv`, compression JSON, stage reports, and - recursive `summary.json` report index. - -## 2026-09-04 — Expanded sample representation naming - -- Renamed the default per-sample genotype graph strategy to `expanded`, while - retaining `condensed` as the alternative strategy. -- Updated the CLI default and accepted values, internal workflow/emitter names, - tests, RML comments, and user documentation. -- The emitted profile IRI is now - `vcfr:representationProfile vcfr:ExpandedRepresentation`. - -## 2026-09-04 — Input-labelled, container-complete metrics layout - -- Replaced timestamp-only run-metrics directories with - `run_metrics/__/`. A single VCF/RDF/representation input - uses its source stem (for example, `1000G_phase3_chr20`); a directory input - with multiple VCFs receives a deterministic batch label. -- Added `run.json` and `summary.json` as the stable entry points for every - mode. The manifest records inputs, requested configuration, output root, and - resolved image; the summary records final status, wrapper runtime, tabular - rows, and an index of stage reports and logs. -- Organized metrics by purpose: `logs/`, `timings/`, `stages/`, and `reports/`. - TSV, RML conversion, compression, decompression, indexing, and warnings now - have predictable locations instead of unrelated `raw_metrics`, - `*_metrics`, and timestamp-nested directories. -- Compression-only, decompression, and standalone index modes now persist - container wall/CPU/system time, peak RSS, exit code, input/output sizes, and - structured stage reports. Full and TSV modes retain the same detail. -- Preserved the complete partitioned-compression runner handoff under - `stages/partitioned/`, including every chunk build, merge, validation, - temporary-workspace free-space sample, timing, peak RSS, and bounded stderr - diagnostic. The report is retained on both successful and failed runs before - the temporary Docker volume is removed. - -## 2026-09-03 — Streaming COTTAS merge for condensed cohorts - -- Replaced the final COTTAS merge/reindex implementation with a PyArrow k-way - merge of the already `spo`-sorted Parquet chunks. It keeps one configurable - batch per input (`COTTAS_MERGE_BATCH_ROWS`, default `2048`), deduplicates - adjacent equal triples, and writes the final COTTAS artifact incrementally. - It preserves RDF set semantics and the embedded COTTAS index without a - graph-wide DuckDB hash table or external-sort spill area. -- This supersedes the prior DuckDB merge variants. A 52-chunk, 36-million-triple - cohort exhausted 41.4 GiB of Docker workspace while DuckDB attempted an - external sort; the streaming merge has no proportional temporary-sort file. -- Added PyArrow 22.0.0 to the image and third-party notices for deterministic - Parquet reader/writer support. -- COTTAS merge progress now reports source triples processed and distinct - triples written. Rich interactive displays remain available, and redirected - terminals receive compact line-based progress updates. -- Retained the bounded stderr tail in failures so malformed COTTAS input, - incompatible index metadata, output disk errors, and other code-1 failures - remain actionable after the ephemeral workspace is removed. - -### Rerun guidance - -Build an image from this revision before retrying. The preserved raw `.nt.gz` -can be retried with `--mode compress`, avoiding another RDF run. The merge -requires normal space for the COTTAS chunks and final artifact, but no longer -requires a large DuckDB spill directory. - -## 2026-09-03 — COTTAS SIGKILL/OOM handling - -- Classified `exit_code=-9` in partitioned index warnings as a child process - killed by `SIGKILL` (normally the Linux kernel/Docker OOM killer). Shell - wrappers can surface the same condition as exit code `137`. -- Initially changed the production COTTAS merge to a bounded pairwise tree. - This reduced input fan-in but did not bound the final graph-wide - `DISTINCT`/`ORDER BY`; the disk-backed replacement above supersedes it. -- Added the failing stage's `max_rss_kb` to `index_warnings.json`, alongside - exit code, stderr tail, and workspace free-space samples, so OOM diagnosis - can be compared directly with the container memory limit. -- The `merge-many` adapter remains available for explicit experiments, but is - no longer selected by the default large-file workflow. - -### Rerun guidance - -Rebuild the image and rerun the full conversion (or use `--mode compress` with -the retained raw `.nt.gz`). The current streaming k-way merge supersedes this -earlier pairwise guidance. - -When COTTAS is optional, `--representations hdt` avoids the COTTAS merge -resource requirement while retaining a queryable representation. - -## 2026-09-02 — COTTAS large-input merge reliability - -- Diagnosed the previous generic `COTTAS merge/index creation failed` warning: - the warning is emitted only after per-chunk COTTAS conversion succeeds and - the final indexed merge returns a non-zero status. HDT generation/indexing - is independent and is not implicated by that warning. -- Added a multi-input `pycottas.cat` adapter for explicit use. The production - workflow now uses the bounded pairwise strategy documented in the 2026-09-03 - entry because the merge API can require substantial in-memory buffers. -- Added cleanup of all COTTAS chunks and merge outputs after a failed COTTAS - attempt so stale intermediates cannot consume the Docker workspace. -- Preserved a bounded stderr tail and exit code for every partitioned stage. - Full-run `index_warnings.json` entries now identify resource/OOM signals, - include before/after Docker-workspace free-space samples, and retain the - underlying COTTAS/DuckDB diagnostic when available. -- Pinned the Docker image to `pycottas==1.1.0` for reproducible behavior. -- Added regression tests for the multi-input merge command, stage diagnostics, - and isolated COTTAS scratch cleanup. - -### Rerun guidance - -Rebuild the image and rerun the same full command. If the COTTAS stage still -fails, inspect `stages/partitioned/.json` and the wrapper log for the -failing `cottas-merge-r*` stage and its -`stderr_tail`. For a resource error, lower `--chunk-target-bytes` and -`--chunk-max-bytes`, or enlarge the Docker data-volume allocation. The raw -`.nt.gz` retained by the failed run is a valid recovery source. diff --git a/conda-recipe/README.md b/conda-recipe/README.md index dfd9a3f..9986fde 100644 --- a/conda-recipe/README.md +++ b/conda-recipe/README.md @@ -7,11 +7,11 @@ do not submit this package to `staged-recipes`. ## Before submitting to conda-forge -1. Commit the version bump, then create and push a Git tag (for example `v2.1.0`). +1. Commit the version bump, then create and push a Git tag (for example `v3.0.0`). 2. Download the source tarball and compute sha256: ```bash curl -L -o vcf-rdfizer.tar.gz \ - https://github.com/ecrum19/VCF-RDFizer/archive/refs/tags/v2.1.0.tar.gz + https://github.com/ecrum19/VCF-RDFizer/archive/refs/tags/v3.0.0.tar.gz shasum -a 256 vcf-rdfizer.tar.gz ``` 3. Replace `version` and `sha256` in the feedstock's `recipe/meta.yaml`. diff --git a/conda-recipe/meta.yaml b/conda-recipe/meta.yaml index 5cd1663..3af2ed0 100644 --- a/conda-recipe/meta.yaml +++ b/conda-recipe/meta.yaml @@ -1,5 +1,5 @@ {% set name = "vcf-rdfizer" %} -{% set version = "2.1.0" %} +{% set version = "3.0.0" %} package: name: {{ name|lower }} @@ -15,6 +15,7 @@ build: script: {{ PYTHON }} -m pip install . -vv entry_points: - vcf-rdfizer=vcf_rdfizer:main + - vcf-rdfizer-link=vcf_rdfizer_link:main requirements: host: @@ -25,24 +26,33 @@ requirements: run: - python >={{ python_min }} - rich >=13.7.0 + - rdflib >=7.0.0 test: requires: - python {{ python_min }} commands: - vcf-rdfizer --help + - vcf-rdfizer-link list imports: - vcf_rdfizer + # The vocabulary terms, VCF-version model and lexical parsers. Importing it + # here catches a packaging miss that would otherwise only surface at + # conversion time, since vcf_rdfizer imports it at module load. + - vcf_rdfizer_vocab about: home: https://github.com/ecrum19/VCF-RDFizer - summary: Docker-first VCF to RDF conversion wrapper with compression and decompression modes + summary: Docker-first VCF to RDF conversion targeting the VCF Core vocabulary, with compressed queryable representations (HDT, COTTAS), semantic validation, and data linking license: MIT license_family: MIT license_file: LICENSE description: | VCF-RDFizer provides a Python CLI wrapper around a Dockerized VCF to RDF conversion pipeline. - It requires a working Docker installation at runtime for conversion/compression commands. + It converts VCF files into RDF in the VCF Core vocabulary, optionally into compressed and + queryable representations (HDT, COTTAS), and can validate that the result still reproduces + the source VCF's semantics. It requires a working Docker installation at runtime for + conversion, compression, indexing and validation commands. doc_url: https://github.com/ecrum19/VCF-RDFizer/blob/main/README.md extra: diff --git a/docs/README.md b/docs/README.md index 6cabb44..8d6e44d 100644 --- a/docs/README.md +++ b/docs/README.md @@ -45,10 +45,10 @@ deciding whether the tool fits your problem, read - **[Custom RML mappings](rml-mappings.md)** — the `--rules` contract, the `vcf-rdfizer-rules` CLI, and an honest account of what a custom mapping does *not* control and what it costs in validation. -- **[Data linking design](datalinking-design.md)** — *proposal, not - implemented.* A plug-in architecture for connecting the graph to external - resources (rsIDs, genes, clinical assertions) with declarative linkers, - reference bundles, live-API safeguards, and provenance. +- **[Data linking](datalinking.md)** — three runnable plug-in tiers, full and + post-hoc usage, the authoring CLI, reference/assembly checks, network policy, + provenance, and measured implementation limits. The broader + **[design](datalinking-design.md)** records what remains planned. - **[Privacy policy design](privacy-policy-design.md)** — *proposal, not implemented.* Granular, machine-readable disclosure control over parts of the graph: an ODRL profile with graph selectors, three enforcement tiers, and @@ -62,7 +62,7 @@ deciding whether the tool fits your problem, read and an assessment of query-time decoding options. - **[VCF coverage matrix](vcf-coverage.md)** — element by element: is it represented, and would a corruption of it be detected. Includes the current - mutation score and the vocabulary alignment gaps. + mutation score and the vocabulary alignment status. ### Whether to trust it @@ -72,6 +72,9 @@ deciding whether the tool fits your problem, read - **[Validation methodology](validation-methodology.md)** — how coverage is *measured* rather than asserted: the mutation-testing harness, why known gaps are assertions rather than comments, and how to reproduce the score. +- **[Validation migration notes](validation-migration-notes.md)** — what the + move to the VCF Core vocabulary leaves to do in the validation suite, and how + the converter's SHACL conformance was verified in the meantime. ### Where it is going @@ -98,7 +101,7 @@ deciding whether the tool fits your problem, read [Custom RML mappings](rml-mappings.md) **"I want to add my own domain's links."** -[Data linking design](datalinking-design.md) → +[Data linking](datalinking.md) → [Data linking design](datalinking-design.md) → [Custom RML mappings](rml-mappings.md) → [Validation methodology](validation-methodology.md) @@ -131,5 +134,6 @@ Three conventions, kept deliberately: Related files outside `docs/`: [`rules/README.md`](../rules/README.md) (the mapping directory), [`test/README.md`](../test/README.md) (the test suite), -[`changelog.md`](../changelog.md), [`scripts/RELEASING.md`](../scripts/RELEASING.md), -and [`ACKNOWLEDGEMENTS.md`](../ACKNOWLEDGEMENTS.md). +[`scripts/RELEASING.md`](../scripts/RELEASING.md), and +[`ACKNOWLEDGEMENTS.md`](../ACKNOWLEDGEMENTS.md). Per-version release notes are +published on the [releases page](https://github.com/ecrum19/VCF-RDFizer/releases). diff --git a/docs/architecture.md b/docs/architecture.md index 1932b1f..1115940 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -24,6 +24,7 @@ writes. ├─ docker run ────────────────────▶ src/vcf_as_tsv.sh (awk) │ src/run_conversion.sh (RMLStreamer/Flink) ├─ append direct RDF (see §4) + ├─ optional host-side linking ─────▶ .links.nt (separate side-graph) ├─ docker run ────────────────────▶ src/partitioned_compression.py │ src/cottas_tool.py │ src/ensure_hdt_index.sh @@ -37,8 +38,10 @@ awkward dependencies — RMLStreamer 2.5.0 on Flink, a Rust `hdtc` 1.1.0 build, `pycottas`, Comunica 5.3.0, QLever, `pyshacl`, `cyvcf2`, `bcftools`. Asking a user to assemble that on their own machine is asking for irreproducible results. Pinning it in one image means a run on a laptop and a run on a cluster execute -the same binaries. The price is that **Docker is a hard requirement** and the -image is large; see [`limitations.md`](limitations.md). +the same binaries. The price is that **Docker is a hard requirement for +conversion and representation operations** and the image is large; see +[`limitations.md`](limitations.md). Post-hoc linking and the authoring CLIs +run entirely on the host. ## 2. Host responsibilities @@ -51,7 +54,8 @@ image is large; see [`limitations.md`](limitations.md). | Docker orchestration | Mounts, user mapping, environment forwarding, permission auto-fix | | Progress rendering | Containers write a JSONL sidecar; the host polls and renders it | | Metrics assembly | Every stage's JSON is collected into one `run_metrics/` tree | -| Interrupt handling | `Ctrl+C` triggers best-effort cleanup of tracked intermediates | +| Interrupt handling | `Ctrl+C` stops this run's labelled containers, then cleans tracked intermediates, then writes a checkpoint | +| Optional data linking | Python plug-ins, reference/response caches and an atomic side-graph share one host runner for full and post-hoc mode | ## 3. Container responsibilities @@ -67,16 +71,32 @@ image is large; see [`limitations.md`](limitations.md). ## 4. Where the split leaks, and why -The design intent is "all data processing happens in the container". There are -**three deliberate exceptions**, all in `vcf_rdfizer.py`, all appending directly +The conversion design intent is "all data processing happens in the container". +There are **three direct-emission exceptions**, all in `vcf_rdfizer.py`, appending directly to the RDF aggregate between the conversion container and the compression -container: +container. + +One rule decides what goes where: + +> **The RML mapping carries every field whose RDF datatype is the same for every +> row. The wrapper carries everything else.** + +Two kinds of field fall on the wrapper's side. A value that may be the VCF +missing token `.` needs `"."^^vcfc:Null` on some rows and a typed literal on +others, and RML cannot switch an object's datatype per row. A value that has to +be decomposed into several resources would need a materialized helper table +whose size is the product of its dimensions. | Emitter | Emits | Why not RML | | --- | --- | --- | -| `emit_sample_representation` | `SampleCall`/`FormatFieldValue`, or `SampleSet`/`CohortCallMatrix`/`FormatValueVector` | RML would first have to materialize a helper table of variants × samples (× FORMAT keys) — the largest intermediate the pipeline can produce | -| `append_header_representation_rdf` | Header-line subclasses, FILTER/ALT/contig attributes, INFO/FORMAT declarations | RML cannot choose a class per row from a parsed attribute string | -| `emit_record_detail` | `QUAL` (typed per value) and structured `InfoFieldValue` nodes | RML cannot switch a literal's datatype based on a declared `Type`, and structured INFO would need a variants × INFO-keys helper table | +| `emit_sample_representation` | Expanded: `SampleCall`/`FormatFieldValue`/`Genotype`/`PhaseSet`/`LocalAlleleSet`/`BaseModification`. Condensed: `SampleSet`/`CohortCallMatrix`/`FormatValueVector`. `SampleSet` and `VCFSample` in both. | RML would first have to materialize a helper table of variants × samples (× FORMAT keys) — the largest intermediate the pipeline can produce | +| `append_header_representation_rdf` | Header-line subclasses, the ordered `HeaderAttribute` resources every structured line needs, and the INFO/FORMAT/FILTER/ALT/contig/META/SAMPLE/PEDIGREE declaration properties | RML cannot choose a class per row, nor decompose one angle-bracketed cell into an ordered set of resources | +| `emit_record_detail` | `ID`, `ALT`, `QUAL`, `FILTER`, `infoRaw` (each may be the missing token); the allele layer (`ReferenceAllele`/`AltAllele`, breakends, symbolic types); structured `InfoFieldValue` with its `FieldValueItem` decomposition; and the SV carriers | RML cannot type the missing token per row, cannot switch a literal's datatype on a declared `Type`, and the allele and value-item layers would need variants × alleles and variants × INFO-keys helper tables | + +The exception count did not change with the move to the VCF Core vocabulary, but +what each emitter covers grew considerably: the vocabulary models the VCF 4.5 +logical model in full, and almost all of the additions are decompositions of a +single source cell, which is exactly the category RML cannot express. This is worth stating plainly because it has consequences: @@ -92,6 +112,14 @@ This is worth stating plainly because it has consequences: `_append_rdf_atomically`, so an interrupted append does not leave a half-written aggregate. +Optional data linking is a separate host-side stage in `vcf_rdfizer_linking/`. +It reads the completed aggregate, stages only join fields/keys in SQLite, +resolves the selected plug-ins, and atomically publishes a side-graph. That +extra graph scan preserves actual subject identities and makes full and +post-hoc linking share exactly one implementation. The reference interval +index lives in memory; HTTP and reference bytes are cached on disk. See +[`datalinking.md`](datalinking.md) for this extension's limits. + ## 5. Data flow in full mode ```text @@ -105,6 +133,8 @@ This is worth stating plainly because it has consequences: │ host-side direct emitters (§4) append in place ▼ complete aggregate + │ optional: --link (host-side; aggregate unchanged) + ├──────────────▶ .links.nt │ optional: --rdf-compression gzip,brotli ├──────────────▶ .nt.gz / .nt.br │ optional: --representations hdt,cottas (chunked) diff --git a/docs/cli-reference.md b/docs/cli-reference.md index 9e3c716..f760610 100644 --- a/docs/cli-reference.md +++ b/docs/cli-reference.md @@ -5,8 +5,8 @@ the constraints that are enforced. `vcf-rdfizer --help` is the authority; this page adds the *why* and the interactions. Two companion CLIs are installed alongside: `vcf-rdfizer-rules` (see -[`rml-mappings.md`](rml-mappings.md)) and, once implemented, -`vcf-rdfizer-link` (see [`datalinking-design.md`](datalinking-design.md)). +[`rml-mappings.md`](rml-mappings.md)) and `vcf-rdfizer-link` (see +[`datalinking.md`](datalinking.md)). --- @@ -24,6 +24,7 @@ artifacts, run metrics and logs, and hidden intermediates all live beneath it. | `compress` | Compress an existing `.nt` / `.nt.gz` | `--rdf` | | `decompress` | Decode a compressed or indexed artifact back to N-Triples | `-C/--compressed-input` | | `validation` | Compare a source VCF against its RDF | `-i/--input` **and** `--rdf` | +| `link` | Write a side-graph from an existing `.nt` / `.nt.gz`, without Docker | `--rdf` **and** `--link` | | `index` | Regenerate an artifact's query index in place | exactly one of `-H/--hdt` or `--cottas` | ## Inputs and outputs @@ -31,7 +32,7 @@ artifacts, run metrics and logs, and hidden intermediates all live beneath it. | Flag | Meaning | | --- | --- | | `-i, --input` | VCF file or directory. Only `*.vcf` and `*.vcf.gz` are recognised; a directory is enumerated one level deep and snapshotted at run start | -| `--rdf` | RDF input for `compress`, or the artifact to check in `validation` | +| `--rdf` | RDF input for `compress`/`link`, or the artifact to check in `validation` | | `-C, --compressed-input` | `.nt.gz`, `.nt.br`, `.hdt`, `.cottas`, `.cottas.gz`, `.cottas.br` | | `-H, --hdt` / `--cottas` | Existing artifact for `--mode index` | | `-d, --decompress-out` | Explicit output `.nt` path; must be inside `--out` | @@ -46,6 +47,7 @@ artifacts, run metrics and logs, and hidden intermediates all live beneath it. | `--sample-representation` | `expanded`, `condensed` | `expanded` | Genotype graph shape | | `--info-representation` | `structured`, `raw` | `structured` | `structured` adds typed `InfoFieldValue` nodes alongside `infoRaw` | | `--header-representation` | `structured`, `basic` | `structured` | `structured` types each `##` line and lifts its attributes | +| `--vcf-version` | `auto`, `4.1`–`4.5` | `auto` | `auto` reads each input's `##fileformat` line; an explicit version overrides a missing or wrong declaration | There is **no automatic sample-count threshold**: the same command always produces the same graph shape, so a downstream consumer can rely on the contract @@ -54,11 +56,28 @@ it selected. QUAL is always emitted regardless of these options. `condensed` rejects a custom mapping that consumes the materialized sample helper tables, because that would emit both genotype representations at once. +## Data linking (`full` / `link`) + +| Flag | Meaning | +| --- | --- | +| `--link` | Comma-separated installed IDs; examples: `rsid-dbsnp`, `gene-demo`, `rsid-ensembl` | +| `--linker-path` | Additional linker directory or parent search directory; repeatable; also `VCF_RDFIZER_LINKER_PATH` | +| `--links-cache` | Reference/response cache; default `~/.cache/vcf-rdfizer/linkers` | +| `--offline` / `--links-cache-only` | Disable linker network access; require local reference bytes or cached responses | +| `--assembly` | Supply missing/unrecognised input assembly metadata; cannot override a recognised mismatch | +| `--links-contact-email` | Contact address for live resolver User-Agent headers | + +The side-graph is `.links.nt`. `gene-demo` contains synthetic intervals; +`rsid-ensembl` requires a real contact address before a live cache miss. The +companion CLI provides `list`, `keys`, `init`, `check`, `dry-run`, and `run`. +See [Data linking](datalinking.md) for commands and current limits. There is no +`--merge-links` in this initial implementation. + ## Compression plan | Flag | Values | Default | | --- | --- | --- | -| `--rdf-storage-mode` | `plain`, `space-optimized` | **required in full mode** | +| `--rdf-storage-mode` | `plain`, `space-optimized` | `space-optimized` | | `--rdf-compression` | `gzip`, `brotli`, `none` | `gzip,brotli` | | `--representations` | `hdt`, `cottas`, `none` | `hdt` | | `--artifact-compression` | `gzip`, `brotli`, `none` | `none` | @@ -67,11 +86,23 @@ helper tables, because that would emit both genotype representations at once. | `--chunk-min-bytes` | bytes | 128 MiB | | `--chunk-max-bytes` | bytes | 1 GiB | +`--rdf-storage-mode` defaults to `space-optimized`: it reaches the same triples +as `plain` at a much smaller peak workspace footprint and no practically +important compute penalty, so the low-disk path is what a caller gets without +having to ask for it. Pass `plain` when you want one uncompressed `.nt` +aggregate on disk, which is also what `--hdt-strategy single` requires. + Constraints that are enforced rather than documented-and-hoped: - Each selector takes a comma-separated list; `none` must appear alone. - `--artifact-compression` requires at least one selected representation. -- `--hdt-strategy single` cannot consume a `space-optimized` gzip stream. +- `--hdt-strategy single` cannot consume a `space-optimized` gzip stream, and + is rejected rather than silently downgraded. Since `space-optimized` is now + the default, `single` needs an explicit `--rdf-storage-mode plain`. +- `--hdt-strategy single` is also rejected when `cottas` is among the + `--representations`. COTTAS always needs bounded chunks, so the partitioned + path would run for both representations and the strategy would have no + effect; earlier releases ignored the flag silently in this case. `-c, --compression` is a hidden legacy alias retained for backward compatibility. Use the three explicit selectors. @@ -151,7 +182,7 @@ volume**, not the output filesystem. | --- | --- | | `0` | Success (possibly with recorded index warnings) | | `1` | One or more inputs failed, including a semantic validation failure | -| `130` | Interrupted with `Ctrl+C`; progress written and tracked intermediates cleaned up | +| `130` | Interrupted with `Ctrl+C`; containers stopped, tracked intermediates cleaned up, `interrupt-checkpoint.json` written | --- diff --git a/docs/conversion.md b/docs/conversion.md index a691188..2eda0bb 100644 --- a/docs/conversion.md +++ b/docs/conversion.md @@ -101,48 +101,106 @@ does not change the output, only how many parts are produced before the merge. Three classes of triple cannot come from RML, and are appended to the aggregate by the host process before compression. See [`architecture.md`](architecture.md#4-where-the-split-leaks-and-why) for why. +The rule is: **RML carries every field whose datatype is the same for every row; +the wrapper carries everything else** — anything that may be the missing token, +and anything that has to be decomposed out of a single source cell. -### `emit_record_detail` — QUAL and structured INFO +### `emit_record_detail` — the row-dependent fixed fields, alleles, INFO -`QUAL` is **always** emitted. The published SHACL shape is -`sh:or([sh:datatype xsd:decimal] [sh:datatype vcfr:Null])`, and RML cannot pick -a datatype per row: +Four fixed fields and the raw INFO string are **always** emitted here, because +each may be the VCF missing token and the vocabulary requires that as +`"."^^vcfc:Null`: + +| Field | Emitted as | +| --- | --- | +| `ID` | the lexical value, or `"."^^vcfc:Null` | +| `ALT` | the lexical value, or `"."^^vcfc:Null` | +| `FILTER` | the lexical value, or `"."^^vcfc:Null` | +| `INFO` (`vcfc:infoRaw`) | the lexical value, or `"."^^vcfc:Null` | +| `QUAL` | see below | + +`QUAL` has its own table because its shape is a disjunction over the whole VCF +Float lexical space: | QUAL value | Emitted as | | --- | --- | -| a number | `""^^xsd:decimal` — the source lexical form, so no precision is gained or lost | -| `.` or empty | `"."^^vcfr:Null` | +| a finite number | `""^^xsd:decimal` — the source lexical form, so no precision is gained or lost | +| `INF` / `INFINITY` / `NAN`, any case | `""^^vcfc:VCFFloat` — these are in the VCF Float lexical space but in no XSD numeric one | +| `.` or empty | `"."^^vcfc:Null` | | anything else | a plain string literal, deliberately kept rather than dropped, and reported by the SHACL layer | -`--info-representation structured` (the default) additionally emits one -`vcfr:InfoFieldValue` per record and key at -`…#call/{ROW_ID}/info/{KEY}`, linked to the `##INFO` declaration through -`vcfr:declaredBy`. A single-valued field (`Number=1`) whose declared `Type` is -`Integer` or `Float` also gets a typed `fieldValueInteger` / `fieldValueDecimal`; -a `Flag` gets `vcfr:fieldValueBoolean true`. A multi-valued field -(`Number=A/R/G/.`) keeps only the lexical value, because the vocabulary's IRI -template gives one node per key, not per value — a real modelling gap, recorded -in [`limitations.md`](limitations.md). - -`--info-representation raw` emits only the opaque `vcfr:infoRaw` string. +`--info-representation structured` (the default) additionally emits: + +- **The allele layer.** `REF` and each `ALT` item become an ordered + `vcfc:ReferenceAllele` / `vcfc:AltAllele` with `alleleIndex` (0 for REF, then + 1..n in source order), `alleleValue` and `alleleKind`. The kind is the VCF + syntactic category: a base sequence, the overlapping-deletion `*`, the missing + `.`, an angle-bracketed symbolic ID, the unspecified `<*>`/``, or a + breakend. A symbolic allele is joined to its `##ALT` declaration with + `declaredByAlt` and, when its ID is one of the reserved codes, given a + `vcfc:svType`. A breakend is parsed into its orientation, replacement string + and mate position. The record is also linked to the contig declaration its + `CHROM` names, with `vcfc:chromosome`. +- **Structured INFO.** One `vcfc:InfoFieldValue` per record and key at + `…#call/{ROW_ID}/info/{KEY}`, linked to the `##INFO` declaration through + `vcfc:declaredBy`. A single-valued field whose declared `Type` is `Integer` or + `Float` also gets a typed `fieldValueInteger` / `fieldValueDecimal`; a `Flag` + gets `vcfc:fieldValueBoolean true`. +- **Value items.** A field whose declared `Number` gives its positions meaning — + `A`, `R`, `LA`, `LR` (one per allele), `G`, `LG` (one per genotype), `P` (one + per GT allele) — is decomposed into ordered `vcfc:FieldValueItem` resources, + each joined to the allele it describes with `vcfc:forAllele`, or carrying its + genotype/GT ordinal. Keys that flatten fixed-width tuples (`CIPOS`, `MEINFO`, + …) record the width with `vcfc:tupleArity` so a consumer can regroup them. + This closes the multi-valued-INFO gap that earlier versions recorded in + `limitations.md`. +- **The SV carriers.** `SVLEN`, `SVCLAIM`, `IMPRECISE`, `NOVEL`, `END`, `EVENT` + with `EVENTTYPE`, the confidence intervals (`CIPOS`/`CIEND` through FALDO + `InRangePosition`, the rest as `vcfc:ConfidenceInterval`), and the + `RN`/`RUS`/`RUL`/`RUC`/`RB` tandem-repeat structure. + +Those carriers are emitted once per record, after the INFO values, rather than +key by key — several of them need more than one key to be well formed. A +`vcfc:VariantEvent` needs both `EVENT` and `EVENTTYPE`; a reference block needs +both `END` and `POS`; a tandem repeat needs `RN` before its repeat values have a +grouping. Where a record does not supply the whole set, the carrier is not +emitted and the values stay available as ordinary INFO values, rather than +producing a resource that would fail its SHACL shape. + +`--info-representation raw` emits only the opaque `vcfc:infoRaw` string, and +turns off the allele layer with it (the value items join to the alleles). ### `append_header_representation_rdf` — structured headers `--header-representation structured` (the default) types each `##` line with its -vocabulary subclass (`INFOHeaderLine`, `ContigHeaderLine`, `FilterDefinition`, -`AltDefinition`, …) and lifts the `` -attributes into their own properties. The attribute parser respects quoting and -backslash escapes, so a `Description` containing a comma is handled correctly. - -An **unrecognised** `##` key keeps only the base `vcfr:HeaderLine` type. -Inventing a subclass would put a term in the graph that the vocabulary does not -define. +vocabulary subclass (`INFOHeaderLine`, `ContigHeaderLine`, `FILTERHeaderLine`, +`ALTHeaderLine`, `MetaHeaderLine`, `SampleHeaderLine`, `PedigreeHeaderLine`, +`AssemblyHeaderLine`, `PedigreeDBHeaderLine`, …) and lifts its attributes into +their own properties. The attribute parser respects quoting and backslash +escapes, so a `Description` containing a comma is handled correctly. + +Every **structured** line also gets one ordered `vcfc:HeaderAttribute` per source +attribute, carrying `attributeKey`, `attributeValue` and a one-based +`attributeIndex`. This is not optional decoration: +`vcfc:StructuredHeaderLineShape` requires at least one `vcfc:hasAttribute`, so a +line typed as one of the structured subclasses without them is non-conformant. +It also keeps implementation-defined attributes queryable, which the dedicated +properties alone cannot do. + +An **unrecognised** `##` key is still typed as `vcfc:StructuredHeaderLine` or +`vcfc:UnstructuredHeaderLine` according to its value's form, which keeps it +queryable without inventing a subclass the vocabulary does not define. An +unstructured line is only typed when it has a value, because +`vcfc:UnstructuredHeaderLineShape` requires one. `##fileDate` is emitted as `xsd:date` when its form is recognisable, and verbatim as a plain literal otherwise — again preserved rather than dropped, and reported by SHACL. -`--header-representation basic` keeps only the base header-line triples. +`--header-representation basic` keeps only the base header-line triples the RML +mapping emits. It is the fastest and smallest header form, but **it does not +produce a SHACL-conformant graph**, because the structured lines then carry no +attributes and no subclass. ### `emit_sample_representation` — genotypes @@ -153,11 +211,107 @@ version: | Mode | Shape | Growth | | --- | --- | --- | -| `expanded` (default) | one `vcfr:SampleCall` per record × sample, one `vcfr:FormatFieldValue` per FORMAT key | ≈ variants × samples × FORMAT fields | -| `condensed` | one reusable `vcfr:SampleSet`, one `vcfr:CohortCallMatrix` per call, one `vcfr:FormatValueVector` per FORMAT key holding a tab-separated `vcfr:VCFTextVector` | ≈ samples + variants × FORMAT fields | - -The file declares which shape it carries via `vcfr:representationProfile`, so a -consumer can branch on it before querying. +| `expanded` (default) | one `vcfc:SampleCall` per record × sample, one `vcfc:FormatFieldValue` per FORMAT key, and the parsed genotype layer below | ≈ variants × samples × FORMAT fields | +| `condensed` | one reusable `vcfc:SampleSet`, one `vcfc:CohortCallMatrix` per call, one `vcfc:FormatValueVector` per FORMAT key holding a tab-separated `vcfc:VCFTextVector` | ≈ samples + variants × FORMAT fields | + +Both modes emit the file-level `vcfc:SampleSet` and its ordered `vcfc:VCFSample` +members, and link them from the `#CHROM` `vcfc:ColumnHeaderLine` with +`vcfc:hasGenotypeColumns`. That is one resource per sample column for the whole +file, not per record, so it costs nothing at cohort scale — and it gives the +expanded profile's `SampleCall` a reusable identity through `vcfc:forSample` +instead of relying on repeated `sampleId` literals. + +**Expanded mode additionally parses the per-sample values** into the vocabulary's +genotype layer: `GT` becomes a `vcfc:Genotype` with `genotypeString`, `ploidy`, +`phasingStatus` and one `vcfc:GenotypeAlleleCall` per position (each either +pointing at the allele it called or flagged `isNoCall`); `FT` becomes +`sampleFilter`; `PS`/`PSL`/`PSO`/`PSQ` become a `vcfc:PhaseSet`; `LAA` becomes a +`vcfc:LocalAlleleSet`; `CN`/`CNQ`/`CNL`/`CNP` and `HAP`/`AHAP` become their +copy-number and haplotype properties; and the pattern-defined `M`/`DPM`/`ADM` +keys become `vcfc:BaseModification` resources pointing at a ChEBI residue. + +**Condensed mode deliberately stops at the vector.** Decomposing GT per sample is +exactly the per-sample materialization the condensed profile exists to avoid. +Every value stays recoverable by decoding a vector against its FORMAT definition +and the matrix `SampleSet`. + +A `GT` value that is not in the `vcfc:GenotypeString` lexical space produces no +`Genotype` resource at all — the value remains present as the FORMAT field's +`fieldValue`, and inventing a genotype for it would create a resource that fails +`vcfc:GenotypeShape`. + +The file declares which shape it carries via `vcfc:representationProfile`, so a +consumer can branch on it before querying. The SPARQL SHACL profile enforces +that a graph never mixes the two. + +## 4a. VCF versions + +VCF Core ships one SHACL overlay per VCF version (4.1 through 4.5), each scoped +by the file's `vcfc:fileFormat`, plus an optional `vcfc:VCF4xFile` subclass that +activates that version's gate. The conversion follows the version of each input. + +### Detection is automatic + +`##fileformat` is required to be the first line of a conforming VCF, and +[`src/vcf_as_tsv.sh`](../src/vcf_as_tsv.sh) already lifts it into +`file_metadata.tsv`, so detection costs one small read and needs nothing from +you. It happens **per input**, not per run: a directory may hold files of +different versions and each gets its own mapping and emitter behaviour. + +`--vcf-version` overrides it, for a file whose declaration is missing or wrong: + +```bash +vcf-rdfizer --mode full -i ./cohort.vcf.gz --vcf-version 4.3 -o ./results +``` + +The run log states which version was used and how it was chosen — `(detected)`, +`(forced)`, or a warning that the declaration was not recognized. + +**VCF 4.0 is not recognized.** VCF Core claims no 4.0 conformance overlay, so a +4.0 file converts with the newest supported rules — nothing representable is +dropped — but no `vcfc:VCF4xFile` class is emitted, so the graph never claims a +version gate that cannot be checked. The same applies to a missing or malformed +`##fileformat` line. All three cases are reported, never assumed silently. + +### What the version changes + +| | 4.1 | 4.2 | 4.3 | 4.4 | 4.5 | +| --- | --- | --- | --- | --- | --- | +| `Number=R` | — | ✓ | ✓ | ✓ | ✓ | +| `Number=P` (FORMAT) | — | — | — | ✓ | ✓ | +| `Number=LA/LR/LG/M` (FORMAT) | — | — | — | — | ✓ | +| `CIPOS`/`CIEND` | one pair per record | one pair per record | one pair per record | one pair per ALT | one pair per ALT | +| `CILEN`/`CICN` | — | — | — | ✓ | ✓ | +| `EVENT` | one per record | one per record | one per record | one per ALT | one per ALT | +| `EVENTTYPE` | — | — | — | ✓ | ✓ | +| Local alleles (`LAA`) | — | — | — | — | ✓ | +| Base modifications (`M`/`DPM`/`ADM`) | — | — | — | — | ✓ | +| Leading GT phase indicator | — | — | — | ✓ | ✓ | + +Three consequences in the emitted graph: + +- **`vcfc:FieldValueItem` linkage.** Before 4.4 a `CIPOS` list is one pair + describing the record, so its items get a `vcfc:valueIndex` and a + `vcfc:tupleArity` but no `vcfc:forAllele` — there is no single allele to point + at, and each ALT allele of a multi-allelic record shares the interval. From + 4.4 the list repeats per ALT and each item joins its own allele. +- **Tuple items are always materialized.** The version overlays count a tuple + key's value items against the ALT count, so `CIPOS` and friends are decomposed + even though 4.4 and 4.5 declare them `Number=.`, which on its own would say + "no positional meaning". +- **Families a version does not define are not invented.** A `LAA` column in a + 4.4 file, or an `M27551C` column in a 4.3 file, keeps its raw + `vcfc:fieldValue` and gets no `vcfc:LocalAlleleSet` or + `vcfc:BaseModification` resource. + +### What it does *not* change + +The version does not affect parsing leniency. A GT with a leading phase +indicator in a 4.1 file is still parsed into a `vcfc:Genotype`; the version +overlay reports it as non-conformant, which is the right division of labour — +the converter transcribes, the validator judges. The same holds for a `Number=R` +declaration in a 4.1 file, or a reserved key declared with the wrong arity: the +graph records what the file says, and SHACL says whether the file was right. ## 5. The two helper tables @@ -183,14 +337,28 @@ conversion is deterministic and re-running it produces byte-comparable subjects. | `HeaderLine` | `file://{SOURCE_FILE}#header/line/{HEADER_INDEX}` | | `VCFRecord` | `file://{SOURCE_FILE}#record/{ROW_ID}` | | `VariantCall` | `file://{SOURCE_FILE}#call/{ROW_ID}` | +| `ColumnHeaderLine` | `file://{SOURCE_FILE}#header/columns` | +| `HeaderAttribute` | `file://{SOURCE_FILE}#header/line/{HEADER_INDEX}/attribute/{N}` | +| `SampleDeclaration` (`##SAMPLE`/`##PEDIGREE`) | `file://{SOURCE_FILE}#header/sample/{ID}` | | `InfoFieldValue` | `file://{SOURCE_FILE}#call/{ROW_ID}/info/{KEY}` | +| `Allele` (REF at 0, ALT from 1) | `file://{SOURCE_FILE}#record/{ROW_ID}/allele/{N}` | +| `FieldValueItem` | `/value/{N}` | +| `VariantEvent` | `file://{SOURCE_FILE}#record/{ROW_ID}/event/{EVENT}` | +| `SampleSet` (both profiles) | `file://{SOURCE_FILE}#samples` | +| `VCFSample` (both profiles) | `file://{SOURCE_FILE}#samples/{SAMPLE}` | | `SampleCall` (expanded) | `file://{SOURCE_FILE}#sample/{ROW_ID}/{SAMPLE}` | | `FormatFieldValue` (expanded) | `file://{SOURCE_FILE}#sample/{ROW_ID}/{SAMPLE}/fmt/{KEY}` | -| `SampleSet` (condensed) | `file://{SOURCE_FILE}#samples` | -| `VCFSample` (condensed) | `file://{SOURCE_FILE}#samples/{SAMPLE}` | +| `Genotype` (expanded) | `file://{SOURCE_FILE}#sample/{ROW_ID}/{SAMPLE}/genotype` | +| `GenotypeAlleleCall` (expanded) | `…/genotype/call/{N}` | +| `PhaseSet` (expanded) | `file://{SOURCE_FILE}#sample/{ROW_ID}/{SAMPLE}/phaseset` | | `CohortCallMatrix` (condensed) | `file://{SOURCE_FILE}#call/{ROW_ID}/matrix` | | `FormatValueVector` (condensed) | `file://{SOURCE_FILE}#call/{ROW_ID}/matrix/fmt/{KEY}` | +These follow the `vcfc:iriTemplate` patterns the vocabulary recommends for each +class. Where the vocabulary declares no template — the allele-layer, genotype +and SV resources — the IRI is minted beneath the resource it belongs to, so the +whole graph stays derivable from the filename and the row counter alone. + `{SOURCE_FILE}` is the **basename**, not a path, so a graph does not encode where the VCF happened to live. The consequence is that two different VCFs with the same filename mint the same IRIs; if you convert `chr1/data.vcf` and @@ -199,19 +367,21 @@ converting, or keep the graphs separate. ## 7. Missing values -The vocabulary's `vcfr:missingValuePolicy` says a missing token should be -`"."^^vcfr:Null`, and the conversion follows it. `preflight_missing_token_conformance` +The vocabulary's `vcfc:missingValuePolicy` says a missing token should be +`"."^^vcfc:Null`, and the conversion follows it. `preflight_missing_token_conformance` reports a plain `"."` literal as an anomaly, and `--strict-conformance` promotes that from a report to a failure. -This policy **conflicted with the published SHACL shapes** for `alt`, which -constrained it to `sh:datatype xsd:string` so that a record with `ALT=.` could -satisfy neither. Vocabulary v1.1.0 resolves it by drawing the boundary the -policy was missing: ALT and ID admit the missing token in VCF 4.5 and their -shapes now accept `xsd:string` or `vcfr:Null`, while CHROM, POS and REF are -required, have no missing form, and keep an exact datatype. The conversion -already followed the policy and is unchanged. See -[`vcf-coverage.md`](vcf-coverage.md#vocabulary-alignment). +This used to conflict with the published SHACL shapes, which constrained +`vcfc:alt` to `sh:datatype xsd:string` and so rejected every REF-only and +gVCF-style record. **The vocabulary fixed that**: `vcfc:VCFRecordShape` now +accepts `xsd:string` or `vcfc:Null` for `alt`, and `vcfc:VariantCallShape` +accepts the full VCF Float lexical space for `qual`. Emitting the missing token +as `vcfc:Null` is now both the policy and the conformant choice. + +`CHROM`, `POS` and `REF` are the three fixed fields VCF 4.5 forbids from being +the missing token, which is exactly why they can stay in the RML mapping while +`ID`, `ALT`, `QUAL` and `FILTER` cannot. ## 8. What conversion does *not* do @@ -222,14 +392,22 @@ already followed the policy and is unchanged. See the central problem the [data-linking design](datalinking-design.md) has to solve. - **No reference checking.** `REF` is not verified against any genome. -- **No structural-variant modelling.** Symbolic ALTs (``), breakends - (`N[chr2:321[`) and `*` are carried as literals. The validation suite - classifies them (`SYMBOLIC_OR_BREAKEND`) but the vocabulary gives them no - structure, and `INFO` keys such as `END` and `SVTYPE` get no special meaning. -- **No genotype interpretation.** A `GT` value is a lexical token. Phasing - (`|` versus `/`) is preserved in the literal but not modelled. - **No cross-file merging.** Each VCF produces its own graph. +Two entries that used to be in this list no longer belong here, because the move +to the VCF Core vocabulary added both: + +- **Structural variants are modelled.** Symbolic ALTs, breakends and `*` are + classified by `vcfc:alleleKind`, given a `vcfc:svType` where the identifier is + a reserved code, and parsed into their components; `END`, `SVLEN`, `SVCLAIM`, + `EVENT`/`EVENTTYPE`, the `CI*` intervals and the tandem-repeat keys all have + meaning. The raw literals are still there — the structure is added alongside, + not instead. +- **Genotypes are interpreted** in the expanded profile: `GT` is parsed into + ordered allele calls with an explicit `vcfc:phasingStatus`, so `|` versus `/` + is modelled rather than only preserved. The condensed profile keeps them + lexical by design. + --- ## See also diff --git a/docs/datalinking-design.md b/docs/datalinking-design.md index 4cc04db..020ce1a 100644 --- a/docs/datalinking-design.md +++ b/docs/datalinking-design.md @@ -1,14 +1,17 @@ # Data linking: a plug-in architecture -*Status: **design proposal**. Nothing described here is implemented yet. This -document exists to fix the extension contract before code is written, because -third-party linkers are the point and a contract is much harder to change once -people depend on it.* +*Status: **partially implemented**. Runnable examples now cover all three +plug-in tiers, with a shared runner, discovery/authoring CLI, full and post-hoc +entry points, side-graphs, provenance and network safeguards. See +[`datalinking.md`](datalinking.md) for the implemented contract and worked +commands. This document retains the broader proposal: allele normalization, +link merging, and automatic plug-in query/mutation discovery remain planned. +The manifest vocabulary is still provisional.* VCF-RDFizer converts a VCF into RDF that is faithful to the VCF and to nothing else. Every IRI it mints is derived from the source file, so the graph is self-contained, reproducible, and — deliberately — an island. It says -`vcfr:recordId "rs334"` and stops there. +`vcfc:recordId "rs334"` and stops there. Data linking is the step that connects that island to the rest of the linked-data web: the rsID becomes a dbSNP resource, the coordinate becomes an diff --git a/docs/datalinking.md b/docs/datalinking.md new file mode 100644 index 0000000..b0fc323 --- /dev/null +++ b/docs/datalinking.md @@ -0,0 +1,356 @@ +# Data linking: implementations and examples + +VCF-RDFizer can now connect its graph to external resources through three tiers +of linker plug-in. The output is a separate N-Triples linkset: enabling linking +does not change the aggregate, its compression inputs, or the core validation +queries. This is an initial implementation of +[`datalinking-design.md`](datalinking-design.md), with the remaining work listed +in §8 below. + +The examples and the runner are installed with the Python package. Linking runs +on the host; only conversion still needs Docker. RDFLib is now a runtime +dependency for parsing actual Turtle manifests and unordered N-Triples, rather +than approximating either syntax with regular expressions. + +--- + +## 1. Three runnable examples + +| Tier | Installed ID | Join and result | Network | +| --- | --- | --- | --- | +| 1: declarative | `rsid-dbsnp` | `ID` rsID tokens → dbSNP identifier IRIs | None | +| 2: reference bundle | `gene-demo` | Explicit allele REF spans → synthetic GFF3 gene intervals | None with the shipped local bundle | +| 3: live resolver | `rsid-ensembl` | Deduplicated rsID batches → service-confirmed Ensembl variation links | Ensembl HTTPS API, or cached responses | + +The canonical example directories are under +[`vcf_rdfizer_data/linkers/`](../vcf_rdfizer_data/linkers/). Each has its own +README and `linker.ttl`; only Tier 3 has `resolver.py`. + +`gene-demo` is deliberately **synthetic**. Its three genes use `example.org` +identifiers and invented intervals. Its `GRCh38` declaration exercises the +assembly guard; it does not turn the fixture into a biological annotation. +Use a real, digest-pinned reference before interpreting overlaps biologically. + +From a checkout: + +```bash +python -m pip install -e . +vcf-rdfizer-link list +vcf-rdfizer-link keys + +# Two local tiers, no Docker or network needed. +vcf-rdfizer-link run -i examples/linking/example.vcf \ + --link rsid-dbsnp,gene-demo --offline \ + --links-cache ./example-cache -o ./example.links.nt +``` + +This writes **seven links**: three dbSNP links and four gene overlaps, plus +provenance triples. Record 2 overlaps both demo genes A and B. Record 3 has no +rsID and falls beyond the gene interval; it produces no link. The JSON report +is `example.links.json`. Existing graph or report paths are refused. + +The companion `run -i` command derives subjects with the converter's default +IRI templates; it does not create or verify a base graph. To link an actual +conversion, use `--rdf` or one of the two entry points below. + +## 2. Conversion and post-hoc linking + +Full mode inserts the stage after the direct RDF emitters and before +compression: + +```bash +vcf-rdfizer --mode full -i examples/linking/example.vcf \ + --rdf-storage-mode plain --representations none --rdf-compression none \ + --link rsid-dbsnp,gene-demo --offline -o ./results +``` + +The aggregate is `results/example/example.nt`; the side-graph is +`results/example/example.links.nt`. Space-optimized `.nt.gz` aggregates work as +well. If linking fails, that input fails at `data-linking` and compression is +not started for it. The completed base aggregate remains available. + +To enrich an existing aggregate without conversion or Docker: + +```bash +vcf-rdfizer --mode link --rdf ./results/example/example.nt \ + --link rsid-dbsnp,gene-demo --offline -o ./relinked +``` + +The result is `relinked/example.links.nt`, with a normal `run_metrics/` tree. +`vcf-rdfizer-link run --rdf --link -o ` uses the +same runner but writes a single companion JSON report instead. + +Both RDF entry points read the graph's `hasRecord` and `hasCall` edges. They +retain the actual subjects, including custom IRI templates, and reject a call +whose subject is absent. They require the VCF-RDFizer predicates for those +edges and the selected join fields. Arbitrary custom vocabularies cannot be +inferred. N-Triples may be unordered; the input reader stores only relevant +fields in a temporary SQLite index and discards genotype triples after parsing. + +## 3. Tier 1: a manifest is the implementation + +[`rsid-dbsnp/linker.ttl`](../vcf_rdfizer_data/linkers/rsid-dbsnp/linker.ttl) +implements the proposal's token example unchanged in substance: + +- `vcfl:TokenJoin` reads `ID`, or a named INFO value using `INFO/`. +- `vcfl:splitOn` splits literal delimiters, defaulting to `;` for ID and `,` + for INFO. Flags, missing keys, empty tokens, and `.` do not create keys. +- `vcfl:accept` is a case-sensitive Python regular expression matched against + the **whole token**. Duplicate tokens do not duplicate a triple. +- `vcfl:objectTemplate` contains exactly one `{TOKEN}` placeholder. The runner + percent-encodes the value before substitution, then validates the IRI. +- `vcfl:subject` selects `vcfl:VariantCall` or `vcfl:VCFRecord`; the manifest + declares the predicate. The runner owns subject identity and serialization. + +This example asserts a weak identifier connection with `vcfl:sameVariantAs`. +It does not check whether dbSNP still recognises the rsID, resolve retired IDs, +or assert `owl:sameAs`. + +## 4. Tier 2: a reference, a digest, and an assembly + +[`gene-demo/linker.ttl`](../vcf_rdfizer_data/linkers/gene-demo/linker.ttl) +declares `vcfl:IntervalJoin`, a GFF3 reference, its exact SHA-256, and its +assembly. The interval index is the implementation; no plug-in Python is needed. + +The join uses **1-based closed intervals**: +`[POS, POS + len(REF) - 1]`. A position at either boundary overlaps. Nested +and overlapping features are retained. `vcfl:featureType` defaults to `gene`; +`vcfl:idAttribute` defaults to `ID`. GFF3 percent escapes are decoded and the +selected identifier feeds the `{ID}` object template. + +Limits are explicit: chromosome names must match exactly (`1` and `chr1` are +different); there is no contig aliasing or liftover. Symbolic alleles, +breakends, and spanning-deletion `*` records are skipped and counted in +`skipped_records`. The runner does not interpret INFO/END or confidence ranges. +It uses explicit DNA REF spans, including their anchor base, rather than an +inferred biological affected region. + +References may be HTTPS URLs, local `file:` URLs, or relative IRI references +such as `` resolved beside `linker.ttl`. An absolute URL string is +also accepted, as in the original proposal. Fetched/local bundles are cached +under `references/`; the digest is checked on every cache use. For +gzip references the digest covers the **compressed bytes**. The GFF3 index is +rebuilt in memory each invocation; the bundle bytes are cached, not the index. + +Before resolution, the runner compares the input reference against the bundle. +It recognises `GRCh37`, `GRCh38`, and the `hg19`/`hg38` aliases within reference +paths, or an exact declared assembly string. Missing/unrecognised metadata +requires `--assembly` after the user checks the input. That option cannot +override a recognised mismatch. No reference is fetched until the assembly +checks for all selected linkers have passed. + +To adapt the example: + +```bash +vcf-rdfizer-link init --example gene-demo -o my-gene-linker +# Edit id, reference URL, digest, assembly, attribute, and object template. +vcf-rdfizer-link check my-gene-linker --links-cache ./reference-cache +vcf-rdfizer-link dry-run my-gene-linker -i your-small.vcf --limit 100 +``` + +`check` parses the manifest, verifies/acquires the reference and parses its +features. Without an input VCF it cannot certify input assembly compatibility. + +## 5. Tier 3: a narrow Python resolver + +The shipped resolver calls Ensembl's +[batch variation endpoint](https://rest.ensembl.org/documentation/info/variation_post). +It submits one POST per batch (up to 100 rsIDs in this example) and emits links +only for returned variation objects with a name. Missing IDs produce no link; +malformed response objects abort the linker. No clinical assertions or +frequencies are inferred from the response. + +```python +from vcf_rdfizer_linking import Link, LinkKey, LinkerContext + +def resolve(batch, ctx): + response = ctx.session.post( + "https://rest.ensembl.org/variation/homo_sapiens", + json={"ids": [key.token for key in batch]}, + ) + response.raise_for_status() + # Inspect response.json(), then yield Link(key, absolute_object_iri). +``` + +`LinkKey` is immutable (`token`, `chrom`, `start`, `end`). `Link` contains only +`key` and `object`. A resolver cannot supply a subject or change the manifest's +predicate through this protocol, and returning an undispatched key is an error. +The session offers `get(url, params=...)` and `post(url, json=...)`; responses +have `status_code`, `headers`, `content`, `json()`, and `raise_for_status()`. + +Preview first, then supply your actual contact address for a live run: + +```bash +vcf-rdfizer-link init --example rsid-ensembl -o ensembl-example +vcf-rdfizer-link dry-run ensembl-example -i examples/linking/example.vcf + +vcf-rdfizer-link run -i examples/linking/example.vcf --link rsid-ensembl \ + --links-contact-email you@your-institution.org \ + --links-cache ./ensembl-cache -o ./ensembl.links.nt + +# Reuse identical request batches without contacting the service. +vcf-rdfizer-link run -i examples/linking/example.vcf --link rsid-ensembl \ + --offline --links-cache ./ensembl-cache -o ./ensembl-replay.links.nt +``` + +The built-in manifest's placeholder contact blocks a cache-miss live request; +an offline replay can still use its cache. Set `vcfl:contactEmail` in a copied +manifest or use `--links-contact-email`. Selecting a live linker sends its join +keys to the declared service; no sample/genotype payload is passed to this +example's resolver. + +The session implements: + +- A declared HTTPS endpoint origin and path. Redirects are refused, including + reference-download redirects, so hidden requests cannot change hosts. +- A mandatory disk cache under `responses///.json`. + The hash includes method, URL and canonical JSON body; stored bodies have + digests. Only successful responses are cached. Corrupt entries fail closed. +- Full key deduplication **per input aggregate**, before any request. Sorted + keys make batching deterministic. SQLite stores key-to-subject associations + and resolved triples so those do not need to fit in memory. +- A per-host token bucket with capacity one, shared by selected linkers, using + the strictest declared rate. At most one request is in flight per run. + Per-linker request ceilings are shared across inputs in a directory run. +- Up to four HTTP attempts for 429/5xx, with exponential delay and numeric or + HTTP-date `Retry-After` honoured. Retries count toward the run ceiling. This + follows Ensembl's documented + [rate-limit protocol](https://github.com/Ensembl/ensembl-rest/wiki/Rate-Limits). +- A 30-second request timeout and 16 MiB response limit. Other HTTP failures, + connection failures, and exhausted budgets abort the linker. +- `--offline` / `--links-cache-only`: no network, with clear cache-miss errors. + Verified local GFF3 references remain usable on a cold cache. + +The limiter is shared within one invocation, not across separate processes. +Directory inputs are deduplicated individually; identical request batches in +later inputs use the response cache. Different batch membership produces a +different cache key even if some identifiers overlap. Cache entries have no +automatic expiry: use a fresh cache directory or a new plug-in version to +refresh service answers deliberately. + +**Python plug-ins are trusted code.** The session constrains traffic made +through its methods. Importing a third-party resolver is not a security sandbox +and cannot prevent that code from opening its own socket. The shipped resolver +uses only `ctx.session`. `list`, `check`, and `dry-run` do not import resolver +modules; Python entry-point discovery does execute the installed provider. + +Tier 3 remains suitable for filtered variant sets. Genome-wide linking should +use a suitable local reference bundle. The network budget is a hard ceiling, +not a promise that a large run will finish. + +## 6. Discovery, authoring, and previews + +`vcf-rdfizer-link init -o my-linker` copies the annotated Tier 1 example; +`--example` selects any of the three tiers. It refuses an existing destination. +Give copies a new `vcfl:id` before adding them to a search path. Discovery reads +packaged examples, repeatable `--linker-path` directories, the platform-separated +`VCF_RDFIZER_LINKER_PATH`, and `vcf_rdfizer.linkers` Python entry points. + +A path may be a linker directory or a parent containing linker directories. +Duplicate IDs at different locations are rejected rather than shadowed. +Entry points export a zero-argument callable returning an installed directory: + +```toml +[project.entry-points."vcf_rdfizer.linkers"] +my-linker = "my_package:linker_directory" +``` + +```python +from pathlib import Path + +def linker_directory(): + return Path(__file__).parent / "my-linker" +``` + +Package that directory's manifest, resolver and optional local data. `list` +prints versions, tiers, references, licensing and terms; `list --json` provides +the parsed fields. `check --json` is available for automation. IDs and versions +must be safe path components. Unsupported joins, template variables, unknown +`vcfl:` properties, invalid regexes, bad digests and invalid budgets are errors. + +`dry-run -i --limit N` reads the first N records (default 100), +writes no persistent graph/report/cache, and never makes a network request. It +shows up to 20 concrete triples for local tiers, total local link counts, and +Tier 3 unique-key/batch counts and the declared request ceiling. A remote +Tier 2 bundle must already be cached. For Tier 3 it reports a **plan**, not +imagined API results; a general resolver's response-dependent request count +cannot be predicted without executing it. The shipped resolver makes one +request per batch before retries. + +## 7. Output integrity and accounting + +The runner stages all resolutions before serializing into a temporary file +with the wrapper's `_append_rdf_atomically` contract. It publishes the completed +side-graph atomically and refuses existing outputs. Failures preserve the base +aggregate and publish no partial linkset. Successful response/reference caches +from a failed run are retained for retry. + +Each source/linker pair has a `vcfl:Linkset` node, even with zero matches. It +records producer ID/version, source, link count, generation time, and manifest +digest; Tier 2 adds reference digest and assembly; Tier 3 adds resolver and used +response digests. `linkCount` counts distinct links emitted by that linker for +that source and excludes provenance. If two linkers emit the same triple, it +appears once in the side-graph but counts in each linker's own provenance. + +Full and post-hoc wrapper runs record linking results in `run.json` and +`stages/linking/.links.json`, with `RunTracker` logging and the normal +progress sidecar protocol (`--quiet`/`--no-progress` apply). Reports include +unique keys, skipped records, links, requests, cache hits, transferred response +bytes, elapsed time, final HTTP status and cached-response timestamps/digests. +Failure reports carry the error and per-linker status. Main-aggregate triple +counts continue to exclude side-graph triples. `--mode link` summary counts are +the new side-graph's triples, including provenance. + +An offline replay reproduces links from the same cached requests. Its +`generatedAtTime` and run timings are new; the entire side-graph is therefore +not promised byte-identical. + +## 8. What remains from the proposal + +The implemented tiers are examples of the extension contract, not completion +of every feature in the proposal: + +- **No allele join or allele normalization.** Tier 3 uses token keys. No + ClinVar/gnomAD/CADD matching is implied by this implementation. +- **No production gene bundle is shipped.** The synthetic fixture demonstrates + digest checking, indexing and assembly refusal. Users supply real references. +- **No `--merge-links`.** Keeping side-graphs separate leaves core validation + intact; the wrapper's custom-mapping report-only forwarding gap is still open. +- **No plug-in SPARQL/mutation auto-discovery.** Optional `queries/` and + `mutations.py` from the proposal are not executed. The new unit tests cover + linker invariants and failures but do not extend the published core mutation + score. No standalone linkset validator is claimed. +- **No HDT/COTTAS input reader for linking.** Supply `.nt` or `.nt.gz`. +- **No persisted interval index or global cross-process quota.** Reference + features are indexed in memory; temporary SQLite storage still requires disk + proportional to join fields and associations. This implementation has not + been benchmarked on genome-scale cohorts. +- **The manifest vocabulary is provisional.** Third-party authors should pin + the tool version while the remaining joins and validation contract settle. + +## 9. Verification + +[`test/test_linking_unit.py`](../test/test_linking_unit.py) exercises known-answer +token links, INFO splitting/escaping, interval boundaries and nested features, +assembly refusal, digest corruption, unordered and gzip RDF, custom subjects, +empty inputs, full-mode integration, discovery, authoring commands, API batching, +cache replay, host pacing, retries, ceilings and failure atomicity. + +The shipped Ensembl resolver runs in these tests against a fake HTTP transport; +tests never contact Ensembl or require Docker. That verifies its response +handling and the session policy, not the current availability or completeness +of a public service. + +```bash +python -m unittest test.test_linking_unit -v +python -m unittest discover -s test -p 'test_*_unit.py' +``` + +--- + +## See also + +- [Original design](datalinking-design.md) — rationale and remaining ambitions +- [Custom mappings](rml-mappings.md) — subject and vocabulary compatibility +- [Output and metrics](output-and-metrics.md) — the shared run report layout +- [Validation methodology](validation-methodology.md) — the separate core suite diff --git a/docs/limitations.md b/docs/limitations.md index 9af0d37..f2d3f48 100644 --- a/docs/limitations.md +++ b/docs/limitations.md @@ -11,7 +11,9 @@ Each item says what it is, why it is that way, and whether it is fixable. ## 1. Operational -**Docker is mandatory.** There is no pure-Python path. The toolchain — Flink, +**Docker is mandatory for conversion and representation operations.** There +is no pure-Python conversion path. Post-hoc data linking and the authoring CLIs +run on the host without Docker. The conversion toolchain — Flink, RMLStreamer, a Rust `hdtc`, `pycottas`, Comunica, QLever, `pyshacl`, `cyvcf2`, `bcftools` — is pinned in one image, which is what makes results reproducible across machines. The cost is a large image, a Docker daemon requirement, and no @@ -32,9 +34,12 @@ no distributed execution and no work queue. **No incremental update.** Adding variants to a converted dataset means reconverting the VCF and rebuilding every representation from scratch. -**Interrupt cleanup is best-effort.** `Ctrl+C` exits 130 and removes tracked -intermediates, but a `SIGKILL` or a host crash can leave a partial output -directory that the collision check will then refuse to write into. +**Interrupt cleanup is best-effort.** `Ctrl+C` exits 130, stops this run's +containers, removes tracked intermediates and writes +`interrupt-checkpoint.json`. But `SIGKILL` and a host crash both bypass it +entirely: no handler runs, so containers survive and a partial output directory +is left behind, which the collision check will then refuse to write into. After +an ungraceful kill, check `docker ps --filter label=vcf-rdfizer.run` yourself. ## 2. Input handling @@ -71,18 +76,63 @@ joinable with normalized external resources, which is the central problem the **No reference checking.** `REF` is never verified against a genome, and the declared assembly is recorded but not used for anything. -**Structural variants are literals.** Symbolic ALTs (``), breakends and `*` -are carried as strings. The validation suite classifies them -(`SYMBOLIC_OR_BREAKEND`) but the vocabulary gives them no structure, and `END` / -`SVTYPE` carry no special meaning. - -**Multi-valued INFO fields keep only their lexical value.** A field declared -`Number=A/R/G/.` gets no typed per-value nodes, because the vocabulary's IRI -template gives one node per key rather than per value. This is a modelling gap -in the vocabulary, not a bug in the conversion. - -**Genotypes are lexical.** Phasing is preserved inside the literal but not -modelled; ploidy is not interpreted. +**Structural variants and genotypes are no longer lexical-only.** Moving the +target to the VCF Core vocabulary closed three gaps that used to be listed here: +symbolic ALTs, breakends and `*` are now classified and parsed; `Number=A/R/G/P` +values are decomposed into per-allele `vcfc:FieldValueItem` resources; and `GT` +is parsed into ordered allele calls with an explicit phasing status. See +[`conversion.md`](conversion.md#4-stage-three--the-wrappers-own-emitters). + +What remains true: + +**Genotype interpretation is expanded-only.** The condensed profile keeps +per-sample values inside `vcfc:encodedValues` vectors, so there is no +`vcfc:Genotype`, no phase set and no base-modification resource in a condensed +graph. That is the point of the profile — the values stay recoverable by +decoding a vector against its FORMAT definition and the matrix `SampleSet` — but +a SPARQL engine cannot filter inside a vector payload without a decoder. Choose +`expanded` if you need to query genotypes directly. + +**Symbolic ALT types are only recognised for the reserved codes.** A +caller-specific `` gets `vcfc:SymbolicAllele` and a link to its +`##ALT` declaration, but no `vcfc:svType`, because the vocabulary enumerates +only the codes VCF 4.5 reserves. + +**Some SV carriers need more than one INFO key to be emitted.** A +`vcfc:VariantEvent` needs `EVENT` *and* `EVENTTYPE`; a gVCF reference block +needs `END` *and* `POS`; a tandem repeat needs `RN` before `RUS`/`RUL`/`RUC`/`RB` +have a grouping. Where the record supplies only part of the set, the carrier is +omitted and the values remain as ordinary INFO values. Emitting a partial +carrier would produce a resource that fails its SHACL shape, which is worse than +not producing it. + +**VCF 4.0 has no conformance overlay.** VCF Core supplies version overlays for +4.1 through 4.5 only. A 4.0 file converts with the newest supported rules, so +nothing representable is dropped, but it gets no `vcfc:VCF4xFile` class and its +graph is only checked by the version-neutral profiles. The same applies to a +file with a missing or malformed `##fileformat` line. All three cases are +reported in the run log rather than assumed silently. + +**Version detection trusts the file's own declaration.** `##fileformat` is +required to be the first line of a conforming VCF, and that is what the +converter reads. A file whose declared version does not match its actual content +converts against the declared one; `--vcf-version` is the override. The +converter does not sniff content to second-guess the declaration, because a +wrong guess would silently produce a graph that validates against rules the file +was never written to. + +**Parsing stays lenient across versions.** A GT with a leading phase indicator +in a VCF 4.1 file is still parsed into a `vcfc:Genotype`, and a `Number=R` +declaration in a 4.1 file is still recorded. The version overlays report both as +non-conformant. This is deliberate: the converter transcribes, the validator +judges, and a converter that refused to represent a slightly non-conforming file +would be less useful than one that represents it faithfully and lets SHACL say +so. + +**`--header-representation basic` is not SHACL-conformant.** It emits no +header-line subclass and no `vcfc:HeaderAttribute` resources, and +`vcfc:StructuredHeaderLineShape` requires at least one attribute on every +structured line. It remains available as the smallest, fastest header form. **Triples only.** No named graphs anywhere in the pipeline, and no blank nodes — a blank node is treated as a validation failure, because every class in the @@ -93,9 +143,10 @@ vocabulary declares an IRI template. ## 4. The custom-mapping extension point **`--rules` does not control the whole graph.** Three families of triple — -genotypes, structured headers, and QUAL/structured INFO — are emitted by the -wrapper rather than by RML, because RML cannot choose a datatype or class per -row and the alternatives require materializing enormous helper tables. See +genotypes, structured headers, and the record detail (the fixed fields that may +be missing, the allele layer, structured INFO and the SV carriers) — are emitted +by the wrapper rather than by RML, because RML cannot choose a datatype or class +per row and the alternatives require materializing enormous helper tables. See [`architecture.md`](architecture.md#4-where-the-split-leaks-and-why). They can be narrowed with `--sample-representation` / `--header-representation` / `--info-representation`, but not replaced by a mapping. @@ -151,7 +202,7 @@ corruption we thought to write down is caught". It is a lower bound on blindness, not a measure of correctness, and a score that rises without the catalogue growing means nothing. -**`vcfr:contigCount` is counted, not read**, so a wrong derived contig total is +**`vcfc:contigCount` is counted, not read**, so a wrong derived contig total is undetected. **Header line values are not compared** — only how many lines carry each key, their types, and the structured attributes lifted out of them. @@ -164,19 +215,23 @@ argv is recorded in every report so a future divergence is diagnosable. ## 7. Vocabulary -**Condensed vocabulary coverage lands with the vocabulary's next release.** The -tool emits 17 condensed-representation terms that `vcf-rdfizer/vocab#` did not -define; all are defined in the vocabulary repository as of v1.1.0, together with -SHACL shapes for the profile. Until that version is published to -`https://w3id.org/vcf-rdfizer/vocab#`, those terms still do not dereference for -a third-party consumer. +**VCF Core is published at 2.0.0** (`https://w3id.org/vcf-core/vocab#`) and +covers VCF 4.1 through 4.5. The three constraints listed here previously — the +condensed terms being undefined, the `ALT=.` shape contradiction, and the +ordinal datatype disagreement — are all resolved in that release. What remains: + +**VCF 4.0 has no conformance overlay.** The vocabulary claims 4.1–4.5. A 4.0 +file converts with the newest supported rules, so nothing representable is +dropped, but it gets no `vcfc:VCF4xFile` class and only the version-neutral +profiles check it. Reported in the run log, never assumed silently. -**The SHACL / missing-value contradiction is resolved** in the same release. -`vcfr:alt` and `vcfr:recordId` now accept `xsd:string` or `vcfr:Null`, matching -where VCF 4.5 actually permits the missing token, while CHROM, POS and REF stay -exact because they have no missing form. The conversion already followed the -policy and needs no change. See -[`vcf-coverage.md`](vcf-coverage.md#vocabulary-alignment). +**The converter's version table is a copy.** VCF Core publishes +`ontology/versions/registry.json` as the source of truth for version-scoped +behaviour; the converter keeps its own dependency-free copy so it runs without +the vocabulary checked out. +`test_version_model_matches_the_published_registry` cross-checks the two when a +vocabulary checkout is available (`VCF_CORE_VOCABULARY_DIR`), and skips +otherwise — so drift is caught in development but not in a minimal CI. **Condensed mode has no query-time decoder.** Reconstructing sample *i*'s value means splitting a tab-separated literal, which SPARQL cannot do portably. The @@ -186,9 +241,12 @@ none is currently implemented. ## 8. Scope -**No data linking yet.** The graph is self-contained and connects to nothing -external. This is by design so far, and the plan to change it is -[`datalinking-design.md`](datalinking-design.md). +**Data linking is an initial implementation.** Optional plug-ins can produce +separate linksets through token joins, GFF3 intervals or a guarded live session. +The gene example is synthetic; allele normalization and automatic plug-in +validation are not implemented. See [`datalinking.md`](datalinking.md) for the +implemented contract and [`datalinking-design.md`](datalinking-design.md) for +the remaining proposal. The base graph remains a faithful, separate artifact. **No disclosure control.** Conversion is all-or-nothing: every sample, every genotype, every header line and every free-text `Description` goes into the @@ -200,7 +258,7 @@ record what an artifact was permitted to contain. Three consequences today: is a monotonic counter that discloses source ordering. - **Header lines are a leak surface.** `##source`, `##SAMPLE`, `##PEDIGREE` and free-text `Description` fields are transcribed verbatim into - `vcfr:headerValue`. + `vcfc:headerValue`. - **Even if that were fixed, genotypes identify people.** A few dozen independent common variants are enough to single out an individual, so no amount of label removal makes a released genotype graph non-identifying. diff --git a/docs/output-and-metrics.md b/docs/output-and-metrics.md index 0fcccb4..fb10cad 100644 --- a/docs/output-and-metrics.md +++ b/docs/output-and-metrics.md @@ -18,6 +18,7 @@ everything lives beneath it: .hdt + .hdt.index.v1-1 .cottas .hdt.gz | .cottas.br | ... + .links.nt optional --link side-graph run_metrics/__/ per-run reports and logs .intermediate/tsv/ hidden intermediates decompressed/ --mode decompress default target @@ -28,6 +29,11 @@ with its recognized VCF/RDF/representation suffix removed. `--mode compress` on `test-larger.nt` and on `test-larger.nt.gz` therefore both write into `/test-larger/`. +`--mode link` is host-only and writes `/.links.nt`, with the usual +`run_metrics/` directory. Full mode writes its linkset beside the aggregate +inside `//`. Side-graph triples are excluded from the base +conversion's triple counts. See [Data linking](datalinking.md). + **Collision policy.** Before Docker starts, the wrapper computes every artifact the run intends to write and fails if any already exists. It never overwrites a planned pipeline artifact. Choose a new `--out`, or move the conflicting file. @@ -60,17 +66,40 @@ recognisable without opening a timestamp-named folder. | --- | --- | | `run.json` | Source identity, resolved input paths, requested configuration, image selection | | `summary.json` | Final status, wrapper wall time, summary rows, and an index of every stage report and log | -| `metrics.csv` | Analysis-ready per-output table | +| `metrics.csv` | Analysis-ready per-output table, including `vcf_version` and `vcf_version_source` | | `tsv_metrics.csv` | TSV-mode benchmark table | | `wrapper_execution_times.csv` | Host-side stage timings | | `logs/wrapper.log`, `logs/progress.log` | Command log and progress history | | `timings//…` | Raw GNU `time -v` output from inside the relevant container | | `stages/tsv/`, `stages/conversion/`, `stages/compression/`, `stages/compression_operations/`, `stages/decompression/`, `stages/index/`, `stages/validation/` | Structured per-stage results | | `stages/partitioned/.json` | Full handoff from the partitioned-compression container | +| `stages/linking/.links.json` | Linker counts, reference/response digests, network accounting and success/failure; also collected in `run.json` | | `reports/index_warnings.json` | Degraded-index events | | `reports/failed_inputs.csv` | Inputs abandoned during a multi-file run | | `reports/validation//` | Detailed semantic-validation reports | +**Finding a validation run.** Its artifacts are spread across three trees by +design — the stage JSON with the other stages, the detail beside the other +reports, the raw engine output under that — which is convenient for analysis and +awkward when diagnosing. `summary.json` therefore carries a `validation` array +naming all of them per target: + +```json +{"validation": [{ + "validation_id": "chr20", + "status": "PASS", "engine": "qlever", "validation_target": "aggregate", + "wall_seconds": 812.4, + "stage_report": "stages/validation/chr20.json", + "results_dir": "reports/validation/chr20", + "summary": "reports/validation/chr20/summary.json", + "benchmark_csv":"reports/validation/chr20/benchmark.csv", + "engines": ["comunica", "qlever"] +}]} +``` + +Start there rather than globbing. A target validated against a non-aggregate +artifact is suffixed: `chr20__hdt`. + `compression_operations/` preserves the per-RDF operation and round-trip validation reports; `compression/` is the final output-level summary over them. @@ -81,6 +110,20 @@ RSS, exit codes, and bounded `stderr` diagnostics — **and it survives the deletion of the temporary Docker volume**, which is what makes a failed cohort-scale run diagnosable at all. +**Recording the VCF version.** `metrics.csv` and +`stages/conversion/*.json` carry `vcf_version` (the specification version the +input was converted under, e.g. `VCFv4.4`) and `vcf_version_source`: + +| `vcf_version_source` | Meaning | +| --- | --- | +| `declared` | Read from the input's own `##fileformat` line | +| `forced` | `--vcf-version` overrode the declaration | +| `fallback` | The declaration was missing, malformed, or a version with no conformance overlay (VCF 4.0); the newest supported rules were used and no `vcfc:VCF4xFile` class was emitted | + +Benchmarks should group by these. The version selects the SHACL overlay, the +flattened-tuple semantics and which FORMAT families exist, so conversion cost +and graph size are not comparable across versions without them. + ## 3. Input size accounting `input_vcf_size_bytes` in `stages/conversion/*.json` and `metrics.csv` is the @@ -167,10 +210,35 @@ number, and no progress history is retained in memory. ## 7. Interrupts and exit codes -`Ctrl+C` exits with **130**, writes progress to `logs/progress.log`, and -performs best-effort cleanup of tracked intermediates. Raw RDF cleanup on -interrupt follows `--keep-rmlstreamer-rdf-output`: with it, raw RDF is -preserved; without it, tracked raw RDF files are removed. +`Ctrl+C` exits with **130** and runs cleanup in a fixed order: + +1. **Stop this run's containers.** Every container the wrapper starts carries a + `vcf-rdfizer.run=-` label, and cleanup kills exactly the ones + matching its own label — a concurrent run's containers are never touched. +2. **Remove this run's temporary Docker volumes.** Partitioned compression + works in a named volume, and Docker refuses to remove one while a container + still holds it — so this can only happen after step 1, and retries a few + times while the killed container releases it. +3. **Remove tracked intermediates.** Raw RDF follows + `--keep-rmlstreamer-rdf-output`: with it, raw RDF is preserved; without it, + tracked raw RDF files are removed. +4. **Write `interrupt-checkpoint.json`** into the run's metrics directory, + recording the stages that completed, the last progress event, how many + containers were stopped, how many volumes were removed, and what else was + deleted. + +The order matters, twice over. Killing the wrapper does not kill its +containers, and until this existed an interrupted run deleted files that its +own container was still reading — one such orphan ran for three days after the +run had "finished", holding 17 GB and a core, working against inputs that no +longer existed. And a volume cannot be removed before the container using it +stops: each volume's own `finally` ran as the exception unwound, *before* the +containers were killed, so it warned and gave up — leaking roughly 700 MB per +interrupted run, which nothing retried. + +A second `Ctrl+C` during cleanup is ignored, so an impatient operator cannot +abort the step that stops the containers. If cleanup itself fails the wrapper +still exits 130 and records `Interrupt cleanup error` in the progress log. Otherwise the wrapper exits `0` on success and non-zero on failure. In full mode with `--validate`, a validation failure sets a non-zero exit even when the diff --git a/docs/privacy-policy-design.md b/docs/privacy-policy-design.md index 7fe6128..228663a 100644 --- a/docs/privacy-policy-design.md +++ b/docs/privacy-policy-design.md @@ -28,7 +28,7 @@ haplotypes have been used to recover surnames from public genealogy databases Three consequences follow, and they shape everything below: -1. **Removing `vcfr:sampleId` does not anonymize anything.** The genotype vector +1. **Removing `vcfc:sampleId` does not anonymize anything.** The genotype vector that remains is a stronger identifier than the label you deleted. 2. **Access control is not anonymization.** A policy layer governs *who is permitted to receive what*, and creates an audit trail. It does not make the @@ -52,8 +52,8 @@ the shapes the conversion already emits (see | Cut | Addresses | Example | | --- | --- | --- | -| **By class** | all resources of a type | every `vcfr:SampleCall` | -| **By predicate** | all triples with a predicate | every `vcfr:sampleName` | +| **By class** | all resources of a type | every `vcfc:SampleCall` | +| **By predicate** | all triples with a predicate | every `vcfc:sampleName` | | **By sample** | one participant's contribution | everything hanging off `#samples/NA12878` | | **By region** | a genomic interval | `chr19:44,905,791-44,909,393` (*APOE*) | | **By declared field** | one INFO or FORMAT key | `DP` yes, `GT` no | @@ -69,10 +69,10 @@ easy to forget when thinking in terms of genotypes: - **IRIs themselves.** `file://cohort.vcf#sample/1/NA12878` contains the sample name. `#record/{ROW_ID}` is a monotonic counter, so row identifiers disclose the source ordering — and therefore approximate genomic position — even if - `vcfr:pos` is dropped. And `{SOURCE_FILE}` is the VCF's basename, which in + `vcfc:pos` is dropped. And `{SOURCE_FILE}` is the VCF's basename, which in practice is often `patient_12345.vcf`. Filtering triples while leaving IRIs intact leaks membership. See §7. -- **Header lines.** `vcfr:headerValue` and the raw line text carry `##source` +- **Header lines.** `vcfc:headerValue` and the raw line text carry `##source` (pipeline and centre identifiers), `##SAMPLE` and `##PEDIGREE` (family structure, by design), free-text `Description` fields, and `##fileDate`. A policy that covers genotypes and ignores the header section has not covered @@ -132,7 +132,7 @@ Namespace `vcfp:` = `https://w3id.org/vcf-rdfizer/policy#`, declared through ```turtle @prefix odrl: . @prefix vcfp: . -@prefix vcfr: . +@prefix vcfc: . @prefix duo: . <#apoe-locus> a odrl:Asset , vcfp:GraphSelection ; @@ -144,7 +144,7 @@ Namespace `vcfp:` = `https://w3id.org/vcf-rdfizer/policy#`, declared through <#direct-identifiers> a odrl:Asset , vcfp:GraphSelection ; vcfp:selector [ a vcfp:PredicateSelector ; - vcfp:predicate vcfr:sampleName , vcfr:sampleId ] . + vcfp:predicate vcfc:sampleName , vcfc:sampleId ] . <#withdrawn-participants> a odrl:Asset , vcfp:GraphSelection ; vcfp:selector [ a vcfp:SampleSelector ; @@ -293,7 +293,7 @@ For applying a policy to a graph that has already been converted Triple- and predicate-level rules are a single streaming pass. **Region rules are not**, because an N-Triples stream has no ordering guarantee: by the time -you see `<…#sample/4711/NA12878> vcfr:fieldValue "0/1"`, the triple that told +you see `<…#sample/4711/NA12878> vcfc:fieldValue "0/1"`, the triple that told you record 4711 is at `chr19:44906000` may be long gone. This needs two passes: pass one builds a `row_id → (chrom, pos)` map restricted to records that fall in a policy region, pass two filters. Memory scales with the number of **in-scope** @@ -380,7 +380,7 @@ This one is specific to VCF-RDFizer and easy to miss until it produces a leak. In **expanded** mode, one participant's value is its own resource: ```text -<…#sample/4711/NA12878/fmt/GT> vcfr:fieldValue "0/1" . +<…#sample/4711/NA12878/fmt/GT> vcfc:fieldValue "0/1" . ``` Excluding a sample is triple filtering. Straightforward. @@ -389,7 +389,7 @@ In **condensed** mode, all participants' values for one FORMAT key live in **one literal**: ```text -<…#call/4711/matrix/fmt/GT> vcfr:encodedValues "0/1\t0/0\t1/1\t./."^^vcfr:VCFTextVector . +<…#call/4711/matrix/fmt/GT> vcfc:encodedValues "0/1\t0/0\t1/1\t./."^^vcfc:VCFTextVector . ``` There is no triple to remove for one sample. Graph-pattern access control cannot @@ -400,7 +400,7 @@ Two consequences: 1. **Redaction must rewrite the literal.** `vcfp:maskVectorPositions` replaces the masked participant's token with `.`, which is the VCF missing marker and - keeps the vector aligned with `vcfr:sampleIndex` — alignment the format + keeps the vector aligned with `vcfc:sampleIndex` — alignment the format depends on. Dropping a position instead would silently shift every downstream sample's value, which is a data-corruption bug wearing a privacy feature's clothing. @@ -458,7 +458,7 @@ linkset node in [`datalinking-design.md`](datalinking-design.md#5-output-and-pro vcfp:policyDigest "sha256:9f2c…" ; vcfp:duoVersion "2024-11-03" ; vcfp:pseudonymKeyId "release-2026-09-key-3" ; - vcfp:representation vcfr:ExpandedRepresentation ; + vcfp:representation vcfc:ExpandedRepresentation ; vcfp:triplesWithheld 1840221 ; vcfp:samplesWithheld 2 ; vcfp:thresholdApplied 5 ; @@ -489,7 +489,7 @@ release that fails one is not published. This turns "the APOE region was excluded" from a claim into a measurement — the distinction [`validation-methodology.md`](validation-methodology.md) is built around. -**SHACL for shape-level guarantees.** "No `vcfr:sampleName` appears anywhere" is +**SHACL for shape-level guarantees.** "No `vcfc:sampleName` appears anywhere" is naturally a shape constraint, and the `--shacl-shapes` layer already exists. **Mutation testing the redactor.** Add a mutation class to diff --git a/docs/representations.md b/docs/representations.md index 9938280..676a774 100644 --- a/docs/representations.md +++ b/docs/representations.md @@ -11,7 +11,7 @@ Compression is deliberately not one option. It is three: | Decision | Flag | Values | Default | | --- | --- | --- | --- | -| How the aggregate is staged | `--rdf-storage-mode` | `plain`, `space-optimized` | *(required in full mode)* | +| How the aggregate is staged | `--rdf-storage-mode` | `plain`, `space-optimized` | `space-optimized` | | Which raw RDF artifacts to keep | `--rdf-compression` | `gzip`, `brotli`, `none` | `gzip,brotli` | | Which queryable representations to build | `--representations` | `hdt`, `cottas`, `none` | `hdt` | | How to package those representations | `--artifact-compression` | `gzip`, `brotli`, `none` | `none` | @@ -54,9 +54,36 @@ usable without ever expanding a second full raw copy. | `--chunk-max-bytes` | hard ceiling; boundaries stay on complete lines | 1 GiB | `--hdt-strategy` chooses the policy: `auto` (build chunks and merge with native -`hdtc`), `partitioned` (always chunk), `single` (one `rdf2hdt` run). `single` -cannot consume a gzip stream without expanding it, so it is incompatible with -`space-optimized`. +`hdtc`), `partitioned` (always chunk), `single` (one `rdf2hdt` run). + +**`single` is a verification path, not a faster alternative.** There is no size +regime in which it wins: below `--chunk-min-bytes` the partitioned path emits +one chunk and its merge is trivial, so the two converge, and above it +partitioned is the only one that survives cohort scale. What `single` is for is +producing a one-shot HDT of the same graph so the chunk-and-merge result can be +checked against it — identical triple counts and identical query answers are +what justify partitioned generation being the default, rather than it merely +being convenient. + +It applies only to an HDT-only run over an uncompressed aggregate, and both +other cases are refused rather than silently downgraded: + +- a gzip aggregate cannot be read in one pass without expanding a second full + uncompressed copy, so `single` is incompatible with `space-optimized` and + needs an explicit `--rdf-storage-mode plain`; +- `cottas` among the `--representations` always needs bounded chunks, so the + partitioned path would run for both representations and the strategy would + have no effect. Earlier releases ignored the flag silently here, which meant a + `--representations hdt,cottas --hdt-strategy single` run measured partitioned + HDT under a `single` label. + +So the one configuration where `single` does what it says is: + +```bash +vcf-rdfizer --mode full --input cohort.vcf.gz \ + --rdf-storage-mode plain --representations hdt \ + --rdf-compression none --hdt-strategy single --out results +``` The whole partitioned stage runs in an **ephemeral Docker-managed volume**. Chunks, scratch, and merge files never reach the output directory, and the @@ -154,7 +181,8 @@ unwrapped file never appears on the host. | Queryable, single artifact, smallest footprint | `--representations hdt --rdf-compression none --remove-rdf-storage-output` | | Comparing HDT against COTTAS | `--representations hdt,cottas` | | Archival transfer | `--artifact-compression brotli` on top of the chosen representation | -| Memory-constrained host | `space-optimized` + `partitioned` + lower `--chunk-*-bytes` | +| Memory-constrained host | `space-optimized` + `partitioned` (both default) + lower `--chunk-*-bytes` | +| Verifying the partitioned merge | `plain` + `--representations hdt` + `single`, compared against `partitioned` | ## 10. Limitations diff --git a/docs/rml-mappings.md b/docs/rml-mappings.md index 5f43668..4c76062 100644 --- a/docs/rml-mappings.md +++ b/docs/rml-mappings.md @@ -78,20 +78,87 @@ helper-table consumers works in both modes. ## 3. What a custom mapping does *not* control -This is the part most easily missed. Three families of triple are emitted by the -wrapper, not by RML, and they appear in your graph regardless of your mapping: +This is the part most easily missed, and it grew with the move to the VCF Core +vocabulary. Several families of triple are emitted by the wrapper, not by RML, +and they appear in your graph regardless of your mapping. + +The split follows one rule: + +> **The RML mapping carries every field whose RDF datatype is the same for every +> row. The wrapper carries everything else.** + +"Everything else" is two things. A value that may be the VCF missing token `.` +needs `"."^^vcfc:Null` on some rows and a typed literal on others, and RML +cannot switch an object's datatype per row. A value that has to be *decomposed* +— an angle-bracketed header line, an ALT list, a GT string, a comma-separated +`Number=A`/`R`/`G` payload — would need a materialized helper table whose size +is the product of its dimensions. | Triples | Turned off with | | --- | --- | -| Genotypes (`SampleCall`/`FormatFieldValue`, or the condensed equivalents) | nothing — one emitter always runs; choose which with `--sample-representation` | -| `QUAL`, and structured `InfoFieldValue` nodes | `--info-representation raw` still emits QUAL | -| Header-line subclasses and lifted FILTER/ALT/contig/INFO/FORMAT attributes | `--header-representation basic` | +| `ID`, `ALT`, `QUAL`, `FILTER`, `infoRaw` — the fixed fields that may be missing | nothing; they are the record's mandatory content | +| `fileDate` — no mandated format in VCF, so the value must be inspected to type it | nothing | +| Genotypes (`SampleCall`/`FormatFieldValue`/`Genotype`, or the condensed `CohortCallMatrix`/`FormatValueVector`) | nothing — one emitter always runs; choose which with `--sample-representation` | +| Structured `InfoFieldValue` nodes, the allele layer, `FieldValueItem` decomposition, and the SV carriers | `--info-representation raw` (`QUAL`, `ID`, `ALT`, `FILTER` and `infoRaw` are still emitted) | +| Header-line subclasses, `HeaderAttribute` resources, and the lifted INFO/FORMAT/FILTER/ALT/contig/META/SAMPLE/PEDIGREE attributes | `--header-representation basic` | +| The `vcfc:VCF4xFile` version class | nothing directly; `--vcf-version` selects which one, and a mapping without the sentinel emits none | + +Two of those are worth calling out because they are new: + +- **The allele layer travels with `--info-representation structured`** (the + default). `vcfc:hasReferenceAllele` and `vcfc:hasAltAllele` are minted there + because the `Number=A`/`R`/`G` value items join to them; turning INFO down to + `raw` turns the alleles off too. +- **`--header-representation basic` no longer produces a SHACL-conformant + graph.** `vcfc:StructuredHeaderLineShape` requires at least one + `vcfc:hasAttribute` on every structured header line, and `basic` emits no + subclass and no attributes at all. It remains available as the minimal, + fastest header form; it is simply not a conforming one. + +The reasons are in [`architecture.md`](architecture.md#4-where-the-split-leaks-and-why). +It is a real constraint on the extension point, not an oversight, but a mapping +author who assumes `--rules` is the sole source of truth for the graph will be +surprised. + +## 3a. The version sentinel + +The mapping carries one placeholder the wrapper resolves per input: + +```turtle +rr:subjectMap [ + rr:template "file://{SOURCE_FILE}" ; + rr:class vcfc:VCFFile ; + rr:class vcfc:VCFVersionFile # rewritten per input +] ; +``` -The reasons are in [`architecture.md`](architecture.md#4-where-the-split-leaks-and-why): -RML cannot choose a datatype or a class per row, and the alternatives would -require materializing helper tables. It is a real constraint on the extension -point, not an oversight, but a mapping author who assumes `--rules` is the sole -source of truth for the graph will be surprised. +`vcfc:VCFVersionFile` is not a vocabulary term. The wrapper rewrites it, exactly +the way it rewrites the five `csvw:url` paths, to the `vcfc:VCF4xFile` subclass +for the version that input's `##fileformat` line declares — which activates that +version's SHACL overlay gate. When the version has no conformance overlay +(VCFv4.0, or a missing or malformed line) it is rewritten to `vcfc:VCFFile`, +which the subject map already asserts, so the graph never claims a gate that +cannot be checked and never gains an undefined term. + +One mapping is therefore the single source of truth for all five versions, +rather than five near-identical copies that would drift apart. + +`vcf-rdfizer-rules check` reports whether your mapping has the sentinel. A +mapping without it still converts, and the version-dependent *emitter* behaviour +still follows each input's own `##fileformat` line — only the class is missing, +so the version overlay's gate stays inactive: + +``` +Warnings: + - No vcfc:VCFVersionFile sentinel in the file subject map. … +``` + +To render a materialized per-version mapping — to read it, or to hand-edit one +version's rules — render it for a specific version and inspect the result: + +```bash +vcf-rdfizer-rules init -o my_rules.ttl # keeps the sentinel +``` ## 4. What a custom mapping costs in validation @@ -144,8 +211,11 @@ Recommendations that come from experience rather than from the checker: 1. **Preserve the stable subject templates** (`file://{SOURCE_FILE}`, `file://{SOURCE_FILE}#record/{ROW_ID}`, `file://{SOURCE_FILE}#call/{ROW_ID}`) if you want joins with the wrapper-emitted triples to keep working. The - genotype, QUAL, INFO and header emitters all mint IRIs against those - templates and will not follow a mapping that changes them. + genotype, record-detail, allele and header emitters all mint IRIs against + those templates and will not follow a mapping that changes them. The allele + and genotype layers additionally depend on + `file://{SOURCE_FILE}#record/{ROW_ID}/allele/{n}` and + `file://{SOURCE_FILE}#samples/{SAMPLE_URI_ID}`. 2. **Add, don't replace.** A mapping that extends the default with additional `rr:TriplesMap` blocks keeps the full validation suite meaningful for the parts it did not change; one that rebuilds the graph from scratch does not. @@ -154,11 +224,30 @@ Recommendations that come from experience rather than from the checker: ## 6. SHACL -The SHACL constraints are maintained in the vocabulary repository, not here: -[`vcf-rdfizer-vocabulary.shacl.ttl`](https://github.com/ecrum19/VCF-RDFizer-vocabulary/blob/main/shacl/vcf-rdfizer-vocabulary.shacl.ttl). +The SHACL constraints are maintained in the vocabulary repository, not here. +VCF Core ships two profiles: + +- `shacl/vcf-core-vocabulary.shacl.ttl` — the portable profile: required + structural links, VCF 4.5 lexical patterns, and field type/arity constraints. +- `shacl/vcf-core-vocabulary-sparql.shacl.ttl` — cross-resource checks that need + SPARQL: header and record ordering, sample uniqueness, GT-first FORMAT order, + and the rule that a graph must not mix expanded sample calls with condensed + call matrices. + Point `--shacl-shapes` at them to check a graph structurally, independently of -the VCF. Note the known contradiction inside the published shapes, recorded in -[`vcf-coverage.md`](vcf-coverage.md#an-open-conflict-inside-the-vocabulary). +the VCF. Both profiles need the bundled ontology as an ontology graph and RDFS +inference enabled — subclass-based targets and `sh:class` constraints do not +resolve without it (a concrete `INFOHeaderLine` is also an +`InfoFieldDefinition`, and only inference makes that true). + +The default mapping's output validates against both profiles with zero +violations in both representation profiles. One inconsistency inside the +published vocabulary is worth knowing about: the ontology declares +`rdfs:range xsd:positiveInteger` on the ordinal properties while every SHACL +shape constrains them with `sh:datatype xsd:integer`, and a SHACL datatype +constraint compares the datatype IRI exactly. This converter emits +`xsd:integer`, which satisfies both. See +[`validation-migration-notes.md`](validation-migration-notes.md#3-ordinals-are-now-xsdinteger-not-xsdpositiveinteger). --- @@ -168,3 +257,5 @@ the VCF. Note the known contradiction inside the published shapes, recorded in - [Architecture](architecture.md) — why some triples bypass RML - [Validation](validation.md) — what changes under a custom mapping - [`rules/README.md`](../rules/README.md) — the mapping directory itself +- [Validation migration notes](validation-migration-notes.md) — what the move to + VCF Core leaves to do in the validation suite diff --git a/docs/roadmap.md b/docs/roadmap.md index 0bf7b95..6b73f65 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -9,34 +9,51 @@ block something else, not by effort. --- +## Resolved by the move to the VCF Core vocabulary + +Both items that previously blocked publication are closed. They are kept here +briefly because the reasoning is referenced elsewhere. + +### ~~1. Vocabulary coverage for the condensed representation~~ — **done** + +The 17 condensed-representation terms the conversion emitted without an +ontology behind them — `CohortCallMatrix`, `FormatValueVector`, `SampleSet`, +`VCFSample`, `VCFTextVector`, `representationProfile`, `sampleIndex` and the +rest — are all defined in VCF Core, with SHACL shapes. Condensed graphs are +ontology-backed and citable as linked data. + +### ~~2. The SHACL / missing-value contradiction~~ — **done** + +`vcfc:VCFRecordShape` now accepts `xsd:string` or `vcfc:Null` for `vcfc:alt`, so +a record with `ALT=.` satisfies both the shape and `vcfc:missingValuePolicy`. +`vcfc:VariantCallShape` likewise accepts the full VCF Float lexical space for +`qual`. + ## Blocking publication -### 1. Vocabulary coverage for the condensed representation — **done, pending release** +### ~~1. The validation suite has not been migrated~~ — **done** -The conversion emitted 17 terms that `https://w3id.org/vcf-rdfizer/vocab#` did -not define — `CohortCallMatrix`, `FormatValueVector`, `SampleSet`, `VCFSample`, -`VCFTextVector`, `representationProfile`, `sampleIndex` and the rest — so -condensed graphs were not ontology-backed. +The suite passes: 415 tests, mutation score 96/113 (85%) across 60 mutations. +The runner and the wrapper now share one vocabulary module instead of mirroring +each other, and the census derives every new resource family from the VCF. Ten +of the eighteen new mutations are recorded gaps, each with what would close it. +What changed and what remains: +[`validation-migration-notes.md`](validation-migration-notes.md). -All 17 are now defined in the vocabulary repository (v1.1.0), with two -superclasses giving the enumerations a range, SHACL shapes for the condensed -profile, and a worked example. What remains is publication: the terms only -dereference for a third-party consumer once v1.1.0 is deployed to the w3id -namespace. Nothing further is required in this repository. -Detail: [`vcf-coverage.md`](vcf-coverage.md#vocabulary-alignment). +### 2. SHACL conformance is not a test -### 2. The SHACL / missing-value contradiction — **resolved** +Checking the output against the vocabulary's own SHACL profiles found four real +modelling bugs during the migration, all of the same kind: a resource typed into +a class whose shape requires a property the record did not supply. That is a +cheap, strong invariant and it should not be a manual step. It needs `pyshacl` +and `rdflib` as dev dependencies and access to the vocabulary's `ontology/` and +`shacl/` directories. -`vcfr:missingValuePolicy` required `"."^^vcfr:Null` for a missing token while -`VCFRecordShape` constrained `vcfr:alt` to `sh:datatype xsd:string`, so a record -with `ALT=.` could satisfy neither and the tool could not be conformant. +### ~~3. Ordinal datatype inconsistency in the vocabulary~~ — **done** -Vocabulary v1.1.0 resolves it by stating where VCF actually permits the token: -`alt` and `recordId` now accept `xsd:string` or `vcfr:Null`, as `qual` already -did, while `chrom`, `pos` and `ref` keep an exact datatype because they are -required and have no missing form. The conversion already followed the policy, -so no change was needed here. -Detail: [`vcf-coverage.md`](vcf-coverage.md#vocabulary-alignment). +VCF Core now supplies `vcfc:IntegerLiteralShape`, which accepts `xsd:integer` +and every integer-derived XSD datatype with the bound applied numerically. The +ontology range and the shapes no longer disagree. ## Known defects @@ -57,11 +74,11 @@ Small change; it makes the custom-mapping extension point actually usable. From [`vcf-coverage.md`](vcf-coverage.md#remaining-gaps): -- **`vcfr:contigCount` is counted, not read.** A wrong derived contig total is +- **`vcfc:contigCount` is counted, not read.** A wrong derived contig total is undetected. Needs the value in a comparison, not just the predicate in the census. - **Header line values are not compared.** `q08` compares how many lines carry - each key and `q10` their types; `vcfr:headerValue` itself is only counted. The + each key and `q10` their types; `vcfc:headerValue` itself is only counted. The structured attributes that matter are already covered. Both are query-set additions with matching mutation-catalogue entries, so @@ -72,14 +89,16 @@ closing either one will *fail* the mutation harness until ### 5. Data linking as a plug-in system -The largest planned addition: let users connect the graph to external resources -(rsIDs, genes, clinical assertions, frequencies) through declarative, -distributable linker plugins rather than by patching the tool. +An initial implementation now connects the graph through declarative, +distributable linker plug-ins, with examples of all three tiers: dbSNP token +links, a synthetic GFF3 interval bundle, and an Ensembl API resolver. Full design, including the three join strategies, the three plugin tiers, the network safeguards, the provenance model and the build order, is in -[`datalinking-design.md`](datalinking-design.md). Nothing is implemented yet; -the design exists to fix the extension contract before code depends on it. +[`datalinking-design.md`](datalinking-design.md). The implemented contract and +commands are in [`datalinking.md`](datalinking.md). Allele joins/normalization, +link merging, and plug-in validation/mutation auto-discovery remain planned; +the manifest vocabulary is provisional. The hardest part is not the plugin system — it is allele normalization (§9 of that document), because an under-matching join looks exactly like a true @@ -162,4 +181,4 @@ Stated so the absence reads as a decision rather than an oversight. - [Data linking design](datalinking-design.md) — the largest planned addition - [Privacy policy design](privacy-policy-design.md) — governed release over parts of the graph - [Validation methodology](validation-methodology.md) — how coverage is measured, so gaps stay falsifiable -- [`changelog.md`](../changelog.md) — what has actually shipped +- [Releases](https://github.com/ecrum19/VCF-RDFizer/releases) — what has actually shipped diff --git a/docs/sample-representation-guide.md b/docs/sample-representation-guide.md index 287c112..60fc8dd 100644 --- a/docs/sample-representation-guide.md +++ b/docs/sample-representation-guide.md @@ -45,24 +45,41 @@ that is much more economical for large cohorts. ## 2. Expanded representation: one RDF object per sample value Expanded mode preserves the original, explicit vocabulary model. For every -variant/sample pair, VCF-RDFizer creates a `vcfr:SampleCall`. For every FORMAT -field in that pair, it creates a `vcfr:FormatFieldValue`. +variant/sample pair, VCF-RDFizer creates a `vcfc:SampleCall`. For every FORMAT +field in that pair, it creates a `vcfc:FormatFieldValue`. For the example above, the conceptual graph includes resources like these (the full output has one line per RDF statement): ```text - vcfr:hasSampleCall . + vcfc:hasSampleCall . - a vcfr:SampleCall ; - vcfr:sampleId "SAMPLE_A" ; - vcfr:hasFormatValue . + a vcfc:SampleCall ; + vcfc:sampleId "SAMPLE_A" ; + vcfc:forSample ; + vcfc:hasFormatValue ; + vcfc:hasGenotype . - a vcfr:FormatFieldValue ; - vcfr:fieldValue "0/1" . + a vcfc:FormatFieldValue ; + vcfc:declaredBy ; + vcfc:fieldValue "0/1" . + + + a vcfc:Genotype ; + vcfc:genotypeString "0/1"^^vcfc:GenotypeString ; + vcfc:ploidy 2 ; + vcfc:phasingStatus vcfc:Unphased ; + vcfc:hasAlleleCall , + . + + + a vcfc:GenotypeAlleleCall ; + vcfc:callIndex 1 ; + vcfc:isNoCall false ; + vcfc:calledAllele . ``` The same structure is repeated for `DP` and `AD`, and then repeated again for @@ -73,14 +90,22 @@ The same structure is repeated for `DP` and `AD`, and then repeated again for Expanded mode makes each value easy to address directly: - a consumer can find one sample call without decoding a vector; -- a consumer can ask for one FORMAT field as one RDF resource; -- existing consumers that expect `SampleCall` and `FormatFieldValue` can use - the graph directly; -- the sample identifier is attached to every sample-call resource. +- a consumer can ask for one FORMAT field as one RDF resource, and follow + `vcfc:declaredBy` to the `##FORMAT` declaration that defines it; +- the sample identifier is attached to every sample-call resource, and + `vcfc:forSample` additionally gives it a reusable, file-scoped identity, so a + sample is a resource rather than a repeated literal; +- **genotypes are queryable directly.** `GT` is parsed into an ordered + `vcfc:Genotype`: a consumer can filter on `vcfc:phasingStatus`, count + `vcfc:ploidy`, or join `vcfc:calledAllele` straight to the record's allele + resources — without string-matching `"0/1"` or `"0|1"` in SPARQL. `FT`, + `PS`/`PSL`, `LAA`, the copy-number and haplotype keys, and the `M`/`DPM`/`ADM` + base modifications get their own resources on the same principle. The VCF file is marked with -`vcfr:representationProfile vcfr:ExpandedRepresentation`, so a consumer can -identify the graph shape explicitly. +`vcfc:representationProfile vcfc:ExpandedRepresentation`, so a consumer can +identify the graph shape explicitly. The SPARQL SHACL profile enforces that a +graph never carries both profiles at once. ### Expanded growth @@ -97,15 +122,18 @@ V × S SampleCall resources V × S × F FormatFieldValue resources ``` -In the simplified case where each sample call has three structural statements -(link, type, and sample ID), and each FORMAT value has three statements (link, -type, and value), the sample portion of the graph contains approximately: +In the simplified case where each sample call has four structural statements +(link, type, sample ID, and `forSample`), and each FORMAT value has four +statements (link, type, `declaredBy`, and value), the sample portion of the +graph contains approximately: ```text -3 × (V × S) + 3 × (V × S × F) +4 × (V × S) + 4 × (V × S × F) ``` -statements, before counting the rest of the VCF graph. +statements, before counting the parsed genotype layer and the rest of the VCF +graph. A diploid `GT` adds roughly nine more statements per sample call: five +for the `vcfc:Genotype` and four for each `vcfc:GenotypeAlleleCall`. For the worked example (`V=1`, `S=3`, `F=3`): @@ -113,9 +141,16 @@ For the worked example (`V=1`, `S=3`, `F=3`): |---|---:| | `SampleCall` resources | 3 | | `FormatFieldValue` resources | 9 | -| Approximate sample-related statements | 36 | +| `Genotype` resources | 3 | +| `GenotypeAlleleCall` resources | 6 | +| Approximate sample-related statements | 75 | +| File-level `SampleSet` + `VCFSample` statements | 18 | | Representation-profile statement | 1 | +The genotype layer makes expanded mode more expensive than it was, which sharpens +rather than changes the trade-off below: expanded mode buys direct queryability +at a cost that multiplies by both samples and FORMAT fields. + The exact number can vary when values are missing or when custom mappings add properties, but the important pattern is the multiplication by both samples and FORMAT fields. @@ -130,15 +165,15 @@ For the same example, the graph first declares a reusable sample set: ```text - a vcfr:SampleSet ; - vcfr:hasSample ; - vcfr:hasSample ; - vcfr:hasSample . + a vcfc:SampleSet ; + vcfc:hasSample ; + vcfc:hasSample ; + vcfc:hasSample . - a vcfr:VCFSample ; - vcfr:sampleName "SAMPLE_A" ; - vcfr:sampleIndex "1" . + a vcfc:VCFSample ; + vcfc:sampleName "SAMPLE_A" ; + vcfc:sampleIndex "1" . ``` The variant then has one cohort matrix. The matrix points back to the sample @@ -146,14 +181,14 @@ set and has one vector for each FORMAT key: ```text - a vcfr:CohortCallMatrix ; - vcfr:appliesToSampleSet ; - vcfr:hasFormatValueVector . + a vcfc:CohortCallMatrix ; + vcfc:appliesToSampleSet ; + vcfc:hasFormatValueVector . - a vcfr:FormatValueVector ; - vcfr:valueEncoding vcfr:VCFTextVector ; - vcfr:encodedValues "0/1\t0/0\t1/1" . + a vcfc:FormatValueVector ; + vcfc:valueEncoding vcfc:VCFTextVector ; + vcfc:encodedValues "0/1\t0/0\t1/1" . ``` The three tab-separated items in `encodedValues` are in `sampleIndex` order: @@ -170,7 +205,7 @@ it is not treated as a sample separator. If a value is absent, the vector uses `.` to keep every sample position aligned. The file is marked with -`vcfr:representationProfile vcfr:CondensedRepresentation`. +`vcfc:representationProfile vcfc:CondensedRepresentation`. ### What condensed mode provides @@ -187,6 +222,18 @@ Condensed mode therefore moves repetition from RDF structure into the literal payload of a vector. It is a storage and graph-shape optimization, not a loss of genotype information. +**What it does not carry.** Condensed mode stops at the vector: it derives no +`vcfc:Genotype`, `vcfc:PhaseSet`, `vcfc:LocalAlleleSet` or +`vcfc:BaseModification`. Those are one resource per sample per record, which is +precisely the growth this profile exists to prevent. The values are still all +there — inside the vectors — but a SPARQL engine cannot filter on them without +a decoder or an application-level vector function. If you need to query +genotypes directly, that is the argument for expanded mode. + +Everything *not* per-sample is identical in both profiles: the header layer, the +allele layer, structured INFO with its value items, and the SV carriers are all +emitted the same way. + ### Condensed growth Using the same variables, condensed mode creates approximately: @@ -369,7 +416,7 @@ text values. See the [GeoSPARQL function table](https://graphdb.ontotext.com/doc and [GraphDB GeoSPARQL extensions](https://graphdb.ontotext.com/documentation/11.5/sparql-ext-functions-reference.html#geosparql-extension-functions). Our condensed value is instead a normal RDF string associated with -`vcfr:VCFTextVector`, for example: +`vcfc:VCFTextVector`, for example: ```text "0/1\t0/0\t1/1" @@ -390,13 +437,13 @@ each split item. The implementation is described as using Java's could look like this: ```sparql -PREFIX vcfr: +PREFIX vcfc: PREFIX spif: SELECT ?vector ?token WHERE { - ?vector a vcfr:FormatValueVector ; - vcfr:valueEncoding vcfr:VCFTextVector ; - vcfr:encodedValues ?encoded . + ?vector a vcfc:FormatValueVector ; + vcfc:valueEncoding vcfc:VCFTextVector ; + vcfc:encodedValues ?encoded . ?token spif:split (STR(?encoded) "\t") . } ``` @@ -415,7 +462,7 @@ generate a sequence of integers, but the reference does not provide a direct that `spif:split` alone is not a complete, reliable sample lookup mechanism. The page also lists RDF-list functions such as `list:index` and -`list:length`. Those operate on RDF Collections. `vcfr:encodedValues` is a +`list:length`. Those operate on RDF Collections. `vcfc:encodedValues` is a single string literal, not an RDF Collection, so these functions do not apply unless the data model is changed to materialize every vector item as list structure. That would reintroduce much of the per-item RDF overhead that @@ -433,7 +480,7 @@ they do not automatically parse a persisted VCF vector literal. | `spif:for` | Generate expected sample positions | Helpful support function, but does not pair positions with split tokens | | `list:index` / `list:length` | Index RDF Collections | Not applicable to the current string encoding | | `helper:tuple` / `helper:iterate` | Work with internal query lists | Not a persisted-vector parser | -| A custom `vcfr:` vector function | Return value at a sample index | Most direct future SPARQL integration | +| A custom `vcfc:` vector function | Return value at a sample index | Most direct future SPARQL integration | | Application-side parsing | Decode one selected vector after retrieval | Best portable near-term approach | GraphDB explicitly labels these as extensions beyond the W3C SPARQL @@ -453,16 +500,16 @@ returns both the position and the value. Conceptually, it could have a function or magic-predicate contract like: ```text -vcfr:vectorValue(?vector, ?sampleIndex) → ?value +vcfc:vectorValue(?vector, ?sampleIndex) → ?value ``` or: ```text -?vector vcfr:valueAt (?sampleIndex ?value) +?vector vcfc:valueAt (?sampleIndex ?value) ``` -For a sample-aware form, the function could accept the `vcfr:VCFSample` IRI +For a sample-aware form, the function could accept the `vcfc:VCFSample` IRI instead of an integer, resolve that resource's `sampleIndex`, and then extract the matching token. Returning the index as well as the value would make it possible to validate that a vector has the expected number of positions. @@ -523,7 +570,7 @@ Large multi-sample cohort + storage/scale focus → condensed ``` If a consumer is unsure which graph it received, inspect the file’s -`vcfr:representationProfile` value before querying. That profile is the +`vcfc:representationProfile` value before querying. That profile is the explicit signal that tells the consumer whether sample values are represented as individual resources or as ordered vectors. diff --git a/docs/validation-methodology.md b/docs/validation-methodology.md index 47000a6..adc3cc5 100644 --- a/docs/validation-methodology.md +++ b/docs/validation-methodology.md @@ -94,7 +94,7 @@ clean result. The SHACL layer earned its place immediately: it found three conformance violations the other two could not see, two of which were fixed (QUAL typed as -a plain literal instead of `xsd:decimal`/`vcfr:Null`; `##fileDate` untyped +a plain literal instead of `xsd:decimal`/`vcfc:Null`; `##fileDate` untyped instead of `xsd:date`) and one of which is a contradiction inside the vocabulary itself, recorded in [`vcf-coverage.md`](vcf-coverage.md). diff --git a/docs/validation-migration-notes.md b/docs/validation-migration-notes.md new file mode 100644 index 0000000..6f70518 --- /dev/null +++ b/docs/validation-migration-notes.md @@ -0,0 +1,178 @@ +# Validation and the VCF Core migration + +The conversion targets the **VCF Core vocabulary** +(`https://w3id.org/vcf-core/vocab#`, prefix `vcfc:`), replacing the retired +VCF-RDFizer vocabulary (`https://w3id.org/vcf-rdfizer/vocab#`, prefix `vcfr:`), +and follows each input's declared VCF version. The validation suite has been +migrated with it. This file records what changed, what it found, and what is +still open. + +**Status: the suite passes.** 415 tests, no failures. The mutation score is +96/113 (85%) across 60 mutations, each listed with its expected detector. + +--- + +## What the migration preserved + +The suite's design is what made this tractable, so it was kept intact: + +- **One declarative fixture, three derived artifacts.** + `test/validation_fixtures.py` still declares one VCF and derives the graph and + the oracle from it, so a fixture change cannot make the two disagree. +- **Coverage measured, not asserted.** A mutation the suite cannot catch is + recorded with `known_undetected`, and the harness asserts it is *still* not + caught — so closing a gap fails a test and forces the catalogue and the + coverage documentation to be updated together. +- **Expectations derived from the VCF, never from the mapping.** Deriving them + from `default_rules.ttl` would make the mapping test itself, which is how QUAL + once stayed invisible. `base_triples()` still hand-writes what the mapping + produces, so a mapping change the fixture does not follow shows up as a + mismatch rather than being absorbed. + +## What changed + +### 1. One vocabulary module, not two mirrored copies + +`src/validation/validation_runner.py` carried its own copies of +`HEADER_LINE_CLASSES`, `rml_uri_component`, `parse_structured_header_fields` and +`parse_info_entries`, with unit tests asserting the copies stayed identical to +the wrapper's. They are gone: both import `vcf_rdfizer_vocab`, and the +`Dockerfile` copies that module next to the runner so the import resolves the +same way inside the container and on the host. + +That removes the drift the tests were policing rather than continuing to police +it. `test_header_class_map_matches_the_wrapper` now compares a module against +itself; it is kept as a guard against someone reintroducing a second copy. + +### 2. The census learned the new layers + +`expected_census` asserts an exact inventory — *these* predicates and classes +with *these* counts and nothing else — so every family the VCF Core emitters +added had to become derivable from the VCF. Two shared functions do that: + +- `emitted_header_counters(header_lines)` — header-line subclasses, + `vcfc:HeaderAttribute` resources, the INFO/FORMAT/FILTER/ALT/contig/META/ + SAMPLE/PEDIGREE declaration properties, `assemblyUrl`, `pedigreeDbUrl`. +- `emitted_record_counters(rows, samples, …)` — the allele layer, the + `vcfc:FieldValueItem` decomposition, the breakend components, and — reported + separately, because it is expanded-only — the parsed genotype layer. + +Both are called by `parse_vcf` inside the container and by the fixture's +`parser_summary` on the host, so the two oracles cannot disagree about what the +graph should contain. `parse_vcf` stays representation-independent, which is +what lets one parse serve both censuses. + +### 3. `vcfc:VCF4xFile` is part of the expectation + +The mapping's version sentinel resolves to the subclass for the version the +input declares, so the census expects that class. The fixture derives it from +its own `FILE_FORMAT` rather than hard-coding it, so changing the fixture's +declared version changes the expected graph with it. + +### 4. Fields that moved from RML to the wrapper + +`ID`, `ALT`, `FILTER` and `infoRaw` left `base_triples()`: each may be the VCF +missing token, which the vocabulary requires as `"."^^vcfc:Null`, and RML cannot +switch an object's datatype per row. The counts are unchanged — one per record — +so only the fixture's account of *which half emits them* moved. + +`preflight_missing_token_conformance` is consequently expected to return zero +rows for any input now. Before the migration, a record with `ID=.` produced a +plain `"."` literal and the check reported it. + +### 5. Ordinals are `xsd:integer` + +The fixture's `XSD_POSITIVE_INTEGER` is gone. VCF Core's +`vcfc:IntegerLiteralShape` accepts `xsd:integer` and every integer-derived +datatype with the bound applied numerically, so the ontology range and the +shapes no longer disagree. + +## What the suite found + +Running it against the migrated emitters surfaced a real conversion defect that +neither the unit tests nor SHACL had caught: + +> **Every declared INFO and FORMAT definition was emitted twice.** The header +> emitter produced a declaration's `fieldId`/`fieldNumber`/`fieldType`/ +> `fieldDescription` from the `##` line, and the value emitters produced the +> same four triples at the same header-line IRI when the key was first +> referenced — twenty duplicate triples in the fixture graph alone. + +`preflight_duplicate_triples` caught it: a check that compares the parsed triple +count against the distinct count. No shape can express that, because a duplicate +triple is not a distinct RDF statement at all — SHACL validated the graph +happily, since the duplicate simply vanishes on parse. It is a good argument for +keeping the graph-integrity preflights alongside the shape layer rather than +treating SHACL as a replacement for them. + +The fix gives the header emitter sole ownership of any declaration derived from +a `##` line. The value emitters cite it with `vcfc:declaredBy` and invent a +definition only for a key no header declared. +`test_declared_field_definitions_are_emitted_exactly_once` pins it. + +## Recorded gaps + +Ten of the 18 new mutations are undetected, each with what would close it. They +fall into three groups. + +**Values counted but not compared.** `corrupt_allele_value`, +`corrupt_allele_kind`, `corrupt_attribute_value`, `corrupt_value_item_allele`, +`corrupt_called_allele`, `flip_phasing_status`, `wrong_declaration_owner`. +Dropping any of these resources is caught by the census; corrupting one inside +is not. Closing them means digests over those resources, the way `q11` covers +the record fields. Several are already checked by the vocabulary's own +`vcf-core-consistency.shacl.ttl`, which verifies raw/parsed agreement — so the +cheapest route is probably to run that profile rather than to reimplement it in +SPARQL. + +**Ordering, covered in a different layer.** `corrupt_record_index` is undetected +by the queries. The SPARQL SHACL profile enforces index uniqueness, nondecreasing +POS within a CHROM block, and contiguous CHROM blocks. + +**One profile-specific asymmetry, worth stating.** `corrupt_sample_index` is +detected in the condensed profile and not in the expanded one, and the catalogue +carries it as two entries for that reason. In the condensed profile the ordinal +is what associates a vector position with a sample, so corrupting it moves +genotypes between samples and `q05`/`q06` shift. In the expanded profile each +`SampleCall` carries its own `vcfc:sampleId`, so nothing reads the ordinal. That +is a real difference in what the two profiles depend on, not a testing +oversight. + +## Still open + +**SHACL conformance is not a test.** Validating the output against the +vocabulary's profiles found seven modelling bugs during this work and is a +stronger, cheaper invariant than a hand-derived oracle. It needs `pyshacl` and +`rdflib` as dev dependencies plus a path to the vocabulary's `ontology/`, +`ontology/versions/` and `shacl/` directories, and it is slow — roughly ten +seconds per graph with the full profile set — so it belongs in a marked slow +suite rather than the default unit run. + +**The fixture declares one VCF version.** Version-dependent behaviour is covered +by unit tests and by the version overlays, not by the mutation harness. The +vocabulary repository ships one example VCF per version under +`examples/vcf-versions/`, a ready-made corpus for a per-version fixture matrix +if that coverage is wanted. + +**The SV carriers have no mutation.** They are represented and conformant, and +the census counts the families the fixture exercises, but the fixture contains +no structural variants, so `emitted_record_counters` is untested against +breakends, tandem repeats and reference blocks. Adding one SV record to the +fixture would exercise them through the existing machinery. + +## How conformance was verified + +Separately from the suite, the converter's output was validated against the +**complete** published profile set — the portable, SPARQL and consistency +profiles plus all five version overlays — with pySHACL, the bundled ontology +plus every `ontology/versions/*.ttl`, and RDFS inference. That is the +configuration the vocabulary's own `tests/validate_shacl.py` uses. + +Inputs: all ten example VCFs from the vocabulary repository (4.1 through 4.5, +including the breakend, gVCF, repeat and condensed-cohort fixtures), and a +hand-written stress file rendered for every version plus a 4.0 variant, in both +representation profiles. + +Result: **zero violations** across 19 graphs. The only results are `sh:Warning`s +that VCF 4.5 recommends `Source` and `Version` on `##INFO` declarations — a +property of the input files, which the converter correctly does not fabricate. diff --git a/docs/validation.md b/docs/validation.md index e1fb75e..ba1b45b 100644 --- a/docs/validation.md +++ b/docs/validation.md @@ -24,6 +24,99 @@ Raptor syntax validation and for the SPARQL queries, and removes any temporary decode before the container exits. No decoded RDF is written beneath `--out`; only reports are retained. +## Choosing an engine, and what happens when it cannot keep up + +**Every engine is now set up once per target**, and all 27 queries reuse that +setup. Setup is timed separately as `setupSeconds` so a benchmark can attribute +it correctly instead of smearing it across 27 query timings. + +| Engine | What is paid once | +| --- | --- | +| `comunica` | one `comunica-sparql-file-http` process: Node startup and engine init | +| `hdt` | one `comunica-sparql-hdt-http` process, holding the memory-mapped `.hdt` | +| `qlever` | an on-disk index build, then its own HTTP server | +| `cottas` | one `COTTASStore`-backed rdflib graph | + +Earlier releases spawned the one-shot `comunica-sparql-file` and +`comunica-sparql-hdt` CLIs **once per query**, paying startup 27 times. +Measured in this image on 1.5M N-Triples, per `SELECT (COUNT(*))`: + +| | per query | +| --- | --- | +| one-shot `comunica-sparql-file` | 26.0s | +| `comunica-sparql-file-http` endpoint | 10.3s | + +**Comunica still has no index, and the endpoint does not change that.** It +streams the source for every query: `LIMIT 1` answers in 0.59s while `COUNT(*)` +takes 10s on every repeat. So a full-scan query still re-reads the whole graph +each time — the endpoint removes per-query *startup*, not per-query *scanning*. + +Materializing the graph into an in-memory store instead was measured and +rejected: an `N3.Store` of the same 1.5M triples cost 2.45 GiB and made `COUNT` +*slower* (15.0s), which extrapolates to roughly 145 GiB for an 88M-triple +cohort graph. Above fixture scale the answer is an engine that indexes, not a +bigger heap. + +| Graph size | Recommended | +| --- | --- | +| Fixtures, small VCFs | `--validation-engine comunica` (default; no index build) | +| Anything above a few GiB of N-Triples | `--validation-engine qlever`, or validate the indexed artifact with `--validate-artifacts hdt` | + +Above 4 GiB the runner warns before it starts and names the alternatives. + +### If validation seems to hang + +First check `setupSeconds` in the engine report. Slow *setup* is a scale +problem (an index build, or a source the endpoint struggles to read); a slow +*query* is a `--validation-query-timeout` problem (default 3600s). + +**Every query runs, always.** A query that fails, times out, or returns the +wrong answer is recorded and the suite moves on — a validation run is a +coverage report, and "26 of 27 passed, q11 disagreed" is a result while +"stopped at q01" is not. `--validation-stop-after-query-timeout` opts into +abandoning an engine's remaining queries after the first timeout, which caps +the worst case at one timeout per engine rather than one per query: + +```json +{"status": "EXECUTION_FAILED", + "abandoned": "q01_record_density_1mb exceeded the 3600s per-query timeout; skipped 26 further queries …", + "queriesRun": 1, "queriesPlanned": 27} +``` + +Options for a graph that is genuinely too large: + +```bash +# Build an index once and query from it. +vcf-rdfizer --mode full -i cohort.vcf.gz --validate --validation-engine qlever -o ./out + +# Or validate the HDT artifact, which is already indexed. +vcf-rdfizer --mode full -i cohort.vcf.gz --validate --validate-artifacts hdt -o ./out + +# Or allow longer per query, with a ceiling on the whole engine. +vcf-rdfizer --mode full -i cohort.vcf.gz --validate \ + --validation-query-timeout 14400 --validation-time-budget 86400 -o ./out +``` + +### A decode that reports success and writes garbage + +Every decode into N-Triples is now checked structurally before anything reads +it: non-empty, newline-terminated, with a parseable first statement. A step +that exits 0 but writes errors to stderr is treated as failed. + +This exists because hdt-cpp's `hdt2rdf` **file** serializer is broken: it emits +`error: :0:0: write error` once per triple, writes truncated subject IRIs with +no terminators and no newlines at all, and still exits 0. Its stdout serializer +is correct, so the decode is taken from stdout and redirected. Left unchecked, +the malformed dump reached `rapper`, which saw one multi-gigabyte "line", +degraded to quadratic parsing, and spent 38 hours reaching 23% of a 4.4 GiB +file — a hang, in every way that matters, produced by a tool that reported +success. + +Queries run in their own process session, so a timeout kills the engine along +with its `/usr/bin/time` wrapper. Before that fix the engine survived the +timeout still holding the whole graph in memory, and enough of those pushed the +host into swap — which is what made a slow step look like a stuck one. + ## Run it Use the representation selected when the RDF was created. @@ -376,7 +469,7 @@ anomaly count. Reports carry both, plus `sampleTruncated`, so a graph with ten million anomalies is never confused with one that has a hundred. `--strict-conformance` promotes a missing-token conformance failure (a plain -`"."` literal not typed as `vcfr:Null`) from a report-only observation to a +`"."` literal not typed as `vcfc:Null`) from a report-only observation to a validation failure. ### SHACL @@ -416,7 +509,7 @@ backed by a named mutation in [`test/validation_mutations.py`](../test/validation_mutations.py), so a change that closes a gap fails a test rather than passing silently. -**`vcfr:contigCount` is counted, not read.** The census asserts the predicate is +**`vcfc:contigCount` is counted, not read.** The census asserts the predicate is present the expected number of times; nothing compares its *value*. A derived contig total that is simply wrong passes. This is the one mutation in the catalogue that is still undetected. @@ -424,7 +517,7 @@ catalogue that is still undetected. **Header line values are compared only through their structured form.** The attributes that matter - `filterId`, `altId`, `contigId`, contig length/md5/ assembly, and the INFO/FORMAT declarations - are lifted into their own -properties and compared. The raw `vcfr:headerValue` literal itself is counted +properties and compared. The raw `vcfc:headerValue` literal itself is counted but never read. **Multi-valued INFO fields keep only their lexical value.** A `Number=1` INFO @@ -439,12 +532,13 @@ report-only and the run rests on the aggregate comparisons alone. That is correct behaviour, not a bug, but it means a custom mapping is validated more weakly than the default one. -**Condensed SHACL coverage depends on the vocabulary's next release.** The -condensed representation used 17 terms the published vocabulary did not define, -so SHACL could not meaningfully check it. All 17 are defined in vocabulary -v1.1.0, which also ships condensed shapes, but a third-party consumer only sees -them once that version is published. See -[`vcf-coverage.md`](vcf-coverage.md#vocabulary-alignment). +**SHACL conformance is checked by hand, not by this suite.** Both +representation profiles are fully ontology-backed as of published VCF Core +2.0.0, and the converter's output validates against the complete profile set — +portable, SPARQL, consistency and all five version overlays — with zero +violations. That check is currently a manual step rather than a test; wiring it +in is tracked in +[`validation-migration-notes.md`](validation-migration-notes.md). **A high mutation score is a lower bound on blindness, not a proof.** It says "almost every corruption we thought to write down is caught". Corruptions nobody @@ -472,23 +566,62 @@ Results are written to the run's canonical metrics tree: /run_metrics/__/reports/validation// ``` -Detailed results live beneath `reports/validation/`, so they are indexed by -the same `summary.json` used for conversion and compression metrics. - -Important files include `summary.json`, `manifest.json`, `parser.json`, -`rdf-validation.json`, `materialization.json`, `preflight.json`, `sparql.json`, -`comparison.json`, `benchmark.json` and `benchmark.csv`. Raw SPARQL Results -JSON, stderr, and query resource logs are in `raw//`; normalized results -are in `normalized/`. `manifest.json` records the engine that ran (including -QLever's exact argv), the source artifact format and checksum, and every decode -step; `materialization.json` records how a compressed or indexed artifact was -turned into N-Triples and how many triples that yielded. - -A multi-engine run adds `engines//` - one `preflight.json`, -`sparql.json`, `comparison.json` and `query-executions.json` per engine - and -`engine-agreement.json`, which records whether every engine returned the same -normalized results and, if not, which queries differed. The top-level reports -remain those of the primary (first-named) engine. +### The tree + +```text +/run_metrics/__/ +├── summary.json ← start here: a `validation` array indexes every target +├── stages/validation/.json ← one compact stage report per target +└── reports/validation// ← the detail for that target + ├── summary.json verdict, engines, pointers + ├── manifest.json provenance + ├── parser.json what the oracle read from the VCF + ├── sparql.json what the queries got from the graph + ├── comparison.json the two, compared per query + ├── preflight.json graph-integrity gates + ├── rdf-validation.json syntax check and parsed triple count + ├── materialization.json how a compressed artifact was decoded + ├── benchmark.json / benchmark.csv timings + ├── shacl.json / shacl-report.txt only with --shacl-shapes + ├── engine-agreement.json only with several engines + ├── engines// only with several engines + ├── raw// raw engine output + └── normalized/ per-query rows the comparison used +``` + +### What each file answers + +| File | Open it when you want to know | +| --- | --- | +| `summary.json` | The verdict, which engines ran and what each returned, and where the rest is. **Start here.** | +| `comparison.json` | *Why* a run is `MISMATCH`: per query, the rows that were missing, extra, or differing | +| `parser.json` | What the oracle independently computed **from the VCF** — the side of the comparison that does not involve RDF | +| `sparql.json` | What the queries returned **from the graph**, normalized — the other side | +| `preflight.json` | Which integrity gate failed on a `BLOCKED_BY_PREFLIGHT`: blank nodes, empty terms, duplicate statements, record cardinality, missing-token conformance, representation profile | +| `rdf-validation.json` | Whether the N-Triples parsed at all (`rapper -c`), and how many statements it holds | +| `materialization.json` | How an `.hdt`/`.cottas`/`.gz` artifact was turned into N-Triples, and how many triples that yielded — the first place to look when a compressed artifact validates differently from the aggregate | +| `manifest.json` | Provenance for a result you want to reproduce or cite: the engine and its exact argv (including QLever's), the source artifact's format and checksum, every decode step | +| `benchmark.json`, `benchmark.csv` | Timings. The CSV is long-format — one row per engine per query — and is the file to load for analysis | +| `shacl.json`, `shacl-report.txt` | Shape conformance, when `--shacl-shapes` was given. Independent of the VCF: it checks the graph against the vocabulary, not against the source | +| `engine-agreement.json` | Whether every engine returned identical normalized results, and which queries differed if not | +| `engines//` | The same `preflight`/`sparql`/`comparison` for one engine, plus `query-executions.json` with each query's exit code, wall time and error. The top-level reports are the **primary** (first-named) engine's | +| `raw//` | Raw SPARQL Results JSON, stderr, and `/usr/bin/time` resource logs, per query — for debugging an engine rather than the data | +| `normalized/` | The per-query rows the comparison actually used, after normalization | + +### Reading order when something fails + +Take the `status` from `summary.json` and go straight to the file that explains it: + +| Status | Meaning | Go to | +| --- | --- | --- | +| `PASS` | Every required query and invariant matched | — | +| `MISMATCH` | Both sides ran and disagree | `comparison.json` | +| `BLOCKED_BY_PREFLIGHT` | RDF syntax or core graph structure failed, so the comparison never ran | `preflight.json`, then `rdf-validation.json` | +| `EXECUTION_FAILED` | A parser or an engine could not complete | `engines//query-executions.json` | + +An engine that exceeded `--validation-query-timeout` reports `EXECUTION_FAILED` +with an `abandoned` field naming the query that tripped it and how many were +skipped. See [If validation seems to hang](#if-validation-seems-to-hang). When several artifacts are validated in one full run, each gets its own directory: the aggregate keeps `/`, and the representations use diff --git a/docs/vcf-coverage.md b/docs/vcf-coverage.md index 12d34f2..3feab54 100644 --- a/docs/vcf-coverage.md +++ b/docs/vcf-coverage.md @@ -15,66 +15,135 @@ Two independent questions per row, deliberately kept apart: mutation in [`test/validation_mutations.py`](../test/validation_mutations.py), not by inspection. -Last measured mutation score: **76/78 (97%)** across 42 distinct mutations. -Regenerate with: +Last measured mutation score: **96/113 (85%)** across 60 distinct mutations, +measured against the VCF Core vocabulary. Regenerate with: ```bash VCF_RDFIZER_MUTATION_REPORT=mutation-score.json \ python -m unittest test.test_validation_mutation_unit ``` +The score fell from 97% because the denominator grew, not because anything +regressed: the migration added 18 mutations covering the layers VCF Core +introduced, and 10 of those are recorded gaps. That is the methodology working +as intended — a new capability that nothing checks shows up as a lower score +rather than as silent confidence. Every gap is listed under +[Remaining gaps](#remaining-gaps) with what would close it. + --- ## Fixed fields (the eight mandatory columns) | VCF element | `records.tsv` | RDF term | Represented | Validated by | Mutation | | --- | --- | --- | --- | --- | --- | -| CHROM | `CHROM` | `vcfr:chrom` | yes | `q01`, `q11` | `corrupt_chrom` | -| POS | `POS` | `vcfr:pos` (`xsd:integer`) | yes | `q01`, `q11`, `preflight_position_datatype` | `corrupt_pos`, `drop_pos`, `retype_pos_as_string` | +| CHROM | `CHROM` | `vcfc:chrom` | yes | `q01`, `q11` | `corrupt_chrom` | +| POS | `POS` | `vcfc:pos` (`xsd:integer`) | yes | `q01`, `q11`, `preflight_position_datatype` | `corrupt_pos`, `drop_pos`, `retype_pos_as_string` | | POS ↔ record binding | — | — | yes | `q11_record_digest` | `permute_pos` | -| ID | `ID` | `vcfr:recordId` | yes | `q11_record_digest` | (covered by digest) | -| REF | `REF` | `vcfr:ref` | yes | `q02`, `q03`, `q11` | `corrupt_alt` | -| ALT | `ALT` | `vcfr:alt` | yes | `q02`, `q03`, `q11` | `corrupt_alt` | +| ID | `ID` | `vcfc:recordId` | yes | `q11_record_digest` | (covered by digest) | +| REF | `REF` | `vcfc:ref` | yes | `q02`, `q03`, `q11` | `corrupt_alt` | +| ALT | `ALT` | `vcfc:alt` | yes | `q02`, `q03`, `q11` | `corrupt_alt` | | REF/ALT ↔ record binding | — | — | yes | `q11_record_digest` | `permute_ref_alt` | -| QUAL | `QUAL` | `vcfr:qual` (`xsd:decimal` / `vcfr:Null`) | yes | `q09`, `q11` | `drop_qual`, `drop_all_qual`, `corrupt_qual` | -| FILTER | `FILTER` | `vcfr:filter` | yes | `q04` (exact lexical), `q11` | `drop_filter`, `corrupt_filter_lexical` | -| INFO (raw) | `INFO` | `vcfr:infoRaw` | yes | `q11_record_digest` | `corrupt_info_raw` | -| INFO (structured) | `INFO` | `vcfr:hasInfoValue` → `vcfr:InfoFieldValue` → `vcfr:declaredBy` | yes | `q09`, `q12_info_value_digest` | `drop_info_value`, `corrupt_info_value`, `retype_info_value` | +| QUAL | `QUAL` | `vcfc:qual` (`xsd:decimal` / `vcfc:Null`) | yes | `q09`, `q11` | `drop_qual`, `drop_all_qual`, `corrupt_qual` | +| FILTER | `FILTER` | `vcfc:filter` | yes | `q04` (exact lexical), `q11` | `drop_filter`, `corrupt_filter_lexical` | +| INFO (raw) | `INFO` | `vcfc:infoRaw` | yes | `q11_record_digest` | `corrupt_info_raw` | +| INFO (structured) | `INFO` | `vcfc:hasInfoValue` → `vcfc:InfoFieldValue` → `vcfc:declaredBy` | yes | `q09`, `q12_info_value_digest` | `drop_info_value`, `corrupt_info_value`, `retype_info_value` | -INFO values carry `vcfr:fieldValue` plus a typed `fieldValueInteger` / +INFO values carry `vcfc:fieldValue` plus a typed `fieldValueInteger` / `fieldValueDecimal` when the declaration says `Number=1` and the value parses; -a Flag entry carries `vcfr:fieldValueBoolean true`. A multi-valued field -(`Number=A/R/G/.`) keeps only the lexical value, because the vocabulary's IRI -template gives one node per key. +a Flag entry carries `vcfc:fieldValueBoolean true`. A multi-valued field is no +longer lexical-only: `Number=A`/`R`/`LA`/`LR`/`G`/`LG`/`P` values are decomposed +into ordered `vcfc:FieldValueItem` resources joined to their allele with +`vcfc:forAllele`, or carrying a genotype/GT ordinal. + +### Allele layer + +| VCF element | RDF term | Represented | Validated by | Mutation | +| --- | --- | --- | --- | --- | +| REF as an ordered allele | `vcfc:hasReferenceAllele` → `vcfc:ReferenceAllele` (`alleleIndex` 0) | yes | — | — | +| Each ALT item, in source order | `vcfc:hasAltAllele` → `vcfc:AltAllele` (`alleleIndex` 1..n) | yes | — | — | +| ALT syntactic category | `vcfc:alleleKind` | yes | — | — | +| Symbolic ALT reserved type | `vcfc:svType` → `vcfc:SymbolicAlleleType` | yes | — | — | +| Symbolic ALT ↔ `##ALT` declaration | `vcfc:declaredByAlt` | yes | — | — | +| Breakend structure | `vcfc:Breakend`, `breakendOrientation`, `breakendReplacementString`, `isSingleBreakend` | yes | — | — | +| CHROM ↔ contig declaration | `vcfc:chromosome` | yes | — | — | +| Indexed field values | `vcfc:hasValueItem` → `vcfc:FieldValueItem` (`valueIndex`, `itemValue`, `forAllele`, `tupleArity`) | yes | — | — | + +### Structural variation + +| VCF element | RDF term | Represented | Validated by | Mutation | +| --- | --- | --- | --- | --- | +| `SVLEN` | `vcfc:svLength` on the allele | yes | — | — | +| `SVCLAIM` | `vcfc:svClaim` → `vcfc:SVClaim` | yes | — | — | +| `IMPRECISE`, `NOVEL` | `vcfc:isImprecise`, `vcfc:isNovel` | yes | — | — | +| `EVENT` + `EVENTTYPE` | `vcfc:inEvent` → `vcfc:VariantEvent` with `vcfc:eventType` | yes (both keys required) | — | — | +| `CIPOS`, `CIEND` | `vcfc:posConfidenceInterval` / `endConfidenceInterval` → FALDO `InRangePosition` | yes | — | — | +| `CILEN`, `CICN` | `vcfc:ConfidenceInterval` with `ciLower` / `ciUpper` | yes | — | — | +| `RN` + `RUS`/`RUL`/`RUC`/`RB` | `vcfc:TandemRepeatAllele` → `vcfc:RepeatSequence` | yes (`RN` required) | — | — | +| gVCF `<*>` with `END` | `vcfc:ReferenceBlock`, `endPosition`, `referenceBlockLength` | yes (`END` required) | — | — | ## Genotype fields | VCF element | RDF term | Represented | Validated by | Mutation | | --- | --- | --- | --- | --- | -| FORMAT declaration | `vcfr:formatRaw` | yes | `q09_predicate_census` | — | -| Sample identity | `vcfr:sampleId` / `vcfr:sampleName` | yes | `preflight_sample_gt_inventory`, `q09` | `drop_sample_call` | -| GT values | `vcfr:hasFormatValue` → `vcfr:fieldValue` | yes | `q05`, `q06`, `q13` | `flip_genotype` | -| Non-GT FORMAT (DP, GQ, AD, PL…) | `vcfr:FormatFieldValue` / `vcfr:FormatValueVector` | yes | `q13_format_value_digest` | `drop_format_value_dp`, `corrupt_format_value_dp`, `corrupt_format_vector` | +| FORMAT declaration | `vcfc:formatRaw` | yes | `q09_predicate_census` | — | +| Sample identity | `vcfc:sampleId` / `vcfc:sampleName` | yes | `preflight_sample_gt_inventory`, `q09` | `drop_sample_call` | +| GT values (lexical) | `vcfc:hasFormatValue` → `vcfc:fieldValue` | yes | `q05`, `q06`, `q13` | `flip_genotype` | +| Non-GT FORMAT (DP, GQ, AD, PL…) | `vcfc:FormatFieldValue` / `vcfc:FormatValueVector` | yes | `q13_format_value_digest` | `drop_format_value_dp`, `corrupt_format_value_dp`, `corrupt_format_vector` | +| Reusable sample identity | `vcfc:SampleSet` → `vcfc:VCFSample` (`sampleName`, `sampleIndex`); `vcfc:forSample` from a `SampleCall` | yes, both profiles | — | — | +| `#CHROM` sample columns | `vcfc:ColumnHeaderLine` → `vcfc:hasGenotypeColumns` | yes | — | — | +| FORMAT value ↔ declaration | `vcfc:declaredBy` → `vcfc:FormatFieldDefinition` | yes | — | — | + +**Parsed genotype layer — expanded profile only.** The condensed profile keeps +these values inside `vcfc:encodedValues`, which is the point of the profile. + +| VCF element | RDF term | Represented | Validated by | Mutation | +| --- | --- | --- | --- | --- | +| `GT` parsed | `vcfc:hasGenotype` → `vcfc:Genotype` (`genotypeString`, `ploidy`) | expanded only | — | — | +| Phasing (`\|` vs `/`) | `vcfc:phasingStatus` → `vcfc:Phased` / `vcfc:Unphased` | expanded only | — | — | +| Each GT position | `vcfc:hasAlleleCall` → `vcfc:GenotypeAlleleCall` (`callIndex`, `isNoCall`, `calledAllele`) | expanded only | — | — | +| `FT` | `vcfc:sampleFilter` | expanded only | — | — | +| `PS`/`PSL`/`PSO`/`PSQ` | `vcfc:inPhaseSet` → `vcfc:PhaseSet` | expanded only | — | — | +| `LAA` | `vcfc:hasLocalAlleleSet` → `vcfc:LocalAlleleSet` | expanded only | — | — | +| `CN`/`CNQ`/`CNL`/`CNP` | `vcfc:copyNumber` and friends | expanded only | — | — | +| `HAP`/`AHAP` | `vcfc:haplotypeId`, `vcfc:ancestralHaplotypeId` | expanded only | — | — | +| `M*`/`DPM*`/`ADM*` | `vcfc:BaseModification` with a ChEBI `modifiedResidue` | expanded only | — | — | ## Header section | VCF element | RDF term | Represented | Validated by | Mutation | | --- | --- | --- | --- | --- | -| `##` line count and keys | `vcfr:HeaderLine` + `vcfr:headerKey` | yes | `q08_header_line_census` | `drop_header_line` | +| `##` line count and keys | `vcfc:HeaderLine` + `vcfc:headerKey` | yes | `q08_header_line_census` | `drop_header_line` | | Line type | `FileFormatHeaderLine`, `INFOHeaderLine`, `ContigHeaderLine`, … | yes | `q10_class_census` | `untype_header_line` | -| `##fileformat` | `vcfr:fileFormat` | yes | `q07_file_metadata` | `corrupt_file_metadata` | -| `##reference` | `vcfr:referenceGenome` | yes | `q07_file_metadata` | `drop_reference_genome` | -| `##source` | `vcfr:sourceSoftware` | yes | `q07_file_metadata` | — | -| `##fileDate` | `vcfr:fileDate` (`xsd:date` when the form allows) | yes | `q09_predicate_census` | `drop_file_date` | -| `##FILTER` | `vcfr:FilterDefinition` + `vcfr:filterId` | yes | `q09`, `q10` | `drop_filter_definition` | -| `##ALT` | `vcfr:AltDefinition` + `vcfr:altId` | yes | `q09`, `q10` | `drop_alt_definition` | -| `##contig` | `vcfr:ContigHeaderLine` + `contigId`/`contigLength`/`contigMd5`/`contigAssembly` | yes | `q09`, `q10` | `drop_contig_attribute` | -| `##INFO` / `##FORMAT` definitions | `vcfr:InfoFieldDefinition` / `vcfr:FormatFieldDefinition` with `fieldId`/`fieldNumber`/`fieldType`/`fieldDescription` | yes | `q09`, `q10` | `drop_info_definition` | -| Header line *values* | `vcfr:headerValue` | yes | `q09` (count only) | — | -| `vcfr:contigCount` | derived scalar | yes | **count only, not value** | `corrupt_contig_count` | - -An unrecognized `##` key keeps only the base `vcfr:HeaderLine` type: inventing -a subclass would put a term in the graph that the vocabulary does not define. +| `##fileformat` | `vcfc:fileFormat` | yes | `q07_file_metadata` | `corrupt_file_metadata` | +| `##reference` | `vcfc:referenceGenome` | yes | `q07_file_metadata` | `drop_reference_genome` | +| `##source` | `vcfc:sourceSoftware` | yes | `q07_file_metadata` | — | +| `##fileDate` | `vcfc:fileDate` (`xsd:date` when the form allows) | yes | `q09_predicate_census` | `drop_file_date` | +| `##FILTER` | `vcfc:FilterDefinition` + `vcfc:filterId` | yes | `q09`, `q10` | `drop_filter_definition` | +| `##ALT` | `vcfc:AltDefinition` + `vcfc:altId` | yes | `q09`, `q10` | `drop_alt_definition` | +| `##contig` | `vcfc:ContigHeaderLine` + `contigId`/`contigLength` (`xsd:integer`)/`contigMd5`/`contigAssembly`/`contigUrl` (`xsd:anyURI`) | yes | `q09`, `q10` | `drop_contig_attribute` | +| `##INFO` / `##FORMAT` definitions | `vcfc:InfoFieldDefinition` / `vcfc:FormatFieldDefinition` with `fieldId`/`fieldNumber`/`fieldType`/`fieldDescription` | yes | `q09`, `q10` | `drop_info_definition` | +| `Number` arity and fixed count | `vcfc:fieldArity` → `vcfc:VCFNumberArity`, `vcfc:fieldNumberInteger` | yes | — | — | +| `##INFO` `Source` / `Version` | `vcfc:fieldSource`, `vcfc:fieldVersion` | yes | — | — | +| Structured-line attributes (all of them) | `vcfc:hasAttribute` → `vcfc:HeaderAttribute` (`attributeKey`, `attributeValue`, `attributeIndex`) | yes | — | — | +| `#CHROM` column-header line | `vcfc:ColumnHeaderLine` + `vcfc:hasColumnHeader` | yes | — | — | +| Header line order | `vcfc:lineIndex` (`xsd:integer`) | yes | — | — | +| `##assembly` | `vcfc:AssemblyHeaderLine` + `vcfc:assemblyUrl` | yes | — | — | +| `##pedigreeDB` | `vcfc:PedigreeDBHeaderLine` + `vcfc:pedigreeDbUrl` | yes | — | — | +| `##META` | `vcfc:MetaHeaderLine` / `vcfc:MetaDefinition` + `vcfc:metaAllowedValue` | yes | — | — | +| `##SAMPLE` | `vcfc:SampleHeaderLine` → `vcfc:declaresSample` → `vcfc:SampleDeclaration` | yes | — | — | +| `##PEDIGREE` | `vcfc:PedigreeHeaderLine` + `pedigreeMother`/`pedigreeFather`/`pedigreeOriginal`/`pedigreeAncestor` + `ancestorRole` | yes | — | — | +| Header line *values* | `vcfc:headerValue` | yes | `q09` (count only) | — | +| `vcfc:contigCount` | derived scalar | yes | **count only, not value** | `corrupt_contig_count` | + +An unrecognized `##` key is typed as `vcfc:StructuredHeaderLine` or +`vcfc:UnstructuredHeaderLine` according to the form of its value, which keeps it +queryable without inventing a subclass the vocabulary does not define. + +Every structured line carries its `vcfc:HeaderAttribute` resources whether or +not it also has dedicated properties. `vcfc:StructuredHeaderLineShape` requires +at least one, and they are the only way an implementation-defined attribute +stays queryable. This is why `--header-representation basic` does not produce a +conformant graph. ## Graph-level properties @@ -86,6 +155,8 @@ a subclass would put a term in the graph that the vocabulary does not define. | Missing-token policy | `preflight_missing_token_conformance`, fatal under `--strict-conformance` | `plain_dot_literal` | | Representation profile | `preflight_representation_profile` | `wrong_representation_profile` | | Record count | q01/q02/q04 totals | `drop_record`, `duplicate_record` | +| Record order (`vcfc:recordIndex`) | SPARQL SHACL profile: uniqueness, nondecreasing POS within CHROM, contiguous CHROM blocks | — | +| Expanded/condensed exclusivity | SPARQL SHACL profile | `wrong_representation_profile` | | **No extraneous triples** | `q09_predicate_census` (extra rows) | `spurious_predicate` | | **No blank nodes** | `preflight_blank_nodes` | `introduce_blank_node`, `blank_node_object` | | **No empty or whitespace-only terms** | `preflight_empty_values` | `empty_literal`, `whitespace_only_literal` | @@ -95,71 +166,121 @@ a subclass would put a term in the graph that the vocabulary does not define. ## Remaining gaps -1. **`vcfr:contigCount` is counted, not read.** A wrong derived contig total is +1. **`vcfc:contigCount` is counted, not read.** A wrong derived contig total is not detected. Closing it needs the value in a comparison, not just the predicate in the census. 2. **Header line values are not compared.** `q08` compares how many lines carry - each key and `q10` their types, but `vcfr:headerValue` itself is only + each key and `q10` their types, but `vcfc:headerValue` itself is only counted. The structured attributes that matter (`filterId`, `contigId`, …) *are* covered. 3. **The census assumes the shipped mapping.** A custom `--rules` changes the inventory and IRI templates by design, so `q09`–`q13` fall back to report-only. That is correct, but it means a custom mapping is validated only by the aggregate comparisons. +4. **The new layers are counted but their values are not compared.** The + allele layer, header attributes, value items and the parsed genotype layer + all appear in the predicate and class censuses, so *dropping* one is caught. + Corrupting a value inside one is not: `corrupt_allele_value`, + `corrupt_allele_kind`, `corrupt_attribute_value`, `corrupt_value_item_allele`, + `corrupt_called_allele`, `flip_phasing_status` and `wrong_declaration_owner` + are all recorded as undetected. Closing them needs digests over those + resources, the way `q11` already covers the record fields — or leaning on the + vocabulary's own consistency SHACL profile, which checks several of these + agreements directly. +5. **Ordering is checked by shapes, not by queries.** `vcfc:lineIndex` and + `vcfc:recordIndex` are counted, but `corrupt_record_index` is undetected by + the query layer. The SPARQL SHACL profile enforces uniqueness, nondecreasing + POS within a CHROM block, and contiguous CHROM blocks, so the coverage exists + — in a different layer. +6. **Version overlays have no mutation.** The converter detects each input's VCF + version and follows it, and `drop_version_class` / `wrong_version_class` + cover the file class. The version-dependent *behaviour* — per-ALT versus + per-record `CIPOS` linkage, the families a version does not define — is + covered by unit tests and by the version overlays themselves, not by the + mutation harness, because the fixture declares a single version. +7. **SHACL conformance is not yet a test.** It was checked by hand and found + seven real modelling bugs; wiring it in would make that a standing invariant. + It needs `pyshacl` and `rdflib` as dev dependencies. ## Vocabulary alignment -Both gaps recorded here have been closed in the vocabulary repository -(`https://github.com/ecrum19/VCF-RDFizer-vocabulary`, v1.1.0). They are kept -below because the fix is only live for consumers once that version is published -to `https://w3id.org/vcf-rdfizer/vocab#`. - -### Condensed terms are now defined - -The conversion emitted 17 terms the vocabulary did not define, all belonging to -the condensed representation, so **condensed graphs were not ontology-backed**: - -`CohortCallMatrix`, `CondensedRepresentation`, `ExpandedRepresentation`, -`FormatValueVector`, `SampleSet`, `VCFSample`, `VCFTextVector`, -`appliesToSampleSet`, `encodedValues`, `hasCallMatrix`, `hasFormatValueVector`, -`hasSample`, `hasSampleSet`, `representationProfile`, `sampleIndex`, -`sampleName`, `valueEncoding` - -All 17 are now defined, along with two superclasses (`RepresentationProfile`, -`VectorEncoding`) that give the two enumerations a range. The vocabulary also -ships SHACL shapes for the condensed profile, constraining what positional -decoding depends on: exactly one `sampleIndex` per sample, a sample set on every -matrix, and an encoding and FORMAT declaration on every vector. - -Verified: the project's own expanded and condensed fixture graphs conform -against the new shapes, and ten targeted corruptions of a condensed graph -(dropped index, duplicated index, untyped index, missing encoding, missing -declaration, missing values, missing sample set, unknown profile, empty sample -set, missing name) are each reported as a violation. - -### The SHACL / missing-value contradiction is resolved - -Running SHACL had surfaced a contradiction between two parts of the published -vocabulary that the tool could not satisfy simultaneously: - -- `vcfr:missingValuePolicy` said a missing token SHOULD be `"."^^vcfr:Null`. -- `VCFRecordShape` constrained `vcfr:alt` to `sh:datatype xsd:string`. - -A record with `ALT=.` therefore violated the shape no matter which rule the -conversion followed. The resolution draws the boundary the policy was missing -rather than relaxing every field: **ALT and ID admit the missing token in VCF -4.5, so their shapes now accept `xsd:string` or `vcfr:Null`**, as `qual` already -did. **CHROM, POS and REF are required and have no missing form**, so their -shapes stay exact — a `.` in one of them is malformed input, and the shape -should say so. The policy text now states that boundary explicitly, and adds -that a missing value inside a `FormatValueVector` stays a `.` character at its -sample's position rather than becoming a typed literal. - -The tool needs no change: it already followed the missing-value policy. - -Two shape violations found the same way had already been fixed here: QUAL is -`xsd:decimal`/`vcfr:Null` rather than a plain literal, and `##fileDate` is typed -`xsd:date` when its form allows. +The conversion targets the **VCF Core vocabulary**, +`https://w3id.org/vcf-core/vocab#` (prefix `vcfc:`), **published at 2.0.0** and +covering VCF 4.1 through 4.5. It replaces the retired +`https://w3id.org/vcf-rdfizer/vocab#`, which is **not** redirected: it serves a +deprecation document in which every former term is present, deprecated, and +linked to its successor. Only one term was renamed rather than re-namespaced — +`vcfr:DenseRepresentation` became `vcfc:ExpandedRepresentation`. + +The converter references the namespace, not a version IRI. The vocabulary's +release cadence is independent of this converter's, and pinning a version here +would go stale without adding anything: the namespace is what the emitted IRIs +actually contain. + + + +### VCF version coverage + +VCF Core supplies a conformance overlay per VCF version, and the converter +follows each input's own `##fileformat` line automatically. Coverage: + +| Version | Overlay | Detected | Notes | +| --- | --- | --- | --- | +| 4.0 | none claimed by the vocabulary | reported as unrecognized | converts with 4.5 rules and **no** `vcfc:VCF4xFile` class | +| 4.1 | `vcf-4.1.shacl.ttl` | ✓ `vcfc:VCF41File` | no `Number=R`; `CIPOS` is one pair per record | +| 4.2 | `vcf-4.2.shacl.ttl` | ✓ `vcfc:VCF42File` | `Number=R` added | +| 4.3 | `vcf-4.3.shacl.ttl` | ✓ `vcfc:VCF43File` | | +| 4.4 | `vcf-4.4.shacl.ttl` | ✓ `vcfc:VCF44File` | `Number=P`; tuples become per-ALT; `CILEN`/`CICN`/`EVENTTYPE` added | +| 4.5 | `vcf-4.5.shacl.ttl` | ✓ `vcfc:VCF45File` | `Number=LA/LR/LG/M`; local alleles and base modifications | + +See [`conversion.md`](conversion.md#4a-vcf-versions) for what each difference +changes in the emitted graph. + +### What the converter emits, versus what VCF Core can express + +VCF Core 2.0.0 defines 338 terms. The converter emits from most of them, and its +output is conformant, but the vocabulary can express more than the converter +currently derives. Ten families are entirely unemitted: + +| Family | Terms | What it would add | +| --- | --- | --- | +| Parsed FILTER | `filterStatus`, `FiltersPassed/Failed/NotApplied`, `FilterCode`, `hasFilterCode`, `declaredByFilter` | FILTER as resources joined to their `##FILTER` declarations, instead of only the raw string | +| Padding semantics | `PaddingInterpretation`, `paddingRule`, `paddingSide`, `paddingAnchorPosition`, `paddingBaseCount` and their individuals | Which base of REF/ALT is the VCF padding base, and why | +| Parsed FORMAT keys | `FormatKey`, `hasFormatKey`, `fieldIndex`, `keyPattern` | The FORMAT key list as ordered resources, not only `formatRaw` | +| Parsed record IDs | `RecordIdentifier`, `hasIdentifier`, `identifierValue`, `aliasOf` | The semicolon-separated ID column split into individual identifiers | +| Repeat units | `RepeatUnit`, `hasRepeatUnit`, `repeatUnitBases`, `rucConfidenceInterval`, `rbConfidenceInterval` | `RUB`-level detail below a repeat sequence | +| Breakend mates | `mateBreakend`, `partnerBreakend`, `insertedSequence`, `isTelomereBreakend` | Resolving a breakend to the record describing its mate | +| Local-allele membership | `LocalAlleleMembership`, `localAllele`, `localIndex` | LAA membership with its own index, beside the current `LocalAlleleSet` | +| Mixed phasing | `MixedPhasing`, `phaseIndicator`, `allelePhaseSet` | Per-position phase indicators rather than one status per genotype | +| Reference blocks | `hasReferenceBlock`, `blockAllele` | A record-level link to its gVCF block | +| Raw/decoded values | `rawValue`, `decodedValue`, `percentEncodingPolicy`, `sampleFieldsRaw` | Percent-decoded companions beside the lossless source values | + +**None of these is required for conformance.** Across 19 graphs — every example +VCF the vocabulary ships, plus a stress matrix for 4.0 through 4.5 in both +representation profiles — the complete profile set reports zero violations +without them. They are enrichment the vocabulary supports and the converter does +not yet derive. + +This matters for benchmarking in one direction only: a benchmark of conversion +throughput, graph size or compression is measuring the converter's actual output +and is unaffected. A benchmark of *query* coverage against the vocabulary would +be measuring a subset, and should say so. + +### Conformance + +The default mapping's output validates against the **complete** published +profile set — the portable and SPARQL profiles, the consistency profile, and all +five version overlays — with **zero violations**, in both representation +profiles, checked with pySHACL using the bundled ontology plus every +`ontology/versions/*.ttl` and RDFS inference, which is the configuration the +vocabulary's own `tests/validate_shacl.py` uses. See +[`validation-migration-notes.md`](validation-migration-notes.md#how-this-was-verified) +for exactly what was covered. + +The only results reported are `sh:Warning`s where an input VCF omits the +`Source` and `Version` attributes that VCF 4.5 recommends on `##INFO` +declarations. Those are properties of the input, and the converter deliberately +does not fabricate them. --- diff --git a/examples/linking/README.md b/examples/linking/README.md new file mode 100644 index 0000000..152e2ef --- /dev/null +++ b/examples/linking/README.md @@ -0,0 +1,23 @@ +# Linking example input + +`example.vcf` has four artificial records with a GRCh38 reference declaration. +Its coordinates are chosen to exercise the synthetic `gene-demo` GFF3 intervals; +the rsIDs demonstrate token parsing and are not claims about those coordinates. + +```bash +vcf-rdfizer-link run -i examples/linking/example.vcf \ + --link rsid-dbsnp,gene-demo --offline \ + --links-cache ./example-cache -o ./example.links.nt +``` + +Expected links, excluding provenance: + +| VCF row | dbSNP tokens | Demo genes | +| --- | --- | --- | +| 1 | rs334 | A | +| 2 | rs699, rs334 | A, B | +| 3 | none | none | +| 4 | none | C | + +Seven links in total. Live resolver usage, caching and authoring are documented +in [`docs/datalinking.md`](../../docs/datalinking.md). diff --git a/examples/linking/example.vcf b/examples/linking/example.vcf new file mode 100644 index 0000000..26dbe6f --- /dev/null +++ b/examples/linking/example.vcf @@ -0,0 +1,8 @@ +##fileformat=VCFv4.3 +##reference=GRCh38 +##INFO= +#CHROM POS ID REF ALT QUAL FILTER INFO +1 100 rs334 A G . PASS RS=rs334 +1 190 rs699;rs334 AT A . PASS RS=rs699,rs334 +1 251 . C T . PASS . +2 20 local-id G A . PASS . diff --git a/pyproject.toml b/pyproject.toml index 695dcc5..539a99a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,8 +4,8 @@ build-backend = "setuptools.build_meta" [project] name = "vcf-rdfizer" -version = "2.1.0" -description = "Docker-first VCF to RDF conversion wrapper with compression/decompression modes" +version = "3.0.0" +description = "Docker-first VCF to RDF conversion targeting the VCF Core vocabulary, with compressed queryable representations (HDT, COTTAS), semantic validation, and data linking" readme = "README.md" requires-python = ">=3.10" license = "MIT" @@ -26,6 +26,7 @@ classifiers = [ ] dependencies = [ "rich>=13.7.0", + "rdflib>=7.0.0", ] [project.urls] @@ -35,20 +36,19 @@ Issues = "https://github.com/ecrum19/VCF-RDFizer/issues" [project.scripts] vcf-rdfizer = "vcf_rdfizer:main" vcf-rdfizer-rules = "vcf_rdfizer_rules:main" +vcf-rdfizer-link = "vcf_rdfizer_link:main" [project.optional-dependencies] dev = [ "build>=1.2.2", "twine>=5.1.1", - # Test-only: an in-process SPARQL engine for the validation mutation harness, - # so it needs no Docker. The suite skips those tests when it is absent. - "rdflib>=7.0.0", ] [tool.setuptools] -py-modules = ["vcf_rdfizer", "vcf_rdfizer_gzip", "vcf_rdfizer_rules"] -packages = ["vcf_rdfizer_data", "vcf_rdfizer_data.rules"] +py-modules = ["vcf_rdfizer", "vcf_rdfizer_gzip", "vcf_rdfizer_rules", "vcf_rdfizer_link", "vcf_rdfizer_vocab"] +packages = ["vcf_rdfizer_data", "vcf_rdfizer_data.rules", "vcf_rdfizer_data.linkers", "vcf_rdfizer_linking"] include-package-data = true [tool.setuptools.package-data] "vcf_rdfizer_data.rules" = ["default_rules.ttl"] +"vcf_rdfizer_data.linkers" = ["*/linker.ttl", "*/resolver.py", "*/genes.gff3", "*/README.md"] diff --git a/rules/README.md b/rules/README.md index 32edbe1..b1a3584 100644 --- a/rules/README.md +++ b/rules/README.md @@ -18,7 +18,9 @@ See "Custom RML Mappings" in the top-level `README.md` for the full contract. - `default_rules.ttl` - Active default mapping for this repository. - - Targets the VCF-RDFizer vocabulary (`https://w3id.org/vcf-rdfizer/vocab#`). + - Targets the VCF Core vocabulary (`https://w3id.org/vcf-core/vocab#`, + prefix `vcfc:`), which replaces the retired + `https://w3id.org/vcf-rdfizer/vocab#`. No version IRI is pinned. - Uses template TSV paths: - `/data/tsv/file_metadata.tsv` - `/data/tsv/header_lines.tsv` @@ -60,15 +62,32 @@ after RMLStreamer emits the custom record-level graph. ## SHACL Notes -The related SHACL constraints are maintained in the vocabulary repository: +The related SHACL constraints are maintained in the vocabulary repository, in +two profiles: -- [vcf-rdfizer-vocabulary.shacl.ttl](https://github.com/ecrum19/VCF-RDFizer-vocabulary/blob/main/shacl/vcf-rdfizer-vocabulary.shacl.ttl) +- `shacl/vcf-core-vocabulary.shacl.ttl` — portable: structural links, VCF 4.5 + lexical patterns, field type and arity constraints. +- `shacl/vcf-core-vocabulary-sparql.shacl.ttl` — cross-resource: header and + record ordering, sample uniqueness, and the rule that a graph must not mix + expanded sample calls with condensed call matrices. -The default mapping is structured to align with those classes/properties, especially: +Both need the bundled ontology as an ontology graph and RDFS inference enabled. -- `vcfr:VCFFile` + `vcfr:hasHeader` -- `vcfr:VCFHeader` + `vcfr:hasHeaderLine` -- `vcfr:VCFRecord` core fields (`chrom`, `pos`, `ref`, `alt`) -- `vcfr:VariantCall` with raw call attributes -- expanded `vcfr:SampleCall` / `vcfr:FormatFieldValue` resources, or condensed - `vcfr:CohortCallMatrix` / `vcfr:FormatValueVector` resources +This mapping emits: + +- `vcfc:VCFFile` + `vcfc:hasHeader`, `vcfc:fileFormat`, `vcfc:sourceSoftware`, + `vcfc:referenceGenome` +- `vcfc:VCFHeader` + `vcfc:hasHeaderLine`, and `vcfc:hasColumnHeader` to the + `#CHROM` `vcfc:ColumnHeaderLine` +- `vcfc:HeaderLine` with `headerKey`, `headerValue` and `lineIndex` +- `vcfc:VCFRecord` with `chrom`, `pos`, `ref`, `recordIndex` and `hasCall` +- `vcfc:VariantCall` with `formatRaw` +- the expanded `vcfc:SampleCall` / `vcfc:FormatFieldValue` compatibility maps + +`ID`, `ALT`, `QUAL`, `FILTER` and `infoRaw` are **not** here. Each may be the +VCF missing token, which the vocabulary requires as `"."^^vcfc:Null`, and RML +cannot switch an object's datatype per row — so the wrapper emits them, along +with the header attributes, the allele layer, structured INFO, the SV carriers +and the genotype layer. The rule is stated at the top of `default_rules.ttl`: +**RML carries every field whose datatype is the same for every row; the wrapper +carries everything else.** diff --git a/rules/default_rules.ttl b/rules/default_rules.ttl index d229b3c..a4e0107 100644 --- a/rules/default_rules.ttl +++ b/rules/default_rules.ttl @@ -3,9 +3,17 @@ @prefix ql: . @prefix csvw: . @prefix xsd: . -@prefix vcfr: . +@prefix vcfc: . -# Default VCF-RDFizer RML rules. +# Default VCF-RDFizer RML rules, targeting the VCF Core vocabulary. +# +# Namespace +# --------- +# The target is https://w3id.org/vcf-core/vocab# (prefix vcfc:), which replaces +# the retired https://w3id.org/vcf-rdfizer/vocab# (prefix vcfr:). The retired +# namespace is not redirected: it serves a deprecation document in which every +# former term is linked to its successor. No version IRI is pinned; these rules +# target the namespace, whose release cadence is independent of the converter's. # # Expected TSV inputs produced by src/vcf_as_tsv.sh: # - /data/tsv/.file_metadata.tsv @@ -14,12 +22,60 @@ # - /data/tsv/.sample_calls.tsv (header-only compatibility source) # - /data/tsv/.sample_format_values.tsv (header-only compatibility source) # The wrapper rewrites the template paths below for each input sample. -# The wrapper selects exactly one direct sample emitter: expanded SampleCall / -# FormatFieldValue triples, or condensed SampleSet / CohortCallMatrix / -# FormatValueVector triples. The helper tables stay header-only in either mode. +# +# What is here and what is not +# ---------------------------- +# The division is a single rule: this mapping carries every field whose RDF +# datatype is the same for every row, and the wrapper's direct N-Triples +# emitters carry everything else. RML cannot choose an object datatype per row, +# and it cannot decompose one source cell into several resources without a +# materialized helper table whose size is the product of its dimensions. +# +# Emitted by the wrapper, not here: +# - ID, ALT, QUAL, FILTER and INFO. Each may be the VCF missing token, which the +# vocabulary requires as "."^^vcfc:Null rather than a plain string, so the +# datatype depends on the row. +# - fileDate. The published shape requires xsd:date, but ##fileDate has no +# mandated format in the VCF specification, so the value must be inspected. +# - The structured header representation: header-line subclasses, the +# vcfc:HeaderAttribute resources that vcfc:StructuredHeaderLineShape requires +# on every ##INFO/##FORMAT/##FILTER/##ALT/##contig/##META/##SAMPLE/##PEDIGREE +# line, and the INFO/FORMAT/FILTER/ALT/contig/META declaration properties. +# These all live inside one angle-bracketed cell. +# - The allele layer: vcfc:ReferenceAllele and vcfc:AltAllele resources with +# their index, kind, symbolic type and breakend components, parsed out of the +# REF and ALT columns. +# - The structured INFO representation, its vcfc:FieldValueItem decomposition, +# and the SV carriers derived from reserved INFO keys. +# - Genotype data. The wrapper selects exactly one direct sample emitter: +# expanded SampleCall / FormatFieldValue / Genotype triples, or condensed +# SampleSet / CohortCallMatrix / FormatValueVector triples. The helper tables +# stay header-only in either mode. +# +# Ordinals +# -------- +# vcfc:lineIndex and vcfc:recordIndex are typed xsd:integer, not +# xsd:positiveInteger. The SHACL profiles constrain every ordinal with +# sh:datatype xsd:integer plus a sh:minInclusive bound, and a SHACL datatype +# constraint compares the datatype IRI exactly. An xsd:integer literal +# satisfies both the shape and the ontology's xsd:positiveInteger range, since +# the value 1 belongs to that value space however it is written. +# +# VCF versions +# ------------ +# VCF Core supplies one SHACL overlay per VCF version, each scoped by the file's +# vcfc:fileFormat. Most of this mapping is version-neutral, because the fixed +# columns and the header-line structure do not change between 4.1 and 4.5. The +# one version-dependent thing RML can carry is the optional vcfc:VCF4xFile +# subclass on the file subject, which is the sentinel in <#VCFFileMap> below. +# +# The version-dependent *semantics* -- which INFO keys flatten a tuple, whether +# that tuple repeats per ALT allele, and whether the local-allele and +# base-modification FORMAT families exist -- are decompositions of one source +# cell, so they live in the wrapper's emitters like every other decomposition. # # Output format: -# - The default conversion now targets triples (N-Triples) without named graph terms. +# - The default conversion targets triples (N-Triples) without named graph terms. <#VCFFileMap> a rr:TriplesMap ; @@ -37,25 +93,30 @@ ] ; rr:subjectMap [ rr:template "file://{SOURCE_FILE}" ; - rr:class vcfr:VCFFile + rr:class vcfc:VCFFile ; + # Version overlay. The wrapper rewrites this sentinel per input to the + # optional subclass for the version the file's ##fileformat line declares + # -- vcfc:VCF41File through vcfc:VCF45File -- which activates that version's + # SHACL gate. When the version has no conformance overlay (VCFv4.0, or a + # missing or malformed line) the sentinel is rewritten to vcfc:VCFFile, + # which is already asserted above, so the graph never claims a version gate + # it cannot support and never gains an undefined term. + rr:class vcfc:VCFVersionFile ] ; rr:predicateObjectMap [ - rr:predicate vcfr:hasHeader ; + rr:predicate vcfc:hasHeader ; rr:objectMap [ rr:template "file://{SOURCE_FILE}#header" ] ] ; rr:predicateObjectMap [ - rr:predicate vcfr:fileFormat ; + rr:predicate vcfc:fileFormat ; rr:objectMap [ rml:reference "FILE_FORMAT" ] ] ; - # fileDate is emitted by the wrapper: the published shape requires xsd:date, - # but ##fileDate has no mandated format in the VCF specification, so the - # value has to be inspected before it can be typed. rr:predicateObjectMap [ - rr:predicate vcfr:sourceSoftware ; + rr:predicate vcfc:sourceSoftware ; rr:objectMap [ rml:reference "SOURCE_SOFTWARE" ] ] ; rr:predicateObjectMap [ - rr:predicate vcfr:referenceGenome ; + rr:predicate vcfc:referenceGenome ; rr:objectMap [ rml:reference "REFERENCE_GENOME" ] ] . @@ -75,9 +136,34 @@ ] ; rr:subjectMap [ rr:template "file://{SOURCE_FILE}#header" ; - rr:class vcfr:VCFHeader + rr:class vcfc:VCFHeader + ] ; + rr:predicateObjectMap [ + rr:predicate vcfc:hasColumnHeader ; + rr:objectMap [ rr:template "file://{SOURCE_FILE}#header/columns" ] + ] . + +# The mandatory #CHROM line. Its ordered sample columns are attached by the +# wrapper with vcfc:hasGenotypeColumns, because they live in the records TSV +# header rather than in a row of any source table. +<#ColumnHeaderLineMap> + a rr:TriplesMap ; + rml:logicalSource [ + rml:source [ + a csvw:Table ; + csvw:url "/data/tsv/file_metadata.tsv" ; + csvw:dialect [ + a csvw:Dialect ; + csvw:delimiter "\t" ; + csvw:headerRowCount 1 + ] + ] ; + rml:referenceFormulation ql:CSV ] ; - . + rr:subjectMap [ + rr:template "file://{SOURCE_FILE}#header/columns" ; + rr:class vcfc:ColumnHeaderLine + ] . # NOTE: # Use explicit link maps (template-based) instead of parentTriplesMap joins. @@ -100,7 +186,7 @@ rr:template "file://{SOURCE_FILE}" ] ; rr:predicateObjectMap [ - rr:predicate vcfr:hasRecord ; + rr:predicate vcfc:hasRecord ; rr:objectMap [ rr:template "file://{SOURCE_FILE}#record/{ROW_ID}" ] @@ -124,12 +210,16 @@ rr:template "file://{SOURCE_FILE}#header" ] ; rr:predicateObjectMap [ - rr:predicate vcfr:hasHeaderLine ; + rr:predicate vcfc:hasHeaderLine ; rr:objectMap [ rr:template "file://{SOURCE_FILE}#header/line/{HEADER_INDEX}" ] ] . +# Every '##' line as a base vcfc:HeaderLine. The wrapper adds the subclass, the +# ordered vcfc:HeaderAttribute resources and the declaration properties. +# vcfc:lineIndex is what the SPARQL SHACL profile uses to check that header +# indices are unique and that ##fileformat is line 1. <#HeaderLineMap> a rr:TriplesMap ; rml:logicalSource [ @@ -146,17 +236,24 @@ ] ; rr:subjectMap [ rr:template "file://{SOURCE_FILE}#header/line/{HEADER_INDEX}" ; - rr:class vcfr:HeaderLine + rr:class vcfc:HeaderLine ] ; rr:predicateObjectMap [ - rr:predicate vcfr:headerKey ; + rr:predicate vcfc:headerKey ; rr:objectMap [ rml:reference "HEADER_KEY" ] ] ; rr:predicateObjectMap [ - rr:predicate vcfr:headerValue ; + rr:predicate vcfc:headerValue ; rr:objectMap [ rml:reference "HEADER_VALUE" ] + ] ; + rr:predicateObjectMap [ + rr:predicate vcfc:lineIndex ; + rr:objectMap [ rml:reference "HEADER_INDEX" ; rr:datatype xsd:integer ] ] . +# CHROM, POS and REF are the three fixed fields VCF 4.5 forbids from being the +# missing token, so their datatype is fixed for every row and they belong here. +# ID and ALT may be missing and are emitted by the wrapper. <#VCFRecordMap> a rr:TriplesMap ; rml:logicalSource [ @@ -173,33 +270,32 @@ ] ; rr:subjectMap [ rr:template "file://{SOURCE_FILE}#record/{ROW_ID}" ; - rr:class vcfr:VCFRecord + rr:class vcfc:VCFRecord ] ; rr:predicateObjectMap [ - rr:predicate vcfr:chrom ; + rr:predicate vcfc:chrom ; rr:objectMap [ rml:reference "CHROM" ] ] ; rr:predicateObjectMap [ - rr:predicate vcfr:pos ; + rr:predicate vcfc:pos ; rr:objectMap [ rml:reference "POS" ; rr:datatype xsd:integer ] ] ; rr:predicateObjectMap [ - rr:predicate vcfr:recordId ; - rr:objectMap [ rml:reference "ID" ] - ] ; - rr:predicateObjectMap [ - rr:predicate vcfr:ref ; + rr:predicate vcfc:ref ; rr:objectMap [ rml:reference "REF" ] ] ; rr:predicateObjectMap [ - rr:predicate vcfr:alt ; - rr:objectMap [ rml:reference "ALT" ] + rr:predicate vcfc:recordIndex ; + rr:objectMap [ rml:reference "ROW_ID" ; rr:datatype xsd:integer ] ] ; rr:predicateObjectMap [ - rr:predicate vcfr:hasCall ; + rr:predicate vcfc:hasCall ; rr:objectMap [ rr:template "file://{SOURCE_FILE}#call/{ROW_ID}" ] ] . +# FORMAT is the only call-level column whose datatype is fixed: VCF 4.5 defines +# it as a colon-separated key list with no missing form. QUAL, FILTER and INFO +# are emitted by the wrapper. <#VariantCallMap> a rr:TriplesMap ; rml:logicalSource [ @@ -216,21 +312,10 @@ ] ; rr:subjectMap [ rr:template "file://{SOURCE_FILE}#call/{ROW_ID}" ; - rr:class vcfr:VariantCall - ] ; - # QUAL is emitted by the wrapper, not here: the published SHACL shape - # requires xsd:decimal or vcfr:Null depending on whether the value is - # present, and RML cannot choose a datatype per row. - rr:predicateObjectMap [ - rr:predicate vcfr:filter ; - rr:objectMap [ rml:reference "FILTER" ] - ] ; - rr:predicateObjectMap [ - rr:predicate vcfr:infoRaw ; - rr:objectMap [ rml:reference "INFO" ] + rr:class vcfc:VariantCall ] ; rr:predicateObjectMap [ - rr:predicate vcfr:formatRaw ; + rr:predicate vcfc:formatRaw ; rr:objectMap [ rml:reference "FORMAT" ] ] . @@ -252,7 +337,7 @@ rr:template "file://{SOURCE_FILE}#call/{ROW_ID}" ] ; rr:predicateObjectMap [ - rr:predicate vcfr:hasSampleCall ; + rr:predicate vcfc:hasSampleCall ; rr:objectMap [ rr:template "file://{SOURCE_FILE}#sample/{ROW_ID}/{SAMPLE_URI_ID}" ] @@ -274,11 +359,17 @@ ] ; rr:subjectMap [ rr:template "file://{SOURCE_FILE}#sample/{ROW_ID}/{SAMPLE_URI_ID}" ; - rr:class vcfr:SampleCall + rr:class vcfc:SampleCall ] ; rr:predicateObjectMap [ - rr:predicate vcfr:sampleId ; + rr:predicate vcfc:sampleId ; rr:objectMap [ rml:reference "SAMPLE_ID" ] + ] ; + rr:predicateObjectMap [ + rr:predicate vcfc:forSample ; + rr:objectMap [ + rr:template "file://{SOURCE_FILE}#samples/{SAMPLE_URI_ID}" + ] ] . <#SampleCallToFormatValueLinkMap> @@ -299,7 +390,7 @@ rr:template "file://{SOURCE_FILE}#sample/{ROW_ID}/{SAMPLE_URI_ID}" ] ; rr:predicateObjectMap [ - rr:predicate vcfr:hasFormatValue ; + rr:predicate vcfc:hasFormatValue ; rr:objectMap [ rr:template "file://{SOURCE_FILE}#sample/{ROW_ID}/{SAMPLE_URI_ID}/fmt/{FORMAT_KEY}" ] @@ -321,9 +412,9 @@ ] ; rr:subjectMap [ rr:template "file://{SOURCE_FILE}#sample/{ROW_ID}/{SAMPLE_URI_ID}/fmt/{FORMAT_KEY}" ; - rr:class vcfr:FormatFieldValue + rr:class vcfc:FormatFieldValue ] ; rr:predicateObjectMap [ - rr:predicate vcfr:fieldValue ; + rr:predicate vcfc:fieldValue ; rr:objectMap [ rml:reference "FORMAT_VALUE" ] ] . diff --git a/src/run_conversion.sh b/src/run_conversion.sh index 3affc27..1e81d79 100644 --- a/src/run_conversion.sh +++ b/src/run_conversion.sh @@ -307,7 +307,7 @@ count_triples_json() { # Replace plain VCF null marker literals (`"."`) with typed null literals. # Ontology alignment: -# "." -> "."^^vcfr:Null +# "." -> "."^^vcfc:Null # # The rewrite is per-line and has no cross-line context, so it is applied to # each RMLStreamer part as it is streamed into the aggregate. Running it as a diff --git a/src/validate_compression.py b/src/validate_compression.py index dd46b6d..ca833e9 100644 --- a/src/validate_compression.py +++ b/src/validate_compression.py @@ -32,14 +32,16 @@ def count_nt(path: Path) -> int: def count_decoded(command: list[str]) -> int: """Run an RDF exporter and count its streamed N-Triples output.""" - process = subprocess.Popen( + # The context manager closes the stdout pipe even when counting raises, so a + # failed decode cannot leak a file descriptor into the rest of the run. + with subprocess.Popen( command, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, - ) - assert process.stdout is not None - count = sum(1 for line in process.stdout if is_triple_line(line)) - return_code = process.wait() + ) as process: + assert process.stdout is not None + count = sum(1 for line in process.stdout if is_triple_line(line)) + return_code = process.wait() if return_code != 0: raise RuntimeError(f"decoder exited with status {return_code}: {' '.join(command)}") return count diff --git a/src/validation/queries/common/preflight_blank_nodes.rq b/src/validation/queries/common/preflight_blank_nodes.rq index bc217a3..9edffc2 100644 --- a/src/validation/queries/common/preflight_blank_nodes.rq +++ b/src/validation/queries/common/preflight_blank_nodes.rq @@ -1,8 +1,8 @@ -PREFIX vcfr: +PREFIX vcfc: # Blank nodes anywhere in the graph. # -# Every class in the VCF-RDFizer vocabulary declares an vcfr:iriTemplate, so a +# Every class in the VCF-RDFizer vocabulary declares an vcfc:iriTemplate, so a # blank node means a term map produced no IRI. That is always a defect here: # blank-node identity is scoped to the file, so the record and value digests # could not address such a node, and neither could anything downstream that diff --git a/src/validation/queries/common/preflight_blank_nodes_count.rq b/src/validation/queries/common/preflight_blank_nodes_count.rq index 8cfd553..4e459b8 100644 --- a/src/validation/queries/common/preflight_blank_nodes_count.rq +++ b/src/validation/queries/common/preflight_blank_nodes_count.rq @@ -1,4 +1,4 @@ -PREFIX vcfr: +PREFIX vcfc: # Exact number of triples involving a blank node. SELECT (COUNT(*) AS ?anomalyCount) diff --git a/src/validation/queries/common/preflight_distinct_triple_count.rq b/src/validation/queries/common/preflight_distinct_triple_count.rq index 6b23075..94f4313 100644 --- a/src/validation/queries/common/preflight_distinct_triple_count.rq +++ b/src/validation/queries/common/preflight_distinct_triple_count.rq @@ -1,4 +1,4 @@ -PREFIX vcfr: +PREFIX vcfc: # How many distinct triples the graph contains. # diff --git a/src/validation/queries/common/preflight_empty_values.rq b/src/validation/queries/common/preflight_empty_values.rq index 2d0effb..e811ef4 100644 --- a/src/validation/queries/common/preflight_empty_values.rq +++ b/src/validation/queries/common/preflight_empty_values.rq @@ -1,10 +1,10 @@ -PREFIX vcfr: +PREFIX vcfc: # Empty or whitespace-only terms. # # The pipeline should never materialize one. RML emits no triple for an empty # reference, and every direct emitter either guards on a non-empty value or -# substitutes the VCF missing token, which becomes "."^^vcfr:Null. An empty +# substitutes the VCF missing token, which becomes "."^^vcfc:Null. An empty # literal therefore means a value was lost rather than marked missing, and an # empty IRI means a template substitution collapsed. # diff --git a/src/validation/queries/common/preflight_empty_values_count.rq b/src/validation/queries/common/preflight_empty_values_count.rq index 022c095..780d464 100644 --- a/src/validation/queries/common/preflight_empty_values_count.rq +++ b/src/validation/queries/common/preflight_empty_values_count.rq @@ -1,4 +1,4 @@ -PREFIX vcfr: +PREFIX vcfc: # Exact number of triples carrying an empty or whitespace-only term. SELECT (COUNT(*) AS ?anomalyCount) diff --git a/src/validation/queries/common/preflight_missing_token_conformance.rq b/src/validation/queries/common/preflight_missing_token_conformance.rq index bfbec42..7aaf36e 100644 --- a/src/validation/queries/common/preflight_missing_token_conformance.rq +++ b/src/validation/queries/common/preflight_missing_token_conformance.rq @@ -1,9 +1,9 @@ -PREFIX vcfr: +PREFIX vcfc: SELECT ?s ?p ?o (DATATYPE(?o) AS ?datatype) WHERE { ?s ?p ?o . FILTER(ISLITERAL(?o) && STR(?o) = ".") - FILTER(DATATYPE(?o) != vcfr:Null) + FILTER(DATATYPE(?o) != vcfc:Null) } LIMIT 100 diff --git a/src/validation/queries/common/preflight_missing_token_conformance_count.rq b/src/validation/queries/common/preflight_missing_token_conformance_count.rq index a909af3..f77867e 100644 --- a/src/validation/queries/common/preflight_missing_token_conformance_count.rq +++ b/src/validation/queries/common/preflight_missing_token_conformance_count.rq @@ -1,9 +1,9 @@ -PREFIX vcfr: +PREFIX vcfc: # Exact number of missing tokens serialized as a plain "." literal. SELECT (COUNT(*) AS ?anomalyCount) WHERE { ?s ?p ?o . FILTER(ISLITERAL(?o) && STR(?o) = ".") - FILTER(DATATYPE(?o) != vcfr:Null) + FILTER(DATATYPE(?o) != vcfc:Null) } diff --git a/src/validation/queries/common/preflight_position_datatype.rq b/src/validation/queries/common/preflight_position_datatype.rq index 132a55e..cff817c 100644 --- a/src/validation/queries/common/preflight_position_datatype.rq +++ b/src/validation/queries/common/preflight_position_datatype.rq @@ -1,4 +1,4 @@ -PREFIX vcfr: +PREFIX vcfc: PREFIX xsd: # POS must carry an integer datatype, never a plain string or a fractional type. @@ -11,7 +11,7 @@ PREFIX xsd: # xsd:decimal, xsd:double, a plain literal - is still reported. SELECT ?record ?pos (DATATYPE(?pos) AS ?datatype) WHERE { - ?record a vcfr:VCFRecord ; vcfr:pos ?pos . + ?record a vcfc:VCFRecord ; vcfc:pos ?pos . FILTER(DATATYPE(?pos) NOT IN ( xsd:integer, xsd:int, xsd:long, xsd:short, xsd:byte, xsd:nonNegativeInteger, xsd:positiveInteger, diff --git a/src/validation/queries/common/preflight_position_datatype_count.rq b/src/validation/queries/common/preflight_position_datatype_count.rq index e773e8e..ed89f84 100644 --- a/src/validation/queries/common/preflight_position_datatype_count.rq +++ b/src/validation/queries/common/preflight_position_datatype_count.rq @@ -1,10 +1,10 @@ -PREFIX vcfr: +PREFIX vcfc: PREFIX xsd: # Exact number of POS values outside the XSD integer family. SELECT (COUNT(*) AS ?anomalyCount) WHERE { - ?record a vcfr:VCFRecord ; vcfr:pos ?pos . + ?record a vcfc:VCFRecord ; vcfc:pos ?pos . FILTER(DATATYPE(?pos) NOT IN ( xsd:integer, xsd:int, xsd:long, xsd:short, xsd:byte, xsd:nonNegativeInteger, xsd:positiveInteger, diff --git a/src/validation/queries/common/preflight_record_cardinality.rq b/src/validation/queries/common/preflight_record_cardinality.rq index 5b683cb..c05fd1a 100644 --- a/src/validation/queries/common/preflight_record_cardinality.rq +++ b/src/validation/queries/common/preflight_record_cardinality.rq @@ -1,4 +1,4 @@ -PREFIX vcfr: +PREFIX vcfc: SELECT ?record (COUNT(DISTINCT ?chrom) AS ?chromCount) @@ -7,12 +7,12 @@ SELECT ?record (COUNT(DISTINCT ?alt) AS ?altCount) (COUNT(DISTINCT ?call) AS ?callCount) WHERE { - ?record a vcfr:VCFRecord . - OPTIONAL { ?record vcfr:chrom ?chrom } - OPTIONAL { ?record vcfr:pos ?pos } - OPTIONAL { ?record vcfr:ref ?ref } - OPTIONAL { ?record vcfr:alt ?alt } - OPTIONAL { ?record vcfr:hasCall ?call } + ?record a vcfc:VCFRecord . + OPTIONAL { ?record vcfc:chrom ?chrom } + OPTIONAL { ?record vcfc:pos ?pos } + OPTIONAL { ?record vcfc:ref ?ref } + OPTIONAL { ?record vcfc:alt ?alt } + OPTIONAL { ?record vcfc:hasCall ?call } } GROUP BY ?record HAVING(COUNT(DISTINCT ?chrom) != 1 || COUNT(DISTINCT ?pos) != 1 || COUNT(DISTINCT ?ref) != 1 || COUNT(DISTINCT ?alt) != 1 || COUNT(DISTINCT ?call) != 1) diff --git a/src/validation/queries/common/preflight_record_cardinality_count.rq b/src/validation/queries/common/preflight_record_cardinality_count.rq index 1f25ced..8d4d81b 100644 --- a/src/validation/queries/common/preflight_record_cardinality_count.rq +++ b/src/validation/queries/common/preflight_record_cardinality_count.rq @@ -1,4 +1,4 @@ -PREFIX vcfr: +PREFIX vcfc: # Exact number of records with anomalous property cardinality. # The companion query returns at most 100 example rows for diagnosis; this one @@ -7,12 +7,12 @@ SELECT (COUNT(*) AS ?anomalyCount) WHERE { SELECT ?record WHERE { - ?record a vcfr:VCFRecord . - OPTIONAL { ?record vcfr:chrom ?chrom } - OPTIONAL { ?record vcfr:pos ?pos } - OPTIONAL { ?record vcfr:ref ?ref } - OPTIONAL { ?record vcfr:alt ?alt } - OPTIONAL { ?record vcfr:hasCall ?call } + ?record a vcfc:VCFRecord . + OPTIONAL { ?record vcfc:chrom ?chrom } + OPTIONAL { ?record vcfc:pos ?pos } + OPTIONAL { ?record vcfc:ref ?ref } + OPTIONAL { ?record vcfc:alt ?alt } + OPTIONAL { ?record vcfc:hasCall ?call } } GROUP BY ?record HAVING(COUNT(DISTINCT ?chrom) != 1 || COUNT(DISTINCT ?pos) != 1 || COUNT(DISTINCT ?ref) != 1 || COUNT(DISTINCT ?alt) != 1 || COUNT(DISTINCT ?call) != 1) diff --git a/src/validation/queries/common/q01_record_density_1mb.rq b/src/validation/queries/common/q01_record_density_1mb.rq index e37b999..6b30f44 100644 --- a/src/validation/queries/common/q01_record_density_1mb.rq +++ b/src/validation/queries/common/q01_record_density_1mb.rq @@ -1,9 +1,9 @@ -PREFIX vcfr: +PREFIX vcfc: PREFIX xsd: SELECT ?chrom ?windowIndex (COUNT(DISTINCT ?record) AS ?recordCount) WHERE { - ?record a vcfr:VCFRecord ; vcfr:chrom ?chrom ; vcfr:pos ?pos . + ?record a vcfc:VCFRecord ; vcfc:chrom ?chrom ; vcfc:pos ?pos . BIND(FLOOR((xsd:integer(?pos) - 1) / 1000000) AS ?windowIndex) } GROUP BY ?chrom ?windowIndex diff --git a/src/validation/queries/common/q02_variant_shape_counts.rq b/src/validation/queries/common/q02_variant_shape_counts.rq index 6f29eb0..546f6b3 100644 --- a/src/validation/queries/common/q02_variant_shape_counts.rq +++ b/src/validation/queries/common/q02_variant_shape_counts.rq @@ -1,8 +1,8 @@ -PREFIX vcfr: +PREFIX vcfc: SELECT ?variantClass (COUNT(DISTINCT ?record) AS ?recordCount) WHERE { - ?record a vcfr:VCFRecord ; vcfr:ref ?refLiteral ; vcfr:alt ?altLiteral . + ?record a vcfc:VCFRecord ; vcfc:ref ?refLiteral ; vcfc:alt ?altLiteral . BIND(UCASE(STR(?refLiteral)) AS ?ref) BIND(UCASE(STR(?altLiteral)) AS ?alt) BIND( diff --git a/src/validation/queries/common/q03_titv.rq b/src/validation/queries/common/q03_titv.rq index a768a91..b805791 100644 --- a/src/validation/queries/common/q03_titv.rq +++ b/src/validation/queries/common/q03_titv.rq @@ -1,11 +1,11 @@ -PREFIX vcfr: +PREFIX vcfc: SELECT (COUNT(DISTINCT ?record) AS ?biallelicSnvCount) (SUM(IF((?ref = "A" && ?alt = "G") || (?ref = "G" && ?alt = "A") || (?ref = "C" && ?alt = "T") || (?ref = "T" && ?alt = "C"), 1, 0)) AS ?transitionCount) (SUM(IF((?ref = "A" && ?alt = "G") || (?ref = "G" && ?alt = "A") || (?ref = "C" && ?alt = "T") || (?ref = "T" && ?alt = "C"), 0, 1)) AS ?transversionCount) WHERE { - ?record a vcfr:VCFRecord ; vcfr:ref ?refLiteral ; vcfr:alt ?altLiteral . + ?record a vcfc:VCFRecord ; vcfc:ref ?refLiteral ; vcfc:alt ?altLiteral . BIND(UCASE(STR(?refLiteral)) AS ?ref) BIND(UCASE(STR(?altLiteral)) AS ?alt) FILTER(REGEX(?ref, "^[ACGT]$") && REGEX(?alt, "^[ACGT]$") && ?ref != ?alt) diff --git a/src/validation/queries/common/q04_filter_distribution.rq b/src/validation/queries/common/q04_filter_distribution.rq index 9545410..3bc2ebd 100644 --- a/src/validation/queries/common/q04_filter_distribution.rq +++ b/src/validation/queries/common/q04_filter_distribution.rq @@ -1,9 +1,9 @@ -PREFIX vcfr: +PREFIX vcfc: SELECT ?filterStatus ?filterLexical (COUNT(DISTINCT ?record) AS ?recordCount) WHERE { - ?record a vcfr:VCFRecord ; vcfr:hasCall ?call . - ?call vcfr:filter ?filterLiteral . + ?record a vcfc:VCFRecord ; vcfc:hasCall ?call . + ?call vcfc:filter ?filterLiteral . BIND(STR(?filterLiteral) AS ?filterLexical) BIND(IF(?filterLexical = "PASS", "PASS", IF(?filterLexical = ".", "NOT_APPLIED", "FAILED")) AS ?filterStatus) } diff --git a/src/validation/queries/common/q07_file_metadata.rq b/src/validation/queries/common/q07_file_metadata.rq index 26155d3..03b5f34 100644 --- a/src/validation/queries/common/q07_file_metadata.rq +++ b/src/validation/queries/common/q07_file_metadata.rq @@ -1,4 +1,4 @@ -PREFIX vcfr: +PREFIX vcfc: # The file-level declarations carried over from the VCF meta-information lines. # @@ -10,8 +10,8 @@ SELECT (COALESCE(?rg, "(absent)") AS ?referenceGenome) (COALESCE(?ss, "(absent)") AS ?sourceSoftware) WHERE { - ?file a vcfr:VCFFile . - OPTIONAL { ?file vcfr:fileFormat ?ffLiteral BIND(STR(?ffLiteral) AS ?ff) } - OPTIONAL { ?file vcfr:referenceGenome ?rgLiteral BIND(STR(?rgLiteral) AS ?rg) } - OPTIONAL { ?file vcfr:sourceSoftware ?ssLiteral BIND(STR(?ssLiteral) AS ?ss) } + ?file a vcfc:VCFFile . + OPTIONAL { ?file vcfc:fileFormat ?ffLiteral BIND(STR(?ffLiteral) AS ?ff) } + OPTIONAL { ?file vcfc:referenceGenome ?rgLiteral BIND(STR(?rgLiteral) AS ?rg) } + OPTIONAL { ?file vcfc:sourceSoftware ?ssLiteral BIND(STR(?ssLiteral) AS ?ss) } } diff --git a/src/validation/queries/common/q08_header_line_census.rq b/src/validation/queries/common/q08_header_line_census.rq index 4bfa223..494ace4 100644 --- a/src/validation/queries/common/q08_header_line_census.rq +++ b/src/validation/queries/common/q08_header_line_census.rq @@ -1,11 +1,11 @@ -PREFIX vcfr: +PREFIX vcfc: # One row per '##' meta-information key, with how many lines carry it. # Counting per key rather than in total means a dropped INFO or FILTER # declaration is localized instead of only shifting a single number. SELECT ?headerKey (COUNT(DISTINCT ?line) AS ?lineCount) WHERE { - ?line a vcfr:HeaderLine ; vcfr:headerKey ?headerKeyLiteral . + ?line a vcfc:HeaderLine ; vcfc:headerKey ?headerKeyLiteral . BIND(STR(?headerKeyLiteral) AS ?headerKey) } GROUP BY ?headerKey diff --git a/src/validation/queries/common/q09_predicate_census.rq b/src/validation/queries/common/q09_predicate_census.rq index cd62500..ced0b7b 100644 --- a/src/validation/queries/common/q09_predicate_census.rq +++ b/src/validation/queries/common/q09_predicate_census.rq @@ -1,4 +1,4 @@ -PREFIX vcfr: +PREFIX vcfc: # Every predicate in the graph and how many triples use it. # diff --git a/src/validation/queries/common/q10_class_census.rq b/src/validation/queries/common/q10_class_census.rq index 872a642..72f2064 100644 --- a/src/validation/queries/common/q10_class_census.rq +++ b/src/validation/queries/common/q10_class_census.rq @@ -1,4 +1,4 @@ -PREFIX vcfr: +PREFIX vcfc: # Every asserted rdf:type and how many resources carry it. Complements the # predicate census: together they bound both the properties and the resources diff --git a/src/validation/queries/common/q11_record_digest.rq b/src/validation/queries/common/q11_record_digest.rq index 69d686b..d59e903 100644 --- a/src/validation/queries/common/q11_record_digest.rq +++ b/src/validation/queries/common/q11_record_digest.rq @@ -1,4 +1,4 @@ -PREFIX vcfr: +PREFIX vcfc: # Per-record identity, aggregated into a fixed-size histogram. # @@ -17,16 +17,16 @@ PREFIX vcfr: # made to collide by shifting a field boundary. SELECT ?bucket (COUNT(*) AS ?recordCount) WHERE { - ?record a vcfr:VCFRecord ; - vcfr:chrom ?chrom ; - vcfr:pos ?pos ; - vcfr:recordId ?recordIdLiteral ; - vcfr:ref ?ref ; - vcfr:alt ?alt ; - vcfr:hasCall ?call . - ?call vcfr:qual ?qual ; - vcfr:filter ?filter ; - vcfr:infoRaw ?info . + ?record a vcfc:VCFRecord ; + vcfc:chrom ?chrom ; + vcfc:pos ?pos ; + vcfc:recordId ?recordIdLiteral ; + vcfc:ref ?ref ; + vcfc:alt ?alt ; + vcfc:hasCall ?call . + ?call vcfc:qual ?qual ; + vcfc:filter ?filter ; + vcfc:infoRaw ?info . BIND(SUBSTR(SHA256(CONCAT( STR(?record), "\u001F", STR(?chrom), "\u001F", diff --git a/src/validation/queries/common/q12_info_value_digest.rq b/src/validation/queries/common/q12_info_value_digest.rq index 48b2a4b..be41651 100644 --- a/src/validation/queries/common/q12_info_value_digest.rq +++ b/src/validation/queries/common/q12_info_value_digest.rq @@ -1,4 +1,4 @@ -PREFIX vcfr: +PREFIX vcfc: # Structured INFO values, bound to the node that identifies record and key. # @@ -9,7 +9,7 @@ PREFIX vcfr: # engine - GROUP_CONCAT would not be portable here. SELECT ?bucket (COUNT(*) AS ?valueCount) WHERE { - ?value a vcfr:InfoFieldValue ; vcfr:fieldValue ?lexLiteral . + ?value a vcfc:InfoFieldValue ; vcfc:fieldValue ?lexLiteral . BIND(STR(?lexLiteral) AS ?lex) BIND(SUBSTR(SHA256(CONCAT(STR(?value), "\u001F", STR(?lex))), 1, 2) AS ?bucket) } diff --git a/src/validation/queries/condensed/preflight_representation_profile.rq b/src/validation/queries/condensed/preflight_representation_profile.rq index 362062c..0c51676 100644 --- a/src/validation/queries/condensed/preflight_representation_profile.rq +++ b/src/validation/queries/condensed/preflight_representation_profile.rq @@ -1,9 +1,9 @@ -PREFIX vcfr: +PREFIX vcfc: SELECT ?file ?profile WHERE { - ?file a vcfr:VCFFile . - OPTIONAL { ?file vcfr:representationProfile ?profile } - FILTER(!BOUND(?profile) || ?profile != vcfr:CondensedRepresentation) + ?file a vcfc:VCFFile . + OPTIONAL { ?file vcfc:representationProfile ?profile } + FILTER(!BOUND(?profile) || ?profile != vcfc:CondensedRepresentation) } LIMIT 100 diff --git a/src/validation/queries/condensed/preflight_representation_profile_count.rq b/src/validation/queries/condensed/preflight_representation_profile_count.rq index 2c90126..4edfce7 100644 --- a/src/validation/queries/condensed/preflight_representation_profile_count.rq +++ b/src/validation/queries/condensed/preflight_representation_profile_count.rq @@ -1,9 +1,9 @@ -PREFIX vcfr: +PREFIX vcfc: # Exact number of VCFFile resources not declaring the condensed profile. SELECT (COUNT(*) AS ?anomalyCount) WHERE { - ?file a vcfr:VCFFile . - OPTIONAL { ?file vcfr:representationProfile ?profile } - FILTER(!BOUND(?profile) || ?profile != vcfr:CondensedRepresentation) + ?file a vcfc:VCFFile . + OPTIONAL { ?file vcfc:representationProfile ?profile } + FILTER(!BOUND(?profile) || ?profile != vcfc:CondensedRepresentation) } diff --git a/src/validation/queries/condensed/preflight_sample_gt_inventory.rq b/src/validation/queries/condensed/preflight_sample_gt_inventory.rq index 9fbde64..635b110 100644 --- a/src/validation/queries/condensed/preflight_sample_gt_inventory.rq +++ b/src/validation/queries/condensed/preflight_sample_gt_inventory.rq @@ -1,4 +1,4 @@ -PREFIX vcfr: +PREFIX vcfc: SELECT (COUNT(DISTINCT ?sample) AS ?sampleCount) @@ -6,11 +6,11 @@ SELECT (COUNT(DISTINCT ?gtVector) AS ?gtVectorCount) WHERE { OPTIONAL { - ?sampleSet a vcfr:SampleSet ; vcfr:hasSample ?sample . - ?sample vcfr:sampleName ?sampleId . + ?sampleSet a vcfc:SampleSet ; vcfc:hasSample ?sample . + ?sample vcfc:sampleName ?sampleId . } OPTIONAL { - ?gtVector a vcfr:FormatValueVector ; vcfr:declaredBy ?definition ; vcfr:encodedValues ?encoded . - ?definition vcfr:fieldId "GT" . + ?gtVector a vcfc:FormatValueVector ; vcfc:declaredBy ?definition ; vcfc:encodedValues ?encoded . + ?definition vcfc:fieldId "GT" . } } diff --git a/src/validation/queries/condensed/q05_sample_genotype_counts.rq b/src/validation/queries/condensed/q05_sample_genotype_counts.rq index 32bf44a..39faf14 100644 --- a/src/validation/queries/condensed/q05_sample_genotype_counts.rq +++ b/src/validation/queries/condensed/q05_sample_genotype_counts.rq @@ -1,17 +1,17 @@ -PREFIX vcfr: +PREFIX vcfc: SELECT ?sampleId ?genotypeClass (COUNT(DISTINCT ?record) AS ?callCount) WHERE { - ?file a vcfr:VCFFile ; vcfr:hasRecord ?record ; vcfr:hasSampleSet ?sampleSet . - ?record a vcfr:VCFRecord ; vcfr:hasCall ?call . - ?sampleSet vcfr:hasSample ?sample . - ?sample vcfr:sampleName ?sampleId ; vcfr:sampleIndex ?sampleIndex . + ?file a vcfc:VCFFile ; vcfc:hasRecord ?record ; vcfc:hasSampleSet ?sampleSet . + ?record a vcfc:VCFRecord ; vcfc:hasCall ?call . + ?sampleSet vcfc:hasSample ?sample . + ?sample vcfc:sampleName ?sampleId ; vcfc:sampleIndex ?sampleIndex . OPTIONAL { - ?sample vcfr:sampleIndex ?sampleIndexForVector . - ?call vcfr:hasCallMatrix ?matrix . - ?matrix vcfr:appliesToSampleSet ?sampleSet ; vcfr:hasFormatValueVector ?gtVector . - ?gtVector vcfr:declaredBy ?definition ; vcfr:encodedValues ?encoded . - ?definition vcfr:fieldId "GT" . + ?sample vcfc:sampleIndex ?sampleIndexForVector . + ?call vcfc:hasCallMatrix ?matrix . + ?matrix vcfc:appliesToSampleSet ?sampleSet ; vcfc:hasFormatValueVector ?gtVector . + ?gtVector vcfc:declaredBy ?definition ; vcfc:encodedValues ?encoded . + ?definition vcfc:fieldId "GT" . BIND(?sampleIndexForVector - 1 AS ?zeroIndex) BIND(REPLACE(STR(?encoded), CONCAT("^(?:[^\\t]*\\t){", STR(?zeroIndex), "}([^\\t]*).*$"), "$1") AS ?gtLiteral) } diff --git a/src/validation/queries/condensed/q06_ac_an_distribution.rq b/src/validation/queries/condensed/q06_ac_an_distribution.rq index 5c26b70..7faf472 100644 --- a/src/validation/queries/condensed/q06_ac_an_distribution.rq +++ b/src/validation/queries/condensed/q06_ac_an_distribution.rq @@ -1,4 +1,4 @@ -PREFIX vcfr: +PREFIX vcfc: SELECT ?an ?ac (COUNT(DISTINCT ?record) AS ?siteCount) WHERE { @@ -8,15 +8,15 @@ WHERE { { SELECT DISTINCT ?record ?sample ?gt WHERE { - ?file a vcfr:VCFFile ; vcfr:hasRecord ?record ; vcfr:hasSampleSet ?sampleSet . - ?record a vcfr:VCFRecord ; vcfr:alt ?altLiteral ; vcfr:hasCall ?call . + ?file a vcfc:VCFFile ; vcfc:hasRecord ?record ; vcfc:hasSampleSet ?sampleSet . + ?record a vcfc:VCFRecord ; vcfc:alt ?altLiteral ; vcfc:hasCall ?call . FILTER(STR(?altLiteral) != "." && !CONTAINS(STR(?altLiteral), ",")) - ?sampleSet vcfr:hasSample ?sample . - ?sample vcfr:sampleIndex ?sampleIndex . - ?call vcfr:hasCallMatrix ?matrix . - ?matrix vcfr:appliesToSampleSet ?sampleSet ; vcfr:hasFormatValueVector ?gtVector . - ?gtVector vcfr:declaredBy ?definition ; vcfr:encodedValues ?encoded . - ?definition vcfr:fieldId "GT" . + ?sampleSet vcfc:hasSample ?sample . + ?sample vcfc:sampleIndex ?sampleIndex . + ?call vcfc:hasCallMatrix ?matrix . + ?matrix vcfc:appliesToSampleSet ?sampleSet ; vcfc:hasFormatValueVector ?gtVector . + ?gtVector vcfc:declaredBy ?definition ; vcfc:encodedValues ?encoded . + ?definition vcfc:fieldId "GT" . BIND(?sampleIndex - 1 AS ?zeroIndex) BIND(REPLACE(STR(?encoded), CONCAT("^(?:[^\\t]*\\t){", STR(?zeroIndex), "}([^\\t]*).*$"), "$1") AS ?gtLiteral) BIND(REPLACE(STR(?gtLiteral), "[|]", "/") AS ?gt) diff --git a/src/validation/queries/condensed/q13_format_value_digest.rq b/src/validation/queries/condensed/q13_format_value_digest.rq index c8e4ff6..dbe1350 100644 --- a/src/validation/queries/condensed/q13_format_value_digest.rq +++ b/src/validation/queries/condensed/q13_format_value_digest.rq @@ -1,4 +1,4 @@ -PREFIX vcfr: +PREFIX vcfc: # The condensed equivalent of the expanded per-cell FORMAT digest. # @@ -7,7 +7,7 @@ PREFIX vcfr: # per-cell nodes of the expanded shape. SELECT ?bucket (COUNT(*) AS ?valueCount) WHERE { - ?value a vcfr:FormatValueVector ; vcfr:encodedValues ?lexLiteral . + ?value a vcfc:FormatValueVector ; vcfc:encodedValues ?lexLiteral . BIND(STR(?lexLiteral) AS ?lex) BIND(SUBSTR(SHA256(CONCAT(STR(?value), "\u001F", STR(?lex))), 1, 2) AS ?bucket) } diff --git a/src/validation/queries/expanded/preflight_representation_profile.rq b/src/validation/queries/expanded/preflight_representation_profile.rq index 3adeb40..eef0fa0 100644 --- a/src/validation/queries/expanded/preflight_representation_profile.rq +++ b/src/validation/queries/expanded/preflight_representation_profile.rq @@ -1,9 +1,9 @@ -PREFIX vcfr: +PREFIX vcfc: SELECT ?file ?profile WHERE { - ?file a vcfr:VCFFile . - OPTIONAL { ?file vcfr:representationProfile ?profile } - FILTER(!BOUND(?profile) || ?profile != vcfr:ExpandedRepresentation) + ?file a vcfc:VCFFile . + OPTIONAL { ?file vcfc:representationProfile ?profile } + FILTER(!BOUND(?profile) || ?profile != vcfc:ExpandedRepresentation) } LIMIT 100 diff --git a/src/validation/queries/expanded/preflight_representation_profile_count.rq b/src/validation/queries/expanded/preflight_representation_profile_count.rq index 9174a06..e49b4d4 100644 --- a/src/validation/queries/expanded/preflight_representation_profile_count.rq +++ b/src/validation/queries/expanded/preflight_representation_profile_count.rq @@ -1,9 +1,9 @@ -PREFIX vcfr: +PREFIX vcfc: # Exact number of VCFFile resources not declaring the expanded profile. SELECT (COUNT(*) AS ?anomalyCount) WHERE { - ?file a vcfr:VCFFile . - OPTIONAL { ?file vcfr:representationProfile ?profile } - FILTER(!BOUND(?profile) || ?profile != vcfr:ExpandedRepresentation) + ?file a vcfc:VCFFile . + OPTIONAL { ?file vcfc:representationProfile ?profile } + FILTER(!BOUND(?profile) || ?profile != vcfc:ExpandedRepresentation) } diff --git a/src/validation/queries/expanded/preflight_sample_gt_inventory.rq b/src/validation/queries/expanded/preflight_sample_gt_inventory.rq index aa59ccb..5925a42 100644 --- a/src/validation/queries/expanded/preflight_sample_gt_inventory.rq +++ b/src/validation/queries/expanded/preflight_sample_gt_inventory.rq @@ -1,4 +1,4 @@ -PREFIX vcfr: +PREFIX vcfc: SELECT (COUNT(DISTINCT ?sampleCall) AS ?sampleCallCount) @@ -6,11 +6,11 @@ SELECT (COUNT(DISTINCT ?gtValueNode) AS ?gtValueNodeCount) WHERE { OPTIONAL { - ?sampleCall a vcfr:SampleCall ; vcfr:sampleId ?sampleId . + ?sampleCall a vcfc:SampleCall ; vcfc:sampleId ?sampleId . OPTIONAL { - ?sampleCall vcfr:hasFormatValue ?gtValueNode . + ?sampleCall vcfc:hasFormatValue ?gtValueNode . FILTER(STRENDS(STR(?gtValueNode), "/fmt/GT")) - ?gtValueNode vcfr:fieldValue ?gtLiteral . + ?gtValueNode vcfc:fieldValue ?gtLiteral . } } } diff --git a/src/validation/queries/expanded/q05_sample_genotype_counts.rq b/src/validation/queries/expanded/q05_sample_genotype_counts.rq index bed78a2..c389b94 100644 --- a/src/validation/queries/expanded/q05_sample_genotype_counts.rq +++ b/src/validation/queries/expanded/q05_sample_genotype_counts.rq @@ -1,12 +1,12 @@ -PREFIX vcfr: +PREFIX vcfc: SELECT ?sampleId ?genotypeClass (COUNT(DISTINCT ?sampleCall) AS ?callCount) WHERE { - ?sampleCall a vcfr:SampleCall ; vcfr:sampleId ?sampleId . + ?sampleCall a vcfc:SampleCall ; vcfc:sampleId ?sampleId . OPTIONAL { - ?sampleCall vcfr:hasFormatValue ?gtValueNode . + ?sampleCall vcfc:hasFormatValue ?gtValueNode . FILTER(STRENDS(STR(?gtValueNode), "/fmt/GT")) - ?gtValueNode vcfr:fieldValue ?gtLiteral . + ?gtValueNode vcfc:fieldValue ?gtLiteral . } BIND(IF(BOUND(?gtLiteral), REPLACE(STR(?gtLiteral), "[|]", "/"), "") AS ?gt) BIND(IF(!BOUND(?gtLiteral), "NO_GT_FIELD", diff --git a/src/validation/queries/expanded/q06_ac_an_distribution.rq b/src/validation/queries/expanded/q06_ac_an_distribution.rq index 1a0fc93..8d33589 100644 --- a/src/validation/queries/expanded/q06_ac_an_distribution.rq +++ b/src/validation/queries/expanded/q06_ac_an_distribution.rq @@ -1,4 +1,4 @@ -PREFIX vcfr: +PREFIX vcfc: SELECT ?an ?ac (COUNT(DISTINCT ?record) AS ?siteCount) WHERE { @@ -8,12 +8,12 @@ WHERE { { SELECT DISTINCT ?record ?sampleCall ?gt WHERE { - ?record a vcfr:VCFRecord ; vcfr:alt ?altLiteral ; vcfr:hasCall ?call . + ?record a vcfc:VCFRecord ; vcfc:alt ?altLiteral ; vcfc:hasCall ?call . FILTER(STR(?altLiteral) != "." && !CONTAINS(STR(?altLiteral), ",")) - ?call vcfr:hasSampleCall ?sampleCall . - ?sampleCall vcfr:hasFormatValue ?gtValueNode . + ?call vcfc:hasSampleCall ?sampleCall . + ?sampleCall vcfc:hasFormatValue ?gtValueNode . FILTER(STRENDS(STR(?gtValueNode), "/fmt/GT")) - ?gtValueNode vcfr:fieldValue ?gtLiteral . + ?gtValueNode vcfc:fieldValue ?gtLiteral . BIND(REPLACE(STR(?gtLiteral), "[|]", "/") AS ?gt) } } diff --git a/src/validation/queries/expanded/q13_format_value_digest.rq b/src/validation/queries/expanded/q13_format_value_digest.rq index 647abed..c777293 100644 --- a/src/validation/queries/expanded/q13_format_value_digest.rq +++ b/src/validation/queries/expanded/q13_format_value_digest.rq @@ -1,4 +1,4 @@ -PREFIX vcfr: +PREFIX vcfc: # Every FORMAT value, bound to the node that identifies record, sample and key. # @@ -7,7 +7,7 @@ PREFIX vcfr: # hashing it with the value makes each cell individually accountable. SELECT ?bucket (COUNT(*) AS ?valueCount) WHERE { - ?value a vcfr:FormatFieldValue ; vcfr:fieldValue ?lexLiteral . + ?value a vcfc:FormatFieldValue ; vcfc:fieldValue ?lexLiteral . BIND(STR(?lexLiteral) AS ?lex) BIND(SUBSTR(SHA256(CONCAT(STR(?value), "\u001F", STR(?lex))), 1, 2) AS ?bucket) } diff --git a/src/validation/validation_runner.py b/src/validation/validation_runner.py index ecc2db5..ff9c381 100644 --- a/src/validation/validation_runner.py +++ b/src/validation/validation_runner.py @@ -33,6 +33,7 @@ import re import shlex import shutil +import signal import subprocess import sys import tempfile @@ -55,6 +56,18 @@ VCF = None +def eprint(*args: Any) -> None: + """Print to stderr. + + This runner executes standalone inside the container, so it cannot borrow + the identically named helper from ``vcf_rdfizer.py`` on the host. Both the + oversized-graph advisory and the query-timeout abandon message call this; + before it existed here, either one raised ``NameError`` and killed the run + at the exact moment it was trying to warn the operator. + """ + print(*args, file=sys.stderr, flush=True) + + SCRIPT_DIR = Path(__file__).resolve().parent QUERY_ROOT = SCRIPT_DIR / "queries" CORE_QUERIES = ( @@ -72,6 +85,31 @@ "q12_info_value_digest", "q13_format_value_digest", ) +#: Which oracle phases a query would actually have paid for, had it been the +#: only question asked. The oracle is one pass that fills every query's +#: counters together, so there is no measured per-query slice of it; this is the +#: explicit statement of what a one-question cyvcf2 script would have cost. +#: +#: Every query pays to open the file, scan the records, and assemble its answer. +#: Only the sample-level queries pay for the per-sample genotype/AC-AN block, +#: which is the part whose cost scales with cohort size. Measured on the +#: shipped fixtures, the block is 15.5% of the scan at one sample and 73.5% at +#: 2,504 -- which is the whole reason per-query oracle numbers are worth having. +#: +#: Known coarseness: `assemblySeconds` is charged to every query equally, and on +#: a cohort file it is not small (149 s against a 181 s scan on the 2,504-sample +#: fixture) because it sorts the per-sample counters. Most of that work belongs +#: to the sample-level queries, so this attribution OVER-charges record-level +#: queries on multi-sample inputs -- i.e. it understates the gap it is measuring. +#: Splitting it would mean restructuring the post-loop assembly into per-query +#: builders; until then the bias is in the conservative direction and is stated +#: rather than hidden. +ORACLE_SAMPLE_LEVEL_QUERIES = frozenset({ + "q05_sample_genotype_counts", + "q06_ac_an_distribution", + "q13_format_value_digest", +}) + #: Queries that must return exactly one row, normalized to a dict rather than #: a list so the comparison reads as a field-by-field check. SINGLE_ROW_QUERIES = frozenset({"q03_titv", "q07_file_metadata"}) @@ -330,27 +368,37 @@ def classify_genotype(alleles: tuple[int | None, ...] | None, *, has_gt: bool) - # A custom mapping therefore invalidates them, and the wrapper switches the # policy to report-only in that case. -VCFR = "https://w3id.org/vcf-rdfizer/vocab#" -RDF_TYPE = "http://www.w3.org/1999/02/22-rdf-syntax-ns#type" +# The vocabulary terms, the VCF-version model and the lexical parsers are +# shared with the wrapper rather than mirrored here. They used to be duplicated, +# with unit tests asserting the copies stayed identical; one module removes the +# drift the tests were policing. `Dockerfile` copies it next to this runner, so +# it imports the same way inside the container and on the host. +try: + import vcf_rdfizer_vocab as vocab +except ImportError: + # In the container the module sits next to this file; in a source checkout + # it is at the repository root. Try both before giving up, so the runner + # works as a script in either layout. + for candidate in (SCRIPT_DIR, SCRIPT_DIR.parent.parent): + if (candidate / "vcf_rdfizer_vocab.py").is_file(): + sys.path.insert(0, str(candidate)) + break + import vcf_rdfizer_vocab as vocab + +VCFC = vocab.VCFC_NAMESPACE +RDF_TYPE = vocab.RDF_TYPE_URI def _nonzero(counts: dict[str, int]) -> dict[str, int]: return {key: value for key, value in counts.items() if value} +rml_uri_component = vocab.rml_uri_component + + #: Field separator for the record digest. U+001F cannot appear in a VCF field, #: so no shift of a field boundary can make two different records collide. The #: query in q11_record_digest.rq writes it as a SPARQL \\u001F escape. -def rml_uri_component(value: str) -> str: - """Encode a template value the way RMLStreamer 2.5.0 does. - - Mirrors ``_rml_uri_component`` in ``vcf_rdfizer.py``; that module is not on - the container's import path, and ``test_validation_logic_unit.py`` asserts - the two agree so they cannot drift. - """ - encoded = quote_plus(value, safe="*-._", encoding="utf-8", errors="strict") - # urllib leaves '~' unescaped; java.net.URLEncoder does not. - return encoded.replace("+", "%20").replace("~", "%7E") DIGEST_SEPARATOR = "\u001f" @@ -366,6 +414,351 @@ def record_digest_bucket(fields: list[str]) -> str: return hashlib.sha256(joined.encode("utf-8")).hexdigest()[:DIGEST_BUCKET_CHARS] +# --------------------------------------------------------------------------- +# Emitted-shape counters +# --------------------------------------------------------------------------- +# +# The census asserts an exact inventory: these predicates and classes, with +# these counts, and nothing else. So every resource family the emitters produce +# has to be counted here, from the VCF, without reading the graph or the +# mapping. These two functions are the single derivation: parse_vcf calls them +# with the columns it read, and the test fixture calls them with the columns it +# declares, so the container oracle and the fixture oracle cannot disagree. +# +# They mirror append_header_representation_rdf and append_record_detail_rdf. +# Where a decision depends on a shared table (which '##' keys are structured, +# which INFO keys carry an SV carrier, which Number tokens are positional), the +# table comes from vcf_rdfizer_vocab rather than being restated. + + +def _count_field_definition(counts: Counter, *, number: str, has_type: bool, + has_description: bool) -> None: + """Count the ID/Number/Type/Description quartet of one field definition.""" + counts["fieldId"] += 1 + counts["fieldNumber"] += 1 + if vocab.arity_individual(number) is not None: + counts["fieldArity"] += 1 + if vocab.number_as_integer(number) is not None: + counts["fieldNumberInteger"] += 1 + if has_type: + counts["fieldType"] += 1 + if has_description: + counts["fieldDescription"] += 1 + + +def emitted_header_counters(header_lines: list[tuple[str, str]]) -> dict[str, Any]: + """Counters for everything append_header_representation_rdf emits. + + ``header_lines`` is the ordered ``(key, value)`` list of '##' lines, without + the leading '##' and split on the first '='. + """ + classes: Counter[str] = Counter() + predicates: Counter[str] = Counter() + contigs = 0 + declarations: set[str] = set() + + for key, value in header_lines: + key_lower = key.lower() + structured = vocab.is_structured_header_value(value) + line_class = HEADER_LINE_CLASSES.get(key_lower) + if line_class is None: + # An unrecognized key is still one of the two VCF forms. An + # unstructured line is only typed when it has a value, because + # vcfc:UnstructuredHeaderLineShape requires one. + if structured: + line_class = "StructuredHeaderLine" + elif value: + line_class = "UnstructuredHeaderLine" + else: + continue + classes[line_class] += 1 + + if not structured: + if key_lower == "assembly" and value: + predicates["assemblyUrl"] += 1 + elif key_lower == "pedigreedb" and value: + predicates["pedigreeDbUrl"] += 1 + continue + + attributes = parse_structured_header_attributes(value) + classes["HeaderAttribute"] += len(attributes) + for name in ("hasAttribute", "attributeKey", "attributeValue", "attributeIndex"): + predicates[name] += len(attributes) + + fields = dict(attributes) + identifier = (fields.get("ID") or "").strip() + + if key_lower in {"info", "format"}: + if not identifier: + continue + # An absent Description is synthesized, so the property is always + # present on an INFO or FORMAT declaration. + _count_field_definition( + predicates, number=fields.get("Number") or ".", + has_type=True, has_description=True, + ) + if key_lower == "info": + for attribute, predicate in vocab.INFO_EXTRA_ATTRIBUTES.items(): + if fields.get(attribute): + predicates[predicate] += 1 + elif key_lower == "filter": + if identifier: + predicates["filterId"] += 1 + predicates["fieldDescription"] += 1 + elif key_lower == "alt": + if identifier: + predicates["altId"] += 1 + predicates["fieldDescription"] += 1 + elif key_lower == "contig": + if not identifier: + continue + contigs += 1 + predicates["contigId"] += 1 + for attribute, (predicate, _datatype) in vocab.CONTIG_ATTRIBUTES.items(): + if fields.get(attribute): + predicates[predicate] += 1 + elif key_lower == "meta": + if not identifier: + continue + _count_field_definition( + predicates, number=fields.get("Number") or ".", + has_type=True, has_description=fields.get("Description") is not None, + ) + predicates["metaAllowedValue"] += len(_meta_values(fields.get("Values"))) + elif key_lower == "sample": + if identifier: + predicates["declaresSample"] += 1 + if identifier not in declarations: + declarations.add(identifier) + classes["SampleDeclaration"] += 1 + elif key_lower == "pedigree": + for attribute_key, attribute_value in attributes: + if attribute_key == "ID" or not attribute_value: + continue + predicates[ + vocab.PEDIGREE_ROLE_PROPERTIES.get(attribute_key, "pedigreeAncestor") + ] += 1 + predicates["ancestorRole"] += 1 + if attribute_value not in declarations: + declarations.add(attribute_value) + classes["SampleDeclaration"] += 1 + + if contigs: + predicates["contigCount"] += 1 + return { + "emittedHeaderClasses": dict(classes), + "emittedHeaderPredicates": dict(predicates), + } + + +def _meta_values(raw: str | None) -> list[str]: + """Split a ##META ``Values=[a, b, c]`` list, as the emitter does.""" + if not raw: + return [] + inner = raw.strip() + if inner.startswith("[") and inner.endswith("]"): + inner = inner[1:-1] + return [item.strip() for item in inner.split(",") if item.strip()] + + +def emitted_record_counters( + rows: list[list[str]], + samples: list[str], + *, + version: Any, + contig_ids: set[str], + alt_declaration_ids: set[str], + has_assembly_line: bool = False, + info_numbers: dict[str, str], + format_numbers: dict[str, str], +) -> dict[str, Any]: + """Counters for the allele, INFO, value-item, SV and genotype families. + + ``rows`` are the raw tab-split VCF data columns. Everything is derived from + those and from the version, exactly as the emitters derive it. + """ + classes: Counter[str] = Counter() + predicates: Counter[str] = Counter() + + assembly_contig_ids: set[str] = set() + reference_alleles = alt_alleles = 0 + value_items = value_item_alleles = tuple_items = 0 + genotypes = genotype_calls = called_alleles = 0 + + for row in rows: + chrom = row[0] if len(row) > 0 else "" + ref = row[3] if len(row) > 3 else "" + alt = row[4] if len(row) > 4 else "" + info = row[7] if len(row) > 7 else "" + + # A bracketed CHROM names a breakpoint-assembly contig instead of a + # declared reference sequence; the two are mutually exclusive. + assembly_id = vocab.parse_bracketed_chrom(chrom) + if assembly_id is not None: + predicates["chromAssemblyContig"] += 1 + if assembly_id not in assembly_contig_ids: + assembly_contig_ids.add(assembly_id) + classes["AssemblyContig"] += 1 + predicates["assemblyContigId"] += 1 + if has_assembly_line: + predicates["declaredInAssembly"] += 1 + elif chrom in contig_ids: + predicates["chromosome"] += 1 + + alleles = vocab.parse_alt_alleles(ref, alt) + allele_uris = {allele.index for allele in alleles} + alt_count = sum(1 for allele in alleles if allele.index >= 1) + for allele in alleles: + if allele.index == 0: + reference_alleles += 1 + else: + alt_alleles += 1 + if allele.symbolic_id and allele.symbolic_id in alt_declaration_ids: + predicates["declaredByAlt"] += 1 + if allele.symbolic_type: + predicates["svType"] += 1 + if allele.breakend is not None: + classes["Breakend"] += 1 + if allele.breakend.orientation is not None: + predicates["breakendOrientation"] += 1 + if allele.breakend.replacement: + predicates["breakendReplacementString"] += 1 + if allele.breakend.is_single: + predicates["isSingleBreakend"] += 1 + + entries = parse_info_entries(info) + for key, value in entries: + if value is None: + continue + number = info_numbers.get(key, ".") + if not (allele_uris and version.is_positional(key, number)): + continue + items = vocab.split_value_items(value) + value_items += len(items) + arity = version.tuple_arity(key) + if arity is not None: + tuple_items += len(items) + for index in range(len(items)): + link = version.value_item_link(key, number, index) + if link.allele_index is not None and link.allele_index in allele_uris: + value_item_alleles += 1 + elif link.genotype_index is not None: + predicates["forGenotypeIndex"] += 1 + elif link.gt_allele_index is not None: + predicates["forGTAlleleIndex"] += 1 + + if samples: + format_keys = (row[8].split(":") if len(row) > 8 and row[8] else []) + payloads = row[9 : 9 + len(samples)] + if "GT" in format_keys: + gt_index = format_keys.index("GT") + for payload in payloads: + fields = payload.split(":") if payload else [] + raw_gt = fields[gt_index] if gt_index < len(fields) else "" + parsed = vocab.parse_genotype(raw_gt) + if parsed is None: + continue + genotypes += 1 + genotype_calls += len(parsed.calls) + called_alleles += sum( + 1 for call in parsed.calls + if call.allele_index is not None and call.allele_index <= alt_count + ) + + if reference_alleles: + classes["ReferenceAllele"] = reference_alleles + predicates["hasReferenceAllele"] = reference_alleles + if alt_alleles: + classes["AltAllele"] = alt_alleles + predicates["hasAltAllele"] = alt_alleles + total_alleles = reference_alleles + alt_alleles + for name in ("alleleIndex", "alleleValue", "alleleKind"): + predicates[name] += total_alleles + + if value_items: + classes["FieldValueItem"] += value_items + for name in ("hasValueItem", "valueIndex", "itemValue"): + predicates[name] += value_items + predicates["forAllele"] += value_item_alleles + predicates["tupleArity"] += tuple_items + + # The parsed genotype layer is expanded-only, so it is reported separately + # and merged by expected_census for that profile alone. Keeping parse_vcf + # representation-independent is what lets one parse serve both censuses. + genotype_classes: Counter[str] = Counter() + genotype_predicates: Counter[str] = Counter() + if genotypes: + genotype_classes["Genotype"] += genotypes + genotype_classes["GenotypeAlleleCall"] += genotype_calls + for name in ("hasGenotype", "genotypeString", "ploidy", "phasingStatus"): + genotype_predicates[name] += genotypes + for name in ("hasAlleleCall", "callIndex", "isNoCall"): + genotype_predicates[name] += genotype_calls + genotype_predicates["calledAllele"] += called_alleles + + return { + "emittedRecordClasses": dict(classes), + "emittedRecordPredicates": dict(predicates), + "emittedGenotypeClasses": dict(genotype_classes), + "emittedGenotypePredicates": dict(genotype_predicates), + "alleleCount": total_alleles, + "valueItemCount": value_items, + "genotypeCount": genotypes, + "genotypeAlleleCallCount": genotype_calls, + } + + +def synthesized_definition_numbers( + rows: list[list[str]], *, declared_info: set[str], declared_format: set[str] +) -> dict[str, list[str]]: + """The Number token of every definition the value emitters have to invent. + + A key declared by a '##INFO' or '##FORMAT' line is owned by the header + emitter, which emits its ID/Number/Type/Description at the header line's + own IRI. Only an undeclared key gets a definition invented at value time, + with Number "0" for a Flag and "." otherwise. + """ + info: list[str] = [] + format_keys: list[str] = [] + seen_info: set[str] = set() + seen_format: set[str] = set() + for row in rows: + for key, value in parse_info_entries(row[7] if len(row) > 7 else ""): + if key in declared_info or key in seen_info: + continue + seen_info.add(key) + info.append("0" if value is None else ".") + declared_keys = row[8].split(":") if len(row) > 8 and row[8] else [] + payloads = [p.split(":") if p else [] for p in row[9:]] + width = max([len(declared_keys), *(len(p) for p in payloads)], default=0) + for index in range(width): + key = (declared_keys[index] + if index < len(declared_keys) and declared_keys[index] + else f"FIELD_{index + 1}") + if key in declared_format or key in seen_format: + continue + seen_format.add(key) + format_keys.append(".") + return {"synthesizedInfoNumbers": info, "synthesizedFormatNumbers": format_keys} + + +def _count_definitions(predicates: dict[str, int], numbers: list[str]) -> None: + """Count the field* triples of each synthesized field definition. + + A synthesized definition always carries ID, Number, Type and Description, + plus the arity individual or the fixed count its Number token implies. + """ + for number in numbers: + for name in ("fieldId", "fieldNumber", "fieldType", "fieldDescription"): + predicates[f"{VCFC}{name}"] = predicates.get(f"{VCFC}{name}", 0) + 1 + if vocab.arity_individual(number) is not None: + predicates[f"{VCFC}fieldArity"] = predicates.get(f"{VCFC}fieldArity", 0) + 1 + if vocab.number_as_integer(number) is not None: + predicates[f"{VCFC}fieldNumberInteger"] = ( + predicates.get(f"{VCFC}fieldNumberInteger", 0) + 1 + ) + + def expected_census( parser: dict[str, Any], representation: str, *, info_representation: str = "structured", header_representation: str = "structured", @@ -376,113 +769,141 @@ def expected_census( header_lines = parser["headerLineCount"] classes: dict[str, int] = { - f"{VCFR}VCFFile": 1, - f"{VCFR}VCFHeader": 1, - f"{VCFR}HeaderLine": header_lines, - f"{VCFR}VCFRecord": records, - f"{VCFR}VariantCall": records, + f"{VCFC}VCFFile": 1, + f"{VCFC}VCFHeader": 1, + f"{VCFC}ColumnHeaderLine": 1, + f"{VCFC}HeaderLine": header_lines, + f"{VCFC}VCFRecord": records, + f"{VCFC}VariantCall": records, } + # The mapping's version sentinel resolves to the subclass for the version + # the file declares, or back to vcfc:VCFFile when the version has no + # conformance overlay -- in which case it adds no new class. + version = vocab.parse_vcf_version(parser.get("fileFormat")) + if version is not None: + classes[f"{VCFC}{version.file_class}"] = 1 predicates: dict[str, int] = { - f"{VCFR}hasHeader": 1, - f"{VCFR}hasHeaderLine": header_lines, - f"{VCFR}headerKey": header_lines, - f"{VCFR}headerValue": parser["headerValueCount"], - f"{VCFR}hasRecord": records, - f"{VCFR}chrom": records, - f"{VCFR}pos": records, - f"{VCFR}recordId": records, - f"{VCFR}ref": records, - f"{VCFR}alt": records, - f"{VCFR}hasCall": records, - f"{VCFR}qual": records, - f"{VCFR}filter": records, - f"{VCFR}infoRaw": records, - f"{VCFR}formatRaw": parser["recordsWithFormatColumn"], + f"{VCFC}hasHeader": 1, + f"{VCFC}hasColumnHeader": 1, + f"{VCFC}hasHeaderLine": header_lines, + f"{VCFC}headerKey": header_lines, + f"{VCFC}headerValue": parser["headerValueCount"], + f"{VCFC}lineIndex": header_lines, + f"{VCFC}hasRecord": records, + f"{VCFC}chrom": records, + f"{VCFC}pos": records, + f"{VCFC}ref": records, + f"{VCFC}recordIndex": records, + f"{VCFC}hasCall": records, + # ID, ALT, QUAL, FILTER and INFO moved out of the mapping and into the + # wrapper's emitter, because each may be the VCF missing token and RML + # cannot type an object per row. They are still one per record. + f"{VCFC}recordId": records, + f"{VCFC}alt": records, + f"{VCFC}qual": records, + f"{VCFC}filter": records, + f"{VCFC}infoRaw": records, + f"{VCFC}formatRaw": parser["recordsWithFormatColumn"], } # RMLStreamer emits nothing for an absent value, so an undeclared # meta-information line contributes no triple rather than an empty one. for field, predicate in ( - ("fileFormat", f"{VCFR}fileFormat"), - ("referenceGenome", f"{VCFR}referenceGenome"), - ("sourceSoftware", f"{VCFR}sourceSoftware"), + ("fileFormat", f"{VCFC}fileFormat"), + ("referenceGenome", f"{VCFC}referenceGenome"), + ("sourceSoftware", f"{VCFC}sourceSoftware"), ): predicates[predicate] = 0 if parser[field] == METADATA_ABSENT else 1 + # A sites-only VCF still declares a profile: vcfc:RepresentationProfileShape + # requires exactly one on every file, and a file with no genotype columns + # has no genotype data to condense. + predicates[f"{VCFC}representationProfile"] = 1 if samples: - predicates[f"{VCFR}representationProfile"] = 1 + definitions = parser["distinctFormatKeyCount"] + # The reusable sample set is file-scoped, not profile-scoped: both + # profiles emit it, and the #CHROM line links to its members. + classes[f"{VCFC}SampleSet"] = 1 + classes[f"{VCFC}VCFSample"] = samples + classes[f"{VCFC}FormatFieldDefinition"] = definitions + predicates[f"{VCFC}hasSampleSet"] = 1 + predicates[f"{VCFC}hasSample"] = samples + predicates[f"{VCFC}sampleName"] = samples + predicates[f"{VCFC}sampleIndex"] = samples + predicates[f"{VCFC}hasGenotypeColumns"] = samples + # A FORMAT key declared by a '##FORMAT' line gets its + # ID/Number/Type/Description from the header emitter, at the same IRI. + # Only a key the header never declared needs an invented definition + # here, so only those are counted. + _count_definitions(predicates, parser.get("synthesizedFormatNumbers", [])) + if representation == "expanded": - classes[f"{VCFR}SampleCall"] = records * samples - classes[f"{VCFR}FormatFieldValue"] = parser["formatValueSlots"] - predicates[f"{VCFR}hasSampleCall"] = records * samples - predicates[f"{VCFR}sampleId"] = records * samples - predicates[f"{VCFR}hasFormatValue"] = parser["formatValueSlots"] - predicates[f"{VCFR}fieldValue"] = parser["nonEmptyFormatValues"] + for class_name, count in parser.get("emittedGenotypeClasses", {}).items(): + classes[f"{VCFC}{class_name}"] = classes.get(f"{VCFC}{class_name}", 0) + count + for name, count in parser.get("emittedGenotypePredicates", {}).items(): + predicates[f"{VCFC}{name}"] = predicates.get(f"{VCFC}{name}", 0) + count + classes[f"{VCFC}SampleCall"] = records * samples + classes[f"{VCFC}FormatFieldValue"] = parser["formatValueSlots"] + predicates[f"{VCFC}hasSampleCall"] = records * samples + predicates[f"{VCFC}sampleId"] = records * samples + predicates[f"{VCFC}forSample"] = records * samples + predicates[f"{VCFC}hasFormatValue"] = parser["formatValueSlots"] + predicates[f"{VCFC}fieldValue"] = parser["nonEmptyFormatValues"] + # Every expanded FORMAT value now cites its declaration. + predicates[f"{VCFC}declaredBy"] = ( + predicates.get(f"{VCFC}declaredBy", 0) + parser["formatValueSlots"] + ) else: - definitions = parser["distinctFormatKeyCount"] - classes[f"{VCFR}SampleSet"] = 1 - classes[f"{VCFR}VCFSample"] = samples - classes[f"{VCFR}CohortCallMatrix"] = parser["recordsWithFormatKeys"] - classes[f"{VCFR}FormatValueVector"] = parser["formatKeyOccurrences"] - classes[f"{VCFR}FormatFieldDefinition"] = definitions - predicates[f"{VCFR}hasSampleSet"] = 1 - predicates[f"{VCFR}hasSample"] = samples - predicates[f"{VCFR}sampleName"] = samples - predicates[f"{VCFR}sampleIndex"] = samples - predicates[f"{VCFR}hasCallMatrix"] = parser["recordsWithFormatKeys"] - predicates[f"{VCFR}appliesToSampleSet"] = parser["recordsWithFormatKeys"] - predicates[f"{VCFR}hasFormatValueVector"] = parser["formatKeyOccurrences"] - predicates[f"{VCFR}declaredBy"] = parser["formatKeyOccurrences"] - predicates[f"{VCFR}valueEncoding"] = parser["formatKeyOccurrences"] - predicates[f"{VCFR}encodedValues"] = parser["formatKeyOccurrences"] - predicates[f"{VCFR}fieldId"] = definitions - predicates[f"{VCFR}fieldNumber"] = definitions - predicates[f"{VCFR}fieldDescription"] = definitions - - predicates[f"{VCFR}fileDate"] = 0 if parser["fileDate"] == METADATA_ABSENT else 1 + classes[f"{VCFC}CohortCallMatrix"] = parser["recordsWithFormatKeys"] + classes[f"{VCFC}FormatValueVector"] = parser["formatKeyOccurrences"] + predicates[f"{VCFC}hasCallMatrix"] = parser["recordsWithFormatKeys"] + predicates[f"{VCFC}appliesToSampleSet"] = parser["recordsWithFormatKeys"] + predicates[f"{VCFC}hasFormatValueVector"] = parser["formatKeyOccurrences"] + predicates[f"{VCFC}valueEncoding"] = parser["formatKeyOccurrences"] + predicates[f"{VCFC}encodedValues"] = parser["formatKeyOccurrences"] + predicates[f"{VCFC}declaredBy"] = ( + predicates.get(f"{VCFC}declaredBy", 0) + parser["formatKeyOccurrences"] + ) + + predicates[f"{VCFC}fileDate"] = 0 if parser["fileDate"] == METADATA_ABSENT else 1 if header_representation == "structured": - for class_name, count in parser["headerLineClassCounts"].items(): - classes[f"{VCFR}{class_name}"] = classes.get(f"{VCFR}{class_name}", 0) + count - # A FILTER/ALT line carries both its header-line subclass and its - # definition class, so it contributes two rdf:type triples. - classes[f"{VCFR}FilterDefinition"] = parser["filterDefinitionCount"] - classes[f"{VCFR}AltDefinition"] = parser["altDefinitionCount"] - predicates[f"{VCFR}filterId"] = parser["filterDefinitionCount"] - predicates[f"{VCFR}altId"] = parser["altDefinitionCount"] - predicates[f"{VCFR}contigId"] = parser["contigCount"] - predicates[f"{VCFR}contigCount"] = 1 if parser["contigCount"] else 0 - for attribute, predicate in ( - ("length", "contigLength"), ("md5", "contigMd5"), ("assembly", "contigAssembly"), - ): - predicates[f"{VCFR}{predicate}"] = parser["contigAttributeCounts"].get(attribute, 0) - predicates[f"{VCFR}fieldDescription"] = ( - predicates.get(f"{VCFR}fieldDescription", 0) - + parser["describedDefinitionCount"] - ) + # Every structured-header family is counted once, by + # emitted_header_counters, from the same '##' lines the emitter reads. + # vcfc:FilterDefinition and vcfc:AltDefinition are no longer asserted: + # vcfc:FILTERHeaderLine and vcfc:ALTHeaderLine are subclasses of them in + # the vocabulary, so RDFS supplies the type and an explicit triple would + # be redundant. + for class_name, count in parser["emittedHeaderClasses"].items(): + classes[f"{VCFC}{class_name}"] = classes.get(f"{VCFC}{class_name}", 0) + count + for name, count in parser["emittedHeaderPredicates"].items(): + predicates[f"{VCFC}{name}"] = predicates.get(f"{VCFC}{name}", 0) + count if info_representation == "structured": values = parser["infoValueCount"] definitions = parser["infoDefinitionCount"] - classes[f"{VCFR}InfoFieldValue"] = values - classes[f"{VCFR}InfoFieldDefinition"] = definitions - predicates[f"{VCFR}hasInfoValue"] = values - predicates[f"{VCFR}fieldValueBoolean"] = parser["infoFlagCount"] - predicates[f"{VCFR}fieldValueInteger"] = parser["infoTypedIntegerCount"] - predicates[f"{VCFR}fieldValueDecimal"] = parser["infoTypedDecimalCount"] - predicates[f"{VCFR}fieldType"] = definitions - # declaredBy and the field* descriptors are shared with the condensed - # FORMAT definitions, so accumulate rather than overwrite. - for predicate, count in ( - (f"{VCFR}declaredBy", values), - (f"{VCFR}fieldId", definitions), - (f"{VCFR}fieldNumber", definitions), - (f"{VCFR}fieldDescription", definitions), - ): - predicates[predicate] = predicates.get(predicate, 0) + count - predicates[f"{VCFR}fieldValue"] = ( - predicates.get(f"{VCFR}fieldValue", 0) + values - parser["infoFlagCount"] + classes[f"{VCFC}InfoFieldValue"] = values + classes[f"{VCFC}InfoFieldDefinition"] = definitions + predicates[f"{VCFC}hasInfoValue"] = values + predicates[f"{VCFC}fieldValueBoolean"] = parser["infoFlagCount"] + predicates[f"{VCFC}fieldValueInteger"] = parser["infoTypedIntegerCount"] + predicates[f"{VCFC}fieldValueDecimal"] = parser["infoTypedDecimalCount"] + # declaredBy and the field* descriptors are shared with the FORMAT + # definitions, so accumulate rather than overwrite. + predicates[f"{VCFC}declaredBy"] = predicates.get(f"{VCFC}declaredBy", 0) + values + # As above: only an INFO key with no '##INFO' declaration needs its + # definition invented, and only those contribute field* triples here. + _count_definitions(predicates, parser.get("synthesizedInfoNumbers", [])) + predicates[f"{VCFC}fieldValue"] = ( + predicates.get(f"{VCFC}fieldValue", 0) + values - parser["infoFlagCount"] ) + # The allele layer, the value items, the SV carriers and the parsed + # genotype layer travel with the structured INFO representation. + for class_name, count in parser["emittedRecordClasses"].items(): + classes[f"{VCFC}{class_name}"] = classes.get(f"{VCFC}{class_name}", 0) + count + for name, count in parser["emittedRecordPredicates"].items(): + predicates[f"{VCFC}{name}"] = predicates.get(f"{VCFC}{name}", 0) + count + classes = _nonzero(classes) predicates = _nonzero(predicates) # Every typed resource contributes exactly one rdf:type triple here. @@ -515,76 +936,13 @@ def sample_uri_ids(sample_ids: list[str]) -> list[str]: return out -#: '##' key (lower-cased) -> vocabulary subclass. Mirrors HEADER_LINE_CLASSES -#: in vcf_rdfizer.py; the unit tests assert the two stay identical. -HEADER_LINE_CLASSES = { - "fileformat": "FileFormatHeaderLine", - "filedate": "FileDateHeaderLine", - "source": "SourceHeaderLine", - "reference": "ReferenceHeaderLine", - "info": "INFOHeaderLine", - "format": "FORMATHeaderLine", - "filter": "FILTERHeaderLine", - "alt": "ALTHeaderLine", - "contig": "ContigHeaderLine", -} - - -def parse_structured_header_fields(value: str) -> dict[str, str]: - """Parse '' attributes, respecting quoting. - - Mirrors ``_parse_structured_header_fields`` in ``vcf_rdfizer.py``; the unit - tests assert the two agree. - """ - inner = value.strip() - if inner.startswith("<") and inner.endswith(">"): - inner = inner[1:-1] - tokens: list[str] = [] - token: list[str] = [] - in_quotes = escaped = False - for character in inner: - if escaped: - token.append(character) - escaped = False - elif character == "\\" and in_quotes: - token.append(character) - escaped = True - elif character == '"': - token.append(character) - in_quotes = not in_quotes - elif character == "," and not in_quotes: - tokens.append("".join(token)) - token = [] - else: - token.append(character) - tokens.append("".join(token)) - fields: dict[str, str] = {} - for item in tokens: - if "=" not in item: - continue - key, raw_value = item.split("=", 1) - parsed = raw_value.strip() - if len(parsed) >= 2 and parsed[0] == parsed[-1] == '"': - parsed = parsed[1:-1].replace('\\"', '"').replace("\\\\", "\\") - fields[key.strip()] = parsed - return fields - - -def parse_info_entries(info: str) -> list[tuple[str, str | None]]: - """Split an INFO column into ``(key, value)`` pairs; a bare key is a Flag. - - Mirrors ``parse_info_entries`` in ``vcf_rdfizer.py``. The two are asserted - to agree in ``test_validation_logic_unit.py``. - """ - if info in ("", "."): - return [] - entries: list[tuple[str, str | None]] = [] - for item in info.split(";"): - if not item: - continue - key, separator, value = item.partition("=") - entries.append((key, value if separator else None)) - return entries +# Shared with the wrapper through vcf_rdfizer_vocab, so the oracle and the +# emitter cannot disagree about what a '##' line means or how it parses. +HEADER_LINE_CLASSES = vocab.HEADER_LINE_CLASSES +STRUCTURED_HEADER_KEYS = vocab.STRUCTURED_HEADER_KEYS +parse_structured_header_fields = vocab.parse_structured_header_fields +parse_structured_header_attributes = vocab.parse_structured_header_attributes +parse_info_entries = vocab.parse_info_entries def info_declared_types(raw_header: str) -> dict[str, str]: @@ -650,42 +1008,44 @@ def parse_header_metadata(raw_header: str) -> dict[str, Any]: # A repeated declaration keeps the first, matching the awk parser. if field is not None and metadata[field] == METADATA_ABSENT: metadata[field] = value - # Header-representation counters. The class map mirrors HEADER_LINE_CLASSES - # in vcf_rdfizer.py; the two are asserted to agree in the unit tests. - header_classes: Counter[str] = Counter() - filter_definitions = alt_definitions = contigs = described = 0 - contig_attributes: Counter[str] = Counter() + # Header-representation counters, derived by the one shared implementation + # the emitter's shape is described by. + header_lines: list[tuple[str, str]] = [] + contig_ids: set[str] = set() + alt_ids: set[str] = set() + info_numbers: dict[str, str] = {} + format_numbers: dict[str, str] = {} for line in raw_header.splitlines(): if not line.startswith("##"): continue key, _, value = line[2:].partition("=") - class_name = HEADER_LINE_CLASSES.get(key.lower()) - if class_name is None: - continue - header_classes[class_name] += 1 - if key.lower() not in {"filter", "alt", "contig"}: + header_lines.append((key, value)) + lowered = key.lower() + if lowered not in {"contig", "alt", "info", "format"}: continue fields = parse_structured_header_fields(value) - if not (fields.get("ID") or "").strip(): - continue - if key.lower() == "contig": - contigs += 1 - for attribute in ("length", "md5", "assembly"): - if fields.get(attribute): - contig_attributes[attribute] += 1 + identifier = (fields.get("ID") or "").strip() + if not identifier: continue - if key.lower() == "filter": - filter_definitions += 1 + if lowered == "contig": + contig_ids.add(identifier) + elif lowered == "alt": + alt_ids.add(identifier) + elif lowered == "info": + info_numbers.setdefault(identifier, fields.get("Number") or ".") else: - alt_definitions += 1 - if fields.get("Description"): - described += 1 - metadata["headerLineClassCounts"] = dict(header_classes) - metadata["filterDefinitionCount"] = filter_definitions - metadata["altDefinitionCount"] = alt_definitions - metadata["contigCount"] = contigs - metadata["contigAttributeCounts"] = dict(contig_attributes) - metadata["describedDefinitionCount"] = described + format_numbers.setdefault(identifier, fields.get("Number") or ".") + + metadata.update(emitted_header_counters(header_lines)) + metadata["headerLines"] = header_lines + metadata["contigIds"] = sorted(contig_ids) + metadata["hasAssemblyLine"] = any( + key.lower() == "assembly" and value for key, value in header_lines + ) + metadata["altDeclarationIds"] = sorted(alt_ids) + metadata["infoKeyNumbers"] = info_numbers + metadata["formatKeyNumbers"] = format_numbers + metadata["contigCount"] = len(contig_ids) metadata["headerLineCount"] = sum(keys.values()) metadata["headerValueCount"] = sum( 1 for line in raw_header.splitlines() @@ -719,7 +1079,35 @@ def attach_census_expectations( return parser -def parse_vcf(vcf_path: Path, *, filter_oracle: str) -> dict[str, Any]: +def parse_vcf( + vcf_path: Path, *, filter_oracle: str, timing: dict[str, float] | None = None +) -> dict[str, Any]: + """Compute every query's expected value in ONE pass over the VCF. + + ``timing``, when supplied, is filled with the phase breakdown below. It is + opt-in because the sample-block measurement costs two clock reads per + record, which is negligible next to a real parse but pointless when nobody + is going to read it. + + readerOpenSeconds open the file and parse the header + scanSeconds the record loop, in total + sampleBlockSeconds the per-sample genotype/AC-AN portion INSIDE the + scan -- the only part of the pass whose cost + depends on which queries you want + assemblySeconds building the per-query expected structures after + the loop + + Why phases and not per-query totals: this is a single pass that accumulates + every query's counters together, so there is no per-query slice of it to + measure. Attributing the shared scan to individual queries is a decision + about what a one-question script would have paid, not a measurement, so it + is made explicitly in the analysis (see ORACLE_QUERY_PHASES) rather than + silently here. + + Gating the pass per query was considered and rejected: the accumulators in + the sample block feed the expected values, so a gate left on in a + correctness run would corrupt the oracle every equality claim rests on. + """ if VCF is None: raise RuntimeError( "cyvcf2 is unavailable; the validator must run inside the " @@ -731,9 +1119,16 @@ def parse_vcf(vcf_path: Path, *, filter_oracle: str) -> dict[str, Any]: if filter_oracle == "bcftools" and not shutil.which("bcftools"): raise RuntimeError("--filter-oracle=bcftools requested but bcftools is unavailable") filters = filters_with_bcftools(vcf_path) if use_bcftools else Counter() + _open_started = time.monotonic() reader = VCF(str(vcf_path), strict_gt=True) samples = list(reader.samples) header_metadata = parse_header_metadata(read_vcf_header_text(vcf_path)) + if timing is not None: + timing["readerOpenSeconds"] = time.monotonic() - _open_started + # Hoisted out of the loop: a per-record attribute lookup on `timing` would + # be measuring the measurement. + _time_samples = timing is not None + _sample_block_seconds = 0.0 density: Counter[tuple[str, int]] = Counter() shapes: Counter[str] = Counter() genotypes: Counter[tuple[str, str]] = Counter() @@ -751,9 +1146,11 @@ def parse_vcf(vcf_path: Path, *, filter_oracle: str) -> dict[str, Any]: info_definitions: set[str] = set() info_values = info_flags = info_typed_integers = info_typed_decimals = 0 source_component = rml_uri_component(vcf_path.name) + record_rows: list[list[str]] = [] records_with_format_column = records_with_format_keys = 0 format_key_occurrences = format_value_slots = non_empty_format_values = 0 distinct_format_keys: set[str] = set() + _scan_started = time.monotonic() try: for variant in reader: total_records += 1 @@ -781,12 +1178,13 @@ def parse_vcf(vcf_path: Path, *, filter_oracle: str) -> dict[str, Any]: if has_gt: gt_records += 1 - # `vcfr:formatRaw` comes from the FORMAT column itself, so it is + # `vcfc:formatRaw` comes from the FORMAT column itself, so it is # present whenever that column is non-empty - samples or not. if format_keys: records_with_format_column += 1 columns = str(variant).rstrip("\r\n").split("\t") + record_rows.append(columns) # The record digest is computed from the raw line so it matches the # lexical values the mapping puts in the graph, character for # character, with no round-trip through cyvcf2's typed accessors. @@ -872,6 +1270,7 @@ def parse_vcf(vcf_path: Path, *, filter_oracle: str) -> dict[str, Any]: condensed_format_digest[ record_digest_bucket([vector_iri, encoded]) ] += 1 + _sample_started = time.monotonic() if _time_samples else 0.0 alleles = [None] * len(samples) if samples and has_gt: raw_genotypes = list(variant.genotypes) @@ -895,26 +1294,44 @@ def parse_vcf(vcf_path: Path, *, filter_oracle: str) -> dict[str, Any]: if an: ac_an[(an, ac)] += 1 q06_eligible += 1 + if _time_samples: + _sample_block_seconds += time.monotonic() - _sample_started finally: reader.close() + if timing is not None: + timing["scanSeconds"] = time.monotonic() - _scan_started + timing["sampleBlockSeconds"] = _sample_block_seconds + _assembly_started = time.monotonic() if sum(filters.values()) != total_records: raise ValueError("FILTER oracle record count differs from the VCF record count") q05 = [ {"sampleId": sample, "genotypeClass": genotype_class, "callCount": int(count)} for (sample, genotype_class), count in sorted(genotypes.items()) ] - return { + result = { "source": str(vcf_path), "sourceSha256": sha256_file(vcf_path), "filterOracle": "bcftools" if use_bcftools else "cyvcf2-serialization", "headerLineCount": header_metadata["headerLineCount"], "fileDate": header_metadata["fileDate"], - "headerLineClassCounts": header_metadata["headerLineClassCounts"], - "filterDefinitionCount": header_metadata["filterDefinitionCount"], - "altDefinitionCount": header_metadata["altDefinitionCount"], + "emittedHeaderClasses": header_metadata["emittedHeaderClasses"], + "emittedHeaderPredicates": header_metadata["emittedHeaderPredicates"], "contigCount": header_metadata["contigCount"], - "contigAttributeCounts": header_metadata["contigAttributeCounts"], - "describedDefinitionCount": header_metadata["describedDefinitionCount"], + **emitted_record_counters( + record_rows, + samples, + version=vocab.resolve_vcf_version(header_metadata["fileFormat"])[0], + contig_ids=set(header_metadata["contigIds"]), + alt_declaration_ids=set(header_metadata["altDeclarationIds"]), + has_assembly_line=header_metadata["hasAssemblyLine"], + info_numbers=header_metadata["infoKeyNumbers"], + format_numbers=header_metadata["formatKeyNumbers"], + ), + **synthesized_definition_numbers( + record_rows, + declared_info=set(header_metadata["infoKeyNumbers"]), + declared_format=set(header_metadata["formatKeyNumbers"]), + ), "headerValueCount": header_metadata["headerValueCount"], "recordsWithFormatColumn": records_with_format_column, "recordsWithFormatKeys": records_with_format_keys, @@ -981,6 +1398,9 @@ def parse_vcf(vcf_path: Path, *, filter_oracle: str) -> dict[str, Any]: for (an, ac), count in sorted(ac_an.items()) ], } + if timing is not None: + timing["assemblySeconds"] = time.monotonic() - _assembly_started + return result # --------------------------------------------------------------------------- @@ -1031,26 +1451,116 @@ def _resolve_binary(env_var: str, *candidates: str) -> str: raise RuntimeError(f"Required binary not found in container: {candidates[0]}") +#: Substrings that mean a decode step failed even though it exited 0. hdt-cpp's +#: ``hdt2rdf`` is the reason this list exists: its raptor file serializer emits +#: ``error: :0:0: write error`` once per triple and still returns success. +STEP_ERROR_MARKERS = ("write error", "error:", "ERROR:") + + def _run_step( - command: list[str], *, label: str, log_dir: Path, env: dict[str, str] | None = None + command: list[str], + *, + label: str, + log_dir: Path, + env: dict[str, str] | None = None, + stdout_path: Path | None = None, + error_markers: tuple[str, ...] = (), ) -> None: - """Run one decode step, preserving its output for diagnosis on failure.""" + """Run one decode step, preserving its output for diagnosis on failure. + + ``stdout_path`` redirects the command's stdout to that file and keeps only + stderr in the log, which is how a tool that can only serialize correctly to + stdout is driven. + + ``error_markers`` are matched against the log after a *successful* exit. An + exit code is only as trustworthy as the tool that returns it, and a decode + step that reports success while writing garbage is worse than one that + fails: the garbage flows downstream and the run dies somewhere unrelated, + hours later. + """ log_dir.mkdir(parents=True, exist_ok=True) log_path = log_dir / f"{label}.log" - with log_path.open("wb") as log: - result = subprocess.run( - command, check=False, stdout=log, stderr=subprocess.STDOUT, env=env - ) + if stdout_path is not None: + stdout_path.parent.mkdir(parents=True, exist_ok=True) + with stdout_path.open("wb") as out, log_path.open("wb") as log: + result = subprocess.run(command, check=False, stdout=out, stderr=log, env=env) + else: + with log_path.open("wb") as log: + result = subprocess.run( + command, check=False, stdout=log, stderr=subprocess.STDOUT, env=env + ) + + tail = "" + try: + tail = log_path.read_text(encoding="utf-8", errors="replace")[-2000:] + except OSError: + pass + if result.returncode != 0: - tail = "" - try: - tail = log_path.read_text(encoding="utf-8", errors="replace")[-2000:] - except OSError: - pass raise RuntimeError( f"{label} failed with exit code {result.returncode}. Output tail: {tail}" ) + if error_markers: + try: + log_text = log_path.read_text(encoding="utf-8", errors="replace") + except OSError: + log_text = tail + hit = next((m for m in error_markers if m in log_text), None) + if hit is not None: + raise RuntimeError( + f"{label} exited 0 but reported errors on stderr " + f"(matched {hit!r}); treating the step as failed because its " + f"output cannot be trusted. Output tail: {tail}" + ) + + +def _verify_ntriples(path: Path, *, label: str) -> int: + """Reject a materialized N-Triples file that is structurally impossible. + + Cheap enough to run on any size (it reads a prefix and a suffix, never the + whole file) and it catches the failure mode that cost this project days: a + 4.4 GiB "N-Triples" file containing no newline at all, whose every line was + a truncated subject IRI. Raptor accepts that as one enormous line and + degrades to quadratic behaviour, so the run does not fail -- it crawls, at + roughly 7 KiB/s, and looks like a hang rather than a bug. + + Returns the size in bytes. Raises RuntimeError with a diagnosis otherwise. + """ + try: + size = path.stat().st_size + except OSError as error: + raise RuntimeError(f"{label} produced no readable output at {path}: {error}") from error + if size == 0: + raise RuntimeError(f"{label} produced an empty N-Triples file at {path}") + + probe = 1024 * 1024 + with path.open("rb") as handle: + head = handle.read(probe) + if b"\n" not in head: + raise RuntimeError( + f"{label} produced {size} bytes with no newline in the first " + f"{min(probe, size)}; this is not N-Triples. First 200 bytes: " + f"{head[:200]!r}" + ) + first_line = head.split(b"\n", 1)[0].strip() + if not (first_line.startswith(b"<") or first_line.startswith(b"_:")): + raise RuntimeError( + f"{label} produced a first line that does not begin a triple: {first_line[:200]!r}" + ) + if not first_line.endswith(b"."): + raise RuntimeError( + f"{label} produced a first line with no statement terminator: " + f"{first_line[:200]!r}" + ) + handle.seek(max(0, size - 4096)) + if not handle.read().endswith(b"\n"): + raise RuntimeError( + f"{label} produced a file not terminated by a newline; it is " + f"most likely truncated ({size} bytes)" + ) + return size + def materialize_ntriples( source: Path, @@ -1075,6 +1585,7 @@ def materialize_ntriples( if rdf_format == "nt.gz": with gzip.open(source, "rb") as handle, target.open("wb") as out: shutil.copyfileobj(handle, out, length=1024 * 1024) + _verify_ntriples(target, label="gzip-decompress") provenance["steps"].append({"tool": "python-gzip", "output": str(target)}) return target, provenance @@ -1082,13 +1593,31 @@ def materialize_ntriples( brotli = _resolve_binary("BROTLI_BIN", "brotli") _run_step([brotli, "-d", "-c", "-o", str(target), str(source)], label="brotli-decompress", log_dir=log_dir) + _verify_ntriples(target, label="brotli-decompress") provenance["steps"].append({"tool": brotli, "output": str(target)}) return target, provenance if rdf_format == "hdt": hdt2rdf = _resolve_binary("HDT2RDF_BIN", "hdt2rdf", "/usr/local/bin/hdt2rdf") - _run_step([hdt2rdf, str(source), str(target)], label="hdt2rdf", log_dir=log_dir) - provenance["steps"].append({"tool": hdt2rdf, "output": str(target)}) + # hdt-cpp's file-output path is broken: for every output format its + # raptor iostream serializer emits "error: :0:0: write error" per + # triple, writes a truncated subject IRI with no terminator and no + # newline, and still exits 0. Its stdout serializer takes a different + # code path and is correct, verified at 200k triples with empty + # stderr, so the dump is taken from stdout and redirected here. + _run_step( + [hdt2rdf, "-f", "ntriples", str(source), "-"], + label="hdt2rdf", + log_dir=log_dir, + stdout_path=target, + error_markers=STEP_ERROR_MARKERS, + ) + _verify_ntriples(target, label="hdt2rdf") + provenance["steps"].append({ + "tool": hdt2rdf, + "output": str(target), + "mode": "stdout-redirect", + }) return target, provenance if rdf_format in {"cottas", "cottas.gz", "cottas.br"}: @@ -1112,6 +1641,7 @@ def materialize_ntriples( label="cottas-decompress", log_dir=log_dir, ) + _verify_ntriples(target, label="cottas-decompress") provenance["steps"].append({"tool": f"{python_bin} cottas_tool.py decompress", "output": str(target)}) if cottas_input != source: @@ -1254,6 +1784,17 @@ def validate_ntriples(source: Path, results_dir: Path) -> dict[str, Any]: DEFAULT_QLEVER_MEMORY_GB = 4 DEFAULT_QLEVER_STARTUP_TIMEOUT = 900 DEFAULT_QUERY_TIMEOUT = 3600 +DEFAULT_COMUNICA_PORT = 7020 +#: Seconds to wait for comunica-sparql-file-http to bind its port. The graph is +#: not read yet at that point, so this is process startup only and stays short. +DEFAULT_COMUNICA_BIND_TIMEOUT = 120 +#: Seconds allowed for the warm-up query that proves the endpoint can read the +#: source before the suite commits to it. Budgeted separately from a normal +#: query because on a large graph even a LIMIT 1 has to start streaming it. +DEFAULT_COMUNICA_WARMUP_TIMEOUT = 3600 +#: The HDT endpoint runs alongside the N-Triples one within a single run, so +#: they must not contend for a port. +DEFAULT_HDT_ENDPOINT_PORT = 7021 QLEVER_STATUS_FILE = Path("/opt/vcf-rdfizer/qlever-status.txt") @@ -1342,63 +1883,337 @@ def _envelope( return envelope -class ComunicaEngine(QueryEngine): - """Query the N-Triples file directly with comunica-sparql-file.""" +#: Above this decoded-graph size, an engine with no persistent index spends +#: more time parsing than querying. It is not a hard limit -- host memory and +#: query shape decide the real ceiling -- but it is the point at which the +#: per-query parse starts to dominate, and where a 27-query run stops being a +#: reasonable thing to start without knowing that. +UNINDEXED_ENGINE_ADVICE_BYTES = 4 * 1024 * 1024 * 1024 - name = "comunica" - def start(self) -> None: - self.executable = shutil.which("comunica-sparql-file") +def engine_advice(engine_name: str, source: Path, query_count: int, timeout: int) -> str | None: + """Warn before a run commits to an engine that cannot finish it. + + Comunica still has no index. The endpoint removes per-query engine startup + but streams the source for every query, so a full-scan query re-reads the + whole graph each time. On a large graph that is the dominant cost and no + amount of tuning fixes it -- only an engine that indexes will. + """ + if engine_name != "comunica": + return None + try: + size = source.stat().st_size + except OSError: + return None + if size < UNINDEXED_ENGINE_ADVICE_BYTES: + return None + return ( + f"comunica has no index and streams the source for every query, so " + f"each of the {query_count} full-scan queries re-reads all " + f"{size / (1024 ** 3):.1f} GiB. Prefer --engine qlever, or validate an " + f"indexed artifact (--validation-targets hdt)." + ) + + +def run_query_process( + command: list[str], *, stdout_path: Path, stderr_path: Path, timeout: int +) -> tuple[int, str | None]: + """Run one query process, killing its whole process tree on timeout. + + ``subprocess.run(timeout=...)`` signals only the process it started. Every + query here is wrapped in ``/usr/bin/time``, which forks the real engine, so + killing the wrapper leaves the engine running -- holding the entire graph in + memory, because that is what an in-memory SPARQL engine does. A handful of + those orphans is enough to push a host into swap and make the rest of the + run appear to hang rather than fail. + + Starting a new session and signalling the group kills the wrapper and the + engine together. SIGTERM first so the engine can unmap its store, SIGKILL if + it does not go. + """ + with stdout_path.open("wb") as stdout, stderr_path.open("wb") as stderr: + process = subprocess.Popen( + command, stdout=stdout, stderr=stderr, start_new_session=True + ) + try: + return process.wait(timeout=timeout), None + except subprocess.TimeoutExpired: + _terminate_process_group(process) + return 124, f"query exceeded {timeout}s" + + +def _terminate_process_group(process: subprocess.Popen) -> None: + """Signal a process group, escalating to SIGKILL, and reap the leader.""" + for signal_number, grace in ((signal.SIGTERM, 10), (signal.SIGKILL, 5)): + try: + os.killpg(os.getpgid(process.pid), signal_number) + except (ProcessLookupError, PermissionError): + break + try: + process.wait(timeout=grace) + return + except subprocess.TimeoutExpired: + continue + try: + process.wait(timeout=5) + except subprocess.TimeoutExpired: + pass + + +class ComunicaHttpEndpointMixin: + """One long-lived Comunica endpoint, shared by every query in the suite. + + Comunica's one-shot CLIs (``comunica-sparql-file``, ``comunica-sparql-hdt``) + were spawned once per query, so each of the 27 queries paid Node startup + and engine initialisation again. The matching ``*-http`` binaries pay that + once and answer every query from the running process. + + Measured in this image on 1.5M N-Triples, per ``SELECT (COUNT(*))``: + + ========================== ======== + one-shot ``comunica-sparql-file`` 26.0s + this endpoint 10.3s + ========================== ======== + + What it does **not** do is materialize the graph. ``LIMIT 1`` answers in + 0.59s while ``COUNT(*)`` takes 10s on every repeat, which is the signature + of a source streamed per query rather than an in-memory store. Loading it + into one instead was measured and rejected: an ``N3.Store`` of the same + 1.5M triples cost 2.45 GiB and made ``COUNT`` *slower* (15.0s), which + extrapolates to roughly 145 GiB for an 88M-triple cohort graph. For a graph + that large the answer is an engine that indexes -- QLever, or the HDT + artifact -- not a bigger heap. + + So the saving here is per-query engine startup, which is real and repeated + 27 times, and the scan cost is unchanged. Per-query worker recycling + (``--freshWorker``) would give the startup cost straight back and is never + passed. The worker count is pinned to 1: workers do not share state, so + each extra one only adds another process reading the same file. + """ + + #: Endpoint binary, e.g. "comunica-sparql-file-http". + endpoint_binary = "" + #: Key in ``options`` holding this endpoint's port. + endpoint_port_option = "comunica_port" + default_endpoint_port = DEFAULT_COMUNICA_PORT + #: Guidance appended when the binary is missing from the image. + endpoint_missing_hint = "rebuild the image or select --engine qlever" + + def _init_endpoint(self, options: dict[str, Any]) -> None: + self.port = int( + options.get(self.endpoint_port_option) or self.default_endpoint_port + ) + self.bind_timeout = int( + options.get("comunica_bind_timeout") or DEFAULT_COMUNICA_BIND_TIMEOUT + ) + self.warmup_timeout = int( + options.get("comunica_warmup_timeout") + or max(DEFAULT_COMUNICA_WARMUP_TIMEOUT, self.query_timeout) + ) + self.server: subprocess.Popen | None = None + self.executable: str | None = None + self.endpoint: str | None = None + self.warmup_seconds: float | None = None + self.command: list[str] = [] + + def _endpoint_source_argument(self) -> str: + """The source argv entry, in whatever form this engine's binary needs.""" + raise NotImplementedError + + def _candidate_endpoints(self) -> list[str]: + return [ + f"http://127.0.0.1:{self.port}/sparql", + f"http://127.0.0.1:{self.port}/", + ] + + def _require_endpoint_binary(self) -> str: + """Fail before any expensive setup when the binary is missing. + + An engine that cannot possibly run should say so before an artifact is + built for it, not after. + """ + self.executable = shutil.which(self.endpoint_binary) if not self.executable: raise RuntimeError( - "comunica-sparql-file is not installed in this image; rebuild it or " - "select --engine qlever" + f"{self.endpoint_binary} is not installed in this image; " + f"{self.endpoint_missing_hint}" ) + return self.executable - def describe(self) -> dict[str, Any]: + def _start_endpoint(self) -> None: + if not self.executable: + self._require_endpoint_binary() + self.log_dir.mkdir(parents=True, exist_ok=True) + self.command = [ + self.executable, + self._endpoint_source_argument(), + "-p", str(self.port), + # One worker: workers share no state, so extra ones would only add + # more processes reading the same source. + "-w", "1", + # Comunica's own per-query ceiling, aligned with the client side so + # neither silently pre-empts the other. + "-t", str(self.query_timeout), + ] + server_log = self.log_dir / f"{self.name}-server.log" + self.server = subprocess.Popen( + self.command, + stdout=server_log.open("wb"), + stderr=subprocess.STDOUT, + start_new_session=True, + ) + self._await_bind(server_log) + self._await_warm(server_log) + + def _await_bind(self, server_log: Path) -> None: + """Wait for the port to answer at all. The graph is not read yet.""" + deadline = time.monotonic() + self.bind_timeout + last_error = "endpoint never bound" + while time.monotonic() < deadline: + self._assert_alive(server_log) + for endpoint in self._candidate_endpoints(): + try: + self._post(endpoint, "ASK { }", timeout=10) + self.endpoint = endpoint + return + except Exception as error: # noqa: BLE001 - readiness probe + last_error = str(error) + time.sleep(0.5) + raise RuntimeError( + f"comunica endpoint did not bind port {self.port} within " + f"{self.bind_timeout}s: {last_error}" + ) + + def _await_warm(self, server_log: Path) -> None: + """Prove the endpoint can actually read the source, before the suite runs. + + Charged to setup rather than to the first query, so one query is not + billed for readiness that every later query got for free. + """ + assert self.endpoint is not None + started = time.monotonic() + try: + self._post( + self.endpoint, "SELECT * WHERE { ?s ?p ?o } LIMIT 1", + timeout=self.warmup_timeout, + ) + except Exception as error: # noqa: BLE001 - surfaced as engine failure + self._assert_alive(server_log) + raise RuntimeError( + f"{self.name} could not read {self._endpoint_source_argument()} " + f"within {self.warmup_timeout}s (raise --comunica-warmup-timeout, " + f"or use --engine qlever): {error}" + ) from error + self.warmup_seconds = time.monotonic() - started + + def _assert_alive(self, server_log: Path) -> None: + if self.server is not None and self.server.poll() is not None: + tail = "" + try: + tail = server_log.read_text(encoding="utf-8", errors="replace")[-2000:] + except OSError: + pass + raise RuntimeError( + f"{self.name} endpoint exited with code {self.server.returncode}. " + f"Log tail: {tail}" + ) + + def _post(self, endpoint: str, query: str, *, timeout: int) -> bytes: + import urllib.request + + request = urllib.request.Request( + endpoint, + data=urllib.parse.urlencode({"query": query}).encode("utf-8"), + headers={ + "Accept": "application/sparql-results+json", + "Content-Type": "application/x-www-form-urlencoded", + }, + method="POST", + ) + with urllib.request.urlopen(request, timeout=timeout) as response: + return response.read() + + def endpoint_describe(self) -> dict[str, Any]: + """Endpoint-specific report fields, merged by each engine's describe.""" return { - "engine": self.name, - "setupSeconds": self.setup_seconds, "version": tool_version( - ["comunica-sparql-file", "--version"], table_label="Comunica Engine" + [self.endpoint_binary, "--version"], table_label="Comunica Engine" ), - "mode": "in-memory file query", + "mode": "one long-lived endpoint, source streamed per query", + "endpoint": self.endpoint, + "port": self.port, + # Readiness, not a graph load: the source is not materialized. + "warmupSeconds": self.warmup_seconds, + "command": list(self.command), } def execute(self, query_id: str, query_path: Path) -> dict[str, Any]: raw_path = self.raw_dir / f"{query_id}.sparql.json" stderr_path = self.raw_dir / f"{query_id}.stderr.txt" - time_path = self.raw_dir / f"{query_id}.time.txt" - command = [ - self.executable, - str(self.source), - "-f", - str(query_path), - "-t", - "application/sparql-results+json", - ] - timed = ( - ["/usr/bin/time", "-v", "-o", str(time_path), *command] - if tool_version(["/usr/bin/time", "--version"]) - else command - ) started = time.monotonic() + if self.endpoint is None: + message = f"{self.name} endpoint is not running" + stderr_path.write_text(message, encoding="utf-8") + raw_path.write_bytes(b"") + return self._envelope( + query_id, query_path, returncode=1, started=started, + raw_path=raw_path, stderr_path=stderr_path, error=message, + ) try: - with raw_path.open("wb") as stdout, stderr_path.open("wb") as stderr: - result = subprocess.run( - timed, check=False, stdout=stdout, stderr=stderr, - timeout=self.query_timeout, - ) - returncode = result.returncode - error = None - except subprocess.TimeoutExpired: - returncode = 124 - error = f"query exceeded {self.query_timeout}s" + payload = self._post( + self.endpoint, query_path.read_text(encoding="utf-8"), + timeout=self.query_timeout, + ) + except Exception as error: # noqa: BLE001 - reported as EXECUTION_FAILED + stderr_path.write_text(str(error), encoding="utf-8") + raw_path.write_bytes(b"") + return self._envelope( + query_id, query_path, returncode=1, started=started, + raw_path=raw_path, stderr_path=stderr_path, error=str(error), + ) + raw_path.write_bytes(payload) + stderr_path.write_bytes(b"") return self._envelope( - query_id, query_path, returncode=returncode, started=started, - raw_path=raw_path, stderr_path=stderr_path, time_path=time_path, error=error, + query_id, query_path, returncode=0, started=started, + raw_path=raw_path, stderr_path=stderr_path, ) + def stop(self) -> None: + if self.server is not None and self.server.poll() is None: + # The endpoint holds the whole graph in memory; a survivor would + # starve everything that runs after it, so the group is signalled + # rather than just the leader. + _terminate_process_group(self.server) + self.server = None + self.endpoint = None + + +class ComunicaEngine(ComunicaHttpEndpointMixin, QueryEngine): + """Serve the N-Triples file from one load via comunica-sparql-file-http.""" + + name = "comunica" + endpoint_binary = "comunica-sparql-file-http" + endpoint_port_option = "comunica_port" + default_endpoint_port = DEFAULT_COMUNICA_PORT + + def __init__(self, source: Path, *, raw_dir: Path, scratch: Path, options: dict[str, Any]): + super().__init__(source, raw_dir=raw_dir, scratch=scratch, options=options) + self.log_dir = raw_dir / "engine" + self._init_endpoint(options) + + def _endpoint_source_argument(self) -> str: + return str(self.source) + + def start(self) -> None: + self._start_endpoint() + + def describe(self) -> dict[str, Any]: + return { + "engine": self.name, + "setupSeconds": self.setup_seconds, + **self.endpoint_describe(), + } + class QleverEngine(QueryEngine): """Build a QLever index over the graph, then answer queries over HTTP. @@ -1665,11 +2480,30 @@ def describe(self) -> dict[str, Any]: } -class HdtEngine(NativeArtifactEngine): - """Query a .hdt artifact in place with Comunica's HDT engine.""" +class HdtEngine(ComunicaHttpEndpointMixin, NativeArtifactEngine): + """Query a .hdt artifact in place, from one load, over Comunica's endpoint. + + The one-shot ``comunica-sparql-hdt`` CLI was previously spawned per query, + so every query re-opened the artifact and re-initialised the engine. HDT is + memory-mapped rather than parsed, so the per-query cost is far smaller than + Comunica's N-Triples reload -- but it is still paid 27 times for nothing, + and it makes the measured query time include engine startup, which is + exactly what a benchmark must not do. + """ name = "hdt" artifact_format = "hdt" + endpoint_binary = "comunica-sparql-hdt-http" + endpoint_port_option = "hdt_port" + default_endpoint_port = DEFAULT_HDT_ENDPOINT_PORT + endpoint_missing_hint = ( + "so HDT cannot be queried natively. Rebuild the image, or validate the " + "HDT artifact by decoding it (--rdf file.hdt --engine comunica)" + ) + + def __init__(self, source: Path, *, raw_dir: Path, scratch: Path, options: dict[str, Any]): + super().__init__(source, raw_dir=raw_dir, scratch=scratch, options=options) + self._init_endpoint(options) def build_artifact(self, target: Path) -> None: rdf2hdt = _resolve_binary("RDF2HDT_BIN", "rdf2hdt", "/usr/local/bin/rdf2hdt") @@ -1678,41 +2512,25 @@ def build_artifact(self, target: Path) -> None: label="hdt-engine-build", log_dir=self.log_dir, ) + def _endpoint_source_argument(self) -> str: + # Comunica needs the source type declared: a bare path is treated as a + # link to dereference and fails with "could not dereference". + return f"hdt@{self.artifact}" + def start(self) -> None: - self.executable = shutil.which("comunica-sparql-hdt") - if not self.executable: - raise RuntimeError( - "comunica-sparql-hdt is not installed in this image, so HDT cannot " - "be queried natively. Rebuild the image, or validate the HDT " - "artifact by decoding it (--rdf file.hdt --engine comunica)" - ) - super().start() + # Check the binary before resolving the artifact: building an HDT for + # an engine that cannot run wastes the most expensive step in the run. + self._require_endpoint_binary() + # Then resolve (or build) the artifact; the endpoint needs its path. + NativeArtifactEngine.start(self) + self._start_endpoint() - def execute(self, query_id: str, query_path: Path) -> dict[str, Any]: - raw_path = self.raw_dir / f"{query_id}.sparql.json" - stderr_path = self.raw_dir / f"{query_id}.stderr.txt" - command = [ - self.executable, - # Comunica needs the source type declared: a bare path is treated - # as a link to dereference and fails with "could not dereference". - f"hdt@{self.artifact}", - "-f", str(query_path), - "-t", "application/sparql-results+json", - ] - started = time.monotonic() - try: - with raw_path.open("wb") as stdout, stderr_path.open("wb") as stderr: - result = subprocess.run( - command, check=False, stdout=stdout, stderr=stderr, - timeout=self.query_timeout, - ) - returncode, error = result.returncode, None - except subprocess.TimeoutExpired: - returncode, error = 124, f"query exceeded {self.query_timeout}s" - return self._envelope( - query_id, query_path, returncode=returncode, started=started, - raw_path=raw_path, stderr_path=stderr_path, error=error, - ) + def describe(self) -> dict[str, Any]: + return { + **NativeArtifactEngine.describe(self), + **self.endpoint_describe(), + "mode": "native hdt query, no decode, single load served over HTTP", + } class CottasEngine(NativeArtifactEngine): @@ -2159,12 +2977,46 @@ def evaluate_validation( "query_id", "status", "wall_seconds", + # The parser's cost for THIS query, comparable row-wise against + # `wall_seconds`. Empty unless the oracle recorded its phase breakdown. + "oracle_query_seconds", + # The parser's total for ALL queries, repeated on every row so the file + # needs no join. NOT comparable row-wise against `wall_seconds`. "oracle_wall_seconds", "engine_setup_seconds", "artifact_origin", ] +def oracle_query_seconds( + phases: dict[str, float] | None, query_ids: tuple[str, ...] +) -> dict[str, float]: + """Attribute the oracle's measured phases to individual queries. + + Not a measurement of each query in isolation -- see + :data:`ORACLE_SAMPLE_LEVEL_QUERIES` for why that does not exist -- but the + cost a one-question script would have paid, composed from phases that were + each measured directly. + """ + if not phases: + return {} + reader_open = phases.get("readerOpenSeconds") or 0.0 + scan = phases.get("scanSeconds") or 0.0 + sample_block = phases.get("sampleBlockSeconds") or 0.0 + assembly = phases.get("assemblySeconds") or 0.0 + record_level_scan = max(scan - sample_block, 0.0) + + out: dict[str, float] = {} + for query_id in query_ids: + pays_sample_block = query_id in ORACLE_SAMPLE_LEVEL_QUERIES + out[query_id] = ( + reader_open + + (scan if pays_sample_block else record_level_scan) + + assembly + ) + return out + + def build_benchmark( per_engine: dict[str, dict[str, Any]], engine_descriptions: dict[str, Any], @@ -2204,6 +3056,8 @@ def build_benchmark( "queries": queries, } + oracle_phases = oracle_seconds.get("phases") or {} + per_query_oracle = oracle_query_seconds(oracle_phases, query_ids) return { "oracle": { # The parser side of the comparison: what it costs to compute the @@ -2211,6 +3065,16 @@ def build_benchmark( "totalSeconds": oracle_seconds.get("total"), "vcfParseSeconds": oracle_seconds.get("parse"), "censusSeconds": oracle_seconds.get("census"), + # Directly measured phases of the single pass. + "phases": oracle_phases or None, + # Per-query cost a one-question script would have paid, composed + # from those phases. `sampleLevelQueries` names the ones that pay + # for the per-sample block, so the attribution is auditable rather + # than implicit. + "perQuerySeconds": per_query_oracle or None, + "sampleLevelQueries": sorted( + q for q in query_ids if q in ORACLE_SAMPLE_LEVEL_QUERIES + ) or None, }, "preparation": { "materializationSeconds": materialization_seconds, @@ -2234,6 +3098,7 @@ def write_benchmark_csv(path: Path, benchmark: dict[str, Any]) -> Path: """Write one row per engine and query, for direct analysis.""" path.parent.mkdir(parents=True, exist_ok=True) oracle_total = benchmark["oracle"]["totalSeconds"] + per_query = benchmark["oracle"].get("perQuerySeconds") or {} with path.open("w", newline="", encoding="utf-8") as handle: writer = csv.DictWriter(handle, fieldnames=BENCHMARK_CSV_HEADER) writer.writeheader() @@ -2244,6 +3109,7 @@ def write_benchmark_csv(path: Path, benchmark: dict[str, Any]) -> Path: "query_id": query_id, "status": entry["status"], "wall_seconds": entry["wallSeconds"], + "oracle_query_seconds": per_query.get(query_id), # Repeated on each row so the CSV is usable without a join. "oracle_wall_seconds": oracle_total, "engine_setup_seconds": engine["setupSeconds"], @@ -2323,7 +3189,7 @@ def build_manifest( "python": platform.python_version(), "cyvcf2": getattr(cyvcf2, "__version__", None), "bcftools": tool_version(["bcftools", "--version"]), "node": tool_version(["node", "--version"]), - "comunicaQuerySparqlFile": tool_version(["comunica-sparql-file", "--version"], table_label="Comunica Engine"), + "comunicaQuerySparqlFile": tool_version(["comunica-sparql-file-http", "--version"], table_label="Comunica Engine"), "rapper": tool_version(["rapper", "--version"]), }, "queries": {path.stem: {"path": str(path), "sha256": sha256_file(path)} for path in query_paths}, @@ -2382,14 +3248,21 @@ def run_validation(args: argparse.Namespace) -> int: detail="parsing source VCF", ) oracle_started = time.monotonic() - parser = parse_vcf(args.vcf, filter_oracle=args.filter_oracle) + oracle_phases: dict[str, float] = {} + parser = parse_vcf( + args.vcf, filter_oracle=args.filter_oracle, timing=oracle_phases + ) parse_seconds = time.monotonic() - oracle_started census_started = time.monotonic() parser = attach_census_expectations(parser, args.representation) + oracle_phases["censusSeconds"] = time.monotonic() - census_started oracle_seconds = { "parse": parse_seconds, - "census": time.monotonic() - census_started, + "census": oracle_phases["censusSeconds"], "total": time.monotonic() - oracle_started, + # Phase breakdown, so per-query oracle cost can be attributed + # from measured parts rather than guessed. + "phases": dict(oracle_phases), } parser["oracleSeconds"] = oracle_seconds write_json(results_dir / "parser.json", parser) @@ -2413,6 +3286,10 @@ def run_validation(args: argparse.Namespace) -> int: "query_timeout": args.query_timeout, "extra_index_args": list(args.qlever_index_arg), "extra_server_args": list(args.qlever_server_arg), + "comunica_port": args.comunica_port, + "hdt_port": args.hdt_port, + "comunica_bind_timeout": args.comunica_bind_timeout, + "comunica_warmup_timeout": args.comunica_warmup_timeout, } engine_options["artifact_path"] = str(args.rdf) engine_options["artifact_format"] = args.rdf_format @@ -2440,6 +3317,7 @@ def run_validation(args: argparse.Namespace) -> int: # multi-engine run usable as a benchmark. per_engine: dict[str, dict[str, Any]] = {} engine_descriptions: dict[str, Any] = {} + engine_warnings: dict[str, list[str]] = {} for engine_name in args.engines: engine_raw_dir = raw_dir / engine_name engine_raw_dir.mkdir(parents=True, exist_ok=True) @@ -2450,6 +3328,12 @@ def run_validation(args: argparse.Namespace) -> int: progress.emit("progress", completed=0, detail=f"preparing {engine_name} engine") if not quiet: print(f"[{args.dataset_id}] preparing {engine_name} engine", flush=True) + advice = engine_advice( + engine_name, decoded, len(query_ids), args.query_timeout + ) + if advice: + engine_warnings.setdefault(engine_name, []).append(advice) + eprint(f"[{args.dataset_id}] warning: {advice}") try: engine.prepare() except (RuntimeError, OSError) as error: @@ -2461,6 +3345,8 @@ def run_validation(args: argparse.Namespace) -> int: continue executions: dict[str, dict[str, Any]] = {} + abandoned: str | None = None + engine_started = time.monotonic() try: engine_descriptions[engine_name] = engine.describe() for completed, query_id in enumerate(query_ids, start=1): @@ -2481,16 +3367,66 @@ def run_validation(args: argparse.Namespace) -> int: "progress", completed=completed, query_id=query_id, detail=f"{engine_name}: completed {query_id}", ) + + # A query that fails, times out, or returns the wrong + # answer never stops the suite: its verdict is recorded + # and the next query runs. Coverage is the point -- + # "26 of 27 passed, q11 disagreed" is a result, while + # "stopped at q01" is not. + # + # --stop-after-query-timeout opts back into abandoning + # the rest, which caps the worst case at one timeout + # per engine rather than one per query. It is off by + # default now that each engine loads the graph once + # instead of once per query, so the remaining queries + # no longer each re-pay that load. + if ( + args.stop_after_query_timeout + and executions[query_id].get("exitCode") == 124 + ): + abandoned = ( + f"{query_id} exceeded the {engine.query_timeout}s " + f"per-query timeout; skipped " + f"{len(query_ids) - completed} further queries " + f"rather than spending that timeout on each. " + f"Raise --query-timeout, or use an indexed " + f"engine or artifact for a graph this size." + ) + break + + budget = args.validation_time_budget + if budget and time.monotonic() - engine_started > budget: + abandoned = ( + f"engine exceeded the {budget}s validation time " + f"budget after {completed} of {len(query_ids)} " + f"queries. Raise or clear " + f"--validation-time-budget to allow more." + ) + break finally: engine.stop() + if abandoned is not None: + if not quiet: + eprint(f"[{args.dataset_id}] {engine_name}: {abandoned}") + progress.emit( + "progress", completed=len(query_ids), + detail=f"{engine_name}: abandoned", + ) + engine_dir = results_dir / "engines" / engine_name engine_dir.mkdir(parents=True, exist_ok=True) write_json(engine_dir / "query-executions.json", executions) - if any(item["status"] != "PASS" for item in executions.values()): + if abandoned is not None or any( + item["status"] != "PASS" for item in executions.values() + ): per_engine[engine_name] = { "status": "EXECUTION_FAILED", "queryExecutions": executions, } + if abandoned is not None: + per_engine[engine_name]["abandoned"] = abandoned + per_engine[engine_name]["queriesRun"] = len(executions) + per_engine[engine_name]["queriesPlanned"] = len(query_ids) continue verdict = evaluate_validation( @@ -2506,6 +3442,10 @@ def run_validation(args: argparse.Namespace) -> int: verdict["executions"] = executions per_engine[engine_name] = verdict + for engine_name, warnings in engine_warnings.items(): + if engine_name in per_engine: + per_engine[engine_name]["warnings"] = warnings + benchmark = build_benchmark( per_engine, engine_descriptions, oracle_seconds=oracle_seconds, @@ -2635,6 +3575,40 @@ def build_arg_parser() -> argparse.ArgumentParser: f"(choices: {', '.join(SPARQL_ENGINES)}; default: comunica)" ), ) + parser.add_argument( + "--validation-time-budget", + type=int, + default=0, + help=( + "Wall-clock ceiling in seconds for one engine's whole query set " + "(default: 0, no ceiling). A backstop for queries that are slow but " + "never individually time out; the per-query timeout and " + "--stop-after-query-timeout handle the common case." + ), + ) + parser.add_argument( + "--stop-after-query-timeout", + dest="stop_after_query_timeout", + action="store_true", + default=False, + help=( + "Abandon an engine's remaining queries once one exceeds the " + "per-query timeout. Off by default: a validation suite reports " + "coverage, so a run is worth far more when every query has a " + "recorded verdict than when it stops at the first slow one. Turn " + "this on to cap the worst case at one timeout per engine instead " + "of one per query" + ), + ) + parser.add_argument( + "--continue-after-query-timeout", + dest="stop_after_query_timeout", + action="store_false", + help=( + "Run every query even after one times out. This is now the " + "default; the flag is kept so existing callers keep working" + ), + ) parser.add_argument( "--query-timeout", type=int, @@ -2653,6 +3627,40 @@ def build_arg_parser() -> argparse.ArgumentParser: default=DEFAULT_QLEVER_PORT, help=f"Container-local QLever port (default: {DEFAULT_QLEVER_PORT})", ) + parser.add_argument( + "--comunica-port", + type=int, + default=DEFAULT_COMUNICA_PORT, + help=f"Container-local Comunica endpoint port (default: {DEFAULT_COMUNICA_PORT})", + ) + parser.add_argument( + "--hdt-port", + type=int, + default=DEFAULT_HDT_ENDPOINT_PORT, + help=( + "Container-local port for the native HDT endpoint " + f"(default: {DEFAULT_HDT_ENDPOINT_PORT})" + ), + ) + parser.add_argument( + "--comunica-bind-timeout", + type=int, + default=DEFAULT_COMUNICA_BIND_TIMEOUT, + help=( + "Seconds to wait for the Comunica endpoint to bind its port, before " + f"the graph is read (default: {DEFAULT_COMUNICA_BIND_TIMEOUT})" + ), + ) + parser.add_argument( + "--comunica-warmup-timeout", + type=int, + default=DEFAULT_COMUNICA_WARMUP_TIMEOUT, + help=( + "Seconds allowed for the Comunica endpoint's warm-up query, which " + "proves it can read the source before the suite starts " + f"(default: {DEFAULT_COMUNICA_WARMUP_TIMEOUT})" + ), + ) parser.add_argument( "--qlever-startup-timeout", type=int, @@ -2688,7 +3696,7 @@ def build_arg_parser() -> argparse.ArgumentParser: action="store_true", help=( "Treat a missing-token conformance failure (a plain '.' literal not " - "typed as vcfr:Null) as a validation failure instead of a report-only " + "typed as vcfc:Null) as a validation failure instead of a report-only " "observation" ), ) @@ -2789,11 +3797,22 @@ def resolve_args(parser: argparse.ArgumentParser, argv: list[str] | None = None) ("--query-timeout", args.query_timeout), ("--qlever-memory-gb", args.qlever_memory_gb), ("--qlever-startup-timeout", args.qlever_startup_timeout), + ("--comunica-bind-timeout", args.comunica_bind_timeout), + ("--comunica-warmup-timeout", args.comunica_warmup_timeout), ): if value <= 0: parser.error(f"{name} must be a positive integer") + # 0 is meaningful here: no ceiling. + if args.validation_time_budget < 0: + parser.error("--validation-time-budget must be zero or a positive integer") if not 1 <= args.qlever_port <= 65535: parser.error("--qlever-port must be between 1 and 65535") + if not 1 <= args.comunica_port <= 65535: + parser.error("--comunica-port must be between 1 and 65535") + if not 1 <= args.hdt_port <= 65535: + parser.error("--hdt-port must be between 1 and 65535") + if len({args.comunica_port, args.qlever_port, args.hdt_port}) != 3: + parser.error("--comunica-port, --qlever-port and --hdt-port must all differ") if args.shacl_shapes is not None: args.shacl_shapes = args.shacl_shapes.resolve() if not args.shacl_shapes.is_file(): diff --git a/test/README.md b/test/README.md index c66c5d7..4b6fa46 100644 --- a/test/README.md +++ b/test/README.md @@ -4,6 +4,15 @@ This repository uses `unittest` (Python standard library) to isolate orchestrati ## What is covered +- `test/test_linking_unit.py` + - Exercises all three shipped linkers with known-answer token and interval + cases, the actual Ensembl resolver over fake HTTP, offline cache replay, + budgets/retries/host pacing, digest and assembly refusal, and atomic output. + - Covers unordered/plain/gzip RDF, existing subject identity, full/post-hoc + wiring, discovery, scaffolding and previews. No Docker or public API calls. + - These checks do not change the core semantic mutation score; plug-in + SPARQL/mutation auto-discovery remains planned. See `docs/datalinking.md`. + - `test/test_vcf_rdfizer_unit.py` - Verifies wrapper control flow for the 5-step pipeline. - Verifies image/version resolution behavior and error handling. @@ -38,8 +47,9 @@ This repository uses `unittest` (Python standard library) to isolate orchestrati - Mutation testing for the semantic validation suite: corrupts a correct graph in 42 named ways and asserts which corruptions the validator detects, producing a reproducible mutation score (currently 76/78). - - Requires `rdflib` (test-only, in the `dev` extra); the tests skip cleanly - without it. See `docs/validation-methodology.md`. + - Requires `rdflib` (now a runtime dependency for linking); the tests still + skip cleanly without it in source-only environments. See + `docs/validation-methodology.md`. - The fixture derives the VCF, the RDF graph and the parser oracle from one declarative spec, and builds its graph with the project's own emitters, so the two halves cannot drift apart. @@ -81,6 +91,50 @@ This repository uses `unittest` (Python standard library) to isolate orchestrati - Verifies that an unresolvable file falls back rather than reporting a wrong size, including the 32-bit `ISIZE` wrap and multi-member trailers. +## VCF fixtures (`test/test_vcf_files/`) + +`test/test_vcf_files/*` is gitignored with an explicit `!` exception per file, +so a new fixture is invisible to git until `.gitignore` names it. + +| File | Records | Samples | Uncompressed | Purpose | +| --- | --- | --- | --- | --- | +| `test-100.vcf` | 100 | 1 | 6 KB | Fast unit-test input; mapping smoke tests | +| `test-1k.vcf` | 1,000 | 1 | 60 KB | Small end-to-end runs | +| `test-10k.vcf` | 10,000 | 1 | 612 KB | Larger end-to-end runs still fast enough for CI | +| `test-larger.vcf.gz` | 1,155,741 | 1 | 296 MB | **Many records**, one sample: the record-scaling axis | +| `test-larger-multisample.vcf.gz` | 10,000 | 2,504 | 97 MB | **Many samples**, few records: the sample-scaling axis | + +The two `test-larger*` fixtures are deliberately complementary. Conversion cost +is driven by *record count x sample count*, and each isolates one factor: + +- `test-larger.vcf.gz` — 1.16M records x 1 sample = **1.16M sample calls** +- `test-larger-multisample.vcf.gz` — 10k records x 2,504 samples = **25.0M sample calls** + +A single-sample fixture cannot exercise the per-sample fan-out at all, which in +the expanded representation is where the triple count actually comes from. Use +both, and never generalise a scaling result from one to the other. + +### Provenance of `test-larger-multisample.vcf.gz` + +Derived from the 1000 Genomes Project phase 3 chromosome 20 call set (b37, +2,504 samples, 1,812,841 records) by taking **20 contiguous blocks of 500 +records at even intervals** across the source, keeping all 2,504 sample columns +and the original header unmodified. Blocks rather than a random sample, so +neighbouring-variant structure survives; spread across the whole chromosome, so +the allele-frequency spectrum and variant mix are representative rather than +whatever happens to sit at the start of the file. + +The result spans chr20 positions 60,343-60,416,754 and contains 9,575 SNPs, 428 +indels, 5 structural variants and 58 multi-allelic sites. It is +`##fileformat=VCFv4.1` (the source's own version — `test-larger.vcf.gz` is +VCFv4.2, so the pair also covers both header versions), POS-ascending, +single-contig, and verified to parse cleanly with both `bcftools stats` and +`cyvcf2` — the two readers the validation oracle itself uses. Three `##` lines +record its derivation inside the file. + +1000 Genomes is open-consent, freely redistributable data, so the fixture +carries no access restrictions. + ## CI matrix behavior - Windows runners execute: diff --git a/test/test_linking_unit.py b/test/test_linking_unit.py new file mode 100644 index 0000000..156764a --- /dev/null +++ b/test/test_linking_unit.py @@ -0,0 +1,471 @@ +"""Known-answer linking and failure tests. No Docker or public API calls.""" + +from contextlib import redirect_stderr, redirect_stdout +from dataclasses import replace +import gzip +import hashlib +from io import BytesIO, StringIO +import json +from pathlib import Path +import shutil +import sys +import tempfile +import unittest +from unittest import mock +from urllib.error import HTTPError + +# These tests exercise data linking itself, which requires rdflib. The rest of +# the suite runs without it, so this module skips rather than erroring the whole +# discovery run -- the same pattern test_validation_mutation_unit.py uses. +try: + import rdflib + from rdflib import Graph, Literal, RDF, URIRef +except ModuleNotFoundError: # pragma: no cover - exercised only without rdflib + rdflib = None + Graph = Literal = RDF = URIRef = None + +import vcf_rdfizer +import vcf_rdfizer_link +from vcf_rdfizer_linking import Link, LinkKey +from vcf_rdfizer_linking.inputs import Record, Source, read_rdf, read_tsv, read_vcf +from vcf_rdfizer_linking.manifest import VCFL, VCFR, absolute_iri, discover, load_manifest, select +from vcf_rdfizer_linking.reference import IntervalIndex, acquire_reference, check_assembly +from vcf_rdfizer_linking.runner import LinkRunError, keys_for, run_linkers, run_stage +from vcf_rdfizer_linking.session import CachedSession, NetworkPolicy + +ROOT = Path(__file__).resolve().parents[1] +EXAMPLE = ROOT / "examples/linking/example.vcf" + + +def base_graph(path=EXAMPLE): + graph = Graph() + for row in read_vcf(path): + source = URIRef(row.source) + graph.add((source, RDF.type, VCFR.VCFFile)) + graph.add((source, VCFR.referenceGenome, Literal(row.reference))) + if isinstance(row, Source): + continue + record, call = URIRef(row.record), URIRef(row.call) + graph.add((source, VCFR.hasRecord, record)) + graph.add((record, RDF.type, VCFR.VCFRecord)) + graph.add((record, VCFR.hasCall, call)) + graph.add((call, RDF.type, VCFR.VariantCall)) + for key, value in (("chrom", row.chrom), ("pos", row.pos), ("ref", row.ref), ("alt", row.alt), ("recordId", row.id)): + graph.add((record, VCFR[key], Literal(value))) + graph.add((call, VCFR.infoRaw, Literal(row.info))) + return graph + + +class HTTPResponse(BytesIO): + def __init__(self, body, code=200, headers=None): + super().__init__(body if isinstance(body, bytes) else json.dumps(body).encode()) + self.code, self.headers = code, headers or {} + + +class FakeClock: + def __init__(self): + self.now, self.delays = 0, [] + + def __call__(self): + return self.now + + def sleep(self, seconds): + self.delays.append(seconds) + self.now += seconds + + +@unittest.skipIf(rdflib is None, "rdflib is required for the linking tests") +class LinkingTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + self.cache = self.root / "cache" + self.output = self.root / "example.links.nt" + self.manifests = discover() + self.token = self.manifests["rsid-dbsnp"] + self.interval = self.manifests["gene-demo"] + self.live = replace(self.manifests["rsid-ensembl"], contact_email="test@example.org") + self.clock = FakeClock() + + def policy(self, *manifests): + return NetworkPolicy(manifests or [self.live], clock=self.clock, sleep=self.clock.sleep) + + def run_examples(self, manifests=None, records=None, **options): + return run_linkers(records if records is not None else read_vcf(EXAMPLE), + manifests or [self.token, self.interval], self.output, cache_dir=self.cache, **options) + + def copy_manifest(self, manifest, transform=lambda text: text): + directory = self.root / manifest.id + shutil.copytree(manifest.directory, directory) + path = directory / "linker.ttl" + path.write_text(transform(path.read_text())) + return directory + + def session(self, transport, *, manifest=None, offline=False, policy=None): + return CachedSession(manifest or self.live, self.cache, policy or self.policy(manifest or self.live), offline=offline, transport=transport) + + def test_tier_1_and_2_known_answers_provenance_and_unique_terms(self): + report = self.run_examples(offline=True) + graph = Graph().parse(self.output, format="nt") + self.assertEqual(report["records"], 4) + self.assertEqual(report["link_triples"], 7) + self.assertEqual([m["unique_keys"] for m in report["linkers"]], [2, 4]) + self.assertEqual(set(graph.objects(URIRef("file://example.vcf#call/2"), VCFL.overlapsGene)), + {URIRef("https://example.org/genes/demo-gene-A"), URIRef("https://example.org/genes/demo-gene-B")}) + self.assertFalse(list(graph.objects(URIRef("file://example.vcf#call/3"), VCFL.overlapsGene))) + self.assertEqual(len(graph), report["triples"]) + self.assertEqual(len(self.output.read_text().splitlines()), len(graph)) + self.assertTrue(all(isinstance(s, URIRef) and isinstance(o, (URIRef, Literal)) for s, _, o in graph)) + for manifest, expected in ((self.token, 3), (self.interval, 4)): + node = URIRef("file://example.vcf#linkset/" + manifest.id) + self.assertEqual(int(graph.value(node, VCFL.linkCount)), expected) + self.assertEqual(str(graph.value(URIRef("file://example.vcf#linkset/gene-demo"), VCFL.referenceDigest)), "sha256:" + self.interval.reference.sha256) + + def test_info_tokens_missing_values_and_uri_encoding(self): + m = replace(self.token, field="INFO/RS", split_on=",", accept=".+") + record = next(r for r in read_vcf(EXAMPLE) if isinstance(r, Record)) + record = replace(record, info="RS=rs1,rs1,.,a/b?c,,é") + report = self.run_examples([m], [record]) + self.assertEqual(report["link_triples"], 3) + self.assertIn("dbsnp:a%2Fb%3Fc", self.output.read_text()) + self.assertIn("dbsnp:%C3%A9", self.output.read_text()) + self.assertEqual(keys_for(replace(record, info="DB;XX=x"), m), []) + with self.assertRaisesRegex(ValueError, "Duplicate INFO"): + keys_for(replace(record, info="RS=rs1;RS=rs2"), m) + + def test_token_regex_matches_whole_token(self): + record = next(r for r in read_vcf(EXAMPLE) if isinstance(r, Record)) + self.assertEqual(keys_for(replace(record, id="rs334;RS1;xrs2;rs3x;.;"), self.token), [LinkKey(token="rs334")]) + + def test_interval_closed_boundaries_ref_span_and_exact_chromosome(self): + path = acquire_reference(self.interval.reference, self.cache, offline=True) + index = IntervalIndex(path, self.interval.reference) + expected = {99: set(), 100: {"demo-gene-A"}, 200: {"demo-gene-A", "demo-gene-B"}, 250: {"demo-gene-B"}, 251: set()} + for pos, matches in expected.items(): + self.assertEqual(set(index.overlaps(LinkKey(chrom="1", start=pos, end=pos))), matches) + self.assertEqual(set(index.overlaps(LinkKey(chrom="1", start=99, end=100))), {"demo-gene-A"}) + self.assertEqual(list(index.overlaps(LinkKey(chrom="chr1", start=100, end=100))), []) + row = next(r for r in read_vcf(EXAMPLE) if isinstance(r, Record)) + self.assertEqual(keys_for(replace(row, pos="99", ref="AT"), self.interval), [LinkKey(chrom="1", start=99, end=100)]) + for alt in ("", "*", "N]2:20]"): + self.assertEqual(keys_for(replace(row, alt=alt), self.interval), []) + + def test_nested_gff_features_are_not_missed(self): + gff = self.root / "nested.gff3" + gff.write_text("1\tx\tgene\t1\t1000\t.\t+\t.\tID=long\n1\tx\tgene\t5\t10\t.\t+\t.\tID=short\n") + index = IntervalIndex(gff, self.interval.reference) + self.assertEqual(list(index.overlaps(LinkKey(chrom="1", start=999, end=999))), ["long"]) + + def test_assembly_missing_conflicting_and_override(self): + for reference in ("GRCh38", "file:///genomes/Homo_sapiens.GRCh38.dna.fa", "hg38"): + check_assembly(reference, "GRCh38") + check_assembly("unknown.fa", "GRCh38", "GRCh38") + for reference, override in (("GRCh37", None), ("GRCh37", "GRCh38"), ("GRCh38", "GRCh37"), (".", None)): + with self.assertRaises(ValueError): + check_assembly(reference, "GRCh38", override) + + def test_assembly_failure_precedes_any_resolution(self): + records = [replace(row, reference="GRCh37") for row in read_vcf(EXAMPLE)] + with mock.patch("vcf_rdfizer_linking.runner.load_resolver") as resolver: + with self.assertRaisesRegex(LinkRunError, "Assembly mismatch"): + self.run_examples([self.live, self.interval], records) + resolver.assert_not_called() + self.assertFalse(self.output.exists()) + + def test_digest_mismatch_and_corrupt_reference_cache_fail_closed(self): + directory = self.copy_manifest(self.interval) + (directory / "genes.gff3").write_text("tampered") + with self.assertRaisesRegex(ValueError, "digest mismatch"): + acquire_reference(load_manifest(directory).reference, self.cache, offline=True) + path = acquire_reference(self.interval.reference, self.cache, offline=True) + path.write_text("tampered cache") + with self.assertRaisesRegex(ValueError, "digest mismatch"): + acquire_reference(self.interval.reference, self.cache, offline=True) + + def test_https_bundle_stream_cache_gzip_and_offline_miss(self): + raw = gzip.compress((self.interval.directory / "genes.gff3").read_bytes()) + reference = replace(self.interval.reference, url="https://example.org/genes.gff3.gz", sha256=hashlib.sha256(raw).hexdigest()) + with self.assertRaisesRegex(ValueError, "cache miss"): + acquire_reference(reference, self.cache, offline=True) + opener = mock.Mock() + opener.open.return_value = HTTPResponse(raw) + with mock.patch("vcf_rdfizer_linking.reference.build_opener", return_value=opener): + path = acquire_reference(reference, self.cache) + self.assertEqual(path, acquire_reference(reference, self.cache, offline=True)) + self.assertEqual(opener.open.call_count, 1) + self.assertEqual(list(IntervalIndex(path, reference).overlaps(LinkKey(chrom="2", start=20, end=20))), ["demo-gene-C"]) + + def test_tier_3_real_resolver_batches_deduplicates_and_replays_offline(self): + calls = [] + def transport(request, timeout): + calls.append(request) + self.assertEqual(json.loads(request.data), {"ids": ["rs334", "rs699"]}) + return HTTPResponse({"rs334": {"name": "rs334"}, "rs699": {"name": "rs699"}}) + def factory(m, cache, policy, offline): + return CachedSession(m, cache, policy, offline=offline, transport=transport) + report = self.run_examples([self.live], session_factory=factory) + self.assertEqual(len(calls), 1) + self.assertIn("test@example.org", calls[0].get_header("User-agent")) + self.assertEqual(report["link_triples"], 3) + self.assertEqual(report["linkers"][0]["requests"], 1) + first = set(Graph().parse(self.output, format="nt").triples((None, VCFL.sameVariantAs, None))) + self.output.unlink() + report = self.run_examples([self.live], session_factory=factory, offline=True) + self.assertEqual(len(calls), 1) + self.assertEqual(report["linkers"][0]["cache_hits"], 1) + self.assertEqual(first, set(Graph().parse(self.output, format="nt").triples((None, VCFL.sameVariantAs, None)))) + + def test_live_missing_identifier_emits_nothing_and_bad_payload_fails(self): + from vcf_rdfizer_linking.runner import load_resolver + from vcf_rdfizer_linking import LinkerContext + resolver = load_resolver(self.live) + session = mock.Mock() + session.post.return_value.json.return_value = {"rs334": {"name": "rs334"}} + self.assertEqual(len(list(resolver([LinkKey(token="rs334"), LinkKey(token="rs699")], LinkerContext(session)))), 1) + for payload in ([], {"rs334": "bad"}, {"rs334": {}}): + session.post.return_value.json.return_value = payload + with self.assertRaises(ValueError): + list(resolver([LinkKey(token="rs334")], LinkerContext(session))) + + def test_offline_cache_miss_issues_no_request(self): + transport = mock.Mock() + with self.assertRaisesRegex(ValueError, "offline/cache-only"): + self.session(transport, offline=True).get(self.live.endpoint) + transport.assert_not_called() + + def test_retry_after_and_per_run_budget_include_retries(self): + transport = mock.Mock(side_effect=[HTTPResponse({}, 429, {"Retry-After": "4.5"}), HTTPResponse({}, 503), HTTPResponse({})]) + session = self.session(transport) + session.get(self.live.endpoint) + self.assertEqual(session.stats["requests"], 3) + self.assertEqual(self.clock.delays, [4.5, 2.0]) + self.assertEqual(session.stats["final_service_status"], 200) + manifest = replace(self.live, max_requests=1) + session = self.session(mock.Mock(return_value=HTTPResponse({}, 503)), manifest=manifest) + with self.assertRaisesRegex(ValueError, "maxRequestsPerRun=1 exhausted"): + session.get(self.live.endpoint + "/different") + self.assertEqual(session.stats["requests"], 1) + + def test_retry_after_http_date_is_honoured(self): + transport = mock.Mock(side_effect=[HTTPResponse({}, 429, {"Retry-After": "Wed, 21 Oct 2099 07:28:00 GMT"}), HTTPResponse({})]) + self.session(transport).get(self.live.endpoint) + self.assertGreater(self.clock.delays[0], 1_000_000) + + def test_httperror_response_is_retried(self): + error = HTTPError(self.live.endpoint, 429, "slow", {"Retry-After": "3"}, BytesIO(b"{}")) + session = self.session(mock.Mock(side_effect=[error, HTTPResponse({})])) + session.get(self.live.endpoint) + self.assertEqual(session.stats["requests"], 2) + self.assertEqual(self.clock.delays, [3.0]) + + def test_host_rate_shared_across_linkers_and_budget_across_sessions(self): + other = replace(self.live, id="second", requests_per_second=0.5) + policy = self.policy(self.live, other) + self.session(mock.Mock(return_value=HTTPResponse({})), policy=policy).get(self.live.endpoint) + self.session(mock.Mock(return_value=HTTPResponse({})), manifest=other, policy=policy).get(other.endpoint) + self.assertEqual(self.clock.delays, [2.0]) + limited = replace(self.live, max_requests=1) + with self.assertRaisesRegex(ValueError, "exhausted"): + self.session(mock.Mock(), manifest=limited, policy=policy).get(self.live.endpoint + "/other") + + def test_cache_separates_version_method_and_request_body_and_checks_digest(self): + transport = mock.Mock(side_effect=lambda *a, **k: HTTPResponse({})) + for manifest, method, body in ((self.live, "GET", None), (self.live, "POST", {"ids": ["rs1"]}), + (self.live, "POST", {"ids": ["rs2"]}), (replace(self.live, version="2"), "GET", None)): + session = self.session(transport, manifest=manifest) + session.get(manifest.endpoint) if method == "GET" else session.post(manifest.endpoint, json=body) + self.assertEqual(transport.call_count, 4) + files = list(self.cache.rglob("*.json")) + self.assertEqual(len(files), 4) + for file in files: + cached = json.loads(file.read_text()) + cached["sha256"] = "0" * 64 + file.write_text(json.dumps(cached)) + with self.assertRaisesRegex(ValueError, "Corrupt response cache"): + self.session(transport, offline=True).get(self.live.endpoint) + + def test_undeclared_hosts_paths_redirects_and_placeholder_contact(self): + transport = mock.Mock() + for url in ("https://other.org/variation/homo_sapiens", "https://rest.ensembl.org/other", "http://rest.ensembl.org/variation/homo_sapiens"): + with self.assertRaisesRegex(ValueError, "outside declared"): + self.session(transport).get(url) + transport.assert_not_called() + with self.assertRaisesRegex(ValueError, "contact"): + self.session(transport, manifest=self.manifests["rsid-ensembl"]).get(self.live.endpoint) + with self.assertRaisesRegex(ValueError, "HTTP 302"): + self.session(mock.Mock(return_value=HTTPResponse({}, 302))).get(self.live.endpoint) + + def test_resolver_invalid_key_and_iri_leave_no_partial_output(self): + for bad in (Link(LinkKey(token="rs999"), "https://example.org/ok"), Link(LinkKey(token="rs334"), 'https://example.org/> ')): + with mock.patch("vcf_rdfizer_linking.runner.load_resolver", return_value=lambda batch, ctx: [bad]): + with self.assertRaises(LinkRunError): + self.run_examples([self.token, self.live]) + self.assertFalse(self.output.exists()) + + def test_resolver_exception_records_failure_and_never_changes_base_graph(self): + rdf = self.root / "base.nt" + original = base_graph().serialize(format="nt").encode() + rdf.write_bytes(original) + with mock.patch("vcf_rdfizer_linking.runner.load_resolver", return_value=mock.Mock(side_effect=ValueError("broken resolver"))): + with self.assertRaisesRegex(LinkRunError, "broken resolver"): + run_stage(rdf, [self.token, self.live], self.output, metrics_dir=self.root / "metrics", cache_dir=self.cache) + self.assertEqual(rdf.read_bytes(), original) + self.assertFalse(self.output.exists()) + report = json.loads(next((self.root / "metrics/stages/linking").glob("*.json")).read_text()) + self.assertEqual(report["status"], "failed") + self.assertEqual(report["linkers"][1]["status"], "failed") + self.assertIn("linking", json.loads((self.root / "metrics/run.json").read_text())) + + def test_unordered_plain_and_gz_rdf_preserve_custom_subjects(self): + graph = base_graph() + old = URIRef("file://example.vcf#call/1") + custom = URIRef("https://example.org/custom-call") + renamed = Graph() + for s, p, o in graph: + renamed.add((custom if s == old else s, p, custom if o == old else o)) + graph = renamed + nt = "\n".join(reversed(graph.serialize(format="nt").splitlines())) + "\n" + for suffix in (".nt", ".nt.gz"): + with self.subTest(suffix=suffix): + path = self.root / ("base" + suffix) + path.write_bytes(gzip.compress(nt.encode()) if suffix.endswith("gz") else nt.encode()) + report = self.run_examples(records=read_rdf(path), offline=True) + linked = Graph().parse(self.output, format="nt") + self.assertEqual(report["link_triples"], 7) + self.assertTrue(list(linked.objects(custom, VCFL.sameVariantAs))) + self.assertTrue(all(s in set(graph.subjects()) for s, p, o in linked if p in {VCFL.sameVariantAs, VCFL.overlapsGene})) + self.output.unlink() + + def test_rdf_missing_call_subject_and_ambiguous_fields_fail(self): + for mutation in ("call", "id"): + graph = base_graph() + if mutation == "call": + graph.remove((URIRef("file://example.vcf#call/1"), None, None)) + else: + graph.add((URIRef("file://example.vcf#record/1"), VCFR.recordId, Literal("rs999"))) + path = self.root / "bad.nt" + path.write_text(graph.serialize(format="nt")) + with self.assertRaises(ValueError): + list(read_rdf(path)) + + def test_empty_vcf_and_rdf_emit_zero_count_provenance(self): + empty = self.root / "empty.vcf" + empty.write_text("##fileformat=VCFv4.3\n##reference=GRCh38\n#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\n") + rdf = self.root / "empty.nt" + rdf.write_text(base_graph(empty).serialize(format="nt")) + for records in (read_vcf(empty), read_rdf(rdf)): + report = self.run_examples(records=records, offline=True) + self.assertEqual(report["link_triples"], 0) + self.assertEqual(len(list(Graph().parse(self.output, format="nt").subjects(RDF.type, VCFL.Linkset))), 2) + self.output.unlink() + + def test_preview_uses_no_network_no_resolver_import_and_no_cache_writes(self): + for manifest in (self.token, self.interval, self.live): + with mock.patch("vcf_rdfizer_linking.runner.load_resolver", side_effect=AssertionError("must not load")): + report = run_linkers(read_vcf(EXAMPLE, limit=2), [manifest], None, cache_dir=self.cache, dry_run=True) + self.assertEqual(report["records"], 2) + self.assertEqual(report["linkers"][0]["batches"], 1) + self.assertEqual(report["linkers"][0]["status"], "planned") + self.assertFalse(self.cache.exists()) + + def test_output_collision_preserves_existing_file(self): + self.output.write_text("keep me") + with self.assertRaisesRegex(ValueError, "overwrite"): + self.run_examples() + self.assertEqual(self.output.read_text(), "keep me") + + def test_publication_failure_removes_temporary_graph(self): + with mock.patch("vcf_rdfizer_linking.runner.os.link", side_effect=OSError("cannot publish")): + with self.assertRaisesRegex(LinkRunError, "cannot publish"): + self.run_examples([self.token]) + self.assertEqual(list(self.root.glob("*.nt")), []) + + def test_manifest_errors_are_early_and_actionable(self): + changes = [('"rsid-dbsnp"', '"../bad"'), ('"1.0.0"', '"../version"'), ("TokenJoin", "AlleleJoin"), + ('"ID"', '"FORMAT/GT"'), ("{TOKEN}", "{BAD}"), ("sameVariantAs", "bad predicate"), + ("splitOn", "spltiOn"), ('"^rs[0-9]+$"', '"["')] + for before, after in changes: + with self.subTest(change=before): + directory = self.copy_manifest(self.token, lambda t: t.replace(before, after)) + with self.assertRaises(ValueError): + load_manifest(directory) + shutil.rmtree(directory) + for before, after in (("batchSize 100", "batchSize 0"), ("maxRequestsPerRun 100", "maxRequestsPerRun -1"), + ("maxRequestsPerSecond 2", 'maxRequestsPerSecond "nan"'), ("https://rest", "http://rest")): + directory = self.copy_manifest(self.live, lambda t: t.replace(before, after)) + with self.assertRaises(ValueError): + load_manifest(directory) + shutil.rmtree(directory) + + def test_discovery_path_environment_entry_point_and_duplicates(self): + directory = self.copy_manifest(self.token, lambda t: t.replace('"rsid-dbsnp"', '"custom"')) + self.assertIn("custom", discover([directory])) + with mock.patch.dict("os.environ", {"VCF_RDFIZER_LINKER_PATH": str(directory)}): + self.assertEqual(select("custom,custom")[0].id, "custom") + entry = mock.Mock() + entry.load.return_value = lambda: directory + with mock.patch("vcf_rdfizer_linking.manifest.importlib.metadata.entry_points", return_value=[entry]): + self.assertIn("custom", discover()) + (directory / "linker.ttl").write_text((directory / "linker.ttl").read_text().replace('"custom"', '"rsid-dbsnp"')) + with self.assertRaisesRegex(ValueError, "Ambiguous"): + discover([directory]) + + def test_cli_init_check_and_dry_run(self): + directory = self.root / "my-linker" + with redirect_stdout(StringIO()), redirect_stderr(StringIO()): + self.assertEqual(vcf_rdfizer_link.main(["init", "--example", "gene-demo", "-o", str(directory)]), 0) + self.assertEqual(vcf_rdfizer_link.main(["check", str(directory), "--offline", "--links-cache", str(self.cache)]), 0) + self.assertEqual(vcf_rdfizer_link.main(["dry-run", str(directory), "-i", str(EXAMPLE), "--limit", "2", "--links-cache", str(self.cache)]), 0) + self.assertEqual(vcf_rdfizer_link.main(["init", "-o", str(directory)]), 1) + + def test_main_link_mode_has_no_docker_dependency_and_writes_metrics(self): + rdf = self.root / "base.nt" + rdf.write_text(base_graph().serialize(format="nt")) + args = ["vcf-rdfizer", "--mode", "link", "--rdf", str(rdf), "--link", "rsid-dbsnp,gene-demo", "--offline", + "--links-cache", str(self.cache), "-o", str(self.root / "results")] + with mock.patch.object(sys, "argv", args), mock.patch.object(vcf_rdfizer, "check_docker", side_effect=AssertionError("Docker must not run")), redirect_stdout(StringIO()): + self.assertEqual(vcf_rdfizer.main(), 0) + self.assertTrue((self.root / "results/base.links.nt").is_file()) + run = json.loads(next((self.root / "results/run_metrics").rglob("run.json")).read_text()) + self.assertEqual(run["mode"], "link") + self.assertEqual(next(iter(run["linking"].values()))["link_triples"], 7) + + def test_full_mode_links_plain_and_gzip_aggregates_without_changing_them(self): + from test.test_vcf_rdfizer_unit import invoke_main, mocked_triplets + input_dir = self.root / "inputs" + input_dir.mkdir() + vcf = input_dir / "sample.vcf" + shutil.copyfile(EXAMPLE, vcf) + rules = self.root / "rules.ttl" + rules.write_text("@prefix ex: .\n") + original = base_graph(vcf).serialize(format="nt").encode() + for storage in ("plain", "space-optimized"): + out = self.root / storage + def fake_run(cmd, **kwargs): + if "/opt/vcf-rdfizer/run_conversion.sh" in cmd: + directory = out / "sample" + directory.mkdir(parents=True, exist_ok=True) + if storage == "plain": + (directory / "sample.nt").write_bytes(original) + else: + (directory / "sample.nt.gz").write_bytes(gzip.compress(original)) + return 0 + with mock.patch.object(vcf_rdfizer, "run", side_effect=fake_run), \ + mock.patch.object(vcf_rdfizer, "check_docker", return_value=True), \ + mock.patch.object(vcf_rdfizer, "docker_image_exists", return_value=True), \ + mock.patch.object(vcf_rdfizer, "discover_tsv_triplets", return_value=mocked_triplets()), \ + mock.patch.object(vcf_rdfizer, "emit_record_detail", return_value=None), \ + redirect_stdout(StringIO()), redirect_stderr(StringIO()): + code = invoke_main(["--input", str(input_dir), "--rules", str(rules), "--rdf-storage-mode", storage, + "--header-representation", "basic", "--compression", "none", "--out", str(out), + "--keep-tsv", "--link", "rsid-dbsnp,gene-demo", "--offline", "--links-cache", str(self.cache)]) + self.assertEqual(code, 0) + linked = Graph().parse(out / "sample/sample.links.nt", format="nt") + self.assertEqual(len(list(linked.triples((None, VCFL.overlapsGene, None)))), 4) + actual = (out / "sample/sample.nt").read_bytes() if storage == "plain" else gzip.decompress((out / "sample/sample.nt.gz").read_bytes()) + self.assertEqual(actual, original) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/test_partitioned_compression_helpers_unit.py b/test/test_partitioned_compression_helpers_unit.py new file mode 100644 index 0000000..7ed6a04 --- /dev/null +++ b/test/test_partitioned_compression_helpers_unit.py @@ -0,0 +1,434 @@ +"""The partitioned compressor's planning, accounting, and reporting helpers. + +``src/partitioned_compression.py`` splits an aggregate into record-safe chunks, +converts each one, and merges the results pairwise. The existing +``test_partitioned_compression_unit`` covers the merge command lines and the +streaming contract; this file covers the surrounding layer - chunk planning, +progress events, resource accounting, and the failure text the operator +actually sees. + +That reporting layer matters more than its size suggests: once the container +exits, its ephemeral volume is gone, so anything ``failure_message`` drops is +lost for good. Every test here runs on the host with no hdtc, pycottas, or +Docker. +""" + +from __future__ import annotations + +import gzip +import importlib.util +import json +import tempfile +import unittest +from pathlib import Path +from unittest import mock + +from test.helpers import VerboseTestCase + +MODULE_PATH = Path(__file__).resolve().parents[1] / "src" / "partitioned_compression.py" + + +def load_module(): + spec = importlib.util.spec_from_file_location("partitioned_compression_helpers", MODULE_PATH) + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + spec.loader.exec_module(module) + return module + + +P = load_module() + + +def empty_totals() -> dict: + return { + "exit_code": 0, "wall_seconds": 0.0, "user_seconds": 0.0, "sys_seconds": 0.0, + "max_rss_kb": 0, "has_user": False, "has_sys": False, "has_rss": False, + } + + +class ProgressDescriptorTests(VerboseTestCase): + def test_a_chunk_stage_carries_its_chunk_number(self): + """Per-chunk stages drive a counted progress bar, so the index is parsed.""" + self.assertEqual(P.progress_descriptor("hdt-build-7"), ("hdt-chunks", "chunks", 7)) + self.assertEqual(P.progress_descriptor("cottas-build-12"), ("cottas-chunks", "chunks", 12)) + + def test_a_malformed_chunk_number_degrades_to_the_plain_stage(self): + """An unparsable suffix must not crash the run for a progress label.""" + self.assertEqual(P.progress_descriptor("hdt-build-x"), ("hdt", "stage", None)) + + def test_merge_stages_are_reported_separately_from_builds(self): + """Merging is the long tail of a partitioned run and gets its own task.""" + self.assertEqual(P.progress_descriptor("hdt-merge-3"), ("hdt-merge", "stage", None)) + self.assertEqual(P.progress_descriptor("cottas-merge-many"), ("cottas-merge", "stage", None)) + + def test_remaining_stages_fall_back_to_their_representation(self): + """Indexing and other stages still group under the right representation.""" + self.assertEqual(P.progress_descriptor("hdt-index"), ("hdt", "stage", None)) + self.assertEqual(P.progress_descriptor("cottas-index"), ("cottas", "stage", None)) + + def test_an_unrecognised_stage_keeps_its_own_name(self): + """A new stage name is reported verbatim rather than mislabelled.""" + self.assertEqual(P.progress_descriptor("verify"), ("verify", "stage", None)) + + +class ProgressSidecarTests(VerboseTestCase): + def setUp(self): + self._tmp = tempfile.TemporaryDirectory() + self.root = Path(self._tmp.name) + self.addCleanup(self._tmp.cleanup) + + def test_the_sidecar_directory_is_created_on_demand(self): + """The host mount may not exist yet when the container starts writing.""" + path = self.root / "nested" / "progress.jsonl" + P.prepare_progress_path(path) + self.assertTrue(path.parent.is_dir()) + + def test_preparing_no_sidecar_is_a_no_op(self): + """Progress is optional, so None must be accepted everywhere.""" + self.assertIsNone(P.prepare_progress_path(None)) + + def test_an_unwritable_sidecar_location_is_swallowed(self): + """Progress must never break a conversion that is otherwise fine.""" + with mock.patch.object(P.Path, "mkdir", side_effect=OSError("read-only")): + P.prepare_progress_path(self.root / "nested" / "progress.jsonl") + + def test_each_event_is_one_json_object_per_line(self): + """The host tails this file, so every line must parse on its own.""" + path = self.root / "progress.jsonl" + P.emit_progress(path, "hdt", "chunks", completed=1, total=5, unit="chunk") + P.emit_progress(path, "hdt", "stage") + events = [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines()] + self.assertEqual(events[0], { + "stage": "hdt", "phase": "chunks", "completed": 1, "total": 5, "unit": "chunk", + }) + self.assertEqual(events[1], {"stage": "hdt", "phase": "stage"}) + + def test_absent_fields_are_omitted_rather_than_written_as_null(self): + """A null total would render as a progress bar of unknown length.""" + path = self.root / "progress.jsonl" + P.emit_progress(path, "cottas", "stage", detail="merging") + event = json.loads(path.read_text(encoding="utf-8").strip()) + self.assertEqual(set(event), {"stage", "phase", "detail"}) + + def test_emitting_without_a_sidecar_is_a_no_op(self): + """The common case is no progress path at all.""" + self.assertIsNone(P.emit_progress(None, "hdt", "stage")) + + def test_a_failing_write_never_propagates(self): + """A full or read-only mount must not abort the conversion.""" + with mock.patch.object(P.Path, "open", side_effect=OSError("no space")): + P.emit_progress(self.root / "progress.jsonl", "hdt", "stage") + + +class ChunkPlanningTests(VerboseTestCase): + RECORD = b" .\n" + + def setUp(self): + self._tmp = tempfile.TemporaryDirectory() + self.root = Path(self._tmp.name) + self.addCleanup(self._tmp.cleanup) + self.source = self.root / "graph.nt" + self.source.write_bytes(b"".join(self.RECORD % index for index in range(40))) + + def test_chunk_sizes_must_be_positive(self): + """A zero or negative bound would produce an unbounded chunk loop.""" + with self.assertRaises(ValueError) as raised: + P.stream_chunks(self.source, self.root / "c", target_bytes=0, min_bytes=1, max_bytes=2) + self.assertIn("must be positive", str(raised.exception)) + + def test_chunk_sizes_must_be_ordered(self): + """min <= target <= max is what makes a record-safe chunk reachable.""" + for target, minimum, maximum in ((10, 20, 30), (10, 5, 8)): + with self.subTest(target=target, minimum=minimum, maximum=maximum): + with self.assertRaises(ValueError) as raised: + P.stream_chunks( + self.source, self.root / "c", + target_bytes=target, min_bytes=minimum, max_bytes=maximum, + ) + self.assertIn("min <= target <= max", str(raised.exception)) + + def test_every_record_survives_the_split(self): + """Chunking is a partition: no record may be lost, split, or duplicated.""" + paths, plan = P.plan_chunks( + self.source, self.root / "chunks", + target_bytes=200, min_bytes=100, max_bytes=400, + ) + self.assertGreater(len(paths), 1) + self.assertEqual(plan["record_count"], 40) + rejoined = b"".join(path.read_bytes() for path in paths) + self.assertEqual(rejoined, self.source.read_bytes()) + + def test_the_plan_counts_what_it_wrote(self): + """The plan is the run's record of the split and must agree with the files.""" + paths, plan = P.plan_chunks( + self.source, self.root / "chunks", + target_bytes=200, min_bytes=100, max_bytes=400, + ) + self.assertEqual(plan["chunk_count"], len(paths)) + self.assertEqual(len(plan["chunks"]), len(paths)) + self.assertEqual(plan["chunk_input_bytes"], self.source.stat().st_size) + + def test_chunk_metadata_spans_the_source_without_gaps(self): + """Record and byte ranges must be contiguous across consecutive chunks.""" + _paths, plan = P.plan_chunks( + self.source, self.root / "chunks", + target_bytes=200, min_bytes=100, max_bytes=400, + ) + for previous, current in zip(plan["chunks"], plan["chunks"][1:]): + self.assertEqual(previous["end_record"], current["start_record"]) + self.assertEqual( + previous["end_uncompressed_byte"], current["start_uncompressed_byte"] + ) + self.assertEqual(plan["chunks"][0]["start_record"], 0) + self.assertEqual(plan["chunks"][-1]["end_record"], plan["record_count"]) + + def test_a_source_smaller_than_the_target_yields_one_chunk(self): + """Below the minimum, partitioning converges on the single-file case.""" + paths, plan = P.plan_chunks( + self.source, self.root / "chunks-single", + target_bytes=10_000_000, min_bytes=1_000_000, max_bytes=100_000_000, + ) + self.assertEqual(len(paths), 1) + self.assertEqual(plan["record_count"], 40) + + def test_a_gzip_source_partitions_identically_to_a_plain_one(self): + """Aggregate compression is a storage choice, not a chunking one.""" + gzipped = self.root / "graph.nt.gz" + gzipped.write_bytes(gzip.compress(self.source.read_bytes())) + plain_paths, plain_plan = P.plan_chunks( + self.source, self.root / "plain", target_bytes=200, min_bytes=100, max_bytes=400) + gz_paths, gz_plan = P.plan_chunks( + gzipped, self.root / "gz", target_bytes=200, min_bytes=100, max_bytes=400) + self.assertEqual(len(gz_paths), len(plain_paths)) + self.assertEqual(gz_plan["record_count"], plain_plan["record_count"]) + self.assertEqual( + [chunk["record_count"] for chunk in gz_plan["chunks"]], + [chunk["record_count"] for chunk in plain_plan["chunks"]], + ) + + def test_comments_and_blank_lines_are_not_counted_as_records(self): + """The record count feeds the round-trip check and must match the graph.""" + noisy = self.root / "noisy.nt" + noisy.write_bytes(b"# generated\n\n .\n\n# trailing\n") + _paths, plan = P.plan_chunks( + noisy, self.root / "noisy-chunks", + target_bytes=1000, min_bytes=100, max_bytes=10_000, + ) + self.assertEqual(plan["record_count"], 1) + + +class BinaryAndSidecarResolutionTests(VerboseTestCase): + def setUp(self): + self._tmp = tempfile.TemporaryDirectory() + self.root = Path(self._tmp.name) + self.addCleanup(self._tmp.cleanup) + + def test_an_executable_on_the_path_is_resolved_to_its_absolute_location(self): + """Stage commands are logged, so the resolved path must be concrete.""" + resolved = P.resolve_executable(("sh",), "shell") + self.assertTrue(Path(resolved).is_absolute()) + self.assertTrue(Path(resolved).is_file()) + + def test_the_first_available_candidate_wins(self): + """Candidate order encodes preference between builds of the same tool.""" + self.assertEqual( + P.resolve_executable(("definitely-absent-xyz", "sh"), "shell"), + P.resolve_executable(("sh",), "shell"), + ) + + def test_a_missing_tool_names_what_is_missing(self): + """A container build problem must be distinguishable from a data problem.""" + with self.assertRaises(RuntimeError) as raised: + P.resolve_executable(("definitely-absent-xyz",), "hdtc") + self.assertIn("Missing hdtc in container", str(raised.exception)) + + def test_no_index_sidecar_reports_none(self): + """An unindexed HDT is a state to handle, not an error to raise.""" + artifact = self.root / "graph.hdt" + artifact.write_bytes(b"hdt") + self.assertIsNone(P.find_hdt_index_sidecar(artifact)) + + def test_an_empty_index_sidecar_is_not_accepted(self): + """A zero-byte index is a failed index run, not a usable one.""" + artifact = self.root / "graph.hdt" + artifact.write_bytes(b"hdt") + (self.root / "graph.hdt.index.v1-1").write_bytes(b"") + self.assertIsNone(P.find_hdt_index_sidecar(artifact)) + + def test_a_populated_index_sidecar_is_found(self): + """The canonical versioned sidecar is what makes the HDT queryable.""" + artifact = self.root / "graph.hdt" + artifact.write_bytes(b"hdt") + (self.root / "graph.hdt.index.v1-1").write_bytes(b"") + (self.root / "graph.hdt.index.v1-2").write_bytes(b"index") + found = P.find_hdt_index_sidecar(artifact) + self.assertIsNotNone(found) + self.assertEqual(found.name, "graph.hdt.index.v1-2") + + def test_each_merge_gets_an_isolated_workspace(self): + """Concurrent merges must not share a spill directory.""" + left = P.hdtc_merge_temp_dir(self.root, self.root / "merge-a.hdt") + right = P.hdtc_merge_temp_dir(self.root, self.root / "merge-b.hdt") + self.assertNotEqual(left, right) + self.assertEqual(left.parent, self.root) + + +class TimeLogTests(VerboseTestCase): + def setUp(self): + self._tmp = tempfile.TemporaryDirectory() + self.root = Path(self._tmp.name) + self.addCleanup(self._tmp.cleanup) + + def test_a_missing_log_reports_unknown_rather_than_zero(self): + """Zero CPU seconds is a measurement; a missing log is not.""" + self.assertEqual( + P.parse_time_log(self.root / "absent.txt"), + {"user_seconds": None, "sys_seconds": None, "max_rss_kb": None}, + ) + + def test_a_complete_log_yields_every_measurement(self): + """The three fields feed the run's metrics row.""" + log = self.root / "time.txt" + log.write_text( + "\tUser time (seconds): 1.25\n" + "\tSystem time (seconds): 0.50\n" + "\tMaximum resident set size (kbytes): 20480\n", + encoding="utf-8", + ) + self.assertEqual( + P.parse_time_log(log), + {"user_seconds": 1.25, "sys_seconds": 0.5, "max_rss_kb": 20480}, + ) + + def test_an_unrecognised_log_yields_no_measurements(self): + """A truncated or foreign log must not be parsed into plausible numbers.""" + log = self.root / "time.txt" + log.write_text("nothing useful here\n", encoding="utf-8") + self.assertEqual( + P.parse_time_log(log), + {"user_seconds": None, "sys_seconds": None, "max_rss_kb": None}, + ) + + def test_a_partial_log_keeps_what_it_can(self): + """One missing field must not discard the others.""" + log = self.root / "time.txt" + log.write_text("\tUser time (seconds): 2\n", encoding="utf-8") + parsed = P.parse_time_log(log) + self.assertEqual(parsed["user_seconds"], 2.0) + self.assertIsNone(parsed["max_rss_kb"]) + + def test_the_resident_set_size_is_an_integer(self): + """The metrics column is integral kilobytes, not a float.""" + log = self.root / "time.txt" + log.write_text("\tMaximum resident set size (kbytes): 1024\n", encoding="utf-8") + self.assertIsInstance(P.parse_time_log(log)["max_rss_kb"], int) + + +class TotalsTests(VerboseTestCase): + def test_no_stages_report_unknown_rather_than_zero(self): + """A run with nothing measured must not claim zero CPU seconds.""" + self.assertEqual(P.finalize_totals(empty_totals()), { + "exit_code": 0, "wall_seconds": 0.0, + "user_seconds": None, "sys_seconds": None, "max_rss_kb": None, + }) + + def test_wall_and_cpu_time_accumulate_across_stages(self): + """Stage times sum; the run's cost is the whole pipeline's.""" + totals = empty_totals() + P.add_totals(totals, {"exit_code": 0, "wall_seconds": 1.5, "user_seconds": 0.5}) + P.add_totals(totals, {"exit_code": 0, "wall_seconds": 2.0, "user_seconds": 0.25}) + finalized = P.finalize_totals(totals) + self.assertEqual(finalized["wall_seconds"], 3.5) + self.assertEqual(finalized["user_seconds"], 0.75) + + def test_peak_memory_is_a_maximum_not_a_sum(self): + """Stages run sequentially, so the peak is the largest, not the total.""" + totals = empty_totals() + P.add_totals(totals, {"exit_code": 0, "wall_seconds": 0, "max_rss_kb": 100}) + P.add_totals(totals, {"exit_code": 0, "wall_seconds": 0, "max_rss_kb": 50}) + self.assertEqual(P.finalize_totals(totals)["max_rss_kb"], 100) + + def test_the_worst_exit_code_survives(self): + """One failed stage must fail the run even when later stages succeed.""" + totals = empty_totals() + P.add_totals(totals, {"exit_code": 2, "wall_seconds": 0}) + P.add_totals(totals, {"exit_code": 0, "wall_seconds": 0}) + self.assertEqual(P.finalize_totals(totals)["exit_code"], 2) + + def test_a_stage_missing_a_measurement_does_not_zero_the_total(self): + """An unmeasured stage leaves the others' measurements intact.""" + totals = empty_totals() + P.add_totals(totals, {"exit_code": 0, "wall_seconds": 1.0, "user_seconds": 0.5}) + P.add_totals(totals, {"exit_code": 0, "wall_seconds": 1.0}) + finalized = P.finalize_totals(totals) + self.assertEqual(finalized["user_seconds"], 0.5) + self.assertIsNone(finalized["sys_seconds"]) + + def test_absent_numbers_are_treated_as_zero_contributions(self): + """A stage result with null timings must not raise.""" + totals = empty_totals() + P.add_totals(totals, {"exit_code": None, "wall_seconds": None}) + self.assertEqual(P.finalize_totals(totals)["wall_seconds"], 0.0) + + +class FailureMessageTests(VerboseTestCase): + def test_no_stage_result_leaves_the_fallback_untouched(self): + """A failure with no captured result still reports something useful.""" + self.assertEqual(P.failure_message(None, "hdt merge failed"), "hdt merge failed") + self.assertEqual(P.failure_message({}, "hdt merge failed"), "hdt merge failed") + + def test_the_exit_code_is_always_reported(self): + """The numeric code is the one thing every failed stage has.""" + self.assertIn("exit_code=1", P.failure_message({"exit_code": 1}, "failed")) + + def test_an_oom_kill_is_named_rather_than_left_as_a_number(self): + """-9 and 137 both mean the kernel or Docker killed the process.""" + self.assertIn("SIGKILL", P.failure_message({"exit_code": -9}, "failed")) + self.assertIn("OOM", P.failure_message({"exit_code": 137}, "failed")) + + def test_a_termination_is_distinguished_from_a_kill(self): + """SIGTERM is an orderly stop and should not be reported as OOM.""" + message = P.failure_message({"exit_code": 143}, "failed") + self.assertIn("SIGTERM", message) + self.assertNotIn("OOM", message) + + def test_the_stderr_tail_is_preserved_and_whitespace_normalised(self): + """The container's volume is gone by the time this is read.""" + message = P.failure_message( + {"exit_code": 1, "stderr_tail": " DuckDB error:\n no space left "}, "failed") + self.assertIn("stderr=DuckDB error: no space left", message) + + def test_the_stderr_tail_is_bounded(self): + """Top-level CLI output must stay readable however large the traceback.""" + message = P.failure_message( + {"exit_code": 1, "stderr_tail": "x" * 5000}, "failed") + self.assertLess(len(message), 2500) + self.assertIn("x" * 100, message) + + def test_a_non_numeric_exit_code_is_reported_without_interpretation(self): + """An unexpected payload must not raise while building an error message.""" + message = P.failure_message({"exit_code": "weird"}, "failed") + self.assertIn("exit_code=weird", message) + self.assertNotIn("SIGKILL", message) + + def test_an_empty_stderr_tail_adds_no_stderr_clause(self): + """An empty tail should not render as 'stderr='.""" + self.assertNotIn("stderr=", P.failure_message( + {"exit_code": 1, "stderr_tail": " "}, "failed")) + + +class TripleLineTests(VerboseTestCase): + def test_statements_are_recognised_regardless_of_whitespace(self): + """Record counting must not depend on the writer's formatting.""" + self.assertTrue(P.is_triple_line(b" .")) + self.assertTrue(P.is_triple_line(b" . \n")) + + def test_comments_blanks_and_fragments_are_not_statements(self): + """Only terminated, non-comment lines count toward the record total.""" + for line in (b"", b" \n", b"# comment .", b" "): + with self.subTest(line=line): + self.assertFalse(P.is_triple_line(line)) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/test_validate_compression_unit.py b/test/test_validate_compression_unit.py new file mode 100644 index 0000000..8f14fac --- /dev/null +++ b/test/test_validate_compression_unit.py @@ -0,0 +1,314 @@ +"""The compression validator's triple-count round trip. + +``src/validate_compression.py`` is the cheap check that runs during conversion: +it counts N-Triples statements in the source and compares them against what the +generated HDT or COTTAS artifact decodes back to. It is weaker than the semantic +suite in ``src/validation/`` - a graph can survive this check and still be wrong - +so what matters here is that it never reports success it did not establish. + +These tests run on the host: the decoder binaries and ``pycottas`` exist only +inside the image, so every test either stubs them or asserts the behaviour when +they are absent. +""" + +from __future__ import annotations + +import argparse +import gzip +import importlib.util +import json +import sys +import tempfile +import unittest +from pathlib import Path +from unittest import mock + +from test.helpers import VerboseTestCase + +MODULE_PATH = Path(__file__).resolve().parents[1] / "src" / "validate_compression.py" + + +def load_module(): + spec = importlib.util.spec_from_file_location("validate_compression_unit", MODULE_PATH) + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + spec.loader.exec_module(module) + return module + + +M = load_module() + + +def emitter(*lines: str) -> list[str]: + """A command that prints *lines*, standing in for a decoder binary.""" + payload = "".join(f"{line}\n" for line in lines) + return [sys.executable, "-c", "import sys; sys.stdout.write(sys.argv[1])", payload] + + +class TripleLineTests(VerboseTestCase): + def test_a_statement_terminated_by_a_period_counts(self): + """The terminating '.' is what distinguishes a statement from a fragment.""" + self.assertTrue(M.is_triple_line(b" .")) + + def test_surrounding_whitespace_does_not_change_the_verdict(self): + """Decoders differ in indentation; the count must not.""" + self.assertTrue(M.is_triple_line(b" . \n")) + + def test_blank_and_comment_lines_are_not_statements(self): + """N-Triples comments and blank lines carry no triple.""" + for line in (b"", b" ", b"\n", b"# a comment .", b" # indented ."): + with self.subTest(line=line): + self.assertFalse(M.is_triple_line(line)) + + def test_an_unterminated_line_is_not_a_statement(self): + """A line without the terminator is a truncated write, not a triple.""" + self.assertFalse(M.is_triple_line(b" ")) + + +class SourceCountTests(VerboseTestCase): + def setUp(self): + self._tmp = tempfile.TemporaryDirectory() + self.root = Path(self._tmp.name) + self.addCleanup(self._tmp.cleanup) + + def test_plain_ntriples_are_counted(self): + """Only statement lines contribute to the source count.""" + path = self.root / "graph.nt" + path.write_bytes(b"# header\n .\n\n .\n") + self.assertEqual(M.count_nt(path), 2) + + def test_a_gzipped_source_counts_the_same_as_a_plain_one(self): + """.nt.gz is read transparently, so compression is not a semantic choice.""" + plain = self.root / "graph.nt" + plain.write_bytes(b" .\n .\n") + gzipped = self.root / "graph.nt.gz" + gzipped.write_bytes(gzip.compress(plain.read_bytes())) + self.assertEqual(M.count_nt(gzipped), M.count_nt(plain)) + + def test_an_empty_graph_counts_zero(self): + """An empty source is a count, not an error, at this layer.""" + path = self.root / "empty.nt" + path.write_bytes(b"") + self.assertEqual(M.count_nt(path), 0) + + +class DecodedCountTests(VerboseTestCase): + def test_streamed_statements_are_counted(self): + """The decoder's stdout is counted without materializing a file.""" + self.assertEqual(M.count_decoded(emitter(" .")), 1) + + def test_decoder_noise_is_excluded_from_the_count(self): + """Comments and blank lines in the decode must not inflate the count.""" + command = emitter("# generated", "", " .") + self.assertEqual(M.count_decoded(command), 1) + + def test_a_failing_decoder_raises_rather_than_returning_a_count(self): + """A non-zero exit means the count is unusable, not merely low.""" + command = [sys.executable, "-c", "import sys; print(' .'); sys.exit(3)"] + with self.assertRaises(RuntimeError) as raised: + M.count_decoded(command) + self.assertIn("status 3", str(raised.exception)) + + +class BinaryResolutionTests(VerboseTestCase): + def setUp(self): + self._tmp = tempfile.TemporaryDirectory() + self.root = Path(self._tmp.name) + self.addCleanup(self._tmp.cleanup) + + def make_executable(self, name: str) -> Path: + path = self.root / name + path.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8") + path.chmod(0o755) + return path + + def test_the_environment_override_wins(self): + """HDT2RDF_BIN lets the image pin a build without editing the script.""" + binary = self.make_executable("hdt2rdf") + with mock.patch.dict(M.os.environ, {"HDT2RDF_BIN": str(binary)}, clear=False): + self.assertEqual(M.resolve_hdt2rdf(), str(binary)) + + def test_a_non_executable_override_is_not_accepted(self): + """A path that cannot be run is skipped rather than reported as found.""" + path = self.root / "not-executable" + path.write_text("", encoding="utf-8") + path.chmod(0o644) + with mock.patch.dict(M.os.environ, {"HDT2RDF_BIN": str(path)}, clear=False): + with self.assertRaises(RuntimeError): + M.resolve_hdt2rdf() + + def test_an_absent_binary_is_refused_with_a_clear_message(self): + """Missing hdt2rdf is a container build problem, and says so.""" + with mock.patch.dict(M.os.environ, {"HDT2RDF_BIN": str(self.root / "absent")}, clear=False): + with mock.patch.object(M.Path, "is_file", return_value=False): + with self.assertRaises(RuntimeError) as raised: + M.resolve_hdt2rdf() + self.assertIn("Missing hdt2rdf binary", str(raised.exception)) + + +class ValidateTests(VerboseTestCase): + def setUp(self): + self._tmp = tempfile.TemporaryDirectory() + self.root = Path(self._tmp.name) + self.addCleanup(self._tmp.cleanup) + self.source = self.root / "graph.nt" + self.source.write_bytes(b" .\n .\n") + self.artifact = self.root / "graph.hdt" + self.artifact.write_bytes(b"not-really-hdt") + + def args(self, **overrides): + values = { + "source": str(self.source), "artifact": str(self.artifact), "format": "hdt", + "expected_triples": None, "source_triples": None, "skip_index_check": True, + } + values.update(overrides) + return argparse.Namespace(**values) + + def decoding_to(self, count: int): + """Stand in for hdt2rdf, which exists only inside the image.""" + return mock.patch.multiple( + M, resolve_hdt2rdf=mock.Mock(return_value="/usr/local/bin/hdt2rdf"), + count_decoded=mock.Mock(return_value=count), + ) + + def test_a_matching_round_trip_is_valid(self): + """Equal source and decoded counts is the success case.""" + with self.decoding_to(2): + result = M.validate(self.args()) + self.assertTrue(result["valid"]) + self.assertTrue(result["count_match"]) + self.assertEqual(result["validator"], "hdt2rdf") + self.assertNotIn("error", result) + + def test_a_count_mismatch_reports_both_numbers(self): + """A failure must name what was compared, not only that it failed.""" + with self.decoding_to(1): + result = M.validate(self.args()) + self.assertFalse(result["count_match"]) + self.assertIn("source=2", result["error"]) + self.assertIn("decoded=1", result["error"]) + + def test_a_precounted_source_is_trusted_instead_of_rereading(self): + """The conversion already streamed the source, so it is not read twice.""" + with self.decoding_to(7): + result = M.validate(self.args(source_triples=7)) + self.assertEqual(result["source_triples"], 7) + self.assertTrue(result["count_match"]) + + def test_a_source_disagreeing_with_the_conversion_stops_before_decoding(self): + """If the source already lost triples, decoding it proves nothing.""" + decoder = mock.Mock() + with mock.patch.object(M, "count_decoded", decoder): + result = M.validate(self.args(expected_triples=99)) + self.assertFalse(result["valid"]) + self.assertIsNone(result["decoded_triples"]) + self.assertIn("does not match the upstream conversion count", result["error"]) + decoder.assert_not_called() + + def test_an_agreeing_expected_count_proceeds_to_the_decode(self): + """The upstream count is a precondition, not a substitute for decoding.""" + with self.decoding_to(2): + result = M.validate(self.args(expected_triples=2)) + self.assertTrue(result["valid"]) + self.assertTrue(result["count_match"]) + + def test_a_missing_source_is_refused(self): + """Validation cannot proceed without the graph it is validating.""" + with self.assertRaises(FileNotFoundError): + M.validate(self.args(source=str(self.root / "absent.nt"))) + + def test_a_missing_or_empty_artifact_is_refused(self): + """A zero-byte artifact is a failed compression, not a valid empty graph.""" + empty = self.root / "empty.hdt" + empty.write_bytes(b"") + with self.assertRaises(FileNotFoundError): + M.validate(self.args(artifact=str(empty))) + with self.assertRaises(FileNotFoundError): + M.validate(self.args(artifact=str(self.root / "absent.hdt"))) + + def test_the_index_check_is_required_unless_explicitly_skipped(self): + """The helper lives in the image; its absence must fail loudly.""" + with self.assertRaises(RuntimeError) as raised: + M.validate(self.args(skip_index_check=False)) + self.assertIn("HDT index helper", str(raised.exception)) + + def test_a_failing_index_check_stops_the_validation(self): + """An unreadable index is a failure even when the decode would succeed.""" + with mock.patch.object(M.Path, "is_file", return_value=True), \ + mock.patch.object(M.subprocess, "run", return_value=mock.Mock(returncode=4)): + with self.assertRaises(RuntimeError) as raised: + M.validate(self.args(skip_index_check=False)) + self.assertIn("status 4", str(raised.exception)) + + def test_a_missing_cottas_dependency_is_reported_as_such(self): + """pycottas is container-only; its absence must not look like a bad graph.""" + with mock.patch.dict(sys.modules, {"pycottas": None}): + with self.assertRaises(RuntimeError) as raised: + M.validate(self.args(format="cottas")) + self.assertIn("COTTAS dependency is unavailable", str(raised.exception)) + + def test_the_cottas_path_names_its_own_validator(self): + """Reports record which decoder established the count.""" + with mock.patch.dict(sys.modules, {"pycottas": mock.Mock()}), \ + mock.patch.object(M, "count_decoded", return_value=2): + result = M.validate(self.args(format="cottas")) + self.assertEqual(result["validator"], "pycottas.cottas2rdf") + self.assertTrue(result["count_match"]) + + +class MainTests(VerboseTestCase): + def setUp(self): + self._tmp = tempfile.TemporaryDirectory() + self.root = Path(self._tmp.name) + self.addCleanup(self._tmp.cleanup) + self.source = self.root / "graph.nt" + self.source.write_bytes(b" .\n") + self.artifact = self.root / "graph.hdt" + self.artifact.write_bytes(b"not-really-hdt") + self.result_path = self.root / "reports" / "compression.json" + + def run_main(self, *extra): + argv = [ + "validate_compression.py", "--source", str(self.source), + "--artifact", str(self.artifact), "--format", "hdt", + "--skip-index-check", "--result-path", str(self.result_path), *extra, + ] + with mock.patch.object(sys, "argv", argv): + return M.main() + + def test_a_successful_validation_exits_zero_and_writes_its_result(self): + """The JSON sidecar is the record; the exit code is only its summary.""" + with mock.patch.object(M, "resolve_hdt2rdf", return_value="/bin/true"), \ + mock.patch.object(M, "count_decoded", return_value=1): + self.assertEqual(self.run_main(), 0) + result = json.loads(self.result_path.read_text(encoding="utf-8")) + self.assertTrue(result["valid"]) + self.assertTrue(result["count_match"]) + + def test_the_result_directory_is_created_when_absent(self): + """The caller should not have to prepare the reports tree first.""" + self.assertFalse(self.result_path.parent.exists()) + with mock.patch.object(M, "resolve_hdt2rdf", return_value="/bin/true"), \ + mock.patch.object(M, "count_decoded", return_value=1): + self.run_main() + self.assertTrue(self.result_path.is_file()) + + def test_a_mismatch_exits_non_zero(self): + """A decoded count that disagrees fails the run.""" + with mock.patch.object(M, "resolve_hdt2rdf", return_value="/bin/true"), \ + mock.patch.object(M, "count_decoded", return_value=0): + self.assertEqual(self.run_main(), 1) + self.assertFalse(json.loads(self.result_path.read_text())["count_match"]) + + def test_an_exception_is_recorded_in_the_sidecar_rather_than_escaping(self): + """A crashed decoder still leaves a machine-readable explanation behind.""" + with mock.patch.object(M, "resolve_hdt2rdf", side_effect=RuntimeError("no hdt2rdf here")): + self.assertEqual(self.run_main(), 1) + result = json.loads(self.result_path.read_text(encoding="utf-8")) + self.assertFalse(result["valid"]) + self.assertIsNone(result["source_triples"]) + self.assertEqual(result["error"], "no hdt2rdf here") + + +if __name__ == "__main__": + unittest.main() diff --git a/test/test_validation_benchmark_unit.py b/test/test_validation_benchmark_unit.py index 47fe899..5e2989a 100644 --- a/test/test_validation_benchmark_unit.py +++ b/test/test_validation_benchmark_unit.py @@ -157,27 +157,64 @@ def test_hdt_queries_are_addressed_with_comunicas_typed_source_prefix(self): Verified against comunica-sparql-hdt 5.0.1: '/tmp/g.hdt' reports "Could not dereference", while 'hdt@/tmp/g.hdt' answers the query. + The typed prefix now goes to the endpoint at startup rather than into + each query's argv, but it is no less required. """ with tempfile.TemporaryDirectory() as td: tmp_path = Path(td) raw_dir = tmp_path / "raw" raw_dir.mkdir() - query = tmp_path / "q.rq" - query.write_text("SELECT * WHERE { ?s ?p ?o }", encoding="utf-8") engine = self._engine("hdt", tmp_path) - engine.executable = "/usr/bin/comunica-sparql-hdt" engine.artifact = tmp_path / "cohort.hdt" - recorded = {} - - def fake_run(command, **kwargs): - recorded["command"] = command - return mock.Mock(returncode=0) + self.assertEqual( + engine._endpoint_source_argument(), + f"hdt@{tmp_path / 'cohort.hdt'}", + ) - with mock.patch.object(V.subprocess, "run", side_effect=fake_run): - envelope = engine.execute("q01", query) + def test_hdt_loads_the_artifact_once_for_the_whole_query_set(self): + """Every query must be answered from one load, not one process each. - self.assertEqual(recorded["command"][1], f"hdt@{tmp_path / 'cohort.hdt'}") - self.assertIn("application/sparql-results+json", recorded["command"]) + The one-shot CLI was spawned per query, so each of the 27 queries + re-opened the artifact and re-initialised the engine - and every + measured query time silently included that startup. + """ + with tempfile.TemporaryDirectory() as td: + tmp_path = Path(td) + raw_dir = tmp_path / "raw" + raw_dir.mkdir() + query = tmp_path / "q.rq" + query_text = "SELECT * WHERE { ?s ?p ?o }" + query.write_text(query_text, encoding="utf-8") + engine = self._engine("hdt", tmp_path) + engine.artifact = tmp_path / "cohort.hdt" + posted = [] + + def fake_post(self, endpoint, q, *, timeout): + posted.append(q) + return b'{"results": {"bindings": []}}' + + with mock.patch.object( + V.shutil, "which", + return_value="/usr/bin/comunica-sparql-hdt-http", + ), \ + mock.patch.object(V, "tool_version", return_value=None), \ + mock.patch.object( + V.NativeArtifactEngine, "start", lambda self: None + ), \ + mock.patch.object(V.subprocess, "Popen") as popen, \ + mock.patch.object(V.ComunicaHttpEndpointMixin, "_post", fake_post): + popen.return_value.poll.return_value = None + engine.start() + for query_id in ("q01", "q02", "q03"): + envelope = engine.execute(query_id, query) + + # One server process for the whole set. + self.assertEqual(popen.call_count, 1) + command = popen.call_args[0][0] + self.assertIn(f"hdt@{tmp_path / 'cohort.hdt'}", command) + self.assertEqual(command[command.index("-w") + 1], "1") + # Three queries answered from that single load. + self.assertEqual(len([q for q in posted if q == query_text]), 3) self.assertEqual(envelope["status"], "PASS") self.assertEqual(envelope["engine"], "hdt") @@ -296,6 +333,100 @@ def test_the_csv_carries_one_row_per_engine_and_query(self): self.assertEqual({row["oracle_wall_seconds"] for row in rows}, {"0.4"}) +class OracleQuerySecondsTests(VerboseTestCase): + """Per-query oracle cost, attributed from directly measured phases.""" + + PHASES = { + "readerOpenSeconds": 1.0, + "scanSeconds": 100.0, + "sampleBlockSeconds": 90.0, + "assemblySeconds": 2.0, + } + + def test_a_record_level_query_does_not_pay_for_the_sample_block(self): + """The per-sample block is the only phase a query can avoid.""" + per_query = V.oracle_query_seconds(self.PHASES, ("q01_record_density_1mb",)) + # 1.0 open + (100 scan - 90 sample block) + 2.0 assembly + self.assertAlmostEqual(per_query["q01_record_density_1mb"], 13.0) + + def test_a_sample_level_query_pays_for_the_whole_scan(self): + per_query = V.oracle_query_seconds(self.PHASES, ("q05_sample_genotype_counts",)) + # 1.0 open + 100 scan + 2.0 assembly + self.assertAlmostEqual(per_query["q05_sample_genotype_counts"], 103.0) + + def test_the_sample_level_set_is_the_three_genotype_queries(self): + """Q5/Q6/Q13 traverse the sample layer; nothing else does.""" + self.assertEqual( + V.ORACLE_SAMPLE_LEVEL_QUERIES, + frozenset({"q05_sample_genotype_counts", "q06_ac_an_distribution", + "q13_format_value_digest"}), + ) + + def test_no_phases_means_no_per_query_attribution(self): + """Without a measured breakdown there is nothing to attribute.""" + self.assertEqual(V.oracle_query_seconds(None, ("q01",)), {}) + self.assertEqual(V.oracle_query_seconds({}, ("q01",)), {}) + + def test_a_sample_block_longer_than_the_scan_cannot_go_negative(self): + """Clock jitter on a fast scan must not produce a negative cost.""" + per_query = V.oracle_query_seconds( + {"readerOpenSeconds": 0.0, "scanSeconds": 1.0, + "sampleBlockSeconds": 1.5, "assemblySeconds": 0.0}, + ("q01_record_density_1mb",), + ) + self.assertEqual(per_query["q01_record_density_1mb"], 0.0) + + def test_the_benchmark_carries_phases_and_per_query_seconds(self): + benchmark = V.build_benchmark( + {"qlever": _verdict("PASS", {"q01_record_density_1mb": 0.2, + "q05_sample_genotype_counts": 0.3})}, + {"qlever": {"setupSeconds": 0.1}}, + oracle_seconds={"total": 103.0, "parse": 101.0, "census": 2.0, + "phases": dict(self.PHASES)}, + materialization_seconds=None, + shacl_seconds=None, + query_ids=("q01_record_density_1mb", "q05_sample_genotype_counts"), + ) + self.assertEqual(benchmark["oracle"]["phases"], self.PHASES) + self.assertAlmostEqual( + benchmark["oracle"]["perQuerySeconds"]["q01_record_density_1mb"], 13.0) + self.assertAlmostEqual( + benchmark["oracle"]["perQuerySeconds"]["q05_sample_genotype_counts"], 103.0) + # The attribution is auditable: the payload names which queries pay. + self.assertEqual(benchmark["oracle"]["sampleLevelQueries"], + ["q05_sample_genotype_counts"]) + + def test_an_oracle_without_phases_reports_none_not_an_empty_dict(self): + """A run made before phase timing existed must not look like zero cost.""" + benchmark = V.build_benchmark( + {"qlever": _verdict("PASS", {"q01": 0.2})}, + {"qlever": {"setupSeconds": 0.1}}, + oracle_seconds={"total": 0.4, "parse": 0.3, "census": 0.1}, + materialization_seconds=None, shacl_seconds=None, query_ids=("q01",), + ) + self.assertIsNone(benchmark["oracle"]["phases"]) + self.assertIsNone(benchmark["oracle"]["perQuerySeconds"]) + + def test_the_csv_carries_a_per_query_oracle_column(self): + """`oracle_query_seconds` IS row-wise comparable; the total is not.""" + benchmark = V.build_benchmark( + {"qlever": _verdict("PASS", {"q01_record_density_1mb": 0.2, + "q05_sample_genotype_counts": 0.3})}, + {"qlever": {"setupSeconds": 0.1}}, + oracle_seconds={"total": 103.0, "parse": 101.0, "census": 2.0, + "phases": dict(self.PHASES)}, + materialization_seconds=None, shacl_seconds=None, + query_ids=("q01_record_density_1mb", "q05_sample_genotype_counts"), + ) + with tempfile.TemporaryDirectory() as td: + path = V.write_benchmark_csv(Path(td) / "benchmark.csv", benchmark) + rows = {r["query_id"]: r for r in csv.DictReader(path.open(encoding="utf-8"))} + self.assertEqual(rows["q01_record_density_1mb"]["oracle_query_seconds"], "13.0") + self.assertEqual(rows["q05_sample_genotype_counts"]["oracle_query_seconds"], "103.0") + # The whole-run total is still there, and still the same on every row. + self.assertEqual({r["oracle_wall_seconds"] for r in rows.values()}, {"103.0"}) + + class EngineAgreementTests(VerboseTestCase): def test_engines_returning_the_same_rows_agree(self): result = V.compare_engines({ diff --git a/test/test_validation_engines_unit.py b/test/test_validation_engines_unit.py index 45afa60..9a227da 100644 --- a/test/test_validation_engines_unit.py +++ b/test/test_validation_engines_unit.py @@ -11,6 +11,7 @@ import os import subprocess import tempfile +import time import unittest from pathlib import Path from unittest import mock @@ -101,9 +102,9 @@ def test_hdt_is_decoded_with_hdt2rdf(self): scratch.mkdir() calls = [] - def fake_run(command, **kwargs): + def fake_run(command, stdout=None, stderr=None, **kwargs): calls.append(command) - Path(command[2]).write_bytes(TRIPLES) + stdout.write(TRIPLES) return subprocess.CompletedProcess(command, 0) with mock.patch.object(V, "_resolve_binary", return_value="/usr/local/bin/hdt2rdf"), \ @@ -112,9 +113,65 @@ def fake_run(command, **kwargs): source, "hdt", scratch, log_dir=tmp_path / "log" ) self.assertEqual(calls[0][0], "/usr/local/bin/hdt2rdf") - self.assertEqual(calls[0][1], str(source)) + self.assertIn(str(source), calls[0]) + # hdt-cpp's file-output serializer is broken, so the dump is taken + # from stdout: the target must never appear as an argv entry. + self.assertEqual(calls[0][-1], "-") + self.assertNotIn(str(decoded), calls[0]) self.assertEqual(decoded.read_bytes(), TRIPLES) self.assertTrue(provenance["materialized"]) + self.assertEqual(provenance["steps"][0]["mode"], "stdout-redirect") + + def test_hdt2rdf_write_errors_fail_even_when_it_exits_zero(self): + """hdt-cpp reports write errors on stderr and still returns success. + + That is how a 4.4 GiB file of truncated subject IRIs with no newline in + it reached rapper, which spent 38 hours on 23% of it before anyone + noticed. An exit code is only as trustworthy as the tool returning it. + """ + with tempfile.TemporaryDirectory() as td: + tmp_path = Path(td) + source = tmp_path / "cohort.hdt" + source.write_bytes(b"fake-hdt") + scratch = tmp_path / "scratch" + scratch.mkdir() + + def fake_run(command, stdout=None, stderr=None, **kwargs): + stdout.write(b"

.\n

dict: +def parser_fixture(representation: str = "expanded") -> dict: """The canonical fixture's oracle. Shared with the graph-level mutation harness rather than hand-written here, so there is exactly one description of what the fixture VCF contains. """ - return copy.deepcopy(fixtures.parser_summary("expanded")) + return copy.deepcopy(fixtures.parser_summary(representation)) def sparql_fixture() -> dict: @@ -182,8 +182,8 @@ def test_core_query_set_is_pinned(self): def test_digest_covers_every_fixed_field(self): """ID, QUAL and INFO are only covered through the record digest.""" query = (V.QUERY_ROOT / "common" / "q11_record_digest.rq").read_text(encoding="utf-8") - for term in ("vcfr:chrom", "vcfr:pos", "vcfr:recordId", "vcfr:ref", - "vcfr:alt", "vcfr:qual", "vcfr:filter", "vcfr:infoRaw"): + for term in ("vcfc:chrom", "vcfc:pos", "vcfc:recordId", "vcfc:ref", + "vcfc:alt", "vcfc:qual", "vcfc:filter", "vcfc:infoRaw"): self.assertIn(term, query, f"{term} missing from the record digest") def test_runner_and_wrapper_agree_on_template_encoding(self): @@ -319,7 +319,61 @@ def test_census_covers_qual_and_structured_info(self): """The elements added in Phases 2, 4 and 5 are in the expected inventory.""" predicates = {row["predicate"] for row in parser_fixture()["q09_predicate_census"]} for term in ("qual", "hasInfoValue", "fileDate", "contigId", "filterId", "altId"): - self.assertIn(f"{V.VCFR}{term}", predicates, term) + self.assertIn(f"{V.VCFC}{term}", predicates, term) + + def test_census_covers_the_vcf_core_layers(self): + """Every family the VCF Core emitters added is in the inventory too. + + The census asserts an exact set, so a family missing from it would make + a correct graph report extra predicates rather than passing. + """ + summary = parser_fixture() + predicates = {row["predicate"] for row in summary["q09_predicate_census"]} + classes = {row["class"] for row in summary["q10_class_census"]} + + for term in ( + # Ordering, required by the SPARQL SHACL profile. + "lineIndex", "recordIndex", + # Header attributes and the #CHROM line. + "hasAttribute", "attributeKey", "attributeValue", "attributeIndex", + "hasColumnHeader", "hasGenotypeColumns", + # The allele layer. + "hasReferenceAllele", "hasAltAllele", "alleleIndex", "alleleValue", + "alleleKind", "chromosome", + # Indexed values. + "hasValueItem", "valueIndex", "itemValue", "forAllele", + # Reusable sample identity, emitted in both profiles. + "hasSampleSet", "hasSample", "sampleName", "sampleIndex", + # Field declarations. + "fieldArity", "fieldNumberInteger", "fieldType", + ): + self.assertIn(f"{V.VCFC}{term}", predicates, term) + + for name in ( + "ColumnHeaderLine", "HeaderAttribute", "ReferenceAllele", "AltAllele", + "FieldValueItem", "SampleSet", "VCFSample", "FormatFieldDefinition", + ): + self.assertIn(f"{V.VCFC}{name}", classes, name) + + # The mapping's version sentinel resolves to the subclass for the + # version the fixture declares. + self.assertIn(f"{V.VCFC}{fixtures.VERSION_FILE_CLASS}", classes) + + def test_expanded_census_covers_the_parsed_genotype_layer(self): + """The genotype layer is expanded-only, and the census must say so.""" + expanded = { + row["predicate"] for row in parser_fixture("expanded")["q09_predicate_census"] + } + condensed = { + row["predicate"] for row in parser_fixture("condensed")["q09_predicate_census"] + } + for term in ("hasGenotype", "genotypeString", "ploidy", "phasingStatus", + "hasAlleleCall", "callIndex", "isNoCall", "calledAllele", + "forSample"): + self.assertIn(f"{V.VCFC}{term}", expanded, term) + # The condensed profile keeps these values inside its vectors, so + # deriving them there would be exactly the growth it prevents. + self.assertNotIn(f"{V.VCFC}{term}", condensed, term) def test_digest_separator_prevents_field_boundary_collisions(self): """Shifting a field boundary must not be able to forge a matching digest.""" @@ -344,7 +398,7 @@ def test_info_entry_parsers_agree(self): ) def test_qual_is_typed_as_the_published_shape_requires(self): - """The shape demands xsd:decimal or vcfr:Null, never a plain literal.""" + """The shape demands xsd:decimal or vcfc:Null, never a plain literal.""" import vcf_rdfizer self.assertIn("XMLSchema#decimal", vcf_rdfizer._qual_object("12.5")) diff --git a/test/test_validation_oracle_unit.py b/test/test_validation_oracle_unit.py new file mode 100644 index 0000000..1b9cc70 --- /dev/null +++ b/test/test_validation_oracle_unit.py @@ -0,0 +1,652 @@ +"""The validator's oracle layer: how a VCF is summarised before SPARQL runs. + +These tests drive the pure classification and header-parsing functions that +produce the parser-side expectation every SPARQL result is compared against. A +defect here is invisible in the comparison tests, because both sides of the +comparison are derived from a fixture rather than from these functions - so the +oracle would agree with itself while disagreeing with the VCF. + +Everything here runs on the host with no cyvcf2, bcftools, Comunica, or Docker: +each function is fed the plain values those tools would have produced. That is +deliberate - the container-only dependencies are absent in CI, and a test that +skips there measures nothing. + +The ``test_known_gap_*`` cases record behaviour that is currently wrong, in the +style the mutation harness uses: the gap is an assertion, so closing it fails +here and prompts an update rather than passing unnoticed. +""" + +from __future__ import annotations + +import argparse +import contextlib +import importlib.util +import io +import json +import tempfile +import unittest +from pathlib import Path +from unittest import mock + +from test.helpers import VerboseTestCase + +RUNNER_PATH = Path(__file__).resolve().parents[1] / "src" / "validation" / "validation_runner.py" + + +def load_runner(): + spec = importlib.util.spec_from_file_location("validation_runner_oracle", RUNNER_PATH) + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + spec.loader.exec_module(module) + return module + + +V = load_runner() + + +class FakeVariant: + """The two attributes the lexical helpers read off a cyvcf2 variant.""" + + def __init__(self, alt=None, filters=None, line=""): + self.ALT = alt + self.FILTERS = filters + self._line = line + + def __str__(self) -> str: + return self._line + + +class VariantShapeTests(VerboseTestCase): + def test_every_shape_class_is_reachable(self): + """Each q02 shape bucket is produced by a representative REF/ALT pair.""" + cases = { + ("A", "."): "NO_ALT", + ("A", "G,T"): "MULTIALLELIC", + ("A", "*"): "SYMBOLIC_OR_BREAKEND", + ("A", ""): "SYMBOLIC_OR_BREAKEND", + ("A", "A[2:321682["): "SYMBOLIC_OR_BREAKEND", + ("A", "]2:321682]A"): "SYMBOLIC_OR_BREAKEND", + ("A", "G"): "SNV", + ("AT", "GC"): "MNV_OR_EQUAL_LENGTH_SUBSTITUTION", + ("A", "AT"): "INSERTION_SHAPE", + ("AT", "A"): "DELETION_SHAPE", + ("A", "R"): "OTHER", + } + for (ref, alt), expected in cases.items(): + with self.subTest(ref=ref, alt=alt): + self.assertEqual(V.classify_variant_shape(ref, alt), expected) + + def test_classification_is_case_insensitive(self): + """Lower-case bases classify identically to upper-case ones.""" + self.assertEqual(V.classify_variant_shape("a", "g"), "SNV") + self.assertEqual(V.classify_variant_shape("at", "a"), "DELETION_SHAPE") + + def test_no_alt_is_decided_before_the_base_alphabet(self): + """'.' is NO_ALT rather than OTHER, whatever REF contains.""" + self.assertEqual(V.classify_variant_shape("N", "."), "NO_ALT") + + def test_multiallelic_wins_over_the_shape_of_its_members(self): + """A comma makes the record MULTIALLELIC even when every allele is an SNV.""" + self.assertEqual(V.classify_variant_shape("A", "G,T"), "MULTIALLELIC") + + def test_a_non_iupac_base_is_other_not_a_length_comparison(self): + """An unexpected alphabet is refused before insertion/deletion sizing.""" + self.assertEqual(V.classify_variant_shape("A", "GX"), "OTHER") + self.assertEqual(V.classify_variant_shape("XY", "A"), "OTHER") + + def test_an_empty_alt_is_other(self): + """An empty ALT matches no base pattern, so it is not sized as a deletion.""" + self.assertEqual(V.classify_variant_shape("AT", ""), "OTHER") + + +class AltLexicalTests(VerboseTestCase): + def test_an_absent_alt_list_renders_as_the_missing_token(self): + """No ALT and an empty ALT both render '.', which is what the graph carries.""" + self.assertEqual(V.alt_lexical(FakeVariant(alt=None)), ".") + self.assertEqual(V.alt_lexical(FakeVariant(alt=[])), ".") + + def test_multiple_alleles_join_on_comma_in_source_order(self): + """The lexical form is the VCF column, not a normalised set.""" + self.assertEqual(V.alt_lexical(FakeVariant(alt=["G", "T"])), "G,T") + + def test_a_none_member_becomes_the_missing_token(self): + """A null allele inside the list renders '.' rather than 'None'.""" + self.assertEqual(V.alt_lexical(FakeVariant(alt=[None, "G"])), ".,G") + + +class FilterLexicalTests(VerboseTestCase): + def test_filter_status_has_three_buckets(self): + """PASS, the missing token, and everything else are distinguished.""" + self.assertEqual(V.filter_status("PASS"), "PASS") + self.assertEqual(V.filter_status("."), "NOT_APPLIED") + self.assertEqual(V.filter_status("q10"), "FAILED") + + def test_filter_status_is_case_sensitive(self): + """Only the exact token 'PASS' is a pass; VCF does not define 'pass'.""" + self.assertEqual(V.filter_status("pass"), "FAILED") + + def test_an_empty_filter_is_not_treated_as_a_pass(self): + """An empty string is a failure bucket, never silently a PASS.""" + self.assertEqual(V.filter_status(""), "FAILED") + + def test_the_filters_list_is_joined_with_semicolons(self): + """Multiple filters reproduce the VCF column's own separator.""" + self.assertEqual(V.exact_filter_lexical(FakeVariant(filters=["q10", "s50"])), "q10;s50") + + def test_an_empty_filters_list_falls_back_to_the_serialized_record(self): + """cyvcf2 reports no FILTERS for '.', so column 7 is read verbatim.""" + record = "1\t100\t.\tA\tG\t50\t.\tDP=3\tGT\t0/1\n" + self.assertEqual(V.exact_filter_lexical(FakeVariant(filters=[], line=record)), ".") + + def test_the_fallback_recovers_a_real_filter_string(self): + """The fallback reads the same value the FILTERS list would have given.""" + record = "1\t100\t.\tA\tG\t50\tq10;s50\tDP=3\n" + self.assertEqual(V.exact_filter_lexical(FakeVariant(filters=None, line=record)), "q10;s50") + + def test_a_truncated_record_is_refused_rather_than_guessed(self): + """Too few columns raises instead of silently reporting a wrong FILTER.""" + with self.assertRaises(ValueError): + V.exact_filter_lexical(FakeVariant(filters=[], line="1\t100\t.\tA\tG\n")) + + +class GenotypeClassificationTests(VerboseTestCase): + def test_the_phase_flag_is_dropped_from_the_allele_tuple(self): + """cyvcf2 appends a phase boolean, which is not an allele.""" + self.assertEqual(V.genotype_alleles([0, 1, True]), (0, 1)) + self.assertEqual(V.genotype_alleles([0, False]), (0,)) + + def test_a_negative_allele_index_becomes_none(self): + """htslib encodes a missing allele as -1; the oracle records it as unknown.""" + self.assertEqual(V.genotype_alleles([0, -1, False]), (0, None)) + self.assertEqual(V.genotype_alleles([-1, -1, True]), (None, None)) + + def test_no_genotype_array_stays_none(self): + """A record with no GT array yields no allele tuple at all.""" + self.assertIsNone(V.genotype_alleles(None)) + + def test_every_genotype_class_is_reachable(self): + """Each q05 genotype bucket is produced by a representative allele tuple.""" + cases = { + ((0,), True): "HAPLOID_REF", + ((1,), True): "HAPLOID_ALT", + ((0, 0), True): "HOM_REF", + ((2, 2), True): "HOM_ALT", + ((0, 1), True): "HET", + ((0, 1, 1), True): "OTHER_PLOIDY", + } + for (alleles, has_gt), expected in cases.items(): + with self.subTest(alleles=alleles): + self.assertEqual(V.classify_genotype(alleles, has_gt=has_gt), expected) + + def test_a_file_without_a_gt_field_is_distinguished_from_a_missing_call(self): + """NO_GT_FIELD and MISSING are separate buckets, and must not collapse.""" + self.assertEqual(V.classify_genotype((0, 1), has_gt=False), "NO_GT_FIELD") + self.assertEqual(V.classify_genotype(None, has_gt=True), "MISSING") + + def test_a_partially_missing_genotype_is_missing_not_haploid(self): + """'./1' must not be counted as a haploid call by dropping the unknown half.""" + self.assertEqual(V.classify_genotype((None, 1), has_gt=True), "MISSING") + + def test_an_empty_allele_tuple_is_missing(self): + """No alleles at all is a missing call rather than an unclassified record.""" + self.assertEqual(V.classify_genotype((), has_gt=True), "MISSING") + + +class InfoDeclaredTypeTests(VerboseTestCase): + def test_each_declared_info_key_maps_to_its_type(self): + """The INFO Type drives typed-value counting, so it is read per key.""" + header = "\n".join([ + '##INFO=', + '##INFO=', + '##INFO=', + ]) + self.assertEqual( + V.info_declared_types(header), + {"DP": "Integer", "AF": "Float", "SOMATIC": "Flag"}, + ) + + def test_a_description_containing_commas_does_not_split_the_declaration(self): + """A quoted Description is one field, so Type is still recovered.""" + header = '##INFO=' + self.assertEqual(V.info_declared_types(header), {"DP": "Integer"}) + + def test_a_declaration_without_a_type_defaults_to_string(self): + """A malformed declaration is typed rather than dropped.""" + header = '##INFO=' + self.assertEqual(V.info_declared_types(header), {"NOTYPE": "String"}) + + def test_the_first_declaration_of_a_repeated_key_wins(self): + """A duplicate INFO key keeps the first Type, matching the awk parser.""" + header = "\n".join([ + '##INFO=', + '##INFO=', + ]) + self.assertEqual(V.info_declared_types(header), {"DP": "Integer"}) + + def test_non_info_lines_are_ignored(self): + """FORMAT and the column line contribute no INFO types.""" + header = "\n".join([ + "##fileformat=VCFv4.3", + '##FORMAT=', + "#CHROM\tPOS\tID", + ]) + self.assertEqual(V.info_declared_types(header), {}) + + +class HeaderMetadataTests(VerboseTestCase): + HEADER = "\n".join([ + "##fileformat=VCFv4.3", + "##fileDate=20260101", + "##fileDate=20260202", + "##reference=file:///ref/GRCh38.fa", + "##source=myCaller", + "##assembly=GRCh38", + "##contig=", + "##contig=", + '##ALT=', + '##INFO=', + '##FORMAT=', + '##FILTER=', + "#CHROM\tPOS\tID", + ]) + + def test_file_metadata_is_lifted_from_its_declarations(self): + """The four file-level metadata fields are read off the header block.""" + metadata = V.parse_header_metadata(self.HEADER) + self.assertEqual(metadata["fileFormat"], "VCFv4.3") + self.assertEqual(metadata["referenceGenome"], "file:///ref/GRCh38.fa") + self.assertEqual(metadata["sourceSoftware"], "myCaller") + + def test_a_repeated_metadata_declaration_keeps_the_first(self): + """Two ##fileDate lines resolve to the first, matching the awk parser.""" + self.assertEqual(V.parse_header_metadata(self.HEADER)["fileDate"], "20260101") + + def test_an_undeclared_metadata_field_is_explicitly_absent(self): + """A missing field carries the absent sentinel, never an empty string.""" + metadata = V.parse_header_metadata("##contig=\n#CHROM\tPOS") + self.assertEqual(metadata["fileFormat"], V.METADATA_ABSENT) + self.assertEqual(metadata["fileDate"], V.METADATA_ABSENT) + self.assertEqual(metadata["sourceSoftware"], V.METADATA_ABSENT) + + def test_the_column_line_is_excluded_from_every_count(self): + """'#CHROM' is not a meta-information line and emits no HeaderLine.""" + metadata = V.parse_header_metadata(self.HEADER) + self.assertEqual(metadata["headerLineCount"], 12) + self.assertEqual(len(metadata["headerLines"]), 12) + self.assertNotIn( + "CHROM", {key for key, _ in metadata["headerLines"]} + ) + + def test_structured_declarations_are_indexed_by_id(self): + """Contig, ALT, INFO and FORMAT identifiers are collected for the census.""" + metadata = V.parse_header_metadata(self.HEADER) + self.assertEqual(metadata["contigIds"], ["chr1", "chr2"]) + self.assertEqual(metadata["contigCount"], 2) + self.assertEqual(metadata["altDeclarationIds"], ["DEL"]) + self.assertEqual(metadata["infoKeyNumbers"], {"DP": "1"}) + self.assertEqual(metadata["formatKeyNumbers"], {"GT": "1"}) + + def test_an_assembly_line_is_detected_only_when_it_carries_a_value(self): + """A bare '##assembly' declares nothing to link a contig to.""" + self.assertTrue(V.parse_header_metadata(self.HEADER)["hasAssemblyLine"]) + self.assertFalse(V.parse_header_metadata("##assembly=\n#CHROM")["hasAssemblyLine"]) + + def test_valueless_lines_are_counted_separately_from_all_lines(self): + """A bare flag line is a header line but carries no value.""" + metadata = V.parse_header_metadata("##fileformat=VCFv4.3\n##bareflag\n#CHROM") + self.assertEqual(metadata["headerLineCount"], 2) + self.assertEqual(metadata["headerValueCount"], 1) + + def test_the_header_census_is_grouped_by_key_and_sorted(self): + """q08 compares against a deterministic, key-sorted census.""" + census = V.parse_header_metadata(self.HEADER)["q08_header_line_census"] + self.assertEqual(census, sorted(census, key=lambda row: row["headerKey"])) + self.assertIn({"headerKey": "contig", "lineCount": 2}, census) + self.assertIn({"headerKey": "fileDate", "lineCount": 2}, census) + + def test_a_structured_declaration_without_an_id_is_skipped(self): + """An ID-less contig cannot be addressed by IRI, so it is not counted.""" + metadata = V.parse_header_metadata("##contig=\n#CHROM") + self.assertEqual(metadata["contigIds"], []) + self.assertEqual(metadata["contigCount"], 0) + + +class EmittedHeaderCounterTests(VerboseTestCase): + def test_a_sample_declared_twice_yields_one_declaration_resource(self): + """SampleDeclaration is identity-scoped, so a repeat adds no new resource.""" + counters = V.emitted_header_counters([ + ("SAMPLE", ''), + ("SAMPLE", ''), + ]) + self.assertEqual(counters["emittedHeaderClasses"]["SampleDeclaration"], 1) + self.assertEqual(counters["emittedHeaderPredicates"]["declaresSample"], 2) + + def test_a_pedigree_ancestor_declares_the_sample_it_names(self): + """An ancestor mentioned only in a PEDIGREE line still needs a resource.""" + counters = V.emitted_header_counters([("PEDIGREE", "")]) + predicates = counters["emittedHeaderPredicates"] + self.assertEqual(predicates["pedigreeFather"], 1) + self.assertEqual(predicates["pedigreeMother"], 1) + self.assertEqual(predicates["ancestorRole"], 2) + self.assertEqual(counters["emittedHeaderClasses"]["SampleDeclaration"], 2) + + def test_the_pedigree_id_is_not_counted_as_its_own_ancestor(self): + """ID names the subject of the pedigree line, not a parent of it.""" + counters = V.emitted_header_counters([("PEDIGREE", "")]) + self.assertEqual(counters["emittedHeaderPredicates"]["ancestorRole"], 1) + + def test_the_contig_count_resource_is_emitted_once_for_any_number_of_contigs(self): + """contigCount summarises the file, so it is one statement, not one per contig.""" + counters = V.emitted_header_counters([ + ("contig", ""), + ("contig", ""), + ]) + self.assertEqual(counters["emittedHeaderPredicates"]["contigId"], 2) + self.assertEqual(counters["emittedHeaderPredicates"]["contigCount"], 1) + + def test_no_contigs_means_no_contig_count_statement(self): + """An absent contig block emits no summary statement at all.""" + counters = V.emitted_header_counters([("fileformat", "VCFv4.3")]) + self.assertNotIn("contigCount", counters["emittedHeaderPredicates"]) + + def test_a_meta_values_list_splits_into_its_members(self): + """##META Values=[a, b, c] declares three allowed values.""" + self.assertEqual(V._meta_values("[WholeGenome, Exome, Panel]"), + ["WholeGenome", "Exome", "Panel"]) + + def test_meta_values_tolerates_an_absent_or_unbracketed_list(self): + """A missing list is empty, and a bare comma list still splits.""" + self.assertEqual(V._meta_values(None), []) + self.assertEqual(V._meta_values(""), []) + self.assertEqual(V._meta_values("a,b"), ["a", "b"]) + + def test_every_member_of_a_multi_value_meta_list_is_counted(self): + """A ##META ``Values=[a, b]`` list declares one allowed value per member. + + The commas inside the bracketed list separate members, not attributes. + Splitting on them used to drop every member after the first - the + fragments carry no '=', so they were discarded as non-attributes, and the + surviving value kept its opening bracket. + """ + counters = V.emitted_header_counters([ + ("META", ""), + ]) + self.assertEqual(counters["emittedHeaderPredicates"]["metaAllowedValue"], 2) + + def test_a_single_value_meta_list_is_parsed_correctly(self): + """The bracket stripping works whenever the list has no internal comma.""" + counters = V.emitted_header_counters([ + ("META", ""), + ]) + self.assertEqual(counters["emittedHeaderPredicates"]["metaAllowedValue"], 1) + + def test_an_attribute_after_the_values_list_is_not_swallowed(self): + """The list ends at its closing bracket; later attributes still parse.""" + counters = V.emitted_header_counters([ + ("META", ''), + ]) + predicates = counters["emittedHeaderPredicates"] + self.assertEqual(predicates["metaAllowedValue"], 2) + self.assertEqual(predicates["fieldDescription"], 1) + + def test_the_bracketed_list_does_not_change_the_attribute_count(self): + """Values is one HeaderAttribute however many members it carries.""" + one = V.emitted_header_counters([ + ("META", "")]) + many = V.emitted_header_counters([ + ("META", "")]) + self.assertEqual( + one["emittedHeaderClasses"]["HeaderAttribute"], + many["emittedHeaderClasses"]["HeaderAttribute"], + ) + + +class ToolVersionTests(VerboseTestCase): + def test_a_missing_binary_reports_none_rather_than_raising(self): + """Manifest construction must not fail because an optional tool is absent.""" + self.assertIsNone(V.tool_version(["definitely-not-a-real-binary-xyz", "--version"])) + + def test_the_first_output_line_is_the_version(self): + """Multi-line version banners are reduced to their first line.""" + completed = mock.Mock(stdout="tool 1.2.3\nbuilt from source\n") + with mock.patch.object(V.subprocess, "run", return_value=completed): + self.assertEqual(V.tool_version(["tool", "--version"]), "tool 1.2.3") + + def test_empty_output_reports_none(self): + """A tool that prints nothing yields no version string.""" + with mock.patch.object(V.subprocess, "run", return_value=mock.Mock(stdout=" \n")): + self.assertIsNone(V.tool_version(["tool", "--version"])) + + def test_a_labelled_table_row_is_read_by_its_label(self): + """Comunica prints a markdown table; the version is the labelled cell.""" + banner = "| Package | Version |\n| Comunica Engine | 5.3.0 |\n" + with mock.patch.object(V.subprocess, "run", return_value=mock.Mock(stdout=banner)): + self.assertEqual( + V.tool_version(["comunica", "--version"], table_label="Comunica Engine"), + "5.3.0", + ) + + def test_a_label_that_never_appears_falls_back_to_the_first_line(self): + """An unexpected banner shape still records something rather than nothing.""" + with mock.patch.object(V.subprocess, "run", return_value=mock.Mock(stdout="5.3.0\n")): + self.assertEqual( + V.tool_version(["comunica", "--version"], table_label="Comunica Engine"), + "5.3.0", + ) + + def test_a_hanging_tool_is_abandoned_rather_than_blocking_the_run(self): + """A timeout is absorbed, because a version banner is never load-bearing.""" + with mock.patch.object( + V.subprocess, "run", + side_effect=V.subprocess.TimeoutExpired(cmd="tool", timeout=20), + ): + self.assertIsNone(V.tool_version(["tool", "--version"])) + + +class ManifestTests(VerboseTestCase): + def setUp(self): + self._tmp = tempfile.TemporaryDirectory() + self.root = Path(self._tmp.name) + self.addCleanup(self._tmp.cleanup) + self.vcf = self.root / "cohort.vcf" + self.vcf.write_text("##fileformat=VCFv4.3\n#CHROM\tPOS\n", encoding="utf-8") + self.rdf = self.root / "cohort.nt" + self.rdf.write_text(" .\n", encoding="utf-8") + + def build(self, rdf_format="nt", materialized=False): + args = argparse.Namespace( + vcf=self.vcf, rdf=self.rdf, rdf_format=rdf_format, + dataset_id="cohort-1", representation="expanded", + ) + # Version probes shell out to container-only binaries; the manifest's + # shape is what is under test, not which tools this host happens to have. + with mock.patch.object(V, "tool_version", return_value=None): + return V.build_manifest( + args, V.QUERY_ROOT / "expanded", {"sourceSha256": "deadbeef"}, + engine_description={"engine": "comunica"}, + materialization={"materialized": materialized}, + ) + + def test_the_manifest_records_both_inputs_by_digest(self): + """A manifest must identify exactly which bytes were validated.""" + manifest = self.build() + self.assertEqual(manifest["sourceVcf"]["sha256"], "deadbeef") + self.assertEqual(manifest["sourceRdf"]["sha256"], V.sha256_file(self.rdf)) + self.assertEqual(manifest["sourceRdf"]["format"], "nt") + + def test_every_executed_query_is_pinned_by_digest(self): + """A changed query must be visible in the manifest, not silently rerun.""" + queries = self.build()["queries"] + self.assertEqual(len(queries), len(set(V.PREFLIGHT_QUERIES + V.CORE_QUERIES))) + for query_id, entry in queries.items(): + with self.subTest(query=query_id): + self.assertEqual(len(entry["sha256"]), 64) + + def test_a_plain_ntriples_source_declares_no_decode(self): + """Reading .nt in place must not claim a temporary decode happened.""" + manifest = self.build(materialized=False) + self.assertFalse(manifest["temporaryRdf"]["decompressedInsideContainer"]) + self.assertTrue(manifest["temporaryRdf"]["cleanupConfirmed"]) + + def test_a_materialized_source_records_the_decode(self): + """Anything decoded into scratch is declared as such.""" + manifest = self.build(rdf_format="hdt", materialized=True) + self.assertTrue(manifest["temporaryRdf"]["decompressedInsideContainer"]) + + def test_the_legacy_gzip_key_appears_only_for_a_gzip_source(self): + """sourceRdfGzip is a compatibility alias, not a field of every manifest.""" + self.assertNotIn("sourceRdfGzip", self.build(rdf_format="nt")) + gzip_manifest = self.build(rdf_format="nt.gz") + self.assertEqual(gzip_manifest["sourceRdfGzip"], gzip_manifest["sourceRdf"]) + + def test_the_manifest_serializes_as_json(self): + """The manifest is written to disk, so every value must be encodable.""" + json.dumps(self.build()) + + +class ResolvedArgumentTests(VerboseTestCase): + def setUp(self): + self._tmp = tempfile.TemporaryDirectory() + self.root = Path(self._tmp.name) + self.addCleanup(self._tmp.cleanup) + self.vcf = self.root / "cohort.vcf" + self.vcf.write_text("##fileformat=VCFv4.3\n#CHROM\tPOS\n", encoding="utf-8") + self.rdf = self.root / "cohort.nt" + self.rdf.write_text("", encoding="utf-8") + + def base(self, **overrides): + argv = { + "--vcf": str(self.vcf), "--rdf": str(self.rdf), + "--scratch-dir": str(self.root), "--results-dir": str(self.root), + "--representation": "expanded", "--dataset-id": "cohort-1", + } + argv.update(overrides) + return [item for pair in argv.items() for item in pair] + + def resolve(self, argv): + return V.resolve_args(V.build_arg_parser(), argv) + + def rejection(self, argv): + """Return the message argparse exits with, for a rejected argument set.""" + captured = io.StringIO() + with self.assertRaises(SystemExit), contextlib.redirect_stderr(captured): + self.resolve(argv) + return captured.getvalue() + + def test_a_valid_invocation_resolves_paths_and_the_primary_engine(self): + """The default run resolves to one absolute source and one engine.""" + args = self.resolve(self.base()) + self.assertEqual(args.engines, ["comunica"]) + self.assertEqual(args.engine, "comunica") + self.assertTrue(args.vcf.is_absolute()) + self.assertTrue(args.rdf.is_absolute()) + + def test_the_format_is_inferred_from_the_artifact_suffix(self): + """A recognised extension means --rdf-format is optional.""" + self.assertEqual(self.resolve(self.base()).rdf_format, "nt") + + def test_an_unrecognised_suffix_names_the_supported_formats(self): + """An inference failure must tell the operator what to pass instead.""" + artifact = self.root / "cohort.xyz" + artifact.write_text("", encoding="utf-8") + message = self.rejection(self.base(**{"--rdf": str(artifact)})) + self.assertIn("Could not infer an RDF format", message) + for fmt in V.RDF_FORMATS: + self.assertIn(fmt, message) + + def test_the_gzip_alias_selects_its_format_and_collapses_into_rdf(self): + """--rdf-gz is a spelling of --rdf, and implies the gzip format.""" + gzipped = self.root / "cohort.nt.gz" + gzipped.write_text("", encoding="utf-8") + argv = self.base() + argv[argv.index("--rdf")] = "--rdf-gz" + argv[argv.index(str(self.rdf))] = str(gzipped) + args = self.resolve(argv) + self.assertEqual(args.rdf_format, "nt.gz") + self.assertEqual(args.rdf.name, "cohort.nt.gz") + + def test_an_explicit_format_is_not_overridden_by_the_suffix(self): + """A deliberate --rdf-format survives a misleading filename.""" + self.assertEqual(self.resolve(self.base(**{"--rdf-format": "nt.gz"})).rdf_format, "nt.gz") + + def test_all_expands_to_every_supported_engine_in_registry_order(self): + """'all' is a shorthand, and the first engine stays the primary.""" + args = self.resolve(self.base(**{"--engine": "all"})) + self.assertEqual(args.engines, list(V.SPARQL_ENGINES)) + self.assertEqual(args.engine, V.SPARQL_ENGINES[0]) + + def test_an_unknown_engine_is_rejected_with_the_supported_set(self): + """A typo must name the alternatives rather than failing opaquely.""" + message = self.rejection(self.base(**{"--engine": "comunca"})) + self.assertIn("unknown SPARQL engine", message) + self.assertIn("comunica", message) + + def test_a_missing_input_is_named(self): + """Both inputs are checked before any container work starts.""" + self.assertIn("VCF does not exist", self.rejection( + self.base(**{"--vcf": str(self.root / "absent.vcf")}))) + self.assertIn("RDF artifact does not exist", self.rejection( + self.base(**{"--rdf": str(self.root / "absent.nt")}))) + + def test_a_missing_scratch_directory_is_named(self): + """Scratch is a mount point; a missing one is a configuration error.""" + self.assertIn("Scratch directory does not exist", self.rejection( + self.base(**{"--scratch-dir": str(self.root / "absent")}))) + + def test_the_dataset_id_is_restricted_to_iri_safe_characters(self): + """The dataset id reaches IRIs and filenames, so it is constrained.""" + self.assertIn("--dataset-id may contain only", self.rejection( + self.base(**{"--dataset-id": "bad id!"}))) + self.assertEqual(self.resolve(self.base(**{"--dataset-id": "a.b_c-1"})).dataset_id, "a.b_c-1") + + def test_every_duration_must_be_positive(self): + """A zero timeout would abandon a query before it started.""" + for flag in ("--query-timeout", "--qlever-memory-gb", "--qlever-startup-timeout", + "--comunica-bind-timeout", "--comunica-warmup-timeout"): + with self.subTest(flag=flag): + self.assertIn(f"{flag} must be a positive integer", + self.rejection(self.base(**{flag: "0"}))) + + def test_the_time_budget_treats_zero_as_no_ceiling(self): + """Zero is meaningful here, so only a negative budget is refused.""" + self.assertEqual(self.resolve( + self.base(**{"--validation-time-budget": "0"})).validation_time_budget, 0) + self.assertIn("must be zero or a positive integer", + self.rejection(self.base(**{"--validation-time-budget": "-1"}))) + + def test_every_port_must_be_in_range(self): + """An out-of-range port fails here rather than when the server binds.""" + for flag in ("--qlever-port", "--comunica-port", "--hdt-port"): + with self.subTest(flag=flag): + self.assertIn(f"{flag} must be between 1 and 65535", + self.rejection(self.base(**{flag: "0"}))) + self.assertIn(f"{flag} must be between 1 and 65535", + self.rejection(self.base(**{flag: "70000"}))) + + def test_the_three_engine_ports_must_differ(self): + """Two engines sharing a port would silently query the wrong server.""" + message = self.rejection(self.base(**{"--comunica-port": "7100", "--qlever-port": "7100"})) + self.assertIn("must all differ", message) + + def test_a_missing_shacl_shapes_file_is_named(self): + """Requesting conformance against shapes that do not exist is refused.""" + self.assertIn("SHACL shapes file does not exist", self.rejection( + self.base(**{"--shacl-shapes": str(self.root / "absent.ttl")}))) + + def test_supplied_shapes_and_progress_paths_are_resolved(self): + """Both optional paths become absolute for the container's benefit.""" + shapes = self.root / "shapes.ttl" + shapes.write_text("", encoding="utf-8") + args = self.resolve(self.base(**{ + "--shacl-shapes": str(shapes), "--progress-path": str(self.root / "progress.json"), + })) + self.assertTrue(args.shacl_shapes.is_absolute()) + self.assertTrue(args.progress_path.is_absolute()) + + def test_no_progress_path_stays_none(self): + """Progress reporting is opt-in and must not be invented.""" + self.assertIsNone(self.resolve(self.base()).progress_path) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/test_validation_results_unit.py b/test/test_validation_results_unit.py new file mode 100644 index 0000000..56d3049 --- /dev/null +++ b/test/test_validation_results_unit.py @@ -0,0 +1,373 @@ +"""Reading engine output, and refusing to trust it when it is unreadable. + +Between a SPARQL engine writing its result and the comparison layer declaring a +verdict sits a thin reading layer: parse SPARQL Results JSON, recover an exact +anomaly total, reconcile parsed statements against distinct triples, and reject a +materialized N-Triples file that cannot be what it claims to be. + +Every failure here has the same shape, and it is the one that matters most in a +validator: a result that could not be read must never be reported as clean. The +tests below drive each of those paths directly, with no engine, decoder, or +Docker involved - the container-only binaries are stubbed, because they are +absent in CI and a skipped test measures nothing. +""" + +from __future__ import annotations + +import importlib.util +import json +import tempfile +import unittest +from pathlib import Path +from unittest import mock + +from test.helpers import VerboseTestCase + +RUNNER_PATH = Path(__file__).resolve().parents[1] / "src" / "validation" / "validation_runner.py" + + +def load_runner(): + spec = importlib.util.spec_from_file_location("validation_runner_results", RUNNER_PATH) + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + spec.loader.exec_module(module) + return module + + +V = load_runner() + + +class BindingsTests(VerboseTestCase): + def setUp(self): + self._tmp = tempfile.TemporaryDirectory() + self.root = Path(self._tmp.name) + self.addCleanup(self._tmp.cleanup) + + def write(self, payload) -> Path: + path = self.root / "result.json" + path.write_text(json.dumps(payload), encoding="utf-8") + return path + + def test_well_formed_results_are_returned_as_rows(self): + """The happy path is the SPARQL Results JSON bindings array.""" + path = self.write({"results": {"bindings": [{"n": {"value": "1"}}]}}) + self.assertEqual(V.bindings(path), [{"n": {"value": "1"}}]) + + def test_an_empty_result_set_is_valid(self): + """No rows is an answer, not a malformed document.""" + self.assertEqual(V.bindings(self.write({"results": {"bindings": []}})), []) + + def test_a_document_without_bindings_is_rejected(self): + """A response shaped like an error must not read as zero rows.""" + with self.assertRaises(ValueError) as raised: + V.bindings(self.write({"results": {}})) + self.assertIn("Invalid SPARQL Results JSON", str(raised.exception)) + + def test_a_non_list_bindings_value_is_rejected(self): + """An object where an array belongs is refused rather than iterated.""" + with self.assertRaises(ValueError) as raised: + V.bindings(self.write({"results": {"bindings": {}}})) + self.assertIn("not a list", str(raised.exception)) + + def test_a_completely_wrong_document_is_rejected(self): + """A bare list or string is not a results document.""" + with self.assertRaises(ValueError): + V.bindings(self.write([1, 2, 3])) + + def test_an_integer_binding_is_read_from_its_value(self): + """SPARQL wraps every term in an object; the lexical form is what counts.""" + self.assertEqual(V.binding_int({"n": {"value": "42"}}, "n"), 42) + + +class AnomalyCountTests(VerboseTestCase): + def setUp(self): + self._tmp = tempfile.TemporaryDirectory() + self.root = Path(self._tmp.name) + self.addCleanup(self._tmp.cleanup) + + def execution(self, payload, status="PASS") -> dict: + path = self.root / "count.json" + path.write_text(json.dumps(payload), encoding="utf-8") + return {"preflight_blank_nodes_count": {"status": status, "rawResult": str(path)}} + + def test_the_exact_total_is_read_from_the_companion_aggregate(self): + """The sampled query is truncated; the count query is not.""" + executions = self.execution( + {"results": {"bindings": [{"anomalyCount": {"value": "17"}}]}}) + self.assertEqual(V.anomaly_count(executions, "preflight_blank_nodes"), 17) + + def test_an_absent_companion_reports_none_not_zero(self): + """A missing exact count must not be presented as 'no anomalies'.""" + self.assertIsNone(V.anomaly_count({}, "preflight_blank_nodes")) + + def test_a_failed_companion_reports_none(self): + """A query that did not pass cannot supply a trustworthy total.""" + executions = self.execution( + {"results": {"bindings": [{"anomalyCount": {"value": "17"}}]}}, status="FAIL") + self.assertIsNone(V.anomaly_count(executions, "preflight_blank_nodes")) + + def test_an_unreadable_companion_result_reports_none(self): + """Corrupt output degrades to 'unknown' rather than raising.""" + path = self.root / "count.json" + path.write_text("{not json", encoding="utf-8") + executions = {"preflight_blank_nodes_count": {"status": "PASS", "rawResult": str(path)}} + self.assertIsNone(V.anomaly_count(executions, "preflight_blank_nodes")) + + def test_a_missing_result_file_reports_none(self): + """A result path that does not exist is unknown, not zero.""" + executions = {"preflight_blank_nodes_count": { + "status": "PASS", "rawResult": str(self.root / "absent.json")}} + self.assertIsNone(V.anomaly_count(executions, "preflight_blank_nodes")) + + def test_an_aggregate_returning_several_rows_reports_none(self): + """A count query must return exactly one row to be an exact total.""" + executions = self.execution({"results": {"bindings": [ + {"anomalyCount": {"value": "1"}}, {"anomalyCount": {"value": "2"}}, + ]}}) + self.assertIsNone(V.anomaly_count(executions, "preflight_blank_nodes")) + + def test_a_row_without_the_count_variable_reports_none(self): + """A projection change must not be read as a valid total.""" + executions = self.execution({"results": {"bindings": [{"other": {"value": "1"}}]}}) + self.assertIsNone(V.anomaly_count(executions, "preflight_blank_nodes")) + + def test_a_non_numeric_count_reports_none(self): + """A malformed literal is unknown rather than an exception.""" + executions = self.execution( + {"results": {"bindings": [{"anomalyCount": {"value": "many"}}]}}) + self.assertIsNone(V.anomaly_count(executions, "preflight_blank_nodes")) + + +class DuplicateTripleReportTests(VerboseTestCase): + def setUp(self): + self._tmp = tempfile.TemporaryDirectory() + self.root = Path(self._tmp.name) + self.addCleanup(self._tmp.cleanup) + + def executions(self, distinct, status="PASS") -> dict: + path = self.root / "distinct.json" + path.write_text(json.dumps( + {"results": {"bindings": [{"distinctTripleCount": {"value": str(distinct)}}]}}), + encoding="utf-8") + return {"preflight_distinct_triple_count": {"status": status, "rawResult": str(path)}} + + def test_equal_counts_pass_with_no_duplicates(self): + """A store that deduplicated nothing means the source had no repeats.""" + report = V.duplicate_triple_report(self.executions(100), 100) + self.assertEqual(report["status"], "PASS") + self.assertEqual(report["duplicateTripleCount"], 0) + + def test_the_difference_is_the_duplicate_count(self): + """Parsed minus distinct is exactly the number of redundant statements.""" + report = V.duplicate_triple_report(self.executions(90), 100) + self.assertEqual(report["status"], "FAIL") + self.assertEqual(report["parsedTripleCount"], 100) + self.assertEqual(report["distinctTripleCount"], 90) + self.assertEqual(report["duplicateTripleCount"], 10) + + def test_a_missing_parsed_count_is_not_evaluated(self): + """Without the parser's total there is nothing to compare against.""" + report = V.duplicate_triple_report(self.executions(90), None) + self.assertEqual(report["status"], "NOT_EVALUATED") + self.assertIn("statement count", report["reason"]) + + def test_a_query_that_did_not_run_is_not_evaluated(self): + """An absent or failed query cannot establish a clean result.""" + self.assertEqual( + V.duplicate_triple_report({}, 100)["status"], "NOT_EVALUATED") + self.assertEqual( + V.duplicate_triple_report(self.executions(90, status="FAIL"), 100)["status"], + "NOT_EVALUATED") + + def test_an_unreadable_result_is_not_evaluated_and_says_why(self): + """A corrupt result is reported as unknown with its cause attached.""" + path = self.root / "distinct.json" + path.write_text("{not json", encoding="utf-8") + report = V.duplicate_triple_report( + {"preflight_distinct_triple_count": {"status": "PASS", "rawResult": str(path)}}, 100) + self.assertEqual(report["status"], "NOT_EVALUATED") + self.assertIn("unreadable result", report["reason"]) + + def test_an_empty_result_set_is_not_evaluated(self): + """No row means no distinct count, which is not the same as zero.""" + path = self.root / "distinct.json" + path.write_text(json.dumps({"results": {"bindings": []}}), encoding="utf-8") + report = V.duplicate_triple_report( + {"preflight_distinct_triple_count": {"status": "PASS", "rawResult": str(path)}}, 100) + self.assertEqual(report["status"], "NOT_EVALUATED") + + +class MaterializedGraphVerificationTests(VerboseTestCase): + def setUp(self): + self._tmp = tempfile.TemporaryDirectory() + self.root = Path(self._tmp.name) + self.addCleanup(self._tmp.cleanup) + + def verify(self, data: bytes) -> int: + path = self.root / "decoded.nt" + path.write_bytes(data) + return V._verify_ntriples(path, label="hdt2rdf") + + def test_a_well_formed_decode_returns_its_size(self): + """The size is reused for reporting, so it is returned rather than discarded.""" + payload = b" .\n" + self.assertEqual(self.verify(payload), len(payload)) + + def test_a_leading_blank_node_is_accepted(self): + """A graph may legitimately begin with a blank-node subject.""" + self.assertGreater(self.verify(b"_:b1 .\n"), 0) + + def test_an_unreadable_path_is_diagnosed(self): + """A decoder that wrote nothing at all must be named as the cause.""" + with self.assertRaises(RuntimeError) as raised: + V._verify_ntriples(self.root / "absent.nt", label="hdt2rdf") + self.assertIn("produced no readable output", str(raised.exception)) + + def test_an_empty_file_is_rejected(self): + """A zero-byte decode is a failure even when the exit code was zero.""" + with self.assertRaises(RuntimeError) as raised: + self.verify(b"") + self.assertIn("empty N-Triples file", str(raised.exception)) + + def test_a_file_with_no_newline_is_rejected_immediately(self): + """This is the failure that looked like a hang: one enormous line.""" + with self.assertRaises(RuntimeError) as raised: + self.verify(b" .") + self.assertIn("no newline", str(raised.exception)) + + def test_a_first_line_that_does_not_begin_a_triple_is_rejected(self): + """A subject must be an IRI or a blank node, not arbitrary text.""" + with self.assertRaises(RuntimeError) as raised: + self.verify(b"oops .\n") + self.assertIn("does not begin a triple", str(raised.exception)) + + def test_a_first_line_without_a_terminator_is_rejected(self): + """A missing '.' means the line was truncated mid-statement.""" + with self.assertRaises(RuntimeError) as raised: + self.verify(b" \n") + self.assertIn("no statement terminator", str(raised.exception)) + + def test_a_file_not_ending_in_a_newline_is_reported_as_truncated(self): + """The tail check catches a decode that stopped partway through.""" + with self.assertRaises(RuntimeError) as raised: + self.verify(b" .\n .") + self.assertIn("most likely truncated", str(raised.exception)) + + def test_the_diagnosis_quotes_the_offending_bytes(self): + """An operator needs to see what was produced, not only that it was wrong.""" + with self.assertRaises(RuntimeError) as raised: + self.verify(b"garbage line here\n") + self.assertIn("garbage line here", str(raised.exception)) + + +class BinaryResolutionTests(VerboseTestCase): + def test_an_environment_override_wins_when_it_resolves(self): + """The image pins decoder builds through environment variables.""" + with mock.patch.dict(V.os.environ, {"HDT2RDF_BIN": "/bin/sh"}, clear=False): + self.assertEqual(V._resolve_binary("HDT2RDF_BIN", "hdt2rdf"), "/bin/sh") + + def test_an_unresolvable_override_falls_through_to_the_candidates(self): + """A stale override must not mask a working binary on PATH.""" + with mock.patch.dict(V.os.environ, {"HDT2RDF_BIN": "not-a-real-binary-xyz"}, clear=False): + self.assertEqual( + V._resolve_binary("HDT2RDF_BIN", "sh"), V.shutil.which("sh")) + + def test_a_blank_override_is_ignored(self): + """An empty variable is 'unset', not a path to the empty string.""" + with mock.patch.dict(V.os.environ, {"HDT2RDF_BIN": " "}, clear=False): + self.assertEqual(V._resolve_binary("HDT2RDF_BIN", "sh"), V.shutil.which("sh")) + + def test_a_missing_binary_names_the_primary_candidate(self): + """The error must say which tool the image is missing.""" + with mock.patch.dict(V.os.environ, {}, clear=True): + with self.assertRaises(RuntimeError) as raised: + V._resolve_binary("HDT2RDF_BIN", "hdt2rdf-absent-xyz", "also-absent-xyz") + self.assertIn("hdt2rdf-absent-xyz", str(raised.exception)) + + +class EngineAdviceTests(VerboseTestCase): + def setUp(self): + self._tmp = tempfile.TemporaryDirectory() + self.root = Path(self._tmp.name) + self.addCleanup(self._tmp.cleanup) + self.source = self.root / "graph.nt" + self.source.write_bytes(b"x" * 4096) + + def test_an_indexing_engine_needs_no_warning(self): + """The advisory is specifically about comunica's lack of an index.""" + self.assertIsNone(V.engine_advice("qlever", self.source, 13, 600)) + + def test_a_small_graph_needs_no_warning(self): + """Below the threshold, streaming per query is perfectly reasonable.""" + self.assertIsNone(V.engine_advice("comunica", self.source, 13, 600)) + + def test_a_large_graph_warns_and_names_the_alternatives(self): + """The warning must be actionable, not merely alarming.""" + with mock.patch.object(V, "UNINDEXED_ENGINE_ADVICE_BYTES", 1024): + advice = V.engine_advice("comunica", self.source, 13, 600) + self.assertIsNotNone(advice) + self.assertIn("13", advice) + self.assertIn("--engine qlever", advice) + + def test_an_unreadable_source_produces_no_advice(self): + """A missing file is a different error, reported elsewhere.""" + self.assertIsNone(V.engine_advice("comunica", self.root / "absent.nt", 13, 600)) + + +class NtriplesSyntaxCheckTests(VerboseTestCase): + def setUp(self): + self._tmp = tempfile.TemporaryDirectory() + self.root = Path(self._tmp.name) + self.addCleanup(self._tmp.cleanup) + self.source = self.root / "graph.nt" + self.source.write_bytes(b" .\n") + + def test_an_absent_rapper_is_an_execution_failure_not_a_conformance_failure(self): + """A missing tool must never be reported as an invalid graph.""" + with mock.patch.object(V.shutil, "which", return_value=None): + result = V.validate_ntriples(self.source, self.root) + self.assertEqual(result["status"], "EXECUTION_FAILED") + self.assertIn("rapper", result["error"]) + + def test_a_clean_parse_passes_and_records_the_triple_count(self): + """The count makes a decoded artifact comparable against its source.""" + completed = mock.Mock( + returncode=0, stdout="", stderr="rapper: Parsing returned 5 triples\n") + with mock.patch.object(V.shutil, "which", return_value="/usr/bin/rapper"), \ + mock.patch.object(V.subprocess, "run", return_value=completed): + result = V.validate_ntriples(self.source, self.root) + self.assertEqual(result["status"], "PASS") + self.assertEqual(result["tripleCount"], 5) + self.assertEqual(result["exitCode"], 0) + + def test_a_non_zero_exit_fails(self): + """A syntax error in the graph is a conformance failure.""" + completed = mock.Mock(returncode=1, stdout="", stderr="rapper: Error - syntax\n") + with mock.patch.object(V.shutil, "which", return_value="/usr/bin/rapper"), \ + mock.patch.object(V.subprocess, "run", return_value=completed): + result = V.validate_ntriples(self.source, self.root) + self.assertEqual(result["status"], "FAIL") + self.assertEqual(result["exitCode"], 1) + + def test_an_unparsable_banner_leaves_the_count_unknown(self): + """A missing count is None rather than a fabricated zero.""" + completed = mock.Mock(returncode=0, stdout="", stderr="done\n") + with mock.patch.object(V.shutil, "which", return_value="/usr/bin/rapper"), \ + mock.patch.object(V.subprocess, "run", return_value=completed): + result = V.validate_ntriples(self.source, self.root) + self.assertIsNone(result["tripleCount"]) + + def test_the_tool_output_is_written_beside_the_report(self): + """The log is the evidence for the verdict and must survive the run.""" + completed = mock.Mock(returncode=0, stdout="out\n", stderr="err\n") + with mock.patch.object(V.shutil, "which", return_value="/usr/bin/rapper"), \ + mock.patch.object(V.subprocess, "run", return_value=completed): + result = V.validate_ntriples(self.source, self.root) + log = Path(result["log"]) + self.assertTrue(log.is_file()) + self.assertIn("out", log.read_text(encoding="utf-8")) + self.assertIn("err", log.read_text(encoding="utf-8")) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/test_vcf_files/test-larger-multisample.vcf.gz b/test/test_vcf_files/test-larger-multisample.vcf.gz new file mode 100644 index 0000000..bb8db38 Binary files /dev/null and b/test/test_vcf_files/test-larger-multisample.vcf.gz differ diff --git a/test/test_vcf_rdfizer_unit.py b/test/test_vcf_rdfizer_unit.py index 9a7a68f..a6c4f2b 100644 --- a/test/test_vcf_rdfizer_unit.py +++ b/test/test_vcf_rdfizer_unit.py @@ -167,11 +167,25 @@ def run_with_default_aggregate(cmd, cwd=None, env=None): ), "rdf", ) + storage_mode = next( + ( + part.split("=", 1)[1] + for part in cmd + if isinstance(part, str) and part.startswith("RDF_STORAGE_MODE=") + ), + "plain", + ) if out_mount is not None: output_dir = out_mount / output_name output_dir.mkdir(parents=True, exist_ok=True) if not any(output_dir.glob("*.nt")) and not any(output_dir.glob("*.nt.gz")): - (output_dir / f"{output_name}.nt").write_text("

.\n") + # Match the aggregate the wrapper plans for this storage mode: + # space-optimized expects one .nt.gz assembled from the parts. + if storage_mode == "space-optimized": + with gzip.open(output_dir / f"{output_name}.nt.gz", "wt") as handle: + handle.write("

.\n") + else: + (output_dir / f"{output_name}.nt").write_text("

.\n") if result == 0: rendered = str(cmd[-1]) if cmd else "" work_mount = next( @@ -272,6 +286,20 @@ def latest_metrics_run_dir(metrics_root: Path) -> Path: +def _vocabulary_registry_path() -> Path | None: + """Locate the published VCF Core version registry, if it is available.""" + override = os.environ.get("VCF_CORE_VOCABULARY_DIR") + candidates = [Path(override)] if override else [] + repo_root = Path(__file__).resolve().parents[1] + candidates += [ + repo_root.parent / "vcf-core-vocabulary", + repo_root.parent / "vcf-rdfizer-vocabulary", + ] + for candidate in candidates: + registry = candidate / "ontology" / "versions" / "registry.json" + if registry.is_file(): + return registry + return None def stable_progress_env(): """Pin the ambient progress opt-outs for one test. @@ -1737,21 +1765,44 @@ def test_default_sample_maps_stream_without_expanded_helper_tables(self): self.assertEqual(stats["records"], 1) self.assertEqual(stats["sample_calls"], 2) self.assertEqual(stats["format_values"], 4) - self.assertEqual(stats["triples"], 19) + # One GT per sample, each diploid. + self.assertEqual(stats["genotypes"], 2) + self.assertEqual(stats["genotype_calls"], 4) rdf_lines = rdf_path.read_text(encoding="utf-8").splitlines() - self.assertEqual(len(rdf_lines), 20) + self.assertEqual(len(rdf_lines), stats["triples"] + 1) self.assertIn( - " " - " .", + " " + " .", rdf_lines, ) self.assertIn( - " " + " " " .", rdf_lines, ) + # The expanded profile gives each call a reusable sample identity in + # addition to its sampleId literal. + self.assertIn( + " " + " .", + rdf_lines, + ) + # GT is parsed into an ordered genotype, and its no-call positions + # are marked rather than given a fabricated allele. + self.assertIn( + " " + " " + '"0/1"^^ .', + rdf_lines, + ) self.assertIn( - '"."^^ .', + " " + " " + '"true"^^ .', + rdf_lines, + ) + self.assertIn( + '"."^^ .', rdf_lines[-1], ) @@ -1782,9 +1833,18 @@ def test_sample_streaming_accepts_a_csv_field_larger_than_python_default(self): self.assertEqual(stats["sample_calls"], 2504) self.assertEqual(stats["format_values"], 2504) - self.assertEqual(stats["triples"], 1 + (2504 * 6)) + self.assertEqual(stats["samples"], 2504) + # The payload is not in the GT lexical space, so no Genotype is + # invented for it; the value stays as the FORMAT field's literal. + self.assertEqual(stats["genotypes"], 0) + # 3 file-level triples (profile, hasSampleSet, SampleSet type) and 6 + # for the single synthesized FORMAT definition, then per sample: 5 + # for the VCFSample and its column-header link, 4 for the SampleCall + # and 4 for its one FORMAT value. + expected = 3 + 6 + (2504 * 13) + self.assertEqual(stats["triples"], expected) with gzip.open(rdf_path, "rt", encoding="utf-8") as handle: - self.assertEqual(sum(1 for _line in handle), 2 + (2504 * 6)) + self.assertEqual(sum(1 for _line in handle), expected + 1) def test_sample_support_strategy_preserves_custom_helper_consumers(self): """Only the exact canonical helper maps use direct RDF streaming.""" @@ -1859,19 +1919,290 @@ def test_condensed_sample_emitter_uses_shared_samples_and_ordered_vectors(self): self.assertEqual(stats["matrices"], 1) self.assertEqual(stats["format_vectors"], 3) self.assertEqual(stats["format_definitions"], 3) - self.assertEqual(stats["triples"], 45) rdf_text = rdf_path.read_text(encoding="utf-8") + self.assertEqual(len(rdf_text.splitlines()), stats["triples"] + 1) self.assertIn("vocab#CondensedRepresentation", rdf_text) self.assertIn("vocab#SampleSet", rdf_text) - self.assertIn("#samples/S3> ", rdf_text) + # Ordinals are xsd:integer so they satisfy the SHACL profiles, which + # constrain every one of them with sh:datatype xsd:integer. + self.assertIn( + "#samples/S3> " + '"3"^^', + rdf_text, + ) self.assertIn("#call/7/matrix/fmt/GT", rdf_text) self.assertIn('"0/1\\t./.\\t1/1"', rdf_text) self.assertIn('"30,12\\t.\\t."', rdf_text) + # A vector cites its declaration at the header line's own IRI. self.assertIn("#header/line/1> .", rdf_text) - self.assertIn("vocab#fieldNumber> \"1\"", rdf_text) - self.assertIn("vocab#fieldDescription> \"Genotype\"", rdf_text) + self.assertIn( + "vocab#FormatFieldDefinition>", rdf_text, + "the vector's declaration must be typed", + ) + # The declaration's ID/Number/Type/Description belong to the header + # emitter, which owns that IRI. Emitting them here too produced two + # copies of every one of those triples, which the validation + # suite's duplicate-triple preflight rejects. + for owned in ("vocab#fieldId>", "vocab#fieldNumber>", + "vocab#fieldType>", "vocab#fieldDescription>"): + self.assertNotIn(owned, rdf_text, owned) self.assertNotIn("vocab#hasSampleCall", rdf_text) self.assertNotIn("vocab#FormatFieldValue", rdf_text) + # The condensed profile keeps genotype values inside their vectors, + # so no per-sample Genotype is derived. (hasGenotypeColumns, on the + # #CHROM line, is a different property and is expected.) + self.assertNotIn("vocab#hasGenotype>", rdf_text) + self.assertIn("vocab#hasGenotypeColumns>", rdf_text) + + def test_version_model_matches_the_published_registry(self): + """The converter's version table must agree with the vocabulary's own. + + VCF Core publishes ``ontology/versions/registry.json`` as the single + source of truth for its version-scoped artifacts: the SHACL overlays, + the reserved-key snapshots and the VCF4xFile classes are all generated + from it. The converter keeps its own dependency-free copy of the same + facts so it works without the vocabulary checked out; this pins the two + together so a new VCF version, or a corrected arity, cannot land on one + side alone. + + Skipped when the vocabulary is not available. Point + VCF_CORE_VOCABULARY_DIR at a checkout to run it. + """ + registry_path = _vocabulary_registry_path() + if registry_path is None: + self.skipTest( + "VCF Core vocabulary not found; set VCF_CORE_VOCABULARY_DIR to " + "a checkout to cross-check the version model" + ) + registry = json.loads(registry_path.read_text(encoding="utf-8")) + vocab = vcf_rdfizer.vocab + + published = {entry["id"] for entry in registry["versions"]} + self.assertEqual( + published, set(vocab.VCF_VERSIONS), + "the converter and the vocabulary disagree about which VCF " + "versions are supported", + ) + # The newest published version is what an unrecognized file falls back + # to, so that has to track the registry too. + self.assertEqual(vocab.FALLBACK_VERSION.short, registry["current"]) + + for entry in registry["versions"]: + version = vocab.VCF_VERSIONS[entry["id"]] + with self.subTest(version=entry["id"]): + self.assertEqual(version.token, entry["code"]) + self.assertEqual(version.file_class, entry["className"]) + self.assertEqual( + version.leading_phase_indicator, entry["leadingPhaseIndicator"] + ) + # "perAlt" means a tuple repeats once per ALT allele; "fixed" + # means one tuple describes the whole record. + self.assertEqual( + version.tuples_per_alt, entry["svTupleScope"] == "perAlt" + ) + + # numberCodes lists regex alternatives; "[0-9]+" and "[.]" are + # the integer and missing forms the model handles separately. + for scope, is_format in (("info", False), ("format", True)): + codes = { + code for code in entry["numberCodes"][scope] + if code != "[0-9]+" + } + codes = {"." if code == "[.]" else code for code in codes} + allowed = { + code for code in vocab.ARITY_INDIVIDUALS + if version.allows_number(code, is_format=is_format) + } + self.assertEqual( + allowed, codes, + f"{scope} Number codes disagree for {entry['id']}", + ) + + expected_tuples = { + key: group["width"] + for group in entry["svTuples"] for key in group["keys"] + } + self.assertEqual( + version.tuple_keys, expected_tuples, + f"flattened-tuple keys disagree for {entry['id']}", + ) + + def test_vcf_version_is_detected_from_the_fileformat_line(self): + """Auto-detection reads ##fileformat; nothing has to be supplied.""" + with tempfile.TemporaryDirectory() as td: + tmp_path = Path(td) + for declared, expected_short, expected_recognized in ( + ("VCFv4.1", "4.1", True), + ("VCFv4.3", "4.3", True), + ("VCFv4.5", "4.5", True), + # VCF Core claims no 4.0 conformance overlay, so 4.0 is not + # recognized and no version class may be claimed for it. + ("VCFv4.0", "4.5", False), + ("", "4.5", False), + ("nonsense", "4.5", False), + ): + metadata = tmp_path / f"{expected_short}-{declared or 'none'}.tsv" + metadata.write_text( + "SOURCE_FILE\tFILE_FORMAT\tFILE_DATE\tSOURCE_SOFTWARE\t" + "REFERENCE_GENOME\tHEADER_COUNT\tRECORD_COUNT\n" + f"x.vcf\t{declared}\t\t\t\t1\t1\n", + encoding="utf-8", + ) + version, recognized, raw = vcf_rdfizer.read_declared_vcf_version(metadata) + self.assertEqual(version.short, expected_short, declared) + self.assertEqual(recognized, expected_recognized, declared) + self.assertEqual(raw, declared) + + def test_rules_rendering_resolves_the_version_class_sentinel(self): + """The mapping's version sentinel becomes the input's own file class.""" + with tempfile.TemporaryDirectory() as td: + tmp_path = Path(td) + template = tmp_path / "template.ttl" + template.write_text( + "rr:class vcfc:VCFFile ;\n rr:class vcfc:VCFVersionFile\n", + encoding="utf-8", + ) + + rendered = tmp_path / "v43.ttl" + vcf_rdfizer.render_rules_for_triplet( + template, rendered, "r.tsv", "h.tsv", "m.tsv", "sc.tsv", "sf.tsv", + vcf_version=vcf_rdfizer.vocab.VCF_VERSIONS["4.3"], + ) + self.assertIn("rr:class vcfc:VCF43File", rendered.read_text()) + self.assertNotIn("VCFVersionFile", rendered.read_text()) + + # An unrecognized version resolves to the base class the subject map + # already asserts, so the graph never claims a gate that cannot be + # checked and never gains a term the vocabulary does not define. + neutral = tmp_path / "neutral.ttl" + vcf_rdfizer.render_rules_for_triplet( + template, neutral, "r.tsv", "h.tsv", "m.tsv", "sc.tsv", "sf.tsv", + vcf_version=None, + ) + text = neutral.read_text() + self.assertNotIn("VCFVersionFile", text) + self.assertEqual(text.count("rr:class vcfc:VCFFile"), 2) + + def test_confidence_intervals_follow_the_version_tuple_rule(self): + """CIPOS is one pair per record before VCF 4.4 and one per ALT after.""" + vocab = vcf_rdfizer.vocab + early, late = vocab.VCF_VERSIONS["4.3"], vocab.VCF_VERSIONS["4.4"] + + # Both treat CIPOS as a materialized tuple, so its items are emitted + # even though 4.4 declares it Number=. -- the overlay counts them. + self.assertTrue(early.is_positional("CIPOS", "2")) + self.assertTrue(late.is_positional("CIPOS", ".")) + + # Before 4.4 an item has a position but no allele to point at. + self.assertTrue(early.value_item_link("CIPOS", "2", 0).is_empty) + # From 4.4 items 0-1 belong to ALT 1 and items 2-3 to ALT 2. + self.assertEqual( + [late.value_item_link("CIPOS", ".", i).allele_index for i in range(4)], + [1, 1, 2, 2], + ) + + # CILEN and CICN are not tuple keys before 4.4 at all. + self.assertIsNone(early.tuple_arity("CILEN")) + self.assertEqual(late.tuple_arity("CILEN"), 2) + + def test_version_gates_number_codes_and_format_families(self): + """Number codes and the LA/M families arrived in specific versions.""" + vocab = vcf_rdfizer.vocab + v41, v42, v44, v45 = ( + vocab.VCF_VERSIONS[s] for s in ("4.1", "4.2", "4.4", "4.5") + ) + # Number=R arrived in 4.2, P in 4.4, and LA/LR/LG/M in 4.5. + self.assertFalse(v41.allows_number("R", is_format=True)) + self.assertTrue(v42.allows_number("R", is_format=True)) + self.assertFalse(v42.allows_number("P", is_format=True)) + self.assertTrue(v44.allows_number("P", is_format=True)) + self.assertFalse(v44.allows_number("M", is_format=True)) + self.assertTrue(v45.allows_number("M", is_format=True)) + # A plain integer count is legal in every version. + self.assertTrue(v41.allows_number("2", is_format=False)) + + self.assertFalse(v44.local_alleles) + self.assertTrue(v45.local_alleles) + self.assertFalse(v44.base_modifications) + self.assertTrue(v45.base_modifications) + # EVENT is one per record before 4.4 and one per ALT after; EVENTTYPE + # does not exist before 4.4 at all. + self.assertFalse(v42.events_per_alt) + self.assertFalse(v42.event_types) + self.assertTrue(v44.events_per_alt) + self.assertTrue(v44.event_types) + + def test_expanded_emitter_omits_families_the_version_lacks(self): + """A 4.4 file gets no base modifications and no local-allele set.""" + with tempfile.TemporaryDirectory() as td: + tmp_path = Path(td) + records_tsv = tmp_path / "s.records.tsv" + records_tsv.write_text( + "SOURCE_FILE\tROW_ID\tCHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\t" + "INFO\tFORMAT\tS1\n" + "s.vcf\t1\t1\t100\t.\tA\tG\t50\tPASS\t.\tGT:LAA:M27551C\t" + "0/1:1:0.8\n", + encoding="utf-8", + ) + vocab = vcf_rdfizer.vocab + + for short, expect_families in (("4.4", False), ("4.5", True)): + rdf_path = tmp_path / f"{short}.nt" + rdf_path.write_text("

.\n", encoding="utf-8") + stats = vcf_rdfizer.append_expanded_sample_rdf( + records_tsv, rdf_path, None, + version=vocab.VCF_VERSIONS[short], + progress_interval_records=0, + ) + expected = 1 if expect_families else 0 + self.assertEqual(stats["local_allele_sets"], expected, short) + self.assertEqual(stats["base_modifications"], expected, short) + # The raw FORMAT values are present either way; only the + # derived resources the version does not define are withheld. + text = rdf_path.read_text(encoding="utf-8") + self.assertIn('vocab#fieldValue> "1"', text) + self.assertIn('vocab#fieldValue> "0.8"', text) + + + def test_declared_field_definitions_are_emitted_exactly_once(self): + """The header emitter owns a declared definition; value emitters cite it. + + Both used to emit ID/Number/Type/Description at the same header-line + IRI, so every declared INFO and FORMAT key produced a duplicate of each + of those triples. + """ + with tempfile.TemporaryDirectory() as td: + tmp_path = Path(td) + records_tsv = tmp_path / "s.records.tsv" + records_tsv.write_text( + "SOURCE_FILE\tROW_ID\tCHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\t" + "INFO\tFORMAT\tS1\n" + "s.vcf\t1\t1\t100\t.\tA\tG\t50\tPASS\tDP=7\tGT:DP\t0/1:7\n", + encoding="utf-8", + ) + headers_tsv = tmp_path / "s.header_lines.tsv" + headers_tsv.write_text( + "SOURCE_FILE\tHEADER_INDEX\tHEADER_KEY\tHEADER_VALUE\tRAW_LINE\n" + "s.vcf\t1\tfileformat\tVCFv4.5\tx\n" + "s.vcf\t2\tINFO\t\tx\n" + "s.vcf\t3\tFORMAT\t\tx\n" + "s.vcf\t4\tFORMAT\t\tx\n", + encoding="utf-8", + ) + rdf_path = tmp_path / "s.nt" + rdf_path.write_text("", encoding="utf-8") + + vcf_rdfizer.append_expanded_sample_rdf( + records_tsv, rdf_path, headers_tsv, progress_interval_records=0 + ) + vcf_rdfizer.append_header_representation_rdf(headers_tsv, rdf_path) + vcf_rdfizer.append_record_detail_rdf( + records_tsv, headers_tsv, rdf_path, progress_interval_records=0 + ) + + lines = [l for l in rdf_path.read_text(encoding="utf-8").splitlines() if l.strip()] + duplicates = {line for line in lines if lines.count(line) > 1} + self.assertEqual(duplicates, set(), "duplicate triples emitted") def test_structured_format_header_parser_preserves_quoted_commas(self): """FORMAT descriptions with commas and escaped quotes remain one attribute.""" @@ -1883,6 +2214,51 @@ def test_structured_format_header_parser_preserves_quoted_commas(self): self.assertEqual(fields["Number"], "1") self.assertEqual(fields["Description"], 'Genotype, with "quoted" text') + def test_structured_header_parser_keeps_a_bracketed_values_list_whole(self): + """A ##META ``Values=[a, b, c]`` list is one attribute, not three fragments. + + The commas inside the brackets separate allowed values, not attributes. + Splitting on them dropped every member after the first, because the + fragments carry no '=' and were discarded, leaving a value that still + held its opening bracket. + """ + fields = vcf_rdfizer._parse_structured_header_fields( + "" + ) + + self.assertEqual(fields["Values"], "[WholeGenome, Exome, Panel]") + self.assertEqual( + vcf_rdfizer._parse_meta_values(fields["Values"]), + ["WholeGenome", "Exome", "Panel"], + ) + + def test_structured_header_parser_resumes_after_a_bracketed_list(self): + """An attribute following the Values list is still parsed.""" + fields = vcf_rdfizer._parse_structured_header_fields( + '' + ) + + self.assertEqual(fields["Type"], "String") + self.assertEqual(fields["Description"], "an assay, described") + + def test_structured_header_parser_survives_an_unbalanced_bracket(self): + """Malformed brackets must not swallow every remaining attribute.""" + fields = vcf_rdfizer._parse_structured_header_fields( + "" + ) + + self.assertEqual(fields["ID"], "Broken") + self.assertEqual(fields["Type"], "String") + + def test_structured_header_parser_ignores_a_bracket_inside_quotes(self): + """A '[' in a description is text, not the start of a values list.""" + fields = vcf_rdfizer._parse_structured_header_fields( + '' + ) + + self.assertEqual(fields["Description"], "depth [reads], per sample") + self.assertEqual(fields["Number"], "1") + def test_condensed_sample_emitter_rolls_back_on_sample_count_mismatch(self): """Malformed sample alignment fails atomically without a partial condensed graph.""" with tempfile.TemporaryDirectory() as td: @@ -2051,7 +2427,7 @@ def test_help_flag_prints_usage_guide(self): self.assertEqual(exc.exception.code, 0) text = out_buf.getvalue() self.assertIn("Examples:", text) - self.assertIn("-m {full,compress,decompress,tsv,index,validation}", text) + self.assertIn("-m {full,compress,decompress,tsv,index,validation,link}", text) self.assertIn("-i INPUT", text) self.assertIn("--keep-rmlstreamer-rdf-output", text) self.assertIn("--remove-rdf-storage-output", text) @@ -2776,16 +3152,87 @@ def test_main_full_mode_requires_input_argument(self): rc = invoke_main(["--mode", "full"]) self.assertEqual(rc, 2) - def test_main_full_mode_requires_storage_mode_argument(self): - """Full mode fails validation when --rdf-storage-mode is omitted.""" + def test_main_full_mode_defaults_to_space_optimized_storage(self): + """Full mode without --rdf-storage-mode resolves to the space-optimized aggregate.""" + self.assertEqual(vcf_rdfizer.DEFAULT_RDF_STORAGE_MODE, "space-optimized") with tempfile.TemporaryDirectory() as td: tmp_path = Path(td) input_dir, rules_path = prepare_inputs(tmp_path) - rc = invoke_main( - ["--mode", "full", "--input", str(input_dir), "--rules", str(rules_path)], - auto_storage=False, - ) + observed = {} + + def capture_full_mode(**kwargs): + observed.update(kwargs) + return 0 + + old_cwd = os.getcwd() + os.chdir(tmp_path) + try: + with mock.patch.object(vcf_rdfizer, "check_docker", return_value=True), \ + mock.patch.object(vcf_rdfizer, "docker_image_exists", return_value=True), \ + mock.patch.object( + vcf_rdfizer, "run_full_mode", side_effect=capture_full_mode + ): + rc = invoke_main( + [ + "--mode", "full", + "--input", str(input_dir), + "--rules", str(rules_path), + "--out", str(tmp_path / "out"), + ], + auto_storage=False, + ) + finally: + os.chdir(old_cwd) + + self.assertEqual(rc, 0) + self.assertEqual(observed.get("rdf_storage_mode"), "space-optimized") + + def test_main_full_mode_rejects_single_hdt_strategy_with_cottas(self): + """`single` is refused, not silently ignored, when COTTAS forces chunking.""" + with tempfile.TemporaryDirectory() as td: + tmp_path = Path(td) + input_dir, rules_path = prepare_inputs(tmp_path) + stderr = StringIO() + with redirect_stderr(stderr): + rc = invoke_main( + [ + "--mode", "full", + "--input", str(input_dir), + "--rules", str(rules_path), + "--rdf-storage-mode", "plain", + "--representations", "hdt,cottas", + "--rdf-compression", "none", + "--hdt-strategy", "single", + "--out", str(tmp_path / "out"), + ], + auto_storage=False, + ) self.assertEqual(rc, 2) + self.assertIn("HDT-only run", stderr.getvalue()) + + def test_main_full_mode_rejects_single_hdt_strategy_with_space_optimized(self): + """`single` cannot read the now-default gzip aggregate.""" + with tempfile.TemporaryDirectory() as td: + tmp_path = Path(td) + input_dir, rules_path = prepare_inputs(tmp_path) + stderr = StringIO() + with redirect_stderr(stderr): + rc = invoke_main( + [ + "--mode", "full", + "--input", str(input_dir), + "--rules", str(rules_path), + "--representations", "hdt", + "--rdf-compression", "none", + "--hdt-strategy", "single", + "--out", str(tmp_path / "out"), + ], + auto_storage=False, + ) + self.assertEqual(rc, 2) + message = stderr.getvalue() + self.assertIn("gzip aggregate", message) + self.assertIn("--rdf-storage-mode plain", message) def test_removed_legacy_cli_options_are_rejected(self): """Removed layout and compatibility aliases are not accepted by the CLI.""" @@ -2865,6 +3312,272 @@ def test_main_full_mode_keyboard_interrupt_returns_130_and_writes_progress_log(s self.assertTrue(progress_log.exists()) self.assertIn("Run interrupted by user signal", progress_log.read_text()) + def test_every_container_is_labelled_so_an_interrupt_can_find_it(self): + """Cleanup can only stop containers it can identify.""" + labels = set() + for as_user in (True, False): + base = vcf_rdfizer.docker_run_base(as_user=as_user) + self.assertIn("--label", base) + labels.add(base[base.index("--label") + 1]) + # Both argv shapes carry the same label - the as_user=False early + # return used to skip everything added after it. + self.assertEqual(len(labels), 1) + label = labels.pop() + self.assertTrue(label.startswith(f"{vcf_rdfizer.DOCKER_RUN_LABEL_KEY}=")) + # The filter must select exactly the label that was stamped. + self.assertEqual(vcf_rdfizer.docker_run_label_filter(), f"label={label}") + + def test_run_label_is_unique_per_process(self): + """Two concurrent runs must never kill each other's containers.""" + before = vcf_rdfizer.docker_run_label_filter() + vcf_rdfizer.set_docker_run_label("20260910T120000") + after = vcf_rdfizer.docker_run_label_filter() + self.assertNotEqual(before, after) + self.assertIn("20260910T120000", after) + self.assertIn(str(os.getpid()), after) + + def test_kill_run_containers_counts_results_and_never_raises(self): + """An interrupt path that raises leaves the orphans it exists to stop.""" + with tempfile.TemporaryDirectory() as td: + tracker = vcf_rdfizer.RunTracker(Path(td) / "progress.log") + + calls = [] + + def fake_subprocess_run(cmd, **kwargs): + calls.append(cmd) + if "ps" in cmd: + return subprocess.CompletedProcess(cmd, 0, stdout="abc123\ndef456\n", stderr="") + # First kill succeeds, second fails. + code = 0 if cmd[-1] == "abc123" else 1 + return subprocess.CompletedProcess(cmd, code, stdout="", stderr="no such container") + + with mock.patch.object(vcf_rdfizer.subprocess, "run", side_effect=fake_subprocess_run): + killed, failed = vcf_rdfizer.kill_run_containers(run_tracker=tracker) + self.assertEqual((killed, failed), (1, 1)) + # Selection is by this run's label, never "all containers". + self.assertIn(vcf_rdfizer.docker_run_label_filter(), calls[0]) + + # Docker absent entirely: reported, not raised. + with mock.patch.object( + vcf_rdfizer.subprocess, "run", side_effect=OSError("docker not found") + ): + self.assertEqual( + vcf_rdfizer.kill_run_containers(run_tracker=tracker), (0, 0) + ) + tracker.close() + + def test_temp_volume_removal_retries_while_a_container_holds_it(self): + """A container killed a moment ago has not released its volume yet.""" + vcf_rdfizer._TEMP_DOCKER_VOLUMES.clear() + vcf_rdfizer.register_temp_volume("vol-a") + attempts = [] + + # Fail twice (volume still in use), then succeed. + def fake_run(cmd, cwd=None, env=None): + attempts.append(cmd) + return 0 if len(attempts) >= 3 else 1 + + with mock.patch.object(vcf_rdfizer, "run", side_effect=fake_run), \ + mock.patch.object(vcf_rdfizer.time, "sleep"): + self.assertTrue(vcf_rdfizer.remove_temp_volume("vol-a", delay=0)) + self.assertEqual(len(attempts), 3) + self.assertEqual(attempts[0][-4:], ["volume", "rm", "-f", "vol-a"]) + # Success deregisters it, so the later sweep does not retry it. + self.assertNotIn("vol-a", vcf_rdfizer._TEMP_DOCKER_VOLUMES) + + def test_a_volume_that_cannot_be_removed_stays_registered(self): + """It must remain for the interrupt sweep, not be silently forgotten.""" + vcf_rdfizer._TEMP_DOCKER_VOLUMES.clear() + vcf_rdfizer.register_temp_volume("vol-stuck") + with mock.patch.object(vcf_rdfizer, "run", return_value=1), \ + mock.patch.object(vcf_rdfizer.time, "sleep"): + self.assertFalse(vcf_rdfizer.remove_temp_volume("vol-stuck", attempts=2, delay=0)) + self.assertIn("vol-stuck", vcf_rdfizer._TEMP_DOCKER_VOLUMES) + + def test_remove_tracked_volumes_counts_and_never_raises(self): + """Cleanup that raises leaves exactly the leak it exists to prevent.""" + with tempfile.TemporaryDirectory() as td: + tracker = vcf_rdfizer.RunTracker(Path(td) / "progress.log") + vcf_rdfizer._TEMP_DOCKER_VOLUMES.clear() + for name in ("vol-ok", "vol-bad"): + vcf_rdfizer.register_temp_volume(name) + + def fake_remove(name, **kwargs): + if name == "vol-bad": + raise RuntimeError("docker daemon gone") + vcf_rdfizer._TEMP_DOCKER_VOLUMES.discard(name) + return True + + with mock.patch.object(vcf_rdfizer, "remove_temp_volume", side_effect=fake_remove): + removed, failed = vcf_rdfizer.remove_tracked_volumes(run_tracker=tracker) + self.assertEqual((removed, failed), (1, 1)) + self.assertIn("temporary volumes removed=1, failed=1", tracker.log_path.read_text()) + tracker.close() + vcf_rdfizer._TEMP_DOCKER_VOLUMES.clear() + + def test_interrupt_removes_volumes_only_after_stopping_containers(self): + """Docker refuses to remove a volume a live container still holds. + + That is the whole bug: each volume's own `finally` ran as the exception + unwound, before the containers were killed, so removal failed there and + nothing retried -- leaking ~700 MB per interrupted run. + """ + with tempfile.TemporaryDirectory() as td: + tmp_path = Path(td) + input_dir, rules_path = prepare_inputs(tmp_path) + out_dir = tmp_path / "out" + order = [] + + old_cwd = os.getcwd() + os.chdir(tmp_path) + try: + with mock.patch.object(vcf_rdfizer, "check_docker", return_value=True), \ + mock.patch.object(vcf_rdfizer, "docker_image_exists", return_value=True), \ + mock.patch.object(vcf_rdfizer, "run_full_mode", side_effect=KeyboardInterrupt()), \ + mock.patch.object( + vcf_rdfizer, "kill_run_containers", + side_effect=lambda **k: (order.append("kill-containers"), (1, 0))[1], + ), \ + mock.patch.object( + vcf_rdfizer, "remove_tracked_volumes", + side_effect=lambda **k: (order.append("remove-volumes"), (2, 0))[1], + ), \ + mock.patch.object( + vcf_rdfizer, "cleanup_interrupted_full_run", + side_effect=lambda **k: (order.append("remove-temporaries"), (4, 0))[1], + ): + rc = invoke_main([ + "--mode", "full", + "--input", str(input_dir), + "--rules", str(rules_path), + "--rdf-storage-mode", "plain", + "--out", str(out_dir), + ]) + finally: + os.chdir(old_cwd) + + self.assertEqual(rc, 130) + # Containers first, then the volumes they were holding. + self.assertLess(order.index("kill-containers"), order.index("remove-volumes")) + + run_metrics_dir = latest_metrics_run_dir(out_dir / "run_metrics") + data = json.loads((run_metrics_dir / "interrupt-checkpoint.json").read_text()) + self.assertEqual(data["volumes"], {"removed": 2, "failed": 0}) + + def test_interrupt_stops_containers_before_deleting_their_inputs(self): + """Order is the bug: temps were deleted from under a live container. + + An interrupted HG004 run removed its .nt.gz while its validation + container kept running for three more days against a file that no + longer existed. Containers must be stopped first. + """ + with tempfile.TemporaryDirectory() as td: + tmp_path = Path(td) + input_dir, rules_path = prepare_inputs(tmp_path) + out_dir = tmp_path / "out" + order = [] + + def fake_kill(**kwargs): + order.append("kill-containers") + return 2, 0 + + def fake_cleanup(**kwargs): + order.append("remove-temporaries") + return 4, 0 + + old_cwd = os.getcwd() + os.chdir(tmp_path) + try: + with mock.patch.object(vcf_rdfizer, "check_docker", return_value=True), \ + mock.patch.object(vcf_rdfizer, "docker_image_exists", return_value=True), \ + mock.patch.object(vcf_rdfizer, "run_full_mode", side_effect=KeyboardInterrupt()), \ + mock.patch.object(vcf_rdfizer, "kill_run_containers", side_effect=fake_kill), \ + mock.patch.object( + vcf_rdfizer, "cleanup_interrupted_full_run", side_effect=fake_cleanup + ): + rc = invoke_main([ + "--mode", "full", + "--input", str(input_dir), + "--rules", str(rules_path), + "--rdf-storage-mode", "plain", + "--out", str(out_dir), + ]) + finally: + os.chdir(old_cwd) + + self.assertEqual(rc, 130) + self.assertEqual(order, ["kill-containers", "remove-temporaries"]) + + def test_interrupt_writes_a_resumable_checkpoint(self): + """A killed run still produced real results; record where it stopped.""" + with tempfile.TemporaryDirectory() as td: + tmp_path = Path(td) + input_dir, rules_path = prepare_inputs(tmp_path) + out_dir = tmp_path / "out" + + old_cwd = os.getcwd() + os.chdir(tmp_path) + try: + with mock.patch.object(vcf_rdfizer, "check_docker", return_value=True), \ + mock.patch.object(vcf_rdfizer, "docker_image_exists", return_value=True), \ + mock.patch.object(vcf_rdfizer, "run_full_mode", side_effect=KeyboardInterrupt()), \ + mock.patch.object( + vcf_rdfizer, "kill_run_containers", return_value=(3, 1) + ): + rc = invoke_main([ + "--mode", "full", + "--input", str(input_dir), + "--rules", str(rules_path), + "--rdf-storage-mode", "plain", + "--out", str(out_dir), + ]) + finally: + os.chdir(old_cwd) + + self.assertEqual(rc, 130) + run_metrics_dir = latest_metrics_run_dir(out_dir / "run_metrics") + checkpoint = run_metrics_dir / "interrupt-checkpoint.json" + self.assertTrue(checkpoint.exists(), "no checkpoint written") + data = json.loads(checkpoint.read_text()) + self.assertEqual(data["status"], "interrupted") + self.assertEqual(data["exit_code"], 130) + self.assertEqual(data["mode"], "full") + self.assertEqual(data["containers"], {"killed": 3, "failed": 1}) + self.assertIn("stages_completed", data) + self.assertIn("interrupted_at", data) + + def test_interrupt_still_exits_130_when_cleanup_itself_fails(self): + """A cleanup failure must not mask the interrupt or crash the wrapper.""" + with tempfile.TemporaryDirectory() as td: + tmp_path = Path(td) + input_dir, rules_path = prepare_inputs(tmp_path) + out_dir = tmp_path / "out" + + old_cwd = os.getcwd() + os.chdir(tmp_path) + try: + with mock.patch.object(vcf_rdfizer, "check_docker", return_value=True), \ + mock.patch.object(vcf_rdfizer, "docker_image_exists", return_value=True), \ + mock.patch.object(vcf_rdfizer, "run_full_mode", side_effect=KeyboardInterrupt()), \ + mock.patch.object( + vcf_rdfizer, "kill_run_containers", + side_effect=RuntimeError("docker daemon gone"), + ): + rc = invoke_main([ + "--mode", "full", + "--input", str(input_dir), + "--rules", str(rules_path), + "--rdf-storage-mode", "plain", + "--out", str(out_dir), + ]) + finally: + os.chdir(old_cwd) + + self.assertEqual(rc, 130) + run_metrics_dir = latest_metrics_run_dir(out_dir / "run_metrics") + log = (run_metrics_dir / "logs" / "progress.log").read_text() + self.assertIn("Interrupt cleanup error", log) + def test_main_compress_mode_none_skips_compression_commands(self): """Compression mode with method none performs no compression runs.""" with tempfile.TemporaryDirectory() as td: diff --git a/test/validation_fixtures.py b/test/validation_fixtures.py index 458ffc7..e879188 100644 --- a/test/validation_fixtures.py +++ b/test/validation_fixtures.py @@ -21,10 +21,9 @@ from dataclasses import dataclass, field from pathlib import Path -VCFR = "https://w3id.org/vcf-rdfizer/vocab#" +VCFC = "https://w3id.org/vcf-core/vocab#" RDF_TYPE = "http://www.w3.org/1999/02/22-rdf-syntax-ns#type" XSD_INTEGER = "http://www.w3.org/2001/XMLSchema#integer" -XSD_POSITIVE_INTEGER = "http://www.w3.org/2001/XMLSchema#positiveInteger" def _runner(): """Load the shipped validation runner (it lives outside the package path).""" @@ -44,6 +43,12 @@ def _runner(): FILE_DATE = "20260101" SOURCE_SOFTWARE = "vcf-rdfizer-fixture" REFERENCE_GENOME = "GRCh38" +#: The optional vcfc subclass the mapping's version sentinel resolves to for +#: this fixture's declared version. Derived rather than hard-coded so changing +#: FILE_FORMAT above changes the expected graph with it. +VERSION_FILE_CLASS = f"VCF{FILE_FORMAT.removeprefix('VCFv').replace('.', '')}File" +#: The bare version, as vcf_rdfizer_vocab.VCF_VERSIONS keys it. +FIXTURE_VERSION = FILE_FORMAT.removeprefix("VCFv") #: ``##`` meta-information lines, in file order. The ``#CHROM`` line is not one #: of these; ``header_count`` below counts only the ``##`` lines. @@ -163,7 +168,7 @@ def header_lines_tsv_text() -> str: def _literal(value: str) -> str: escaped = value.replace("\\", "\\\\").replace('"', '\\"') if value == ".": - return f'"{escaped}"^^<{VCFR}Null>' + return f'"{escaped}"^^<{VCFC}Null>' return f'"{escaped}"' @@ -184,46 +189,57 @@ def _info_entries(record: "FixtureRecord") -> list[tuple[str, str | None]]: def base_triples(*, include_qual: bool = True, include_info: bool = True) -> list[str]: """The triples RMLStreamer produces from `default_rules.ttl`. - ``include_qual`` is on because the shipped mapping now emits ``vcfr:qual``; - turning it off models the pre-fix mapping, which is how the harness shows - that dropping QUAL is now detected. ``include_info`` models the structured - INFO representation. + This has to track the shipped mapping, and the division it follows is one + rule: RML carries every field whose RDF datatype is the same for every row. + ID, ALT, QUAL, FILTER and INFO are therefore *not* here -- each may be the + VCF missing token, which the vocabulary requires as ``"."^^vcfc:Null``, so + the wrapper's own emitter produces them and ``build_graph`` appends them. + + ``include_qual`` and ``include_info`` are kept because the harness uses them + to model a graph missing those emitter outputs, which is how it shows that + dropping them is detected. """ file_uri = f"file://{SOURCE_FILE}" header_uri = f"{file_uri}#header" + columns_uri = f"{file_uri}#header/columns" out = [ - f"<{file_uri}> <{RDF_TYPE}> <{VCFR}VCFFile> .", - f"<{file_uri}> <{VCFR}fileFormat> {_literal(FILE_FORMAT)} .", - f"<{file_uri}> <{VCFR}sourceSoftware> {_literal(SOURCE_SOFTWARE)} .", - f"<{file_uri}> <{VCFR}referenceGenome> {_literal(REFERENCE_GENOME)} .", - - f"<{file_uri}> <{VCFR}hasHeader> <{header_uri}> .", - f"<{header_uri}> <{RDF_TYPE}> <{VCFR}VCFHeader> .", + f"<{file_uri}> <{RDF_TYPE}> <{VCFC}VCFFile> .", + # The mapping's version sentinel, resolved per input to the subclass for + # the version the file declares. The fixture declares VCFv4.2. + f"<{file_uri}> <{RDF_TYPE}> <{VCFC}{VERSION_FILE_CLASS}> .", + f"<{file_uri}> <{VCFC}fileFormat> {_literal(FILE_FORMAT)} .", + f"<{file_uri}> <{VCFC}sourceSoftware> {_literal(SOURCE_SOFTWARE)} .", + f"<{file_uri}> <{VCFC}referenceGenome> {_literal(REFERENCE_GENOME)} .", + + f"<{file_uri}> <{VCFC}hasHeader> <{header_uri}> .", + f"<{header_uri}> <{RDF_TYPE}> <{VCFC}VCFHeader> .", + # The mandatory #CHROM line. Its ordered sample columns are attached by + # the wrapper, because they live in the records TSV header. + f"<{header_uri}> <{VCFC}hasColumnHeader> <{columns_uri}> .", + f"<{columns_uri}> <{RDF_TYPE}> <{VCFC}ColumnHeaderLine> .", ] for index, (key, value) in enumerate(HEADER_LINES, start=1): line_uri = f"{file_uri}#header/line/{index}" out += [ - f"<{header_uri}> <{VCFR}hasHeaderLine> <{line_uri}> .", - f"<{line_uri}> <{RDF_TYPE}> <{VCFR}HeaderLine> .", - f"<{line_uri}> <{VCFR}headerKey> {_literal(key)} .", - f"<{line_uri}> <{VCFR}headerValue> {_literal(value)} .", + f"<{header_uri}> <{VCFC}hasHeaderLine> <{line_uri}> .", + f"<{line_uri}> <{RDF_TYPE}> <{VCFC}HeaderLine> .", + f"<{line_uri}> <{VCFC}headerKey> {_literal(key)} .", + f"<{line_uri}> <{VCFC}headerValue> {_literal(value)} .", + f'<{line_uri}> <{VCFC}lineIndex> "{index}"^^<{XSD_INTEGER}> .', ] for record in RECORDS: record_uri = f"{file_uri}#record/{record.row_id}" call_uri = f"{file_uri}#call/{record.row_id}" out += [ - f"<{file_uri}> <{VCFR}hasRecord> <{record_uri}> .", - f"<{record_uri}> <{RDF_TYPE}> <{VCFR}VCFRecord> .", - f"<{record_uri}> <{VCFR}chrom> {_literal(record.chrom)} .", - f'<{record_uri}> <{VCFR}pos> "{record.pos}"^^<{XSD_INTEGER}> .', - f"<{record_uri}> <{VCFR}recordId> {_literal(record.record_id)} .", - f"<{record_uri}> <{VCFR}ref> {_literal(record.ref)} .", - f"<{record_uri}> <{VCFR}alt> {_literal(record.alt)} .", - f"<{record_uri}> <{VCFR}hasCall> <{call_uri}> .", - f"<{call_uri}> <{RDF_TYPE}> <{VCFR}VariantCall> .", - f"<{call_uri}> <{VCFR}filter> {_literal(record.filter_value)} .", - f"<{call_uri}> <{VCFR}infoRaw> {_literal(record.info)} .", - f"<{call_uri}> <{VCFR}formatRaw> {_literal(':'.join(record.format_keys))} .", + f"<{file_uri}> <{VCFC}hasRecord> <{record_uri}> .", + f"<{record_uri}> <{RDF_TYPE}> <{VCFC}VCFRecord> .", + f"<{record_uri}> <{VCFC}chrom> {_literal(record.chrom)} .", + f'<{record_uri}> <{VCFC}pos> "{record.pos}"^^<{XSD_INTEGER}> .', + f"<{record_uri}> <{VCFC}ref> {_literal(record.ref)} .", + f'<{record_uri}> <{VCFC}recordIndex> "{record.row_id}"^^<{XSD_INTEGER}> .', + f"<{record_uri}> <{VCFC}hasCall> <{call_uri}> .", + f"<{call_uri}> <{RDF_TYPE}> <{VCFC}VariantCall> .", + f"<{call_uri}> <{VCFC}formatRaw> {_literal(':'.join(record.format_keys))} .", ] return out @@ -256,14 +272,21 @@ def build_graph( "\n".join(base_triples(include_qual=include_qual, include_info=include_info)) + "\n", encoding="utf-8", ) - if include_qual or include_info: - vcf_rdfizer.append_record_detail_rdf( - records_tsv, headers_tsv, graph, - emit_qual=include_qual, emit_info=include_info, + # The fixture declares a VCF version; the emitters follow it exactly as + # a real conversion does, so version-dependent behaviour is exercised + # rather than assumed. + version = vcf_rdfizer.vocab.VCF_VERSIONS[FIXTURE_VERSION] + vcf_rdfizer.append_record_detail_rdf( + records_tsv, headers_tsv, graph, + emit_qual=include_qual, emit_info=include_info, + emit_alleles=include_info, version=version, + progress_interval_records=0, + ) + if representation == "expanded": + vcf_rdfizer.append_expanded_sample_rdf( + records_tsv, graph, headers_tsv, version=version, progress_interval_records=0, ) - if representation == "expanded": - vcf_rdfizer.append_expanded_sample_rdf(records_tsv, graph, progress_interval_records=0) else: vcf_rdfizer.append_condensed_sample_rdf( records_tsv, headers_tsv, graph, progress_interval_records=0 @@ -348,42 +371,80 @@ def _format_shape() -> dict: } -def _header_shape() -> dict: - """Header counters the census needs, derived with the shipped parser.""" +def _declared_numbers(header_key: str) -> dict[str, str]: + """Map each declared INFO or FORMAT ID to its Number token.""" runner = _runner() - classes: Counter[str] = Counter() - filters = alts = contigs = described = 0 - attributes: Counter[str] = Counter() + numbers: dict[str, str] = {} for key, value in HEADER_LINES: - class_name = runner.HEADER_LINE_CLASSES.get(key.lower()) - if class_name is None: - continue - classes[class_name] += 1 - if key.lower() not in {"filter", "alt", "contig"}: + if key.upper() != header_key: continue fields = runner.parse_structured_header_fields(value) - if not (fields.get("ID") or "").strip(): - continue - if key.lower() == "contig": - contigs += 1 - for attribute in ("length", "md5", "assembly"): - if fields.get(attribute): - attributes[attribute] += 1 + ident = (fields.get("ID") or "").strip() + if ident: + numbers.setdefault(ident, fields.get("Number") or ".") + return numbers + + +def _declared_ids(header_key: str) -> set[str]: + """The set of IDs declared by one structured header key.""" + runner = _runner() + ids: set[str] = set() + for key, value in HEADER_LINES: + if key.upper() != header_key: continue - if key.lower() == "filter": - filters += 1 - else: - alts += 1 - if fields.get("Description"): - described += 1 - return { - "headerLineClassCounts": dict(classes), - "filterDefinitionCount": filters, - "altDefinitionCount": alts, - "contigCount": contigs, - "contigAttributeCounts": dict(attributes), - "describedDefinitionCount": described, - } + ident = (runner.parse_structured_header_fields(value).get("ID") or "").strip() + if ident: + ids.add(ident) + return ids + + +def _record_rows() -> list[list[str]]: + """The raw VCF data columns, as the runner's oracle sees them.""" + return [ + [record.chrom, str(record.pos), record.record_id, record.ref, record.alt, + record.qual, record.filter_value, record.info, + ":".join(record.format_keys), *record.sample_payloads] + for record in RECORDS + ] + + +def _header_shape() -> dict: + """Header counters, derived by the shipped runner rather than restated. + + The census asserts an exact inventory, so this has to cover every family the + header emitter produces. Delegating keeps the fixture and the container + oracle on one implementation. + """ + runner = _runner() + shape = dict(runner.emitted_header_counters(list(HEADER_LINES))) + shape["headerValueCount"] = sum(1 for _key, value in HEADER_LINES if value != "") + return shape + + +def _synthesized_definition_numbers() -> dict: + """Delegated to the runner, so the two oracles cannot disagree.""" + return dict(_runner().synthesized_definition_numbers( + _record_rows(), + declared_info=set(_declared_numbers("INFO")), + declared_format=set(_declared_numbers("FORMAT")), + )) + + +def _record_shape() -> dict: + """Allele, value-item, SV and genotype counters, derived the same way.""" + runner = _runner() + return dict(runner.emitted_record_counters( + _record_rows(), + list(SAMPLES), + version=runner.vocab.VCF_VERSIONS[FIXTURE_VERSION], + contig_ids=_declared_ids("CONTIG"), + alt_declaration_ids=_declared_ids("ALT"), + has_assembly_line=any( + key.lower() == "assembly" and value for key, value in HEADER_LINES + ), + info_numbers=_declared_numbers("INFO"), + format_numbers=_declared_numbers("FORMAT"), + )) def _info_shape() -> dict: @@ -486,7 +547,6 @@ def parser_summary( "singleAltRecordCount": single_alt, "q06EligibleSiteCount": q06_eligible, "headerLineCount": len(HEADER_LINES), - "headerValueCount": sum(1 for _key, value in HEADER_LINES if value != ""), **_format_shape(), **_info_shape(), "fileFormat": FILE_FORMAT, @@ -494,6 +554,8 @@ def parser_summary( "sourceSoftware": SOURCE_SOFTWARE, "fileDate": FILE_DATE, **_header_shape(), + **_record_shape(), + **_synthesized_definition_numbers(), "q07_file_metadata": { "fileFormat": FILE_FORMAT, "referenceGenome": REFERENCE_GENOME, diff --git a/test/validation_mutations.py b/test/validation_mutations.py index 21d8285..088737f 100644 --- a/test/validation_mutations.py +++ b/test/validation_mutations.py @@ -16,7 +16,7 @@ from dataclasses import dataclass, field from typing import Callable -VCFR = "https://w3id.org/vcf-rdfizer/vocab#" +VCFC = "https://w3id.org/vcf-core/vocab#" FILE = "file://fixture.vcf" @@ -78,6 +78,11 @@ def swap_objects(text: str, *, subject_a: str, subject_b: str, predicate: str) - new_object=objects[subject_a]) +def drop_line_containing(text: str, needle: str) -> str: + """Remove every line containing ``needle`` (an object IRI, typically).""" + return _join([line for line in _lines(text) if needle not in line]) + + def append_lines(text: str, *new: str) -> str: return _join(_lines(text) + list(new)) @@ -135,7 +140,7 @@ class Mutation: id="drop_pos", description="Remove one record's POS triple.", vcf_element="POS", - apply=lambda t: drop_matching(t, subject=f"{FILE}#record/1", predicate=f"{VCFR}pos"), + apply=lambda t: drop_matching(t, subject=f"{FILE}#record/1", predicate=f"{VCFC}pos"), expected_detected_by="preflight_record_cardinality", ), Mutation( @@ -143,7 +148,7 @@ class Mutation: description="Move a record into a different 1 Mb window.", vcf_element="POS", apply=lambda t: replace_object( - t, subject=f"{FILE}#record/1", predicate=f"{VCFR}pos", + t, subject=f"{FILE}#record/1", predicate=f"{VCFC}pos", new_object='"9100100"^^'), expected_detected_by="q01_record_density_1mb", ), @@ -153,7 +158,7 @@ class Mutation: vcf_element="record identity", apply=lambda t: swap_objects( t, subject_a=f"{FILE}#record/1", subject_b=f"{FILE}#record/2", - predicate=f"{VCFR}pos"), + predicate=f"{VCFC}pos"), expected_detected_by="q11_record_digest", ), Mutation( @@ -162,9 +167,9 @@ class Mutation: vcf_element="record identity", apply=lambda t: swap_objects( swap_objects(t, subject_a=f"{FILE}#record/1", subject_b=f"{FILE}#record/6", - predicate=f"{VCFR}ref"), + predicate=f"{VCFC}ref"), subject_a=f"{FILE}#record/1", subject_b=f"{FILE}#record/6", - predicate=f"{VCFR}alt"), + predicate=f"{VCFC}alt"), expected_detected_by="q11_record_digest", ), Mutation( @@ -172,7 +177,7 @@ class Mutation: description="Change one record's CHROM.", vcf_element="CHROM", apply=lambda t: replace_object( - t, subject=f"{FILE}#record/1", predicate=f"{VCFR}chrom", new_object='"X"'), + t, subject=f"{FILE}#record/1", predicate=f"{VCFC}chrom", new_object='"X"'), expected_detected_by="q01_record_density_1mb", ), Mutation( @@ -180,7 +185,7 @@ class Mutation: description="Change one record's ALT so its shape class changes.", vcf_element="ALT", apply=lambda t: replace_object( - t, subject=f"{FILE}#record/1", predicate=f"{VCFR}alt", new_object='"GGG"'), + t, subject=f"{FILE}#record/1", predicate=f"{VCFC}alt", new_object='"GGG"'), expected_detected_by="q02_variant_shape_counts", ), Mutation( @@ -188,7 +193,7 @@ class Mutation: description="Emit POS as a plain string instead of an integer.", vcf_element="POS datatype", apply=lambda t: replace_object( - t, subject=f"{FILE}#record/1", predicate=f"{VCFR}pos", new_object='"100"'), + t, subject=f"{FILE}#record/1", predicate=f"{VCFC}pos", new_object='"100"'), expected_detected_by="preflight_position_datatype", ), Mutation( @@ -204,7 +209,7 @@ class Mutation: description="Emit a record as a blank node instead of an IRI.", vcf_element="graph integrity", apply=lambda t: append_lines( - t, f'_:orphan <{VCFR}chrom> "20" .'), + t, f'_:orphan <{VCFC}chrom> "20" .'), expected_detected_by="preflight_blank_nodes", ), Mutation( @@ -212,7 +217,7 @@ class Mutation: description="Point a record at a blank node instead of its call resource.", vcf_element="graph integrity", apply=lambda t: replace_object( - t, subject=f"{FILE}#record/1", predicate=f"{VCFR}hasCall", + t, subject=f"{FILE}#record/1", predicate=f"{VCFC}hasCall", new_object="_:call1"), expected_detected_by="preflight_blank_nodes", ), @@ -221,7 +226,7 @@ class Mutation: description="Emit an empty literal where a value was expected.", vcf_element="graph integrity", apply=lambda t: replace_object( - t, subject=f"{FILE}#record/1", predicate=f"{VCFR}chrom", new_object='""'), + t, subject=f"{FILE}#record/1", predicate=f"{VCFC}chrom", new_object='""'), expected_detected_by="preflight_empty_values", ), Mutation( @@ -229,7 +234,7 @@ class Mutation: description="Emit a literal containing only whitespace.", vcf_element="graph integrity", apply=lambda t: replace_object( - t, subject=f"{FILE}#record/2", predicate=f"{VCFR}chrom", new_object='" "'), + t, subject=f"{FILE}#record/2", predicate=f"{VCFC}chrom", new_object='" "'), expected_detected_by="preflight_empty_values", ), Mutation( @@ -237,7 +242,7 @@ class Mutation: description="Emit the same statement twice, as a duplicated RDF part would.", vcf_element="graph integrity", apply=lambda t: append_lines( - t, f'<{FILE}#record/1> <{VCFR}chrom> "20" .'), + t, f'<{FILE}#record/1> <{VCFC}chrom> "20" .'), expected_detected_by="preflight_duplicate_triples", ), Mutation( @@ -252,14 +257,14 @@ class Mutation: description="Add a triple using a predicate the vocabulary does not define.", vcf_element="graph completeness", apply=lambda t: append_lines( - t, f'<{FILE}#record/1> <{VCFR}notARealProperty> "x" .'), + t, f'<{FILE}#record/1> <{VCFC}notARealProperty> "x" .'), expected_detected_by="q09_predicate_census (extra row)", ), Mutation( id="drop_filter", description="Remove one FILTER triple.", vcf_element="FILTER", - apply=lambda t: drop_matching(t, subject=f"{FILE}#call/1", predicate=f"{VCFR}filter"), + apply=lambda t: drop_matching(t, subject=f"{FILE}#call/1", predicate=f"{VCFC}filter"), expected_detected_by="q04_filter_distribution", ), Mutation( @@ -267,15 +272,15 @@ class Mutation: description="Change a FILTER value while keeping its broad status class.", vcf_element="FILTER", apply=lambda t: replace_object( - t, subject=f"{FILE}#call/3", predicate=f"{VCFR}filter", new_object='"q20"'), + t, subject=f"{FILE}#call/3", predicate=f"{VCFC}filter", new_object='"q20"'), expected_detected_by="q04_filter_distribution", ), Mutation( id="plain_dot_literal", - description="Emit a missing token as a plain '.' instead of '.'^^vcfr:Null.", + description="Emit a missing token as a plain '.' instead of '.'^^vcfc:Null.", vcf_element="missing-value policy", apply=lambda t: replace_object( - t, subject=f"{FILE}#record/2", predicate=f"{VCFR}recordId", new_object='"."'), + t, subject=f"{FILE}#record/2", predicate=f"{VCFC}recordId", new_object='"."'), expected_detected_by="preflight_missing_token_conformance (--strict-conformance)", ), Mutation( @@ -283,7 +288,7 @@ class Mutation: description="Change one sample's GT value.", vcf_element="FORMAT/GT", apply=lambda t: replace_object( - t, subject=f"{FILE}#sample/1/HG001/fmt/GT", predicate=f"{VCFR}fieldValue", + t, subject=f"{FILE}#sample/1/HG001/fmt/GT", predicate=f"{VCFC}fieldValue", new_object='"1/1"'), expected_detected_by="q05_sample_genotype_counts", representations=("expanded",), @@ -309,7 +314,7 @@ class Mutation: description="Change a DP value.", vcf_element="FORMAT/DP", apply=lambda t: replace_object( - t, subject=f"{FILE}#sample/1/HG001/fmt/DP", predicate=f"{VCFR}fieldValue", + t, subject=f"{FILE}#sample/1/HG001/fmt/DP", predicate=f"{VCFC}fieldValue", new_object='"999"'), expected_detected_by="q13_format_value_digest", representations=("expanded",), @@ -318,14 +323,14 @@ class Mutation: id="drop_qual", description="Remove a QUAL triple.", vcf_element="QUAL", - apply=lambda t: drop_matching(t, subject=f"{FILE}#call/1", predicate=f"{VCFR}qual"), + apply=lambda t: drop_matching(t, subject=f"{FILE}#call/1", predicate=f"{VCFC}qual"), expected_detected_by="q09_predicate_census", ), Mutation( id="drop_all_qual", description="Emit no QUAL at all, as the mapping did before it was fixed.", vcf_element="QUAL", - apply=lambda t: drop_matching(t, predicate=f"{VCFR}qual", limit=None), + apply=lambda t: drop_matching(t, predicate=f"{VCFC}qual", limit=None), expected_detected_by="q09_predicate_census", ), Mutation( @@ -333,14 +338,14 @@ class Mutation: description="Change a QUAL value.", vcf_element="QUAL", apply=lambda t: replace_object( - t, subject=f"{FILE}#call/1", predicate=f"{VCFR}qual", new_object='"0"'), + t, subject=f"{FILE}#call/1", predicate=f"{VCFC}qual", new_object='"0"'), expected_detected_by="q11_record_digest", ), Mutation( id="drop_info_value", description="Remove a structured INFO value node.", vcf_element="INFO", - apply=lambda t: drop_matching(t, predicate=f"{VCFR}hasInfoValue"), + apply=lambda t: drop_matching(t, predicate=f"{VCFC}hasInfoValue"), expected_detected_by="q09_predicate_census", ), Mutation( @@ -348,7 +353,7 @@ class Mutation: description="Change a structured INFO value.", vcf_element="INFO", apply=lambda t: replace_object( - t, subject=f"{FILE}#call/1/info/AC", predicate=f"{VCFR}fieldValue", + t, subject=f"{FILE}#call/1/info/AC", predicate=f"{VCFC}fieldValue", new_object='"99"'), expected_detected_by="q12_info_value_digest", ), @@ -357,14 +362,14 @@ class Mutation: description="Remove an INFO field declaration resource.", vcf_element="INFO declaration", apply=lambda t: drop_matching( - t, subject=f"{FILE}#header/line/6", predicate=f"{VCFR}fieldType"), + t, subject=f"{FILE}#header/line/6", predicate=f"{VCFC}fieldType"), expected_detected_by="q09_predicate_census", ), Mutation( id="retype_info_value", description="Drop the typed integer form of an INFO value.", vcf_element="INFO typing", - apply=lambda t: drop_matching(t, predicate=f"{VCFR}fieldValueInteger"), + apply=lambda t: drop_matching(t, predicate=f"{VCFC}fieldValueInteger"), expected_detected_by="q09_predicate_census", ), Mutation( @@ -372,7 +377,7 @@ class Mutation: description="Change a record's raw INFO string.", vcf_element="INFO", apply=lambda t: replace_object( - t, subject=f"{FILE}#call/1", predicate=f"{VCFR}infoRaw", new_object='"AC=99"'), + t, subject=f"{FILE}#call/1", predicate=f"{VCFC}infoRaw", new_object='"AC=99"'), expected_detected_by="q11_record_digest", ), Mutation( @@ -380,7 +385,7 @@ class Mutation: description="Change one sample's value inside a condensed FORMAT vector.", vcf_element="FORMAT/DP", apply=lambda t: replace_object( - t, subject=f"{FILE}#call/1/matrix/fmt/DP", predicate=f"{VCFR}encodedValues", + t, subject=f"{FILE}#call/1/matrix/fmt/DP", predicate=f"{VCFC}encodedValues", new_object='"999\t28"'), expected_detected_by="q13_format_value_digest", representations=("condensed",), @@ -405,28 +410,28 @@ class Mutation: id="drop_contig_attribute", description="Remove a contig's declared length.", vcf_element="contig declaration", - apply=lambda t: drop_matching(t, predicate=f"{VCFR}contigLength"), + apply=lambda t: drop_matching(t, predicate=f"{VCFC}contigLength"), expected_detected_by="q09_predicate_census", ), Mutation( id="drop_filter_definition", description="Remove a FILTER declaration's id.", vcf_element="FILTER declaration", - apply=lambda t: drop_matching(t, predicate=f"{VCFR}filterId"), + apply=lambda t: drop_matching(t, predicate=f"{VCFC}filterId"), expected_detected_by="q09_predicate_census", ), Mutation( id="drop_alt_definition", description="Remove a symbolic ALT declaration's id.", vcf_element="ALT declaration", - apply=lambda t: drop_matching(t, predicate=f"{VCFR}altId"), + apply=lambda t: drop_matching(t, predicate=f"{VCFC}altId"), expected_detected_by="q09_predicate_census", ), Mutation( id="drop_file_date", description="Remove the declared file date.", vcf_element="file metadata", - apply=lambda t: drop_matching(t, subject=FILE, predicate=f"{VCFR}fileDate"), + apply=lambda t: drop_matching(t, subject=FILE, predicate=f"{VCFC}fileDate"), expected_detected_by="q09_predicate_census", ), Mutation( @@ -434,7 +439,7 @@ class Mutation: description="Declare the wrong number of contigs.", vcf_element="contig declaration", apply=lambda t: replace_object( - t, subject=FILE, predicate=f"{VCFR}contigCount", + t, subject=FILE, predicate=f"{VCFC}contigCount", new_object='"99"^^'), expected_detected_by="header census", known_undetected=( @@ -447,23 +452,255 @@ class Mutation: description="Change the declared fileformat.", vcf_element="file metadata", apply=lambda t: replace_object( - t, subject=FILE, predicate=f"{VCFR}fileFormat", new_object='"VCFv9.9"'), + t, subject=FILE, predicate=f"{VCFC}fileFormat", new_object='"VCFv9.9"'), expected_detected_by="q07_file_metadata (Phase 1c)", ), Mutation( id="drop_reference_genome", description="Remove the declared reference genome.", vcf_element="file metadata", - apply=lambda t: drop_matching(t, subject=FILE, predicate=f"{VCFR}referenceGenome"), + apply=lambda t: drop_matching(t, subject=FILE, predicate=f"{VCFC}referenceGenome"), expected_detected_by="q07_file_metadata (Phase 1c)", ), + # ------------------------------------------------------------------ + # The layers the VCF Core vocabulary added. Each names one way the new + # emitters could regress; whether the suite catches it is measured, not + # asserted, so a `known_undetected` entry here is a recorded gap. + # ------------------------------------------------------------------ + Mutation( + id="drop_version_class", + description="Remove the vcfc:VCF4xFile class the version sentinel resolves to.", + vcf_element="VCF version", + apply=lambda t: drop_line_containing(t, f"<{VCFC}VCF42File>"), + expected_detected_by="q10_class_census", + ), + Mutation( + id="wrong_version_class", + description="Type the file with a VCF version it does not declare.", + vcf_element="VCF version", + apply=lambda t: t.replace(f"<{VCFC}VCF42File>", f"<{VCFC}VCF45File>"), + expected_detected_by="q10_class_census", + ), + Mutation( + id="drop_line_index", + description="Remove one header line's ordering index.", + vcf_element="header ordering", + apply=lambda t: drop_matching( + t, subject=f"{FILE}#header/line/1", predicate=f"{VCFC}lineIndex"), + expected_detected_by="q09_predicate_census", + ), + Mutation( + id="corrupt_record_index", + description="Renumber one record, breaking the file's record order.", + vcf_element="record ordering", + apply=lambda t: replace_object( + t, subject=f"{FILE}#record/2", predicate=f"{VCFC}recordIndex", + new_object='"99"^^'), + expected_detected_by="nothing yet", + known_undetected=( + "recordIndex is counted by q09 but its values are not compared. " + "The SPARQL SHACL profile checks uniqueness and ordering; the " + "aggregate queries do not." + ), + ), + Mutation( + id="drop_header_attribute", + description="Remove one vcfc:HeaderAttribute from a structured header line.", + vcf_element="header attributes", + apply=lambda t: drop_matching( + t, subject=f"{FILE}#header/line/6/attribute/1", limit=None), + expected_detected_by="q09_predicate_census, q10_class_census", + ), + Mutation( + id="corrupt_attribute_value", + description="Change one header attribute's value.", + vcf_element="header attributes", + apply=lambda t: replace_object( + t, subject=f"{FILE}#header/line/6/attribute/1", + predicate=f"{VCFC}attributeValue", new_object='"WRONG"'), + expected_detected_by="nothing yet", + known_undetected=( + "Attribute values are counted but not compared. Closing this needs " + "a digest over the structured header attributes, the way q11 covers " + "record fields." + ), + ), + Mutation( + id="drop_alt_allele", + description="Remove one ALT allele resource from a multi-allelic record.", + vcf_element="allele layer", + apply=lambda t: drop_matching( + t, subject=f"{FILE}#record/4/allele/2", limit=None), + expected_detected_by="q09_predicate_census, q10_class_census", + ), + Mutation( + id="corrupt_allele_value", + description="Change one parsed ALT allele's lexical value.", + vcf_element="allele layer", + apply=lambda t: replace_object( + t, subject=f"{FILE}#record/1/allele/1", predicate=f"{VCFC}alleleValue", + new_object='"TTT"'), + expected_detected_by="nothing yet", + known_undetected=( + "Allele values are counted but not compared against the raw ALT " + "column. A digest joining vcfc:alleleValue to vcfc:alt would close " + "it; the vocabulary's own consistency SHACL profile already checks " + "this agreement." + ), + ), + Mutation( + id="corrupt_allele_kind", + description="Misclassify an SNV allele as a symbolic one.", + vcf_element="allele layer", + apply=lambda t: replace_object( + t, subject=f"{FILE}#record/1/allele/1", predicate=f"{VCFC}alleleKind", + new_object=f"<{VCFC}SymbolicAllele>"), + expected_detected_by="nothing yet", + known_undetected=( + "alleleKind is counted but its value is not compared. q02 already " + "classifies variant shape from REF/ALT, so cross-checking the two " + "would close this without a new oracle." + ), + ), + Mutation( + id="drop_contig_link", + description="Unlink one record from the contig its CHROM names.", + vcf_element="allele layer", + apply=lambda t: drop_matching( + t, subject=f"{FILE}#record/1", predicate=f"{VCFC}chromosome"), + expected_detected_by="q09_predicate_census", + ), + Mutation( + id="drop_value_item", + description="Remove one parsed item of a Number=A INFO value.", + vcf_element="indexed values", + apply=lambda t: drop_matching( + t, subject=f"{FILE}#call/4/info/AC/value/1", limit=None), + expected_detected_by="q09_predicate_census, q10_class_census", + ), + Mutation( + id="corrupt_value_item_allele", + description="Point a Number=A value item at the wrong ALT allele.", + vcf_element="indexed values", + apply=lambda t: replace_object( + t, subject=f"{FILE}#call/4/info/AC/value/0", predicate=f"{VCFC}forAllele", + new_object=f"<{FILE}#record/4/allele/2>"), + expected_detected_by="nothing yet", + known_undetected=( + "forAllele is counted but the join is not checked. The vocabulary's " + "consistency profile checks item/raw agreement; an equivalent " + "aggregate here would need a per-item digest." + ), + ), + Mutation( + id="drop_sample_set", + description="Remove the file's reusable sample set.", + vcf_element="sample identity", + apply=lambda t: drop_matching(t, subject=f"{FILE}#samples", limit=None), + expected_detected_by="q09_predicate_census, q10_class_census", + ), + # The same corruption is detected in one profile and not the other, which + # is worth recording rather than smoothing over: in the condensed profile + # the ordinal is what associates a vector position with a sample, so + # breaking it moves genotypes between samples and the aggregates shift. In + # the expanded profile each SampleCall carries its own vcfc:sampleId, so the + # set ordinal is decorative and nothing downstream reads it. + Mutation( + id="corrupt_sample_index", + description="Give two samples the same ordinal in the sample set.", + vcf_element="sample identity", + apply=lambda t: replace_object( + t, subject=f"{FILE}#samples/HG002", predicate=f"{VCFC}sampleIndex", + new_object='"1"^^'), + expected_detected_by="q05_sample_genotype_counts, q06_ac_an_distribution", + representations=("condensed",), + ), + Mutation( + id="corrupt_sample_index_expanded", + description="The same ordinal corruption, where nothing decodes by position.", + vcf_element="sample identity", + apply=lambda t: replace_object( + t, subject=f"{FILE}#samples/HG002", predicate=f"{VCFC}sampleIndex", + new_object='"1"^^'), + expected_detected_by="nothing yet", + known_undetected=( + "In the expanded profile the sample set is a convenience: each " + "SampleCall carries vcfc:sampleId and vcfc:forSample, so no query " + "reads the ordinal. The SPARQL SHACL profile rejects duplicate " + "sampleIndex values within a file, so the shape layer covers it." + ), + representations=("expanded",), + ), + Mutation( + id="drop_genotype", + description="Remove one parsed genotype resource.", + vcf_element="genotype layer", + apply=lambda t: drop_matching( + t, subject=f"{FILE}#sample/1/HG001/genotype", limit=None), + expected_detected_by="q09_predicate_census, q10_class_census", + representations=("expanded",), + ), + Mutation( + id="flip_phasing_status", + description="Report a phased genotype as unphased.", + vcf_element="genotype layer", + apply=lambda t: replace_object( + t, subject=f"{FILE}#sample/1/HG001/genotype", + predicate=f"{VCFC}phasingStatus", new_object=f"<{VCFC}Unphased>"), + expected_detected_by="nothing yet", + known_undetected=( + "Phasing is counted but not compared. q05 classifies genotypes from " + "the raw GT string and normalizes '|' to '/', so it cannot see the " + "difference; cross-checking vcfc:phasingStatus against the raw " + "genotypeString would close it." + ), + representations=("expanded",), + ), + Mutation( + id="corrupt_called_allele", + description="Point a genotype allele call at the wrong allele.", + vcf_element="genotype layer", + apply=lambda t: replace_object( + t, subject=f"{FILE}#sample/1/HG001/genotype/call/0", + predicate=f"{VCFC}calledAllele", + new_object=f"<{FILE}#record/1/allele/1>"), + expected_detected_by="nothing yet", + known_undetected=( + "calledAllele is counted but the join is not checked. q05 reads the " + "raw GT, so a parsed call pointing at the wrong allele is invisible " + "to it." + ), + representations=("expanded",), + ), + Mutation( + id="duplicate_field_definition", + description="Emit a declared field definition's ID twice.", + vcf_element="field declarations", + apply=lambda t: append_lines( + t, f'<{FILE}#header/line/6> <{VCFC}fieldId> "AC" .'), + expected_detected_by="preflight_duplicate_triples", + ), + Mutation( + id="wrong_declaration_owner", + description="Move a declared definition's ID onto the wrong header line.", + vcf_element="field declarations", + apply=lambda t: replace_object( + t, subject=f"{FILE}#header/line/6", predicate=f"{VCFC}fieldId", + new_object='"DB"'), + expected_detected_by="nothing yet", + known_undetected=( + "Field IDs are counted but not tied to their header line. q08 " + "counts lines per '##' key and q09 counts fieldId triples, so " + "swapping which line carries which ID changes neither." + ), + ), Mutation( id="wrong_representation_profile", description="Declare the wrong sample representation profile.", vcf_element="representation profile", apply=lambda t: replace_object( - t, subject=FILE, predicate=f"{VCFR}representationProfile", - new_object=f"<{VCFR}ExpandedRepresentation>"), + t, subject=FILE, predicate=f"{VCFC}representationProfile", + new_object=f"<{VCFC}ExpandedRepresentation>"), expected_detected_by="preflight_representation_profile", representations=("condensed",), ), diff --git a/vcf-rdfizer-v1.0.0.tar.gz b/vcf-rdfizer-v1.0.0.tar.gz deleted file mode 100644 index e5d3769..0000000 Binary files a/vcf-rdfizer-v1.0.0.tar.gz and /dev/null differ diff --git a/vcf-rdfizer-v1.1.0.tar.gz b/vcf-rdfizer-v1.1.0.tar.gz deleted file mode 100644 index 969bd4f..0000000 Binary files a/vcf-rdfizer-v1.1.0.tar.gz and /dev/null differ diff --git a/vcf_rdfizer.py b/vcf_rdfizer.py index 3a15403..91b56e3 100644 --- a/vcf_rdfizer.py +++ b/vcf_rdfizer.py @@ -68,13 +68,22 @@ from dataclasses import dataclass from datetime import datetime from pathlib import Path -from urllib.parse import quote_plus try: import vcf_rdfizer_gzip except ImportError: # pragma: no cover - shipped alongside this module vcf_rdfizer_gzip = None +import vcf_rdfizer_vocab as vocab +from vcf_rdfizer_vocab import ( + CONTIG_ATTRIBUTES, + HEADER_LINE_CLASSES, + VCFC_NAMESPACE, + VCF_VALUE_TYPE_INDIVIDUALS, + file_date_object, + parse_info_entries, +) + try: from rich.console import Console from rich.progress import ( @@ -119,6 +128,12 @@ "timestamp", "output_name", "output_dir", + # The VCF specification version this input was converted under, and how it + # was chosen. Benchmarks group by these: the version selects the SHACL + # overlay, the tuple semantics and which FORMAT families exist, so a run's + # cost and graph size are not comparable across versions without them. + "vcf_version", + "vcf_version_source", "exit_code_java", "wall_seconds_java", "user_seconds_java", @@ -271,6 +286,12 @@ HDT_STRATEGY_CHOICES = {"auto", "single", "partitioned"} DEFAULT_HDT_STRATEGY = "auto" RDF_STORAGE_MODES = {"space-optimized", "plain"} +#: Full mode defaults to the space-optimized aggregate. Benchmarking showed it +#: reaches the same triples as `plain` at a much smaller peak workspace +#: footprint and no practically important compute penalty, so the low-disk +#: path is the one a caller gets without having to ask. `plain` remains +#: available and is required for `--hdt-strategy single`. +DEFAULT_RDF_STORAGE_MODE = "space-optimized" DEFAULT_CHUNK_TARGET_BYTES = 512 * 1024 * 1024 DEFAULT_CHUNK_MIN_BYTES = 128 * 1024 * 1024 DEFAULT_CHUNK_MAX_BYTES = 1024 * 1024 * 1024 @@ -303,28 +324,31 @@ ) CANONICAL_SAMPLE_RULE_FRAGMENTS = ( 'rr:template "file://{SOURCE_FILE}#call/{ROW_ID}"', - "rr:predicate vcfr:hasSampleCall", + "rr:predicate vcfc:hasSampleCall", 'rr:template "file://{SOURCE_FILE}#sample/{ROW_ID}/{SAMPLE_URI_ID}"', - "rr:class vcfr:SampleCall", - "rr:predicate vcfr:sampleId", + "rr:class vcfc:SampleCall", + "rr:predicate vcfc:sampleId", 'rml:reference "SAMPLE_ID"', - "rr:predicate vcfr:hasFormatValue", + "rr:predicate vcfc:hasFormatValue", 'rr:template "file://{SOURCE_FILE}#sample/{ROW_ID}/{SAMPLE_URI_ID}/fmt/{FORMAT_KEY}"', - "rr:class vcfr:FormatFieldValue", - "rr:predicate vcfr:fieldValue", + "rr:class vcfc:FormatFieldValue", + "rr:predicate vcfc:fieldValue", 'rml:reference "FORMAT_VALUE"', ) -VCFR_NAMESPACE = "https://w3id.org/vcf-rdfizer/vocab#" -RDF_TYPE_URI = "http://www.w3.org/1999/02/22-rdf-syntax-ns#type" -XSD_POSITIVE_INTEGER_URI = "http://www.w3.org/2001/XMLSchema#positiveInteger" +RDF_TYPE_URI = vocab.RDF_TYPE_URI +#: Ordinals are serialized as xsd:integer, not xsd:positiveInteger: the SHACL +#: profiles constrain every one of them with sh:datatype xsd:integer, and that +#: constraint compares the datatype IRI exactly. See vcf_rdfizer_vocab. +XSD_ORDINAL_URI = vocab.XSD_ORDINAL_URI SAMPLE_RDF_BUFFER_BYTES = 8 * 1024 * 1024 # An N-Triples subject is always an IRI reference or a blank node label, so a # line starting with one of these bytes and ending in " ." is a statement. _NTRIPLES_SUBJECT_STARTS = (b"<", b"_") SAMPLE_REPRESENTATION_CHOICES = {"expanded", "condensed"} # How the INFO column is represented. "raw" is the historical behaviour (an -# opaque vcfr:infoRaw string). "structured" additionally emits one -# vcfr:InfoFieldValue per record and key, which is what makes INFO queryable. +# opaque vcfc:infoRaw string). "structured" additionally emits one +# vcfc:InfoFieldValue per record and key, plus the allele layer and the +# vcfc:FieldValueItem decomposition, which is what makes INFO queryable. INFO_REPRESENTATION_CHOICES = ("raw", "structured") DEFAULT_INFO_REPRESENTATION = "structured" # Semantic validation accepts any artifact the pipeline can produce. Anything @@ -377,7 +401,7 @@ def parse_validation_engines(raw: str) -> list[str]: # This is an internal rules-compatibility value, not a third public # representation. It means that custom helper TSV rows must be materialized. SAMPLE_HELPER_STRATEGY_MATERIALIZED = "expanded" -METRICS_LAYOUT_VERSION = 2 +METRICS_LAYOUT_VERSION = 3 # --------------------------------------------------------------------------- @@ -682,11 +706,19 @@ def __init__(self, log_path: Path): self._handle = self.log_path.open("a", encoding="utf-8") self.intermediate_paths: set[Path] = set() self.raw_rdf_paths: set[Path] = set() + #: Every mark, kept in memory so an interrupt can write a checkpoint + #: without re-reading and re-parsing its own log. + self.events: list[dict] = [] + #: Marks that announced a finished stage, in order. + self.completed_stages: list[str] = [] def mark(self, message: str): timestamp = datetime.now().strftime("%Y-%m-%dT%H:%M:%S") self._handle.write(f"[{timestamp}] {message}\n") self._handle.flush() + self.events.append({"timestamp": timestamp, "message": message}) + if "completed" in message: + self.completed_stages.append(message) def track_intermediate(self, path: Path): self.intermediate_paths.add(path) @@ -811,9 +843,35 @@ def docker_cmd_prefix(*, use_sudo: bool | None = None): return ["sudo", "docker"] if use_sudo else ["docker"] +#: Label key stamped on every container this wrapper starts, so an interrupted +#: run can find and stop its own containers -- and only its own. +DOCKER_RUN_LABEL_KEY = "vcf-rdfizer.run" +#: Unique per wrapper process. Two runs on one host must never kill each +#: other's containers, so the pid is part of the value rather than the run id +#: alone (two runs started in the same second share a run id). +_DOCKER_RUN_LABEL_VALUE = f"{os.getpid()}-{int(time.time())}" + + +def set_docker_run_label(run_id: str) -> None: + """Fold the run id into the container label once it is known. + + Purely for legibility: `docker ps --filter label=vcf-rdfizer.run` should + show which run a stray container belongs to. + """ + global _DOCKER_RUN_LABEL_VALUE + _DOCKER_RUN_LABEL_VALUE = f"{run_id}-{os.getpid()}" + + +def docker_run_label_filter() -> str: + return f"label={DOCKER_RUN_LABEL_KEY}={_DOCKER_RUN_LABEL_VALUE}" + + def docker_run_base(*, as_user: bool = True): """Return base args for `docker run`, optionally mapped to host UID/GID.""" - base = [*docker_cmd_prefix(), "run", "--rm"] + base = [ + *docker_cmd_prefix(), "run", "--rm", + "--label", f"{DOCKER_RUN_LABEL_KEY}={_DOCKER_RUN_LABEL_VALUE}", + ] if not as_user: return base as_user = os.environ.get("VCF_RDFIZER_DOCKER_AS_USER", "1").strip().lower() @@ -1606,6 +1664,155 @@ def remove_path_with_docker_fallback( return True +#: Temporary Docker volumes this process created and has not yet removed. +#: +#: Each volume is already removed in its own `finally`, which is enough on the +#: normal path. On an interrupt it is not: that `finally` runs as the exception +#: unwinds, which is *before* the handler kills the containers, and Docker +#: refuses to remove a volume a container still holds. The removal failed +#: there, warned, and nothing ever retried -- leaking ~700 MB per interrupted +#: run. Registering them here lets the interrupt path sweep once the +#: containers are actually gone. +_TEMP_DOCKER_VOLUMES: set[str] = set() + + +def register_temp_volume(name: str) -> None: + _TEMP_DOCKER_VOLUMES.add(name) + + +def remove_temp_volume(name: str, *, attempts: int = 3, delay: float = 1.0) -> bool: + """Remove one temporary volume, retrying while a container still holds it. + + A container killed a moment ago may not have released the volume yet, so a + single attempt is not enough on the interrupt path. + """ + for attempt in range(max(1, attempts)): + if run([*docker_cmd_prefix(), "volume", "rm", "-f", name]) == 0: + _TEMP_DOCKER_VOLUMES.discard(name) + return True + if attempt + 1 < attempts: + time.sleep(delay) + return False + + +def remove_tracked_volumes(*, run_tracker: RunTracker | None = None) -> tuple[int, int]: + """Remove every temporary volume still outstanding. Returns ``(removed, failed)``. + + Must run *after* the containers are stopped. Best effort by construction: + an interrupt path must never raise. + """ + removed = 0 + failed = 0 + for name in sorted(_TEMP_DOCKER_VOLUMES): + try: + if remove_temp_volume(name): + removed += 1 + else: + failed += 1 + except Exception: # noqa: BLE001 - cleanup must not raise + failed += 1 + if (removed or failed) and run_tracker is not None: + run_tracker.mark( + f"Interrupt cleanup: temporary volumes removed={removed}, failed={failed}" + ) + return removed, failed + + +def kill_run_containers(*, run_tracker: RunTracker | None = None) -> tuple[int, int]: + """Stop every container this wrapper started. Returns ``(killed, failed)``. + + Killing the wrapper does not kill its containers. ``docker run`` forwards a + SIGINT it receives itself, but a wrapper that dies without forwarding -- + SIGTERM, a closed terminal, an exception unwinding past the wait -- leaves + the container running and reparented to init. One such orphan ran for three + days after its run had "finished", holding 17 GB and a core the whole time. + + Worse, cleanup then deleted files the orphan was still reading, so the + container spent those days working against inputs that no longer existed. + That is why this runs *before* any temporary file is removed. + + Best effort by construction: an interrupt path must never raise. Every + failure is counted and reported, never propagated. + """ + killed = 0 + failed = 0 + try: + listing = subprocess.run( + [*docker_cmd_prefix(), "ps", "-q", "--filter", docker_run_label_filter()], + check=False, capture_output=True, text=True, timeout=30, + ) + except (OSError, subprocess.SubprocessError) as error: + if run_tracker is not None: + run_tracker.mark(f"Interrupt cleanup: could not list containers ({error})") + return 0, 0 + + container_ids = [line.strip() for line in listing.stdout.splitlines() if line.strip()] + if not container_ids: + return 0, 0 + + for container_id in container_ids: + try: + result = subprocess.run( + [*docker_cmd_prefix(), "kill", container_id], + check=False, capture_output=True, text=True, timeout=60, + ) + if result.returncode == 0: + killed += 1 + else: + failed += 1 + except (OSError, subprocess.SubprocessError): + failed += 1 + + if run_tracker is not None: + run_tracker.mark( + f"Interrupt cleanup: stopped containers killed={killed}, failed={failed}" + ) + return killed, failed + + +def write_interrupt_checkpoint( + *, + run_tracker: RunTracker, + metrics_dir: Path, + run_id: str, + mode: str, + containers_killed: int, + containers_failed: int, + removed: int | None, + failed: int | None, + volumes_removed: int = 0, + volumes_failed: int = 0, +) -> Path | None: + """Record where an interrupted run stopped, so it can be resumed or read. + + A run that is killed mid-flight still produced real measurements for every + stage it finished. Without a checkpoint those are only recoverable by + reading the progress log by eye, and nothing records *why* the run ended. + """ + checkpoint = { + "run_id": run_id, + "mode": mode, + "status": "interrupted", + "exit_code": 130, + "interrupted_at": datetime.now().strftime("%Y-%m-%dT%H:%M:%S"), + "containers": {"killed": containers_killed, "failed": containers_failed}, + "volumes": {"removed": volumes_removed, "failed": volumes_failed}, + "temporaries": {"removed": removed, "failed": failed}, + "stages_completed": list(run_tracker.completed_stages), + "last_event": run_tracker.events[-1] if run_tracker.events else None, + "tracked_intermediates": sorted(str(p) for p in run_tracker.intermediate_paths), + } + try: + path = metrics_dir / "interrupt-checkpoint.json" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(checkpoint, indent=2) + "\n", encoding="utf-8") + run_tracker.mark(f"Interrupt checkpoint written: {path}") + return path + except OSError as error: + run_tracker.mark(f"Interrupt checkpoint could not be written: {error}") + return None + + def cleanup_interrupted_full_run( *, run_tracker: RunTracker, @@ -1873,45 +2080,19 @@ def _sample_id_to_uri_id(sample_id: str, fallback_index: int) -> str: return candidate or f"sample_{fallback_index}" -# Characters that ``quote_plus(..., safe="*-._")`` leaves untouched. Sample -# ids, FORMAT keys, and numeric row ids are almost always drawn from this set, -# so the fast path below skips percent-encoding entirely for them. -_URI_COMPONENT_PASSTHROUGH_RE = re.compile(r"[A-Za-z0-9*\-._]*\Z") -# Characters that require N-Triples escaping. VCF genotype/FORMAT payloads -# essentially never contain them, so the common case avoids five str.replace -# scans of the value. -_NTRIPLES_ESCAPE_RE = re.compile(r'[\\"\n\r\t]') - - -def _rml_uri_component(value: str) -> str: - """Match RMLStreamer's Java URLEncoder-based template substitution.""" - if _URI_COMPONENT_PASSTHROUGH_RE.match(value) is not None: - return value - encoded = quote_plus(value, safe="*-._", encoding="utf-8", errors="strict") - # urllib follows current RFC rules and always leaves '~' unescaped, whereas - # java.net.URLEncoder (used by RMLStreamer 2.5.0) encodes it. - return encoded.replace("+", "%20").replace("~", "%7E") - - -def _ntriples_string_literal(value: str) -> str: - """Serialize an RDF 1.1 plain/xsd:string literal for N-Triples.""" - if _NTRIPLES_ESCAPE_RE.search(value) is None: - return f'"{value}"' - escaped = ( - value.replace("\\", "\\\\") - .replace('"', '\\"') - .replace("\n", "\\n") - .replace("\r", "\\r") - .replace("\t", "\\t") - ) - return f'"{escaped}"' +# The serialization helpers live in vcf_rdfizer_vocab so the parsers there can +# use them; these names stay because the emitters below and the unit tests +# reference them. +_rml_uri_component = vocab.rml_uri_component +_ntriples_string_literal = vocab.ntriples_string_literal +_ntriples_literal = vocab.ntriples_literal +_ntriples_typed_literal = vocab.ntriples_typed_literal +_ordinal_literal = vocab.ordinal_literal -def _ntriples_literal(value: str) -> str: - literal = _ntriples_string_literal(value) - if value == ".": - literal += f"^^<{VCFR_NAMESPACE}Null>" - return literal +def _vocab(local_name: str) -> str: + """Absolute IRI of one VCF Core term.""" + return f"{VCFC_NAMESPACE}{local_name}" @dataclass(frozen=True) @@ -1929,12 +2110,20 @@ class ParsedSampleRecord: source_file: str row_id: str - #: Raw QUAL and INFO columns, needed by the record-detail emitter. + #: Raw fixed-field columns the record-detail emitter needs: the three whose + #: datatype depends on the row (ID, QUAL, FILTER), the INFO payload, and + #: CHROM/REF/ALT, which the allele layer decomposes. qual: str info: str format_keys: tuple[str, ...] sample_payloads: tuple[str, ...] sample_values: tuple[tuple[str, ...], ...] + chrom: str = "" + pos: str = "" + record_id: str = "" + ref: str = "" + alt: str = "" + filter_value: str = "" class SampleRecordStream: @@ -2020,7 +2209,13 @@ def _parse_row(self, row: list[str]) -> ParsedSampleRecord: f"'{source_file}'" ) row_id = row[1] if len(row) > 1 else "" + chrom_raw = row[2] if len(row) > 2 else "" + pos_raw = row[3] if len(row) > 3 else "" + record_id_raw = row[4] if len(row) > 4 else "" + ref_raw = row[5] if len(row) > 5 else "" + alt_raw = row[6] if len(row) > 6 else "" qual_raw = row[7] if len(row) > 7 else "" + filter_raw = row[8] if len(row) > 8 else "" info_raw = row[9] if len(row) > 9 else "" format_raw = row[10] if len(row) > 10 else "" samples_raw = row[-1] if len(row) >= 12 else "" @@ -2068,6 +2263,12 @@ def _parse_row(self, row: list[str]) -> ParsedSampleRecord: format_keys=format_keys, sample_payloads=tuple(sample_payloads), sample_values=sample_values, + chrom=chrom_raw, + pos=pos_raw, + record_id=record_id_raw, + ref=ref_raw, + alt=alt_raw, + filter_value=filter_raw, ) @@ -2111,22 +2312,272 @@ def emit(line: str): raise +def _emit_sample_set(emit, *, file_uri: str, columns, stats: dict) -> str: + """Emit the file's ordered sample columns once, and return the set IRI. + + ``vcfc:SampleSet`` and ``vcfc:VCFSample`` are file-scoped, not profile- + scoped: the condensed profile needs the set to interpret its vectors, and + the expanded profile uses it to give each ``vcfc:SampleCall`` a reusable + identity through ``vcfc:forSample`` instead of relying on repeated + ``vcfc:sampleId`` literals. Emitting it in both profiles does not mix them, + because what the SPARQL profile forbids mixing is per-sample calls and + cohort matrices, not the sample set itself. + + The ``#CHROM`` column header line also gains ``vcfc:hasGenotypeColumns`` + here, because the sample columns live in the records TSV header rather than + in any row of the header-lines table. + """ + if not columns: + # A sites-only VCF declares no sample columns. vcfc:SampleSetShape + # requires at least one vcfc:hasSample, so an empty set would be worse + # than no set at all; vcfc:hasSampleSet is optional on the file. + return "" + sample_set_uri = f"{file_uri}#samples" + column_header_uri = f"{file_uri}#header/columns" + emit(f"<{file_uri}> <{_vocab('hasSampleSet')}> <{sample_set_uri}> .\n") + emit(f"<{sample_set_uri}> <{RDF_TYPE_URI}> <{_vocab('SampleSet')}> .\n") + for column in columns: + sample_uri = f"{sample_set_uri}/{_rml_uri_component(column.uri_id)}" + emit(f"<{sample_set_uri}> <{_vocab('hasSample')}> <{sample_uri}> .\n") + emit(f"<{sample_uri}> <{RDF_TYPE_URI}> <{_vocab('VCFSample')}> .\n") + emit( + f"<{sample_uri}> <{_vocab('sampleName')}> " + f"{_ntriples_string_literal(column.sample_id)} .\n" + ) + emit( + f"<{sample_uri}> <{_vocab('sampleIndex')}> " + f"{_ordinal_literal(column.index)} .\n" + ) + emit( + f"<{column_header_uri}> <{_vocab('hasGenotypeColumns')}> " + f"<{sample_uri}> .\n" + ) + stats["samples"] += 1 + return sample_set_uri + + +def _emit_format_field_definition( + emit, + *, + definition: "FormatDefinition", + field_id: str, + emitted: set, + stats: dict | None = None, +) -> str: + """Emit one ``vcfc:FormatFieldDefinition`` the first time its key is seen.""" + if definition.uri in emitted: + return definition.uri + emitted.add(definition.uri) + emit( + f"<{definition.uri}> <{RDF_TYPE_URI}> " + f"<{_vocab('FormatFieldDefinition')}> .\n" + ) + # A key declared by a '##FORMAT' line already has its ID/Number/Type/ + # Description from the header emitter, at this very IRI; emitting them + # again would duplicate every one of those triples. + if definition.synthesized: + _emit_field_definition_attributes( + emit, + definition.uri, + field_id=field_id, + number=definition.field_number, + # A FORMAT field may not be a Flag, so an undeclared key defaults + # to String rather than inheriting the INFO fallback. + value_type=definition.value_type or "String", + description=definition.description, + ) + if stats is not None: + stats["format_definitions"] += 1 + return definition.uri + + +def _synthesized_format_definition(file_uri: str, format_key: str, component: str) -> "FormatDefinition": + """Build a definition for a FORMAT key the header never declared.""" + return FormatDefinition( + uri=f"{file_uri}#header/format/{component}", + field_number=".", + description=f"Synthesized definition for undeclared FORMAT key {format_key}", + value_type="String", + synthesized=True, + ) + + +def _emit_genotype( + emit, + *, + sample_uri: str, + record_uri: str, + value: str, + alt_count: int, + stats: dict, +) -> None: + """Emit the parsed GT of one sample call. + + A GT that is absent or outside the vcfc:GenotypeString lexical space + produces no Genotype resource: the value is still present as the FORMAT + field's ``vcfc:fieldValue``, and inventing a Genotype for it would create a + resource that fails ``vcfc:GenotypeShape``. + """ + parsed = vocab.parse_genotype(value) + if parsed is None: + return + genotype_uri = f"{sample_uri}/genotype" + emit(f"<{sample_uri}> <{_vocab('hasGenotype')}> <{genotype_uri}> .\n") + emit(f"<{genotype_uri}> <{RDF_TYPE_URI}> <{_vocab('Genotype')}> .\n") + emit( + f"<{genotype_uri}> <{_vocab('genotypeString')}> " + f"{_ntriples_typed_literal(parsed.genotype_string, vocab.GENOTYPE_STRING_DATATYPE_URI)} .\n" + ) + emit( + f"<{genotype_uri}> <{_vocab('ploidy')}> " + f"{_ordinal_literal(parsed.ploidy)} .\n" + ) + emit( + f"<{genotype_uri}> <{_vocab('phasingStatus')}> " + f"<{_vocab('Phased' if parsed.is_phased else 'Unphased')}> .\n" + ) + for call in parsed.calls: + call_uri = f"{genotype_uri}/call/{call.call_index}" + emit(f"<{genotype_uri}> <{_vocab('hasAlleleCall')}> <{call_uri}> .\n") + emit( + f"<{call_uri}> <{RDF_TYPE_URI}> " + f"<{_vocab('GenotypeAlleleCall')}> .\n" + ) + emit( + f"<{call_uri}> <{_vocab('callIndex')}> " + f"{_ordinal_literal(call.call_index)} .\n" + ) + emit( + f"<{call_uri}> <{_vocab('isNoCall')}> " + f'"{"true" if call.is_no_call else "false"}"^^<{XSD_BOOLEAN_URI}> .\n' + ) + if call.allele_index is not None and call.allele_index <= alt_count: + emit( + f"<{call_uri}> <{_vocab('calledAllele')}> " + f"<{record_uri}/allele/{call.allele_index}> .\n" + ) + stats["genotype_calls"] += 1 + stats["genotypes"] += 1 + + +def _emit_phase_set(emit, *, sample_uri: str, phase_fields: dict, stats: dict) -> None: + """Emit the PS/PSL/PSO/PSQ phase-set carrier of one sample call.""" + identifier = phase_fields.get("PS") + name = phase_fields.get("PSL") + if vocab.is_missing(identifier) and vocab.is_missing(name): + return + phase_set_uri = f"{sample_uri}/phaseset" + genotype_uri = f"{sample_uri}/genotype" + emit(f"<{genotype_uri}> <{_vocab('inPhaseSet')}> <{phase_set_uri}> .\n") + emit(f"<{phase_set_uri}> <{RDF_TYPE_URI}> <{_vocab('PhaseSet')}> .\n") + if not vocab.is_missing(identifier): + emit( + f"<{phase_set_uri}> <{_vocab('phaseSetId')}> " + f"{_ntriples_string_literal(identifier)} .\n" + ) + if not vocab.is_missing(name): + emit( + f"<{phase_set_uri}> <{_vocab('phaseSetName')}> " + f"{_ntriples_string_literal(name)} .\n" + ) + ordinal = phase_fields.get("PSO") + if not vocab.is_missing(ordinal) and ordinal.lstrip("-").isdigit(): + emit( + f"<{phase_set_uri}> <{_vocab('phaseSetOrdinal')}> " + f"{_ntriples_typed_literal(ordinal, XSD_INTEGER_URI)} .\n" + ) + quality = phase_fields.get("PSQ") + if not vocab.is_missing(quality): + emit( + f"<{phase_set_uri}> <{_vocab('phaseSetQuality')}> " + f"{_ntriples_literal(quality)} .\n" + ) + stats["phase_sets"] += 1 + + +def _emit_local_allele_set( + emit, *, sample_uri: str, record_uri: str, value: str, alt_count: int, stats: dict +) -> None: + """Emit the LAA local-allele subset declared for one sample.""" + indices = [index for index in vocab.parse_local_allele_indices(value) if index <= alt_count] + if not indices: + return + set_uri = f"{sample_uri}/localalleles" + genotype_uri = f"{sample_uri}/genotype" + emit(f"<{genotype_uri}> <{_vocab('hasLocalAlleleSet')}> <{set_uri}> .\n") + emit(f"<{set_uri}> <{RDF_TYPE_URI}> <{_vocab('LocalAlleleSet')}> .\n") + for local_index, allele_index in enumerate(indices, start=1): + allele_uri = f"{record_uri}/allele/{allele_index}" + emit(f"<{set_uri}> <{_vocab('hasLocalAllele')}> <{allele_uri}> .\n") + emit( + f"<{allele_uri}> <{_vocab('localAlleleIndex')}> " + f"{_ordinal_literal(local_index)} .\n" + ) + stats["local_allele_sets"] += 1 + + +def _emit_base_modification( + emit, *, sample_uri: str, modification, value: str, stats: dict +) -> None: + """Emit the vcfc:BaseModification carrier for one M/DPM/ADM FORMAT key.""" + if vocab.is_missing(value): + return + modification_uri = f"{sample_uri}/basemod/{_rml_uri_component(modification.key)}" + emit( + f"<{modification_uri}> <{RDF_TYPE_URI}> " + f"<{_vocab('BaseModification')}> .\n" + ) + emit( + f"<{modification_uri}> <{_vocab('modifiedResidue')}> " + f"<{modification.residue_uri}> .\n" + ) + # VCF 4.5 gives the base-modification FORMAT families a per-base offset; + # without an explicit one, position 0 is the modified base named by the key. + emit( + f"<{modification_uri}> <{_vocab('modifiedBaseOffset')}> " + f"{_ordinal_literal(0)} .\n" + ) + emit( + f"<{modification_uri}> <{_vocab(modification.value_property)}> " + f"{_ntriples_literal(value)} .\n" + ) + stats["base_modifications"] += 1 + + def append_expanded_sample_rdf( records_tsv: Path, rdf_path: Path, + header_lines_tsv: Path | None = None, *, + version: "vocab.VCFVersion | None" = None, progress_interval_records: int = 10_000, ) -> dict: - """Append the expanded SampleCall/FormatFieldValue representation. - - This produces the same canonical SampleCall and FormatFieldValue triples as - the default RML maps without materializing V*S and V*S*F helper TSV rows. + """Append the expanded per-sample representation. + + Produces the canonical SampleCall and FormatFieldValue resources without + materializing the V*S and V*S*F helper TSV rows, and adds the genotype layer + the vocabulary defines for this profile: the parsed GT of each sample as a + ``vcfc:Genotype`` with one ``vcfc:GenotypeAlleleCall`` per position, the + FT sample filter, the PS/PSL phase set, the LAA local-allele subset, the + copy-number and haplotype fields, and the M/DPM/ADM base modifications. + + The genotype layer is expanded-only by design. Decomposing GT per sample is + exactly the per-sample materialization the condensed profile exists to + avoid, so the condensed emitter keeps those values inside their vectors. """ stats = { "representation": "expanded", "records": 0, + "samples": 0, "sample_calls": 0, "format_values": 0, + "format_definitions": 0, + "genotypes": 0, + "genotype_calls": 0, + "phase_sets": 0, + "local_allele_sets": 0, + "base_modifications": 0, + "value_items": 0, "triples": 0, "appended_bytes": 0, } @@ -2135,17 +2586,32 @@ def append_expanded_sample_rdf( if not rdf_path.is_file(): raise FileNotFoundError(f"RDF aggregate not found for sample streaming: {rdf_path}") + version = version or vocab.FALLBACK_VERSION + definitions = ( + _load_format_definitions(header_lines_tsv) + if header_lines_tsv is not None + else {} + ) + with SampleRecordStream(records_tsv) as record_stream: - if not record_stream.columns or not record_stream.source_file: + if not record_stream.source_file: return stats def produce(emit): source_component = _rml_uri_component(record_stream.source_file) file_uri = f"file://{source_component}" + # The profile is declared even for a sites-only VCF. + # vcfc:RepresentationProfileShape requires exactly one on every + # vcfc:VCFFile, and a file with no genotype columns has no genotype + # data to condense, so the expanded profile is the honest claim. emit( - f"<{file_uri}> <{VCFR_NAMESPACE}representationProfile> " - f"<{VCFR_NAMESPACE}ExpandedRepresentation> .\n" + f"<{file_uri}> <{_vocab('representationProfile')}> " + f"<{_vocab('ExpandedRepresentation')}> .\n" + ) + _emit_sample_set( + emit, file_uri=file_uri, columns=record_stream.columns, stats=stats ) + # Sample columns and their serialized sampleId literal are fixed for # the whole file; FORMAT keys are drawn from a handful of distinct # values. Encoding them once per file instead of once per @@ -2158,14 +2624,24 @@ def produce(emit): for column in record_stream.columns ] format_components: dict[str, str] = {} + emitted_definitions: set[str] = set() + # Base-modification FORMAT keys are pattern-defined, so each distinct + # key is classified once rather than per sample and record. + base_modifications: dict[str, object] = {} sample_call_count = 0 format_value_count = 0 for record in record_stream: row_component = _rml_uri_component(record.row_id) call_uri = f"{file_uri}#call/{row_component}" + record_uri = f"{file_uri}#record/{row_component}" format_keys = record.format_keys record_sample_values = record.sample_values + alt_count = ( + 0 + if vocab.is_missing(record.alt) + else len(record.alt.split(",")) + ) for sample_index, (sample_component, sample_id_literal) in enumerate( sample_prefixes @@ -2173,14 +2649,19 @@ def produce(emit): sample_uri = f"{file_uri}#sample/{row_component}/{sample_component}" sample_values = record_sample_values[sample_index] - emit(f"<{call_uri}> <{VCFR_NAMESPACE}hasSampleCall> <{sample_uri}> .\n") - emit(f"<{sample_uri}> <{RDF_TYPE_URI}> <{VCFR_NAMESPACE}SampleCall> .\n") + emit(f"<{call_uri}> <{_vocab('hasSampleCall')}> <{sample_uri}> .\n") + emit(f"<{sample_uri}> <{RDF_TYPE_URI}> <{_vocab('SampleCall')}> .\n") emit( - f"<{sample_uri}> <{VCFR_NAMESPACE}sampleId> " + f"<{sample_uri}> <{_vocab('sampleId')}> " f"{sample_id_literal} .\n" ) + emit( + f"<{sample_uri}> <{_vocab('forSample')}> " + f"<{file_uri}#samples/{sample_component}> .\n" + ) sample_call_count += 1 + phase_fields: dict[str, str] = {} for format_index, format_key in enumerate(format_keys): format_value = sample_values[format_index] format_component = format_components.get(format_key) @@ -2188,15 +2669,114 @@ def produce(emit): format_component = _rml_uri_component(format_key) format_components[format_key] = format_component format_uri = f"{sample_uri}/fmt/{format_component}" - emit(f"<{sample_uri}> <{VCFR_NAMESPACE}hasFormatValue> <{format_uri}> .\n") - emit(f"<{format_uri}> <{RDF_TYPE_URI}> <{VCFR_NAMESPACE}FormatFieldValue> .\n") + + definition = definitions.get(format_key) + if definition is None: + definition = _synthesized_format_definition( + file_uri, format_key, format_component + ) + definitions[format_key] = definition + _emit_format_field_definition( + emit, + definition=definition, + field_id=format_key, + emitted=emitted_definitions, + stats=stats, + ) + + emit(f"<{sample_uri}> <{_vocab('hasFormatValue')}> <{format_uri}> .\n") + emit( + f"<{format_uri}> <{RDF_TYPE_URI}> " + f"<{_vocab('FormatFieldValue')}> .\n" + ) + emit( + f"<{format_uri}> <{_vocab('declaredBy')}> " + f"<{definition.uri}> .\n" + ) if format_value: emit( - f"<{format_uri}> <{VCFR_NAMESPACE}fieldValue> " + f"<{format_uri}> <{_vocab('fieldValue')}> " f"{_ntriples_literal(format_value)} .\n" ) + if alt_count: + _emit_value_items( + emit, + parent_uri=format_uri, + field_key=format_key, + value=format_value, + number=definition.field_number, + allele_uris={ + index: f"{record_uri}/allele/{index}" + for index in range(alt_count + 1) + }, + version=version, + stats=stats, + ) format_value_count += 1 + if not format_value: + continue + if format_key == "GT": + _emit_genotype( + emit, + sample_uri=sample_uri, + record_uri=record_uri, + value=format_value, + alt_count=alt_count, + stats=stats, + ) + elif format_key == "FT": + emit( + f"<{sample_uri}> <{_vocab('sampleFilter')}> " + f"{_ntriples_literal(format_value)} .\n" + ) + elif format_key in vocab.PHASE_SET_FORMAT_KEYS: + phase_fields[format_key] = format_value + elif format_key == "LAA" and version.local_alleles: + _emit_local_allele_set( + emit, + sample_uri=sample_uri, + record_uri=record_uri, + value=format_value, + alt_count=alt_count, + stats=stats, + ) + elif format_key in vocab.COPY_NUMBER_FORMAT_KEYS: + emit( + f"<{sample_uri}> " + f"<{_vocab(vocab.COPY_NUMBER_FORMAT_KEYS[format_key])}> " + f"{_ntriples_literal(format_value)} .\n" + ) + elif format_key in vocab.HAPLOTYPE_FORMAT_KEYS: + if format_value.lstrip("-").isdigit(): + emit( + f"<{sample_uri}> " + f"<{_vocab(vocab.HAPLOTYPE_FORMAT_KEYS[format_key])}> " + f"{_ntriples_typed_literal(format_value, XSD_INTEGER_URI)} .\n" + ) + elif version.base_modifications: + if format_key not in base_modifications: + base_modifications[format_key] = ( + vocab.parse_base_modification_key(format_key) + ) + modification = base_modifications[format_key] + if modification is not None: + _emit_base_modification( + emit, + sample_uri=sample_uri, + modification=modification, + value=format_value, + stats=stats, + ) + + if phase_fields: + _emit_phase_set( + emit, + sample_uri=sample_uri, + phase_fields=phase_fields, + stats=stats, + ) + stats["records"] += 1 stats["sample_calls"] = sample_call_count stats["format_values"] = format_value_count @@ -2213,20 +2793,24 @@ def produce(emit): def append_canonical_sample_rdf( records_tsv: Path, rdf_path: Path, + header_lines_tsv: Path | None = None, *, + version: "vocab.VCFVersion | None" = None, progress_interval_records: int = 10_000, ) -> dict: """Backward-compatible name for the expanded sample RDF emitter.""" return append_expanded_sample_rdf( records_tsv, rdf_path, + header_lines_tsv, + version=version, progress_interval_records=progress_interval_records, ) @dataclass(frozen=True) class FormatDefinition: - """Structured attributes and RDF identity for one FORMAT declaration.""" + """Structured attributes and RDF identity for one INFO/FORMAT declaration.""" uri: str field_number: str @@ -2234,46 +2818,21 @@ class FormatDefinition: #: The declared VCF Type (Integer/Float/Flag/Character/String). Defaults to #: String so an undeclared field still has a usable value type. value_type: str = "String" + #: ##INFO recommends Source and Version; the SHACL profile warns when a + #: concrete INFO declaration omits them. + source: str | None = None + version: str | None = None + #: True when no '##' line declared this key and the definition was invented + #: so the value still has something to cite. A declared definition is owned + #: by the header emitter, which emits its ID/Number/Type/Description from + #: the header line; emitting them again here would duplicate every triple. + synthesized: bool = False -def _parse_structured_header_fields(value: str) -> dict[str, str]: - """Parse comma-delimited VCF header attributes while respecting quotes.""" - inner = value.strip() - if inner.startswith("<") and inner.endswith(">"): - inner = inner[1:-1] - - tokens: list[str] = [] - token: list[str] = [] - in_quotes = False - escaped = False - for character in inner: - if escaped: - token.append(character) - escaped = False - elif character == "\\" and in_quotes: - token.append(character) - escaped = True - elif character == '"': - token.append(character) - in_quotes = not in_quotes - elif character == "," and not in_quotes: - tokens.append("".join(token)) - token = [] - else: - token.append(character) - tokens.append("".join(token)) - - fields: dict[str, str] = {} - for item in tokens: - if "=" not in item: - continue - key, raw_value = item.split("=", 1) - parsed_value = raw_value.strip() - if len(parsed_value) >= 2 and parsed_value[0] == parsed_value[-1] == '"': - parsed_value = parsed_value[1:-1] - parsed_value = parsed_value.replace('\\"', '"').replace("\\\\", "\\") - fields[key.strip()] = parsed_value - return fields +#: Kept as module-level names because the emitters and the unit tests use them. +_parse_structured_header_fields = vocab.parse_structured_header_fields +_parse_structured_header_attributes = vocab.parse_structured_header_attributes +_typed_info_object = vocab.typed_field_object def _load_field_definitions( @@ -2283,7 +2842,7 @@ def _load_field_definitions( Serves both ``##FORMAT`` and ``##INFO`` declarations, which share the ```` syntax and the same vocabulary shape - (``vcfr:FieldDefinition`` with ``fieldId``/``fieldNumber``/``fieldType``). + (``vcfc:FieldDefinition`` with ``fieldId``/``fieldNumber``/``fieldType``). """ definitions: dict[str, FormatDefinition] = {} if not header_lines_tsv.is_file(): @@ -2310,6 +2869,8 @@ def _load_field_definitions( or f"{wanted} field {field_id} (source declaration has no Description)" ), value_type=fields.get("Type") or "String", + source=fields.get("Source"), + version=fields.get("Version"), ), ) return definitions @@ -2320,134 +2881,160 @@ def _load_format_definitions(header_lines_tsv: Path) -> dict[str, FormatDefiniti return _load_field_definitions(header_lines_tsv, "FORMAT") -#: Maps a declared VCF INFO/FORMAT Type to its vocabulary value-type class. -VCF_VALUE_TYPE_CLASSES = { - "Integer": "IntegerType", - "Float": "FloatType", - "Flag": "FlagType", - "Character": "CharacterType", - "String": "StringType", -} -#: Typed value predicates used when a single-valued field declares a numeric or -#: flag type. Multi-valued fields (Number=A/R/G/.) keep only the lexical value, -#: because the vocabulary gives one InfoFieldValue node per key. -XSD_INTEGER_URI = "http://www.w3.org/2001/XMLSchema#integer" -XSD_DECIMAL_URI = "http://www.w3.org/2001/XMLSchema#decimal" -XSD_BOOLEAN_URI = "http://www.w3.org/2001/XMLSchema#boolean" +def _load_assembly_line_uri(header_lines_tsv: Path) -> str | None: + """Return the IRI of the ``##assembly`` header line, if the file has one. + + A bracketed CHROM names a contig from that assembly FASTA, so the assembly + contig cites the line that located it. + """ + if not header_lines_tsv.is_file(): + return None + _set_max_csv_field_size() + with header_lines_tsv.open(newline="", encoding="utf-8") as handle: + for row in csv.DictReader(handle, delimiter="\t"): + if (row.get("HEADER_KEY") or "").strip().lower() != "assembly": + continue + source_component = _rml_uri_component(row.get("SOURCE_FILE") or "") + index_component = _rml_uri_component(row.get("HEADER_INDEX") or "") + return f"file://{source_component}#header/line/{index_component}" + return None -def parse_info_entries(info: str) -> list[tuple[str, str | None]]: - """Split an INFO column into ``(key, value)`` pairs. +def _load_contig_line_uris(header_lines_tsv: Path) -> dict[str, str]: + """Map each declared contig ID to the IRI of its ``##contig`` header line. - An entry without ``=`` is a Flag: a presence assertion with no value, which - the vocabulary models as ``vcfr:fieldValueBoolean true``. + ``vcfc:chromosome`` links a record to the contig declaration its CHROM value + names, which is what the SPARQL profile's POS-versus-contig-length check + traverses. """ - if info in ("", "."): - return [] - entries: list[tuple[str, str | None]] = [] - for item in info.split(";"): - if not item: - continue - key, separator, value = item.partition("=") - entries.append((key, value if separator else None)) - return entries + contigs: dict[str, str] = {} + if not header_lines_tsv.is_file(): + return contigs + _set_max_csv_field_size() + with header_lines_tsv.open(newline="", encoding="utf-8") as handle: + for row in csv.DictReader(handle, delimiter="\t"): + if (row.get("HEADER_KEY") or "").strip().lower() != "contig": + continue + fields = _parse_structured_header_fields(row.get("HEADER_VALUE") or "") + contig_id = (fields.get("ID") or "").strip() + if not contig_id: + continue + source_component = _rml_uri_component(row.get("SOURCE_FILE") or "") + index_component = _rml_uri_component(row.get("HEADER_INDEX") or "") + contigs.setdefault( + contig_id, + f"file://{source_component}#header/line/{index_component}", + ) + return contigs -def _typed_info_object(value: str, declared_type: str) -> tuple[str, str] | None: - """Return ``(predicate_local_name, literal)`` for a typed single value.""" - if "," in value or value == ".": - return None - try: - if declared_type == "Integer": - return "fieldValueInteger", f'"{int(value)}"^^<{XSD_INTEGER_URI}>' - if declared_type == "Float": - # Serialize the source lexical form rather than a reparsed float, so - # the graph never gains or loses precision relative to the VCF. - float(value) - return "fieldValueDecimal", f'"{value}"^^<{XSD_DECIMAL_URI}>' - except ValueError: - # A value that contradicts its declared type is kept as a plain literal - # rather than dropped; the conversion must not silently lose data. - return None - return None +#: Retained name for the VCF Type individuals. Since vocabulary 3.0.0 these are +#: owl:NamedIndividual members of vcfc:VCFValueType rather than classes, so they +#: only ever appear as the object of vcfc:fieldType. +VCF_VALUE_TYPE_CLASSES = VCF_VALUE_TYPE_INDIVIDUALS + +XSD_INTEGER_URI = vocab.XSD_INTEGER_URI +XSD_DECIMAL_URI = vocab.XSD_DECIMAL_URI +XSD_BOOLEAN_URI = vocab.XSD_BOOLEAN_URI +XSD_DATE_URI = vocab.XSD_DATE_URI +XSD_ANY_URI = vocab.XSD_ANY_URI +FILE_DATE_PATTERNS = vocab.FILE_DATE_PATTERNS # How the VCF meta-information block is represented. "basic" is the historical -# behaviour: every '##' line becomes an untyped vcfr:HeaderLine carrying its raw +# behaviour: every '##' line becomes an untyped vcfc:HeaderLine carrying its raw # key and value. "structured" additionally types each line with the vocabulary's -# subclass and lifts the attributes of FILTER, ALT and contig declarations into -# their own properties, so the header becomes queryable rather than just present. +# subclass, emits the ordered vcfc:HeaderAttribute resources that +# vcfc:StructuredHeaderLineShape requires, and lifts the INFO, FORMAT, FILTER, +# ALT, contig, META, SAMPLE and PEDIGREE attributes into their own properties, +# so the header becomes queryable rather than just present. +# +# Only "structured" produces a graph that satisfies the published SHACL +# profiles: a header line typed with one of the structured subclasses must +# carry at least one vcfc:hasAttribute, and "basic" emits no subclass at all. HEADER_REPRESENTATION_CHOICES = ("basic", "structured") DEFAULT_HEADER_REPRESENTATION = "structured" -#: '##' key (lower-cased) -> the vocabulary subclass for that line. -HEADER_LINE_CLASSES = { - "fileformat": "FileFormatHeaderLine", - "filedate": "FileDateHeaderLine", - "source": "SourceHeaderLine", - "reference": "ReferenceHeaderLine", - "info": "INFOHeaderLine", - "format": "FORMATHeaderLine", - "filter": "FILTERHeaderLine", - "alt": "ALTHeaderLine", - "contig": "ContigHeaderLine", -} -#: contig attribute -> vocabulary predicate. -CONTIG_ATTRIBUTES = { - "length": "contigLength", - "md5": "contigMd5", - "assembly": "contigAssembly", -} - +# Which VCF specification version the conversion should follow. "auto" reads it +# from the input's own ##fileformat line, which VCF requires to be the first +# line of the file, so the default needs nothing from the user. An explicit +# version is an override for a file whose declaration is missing or wrong. +VCF_VERSION_CHOICES = ("auto", *sorted(vocab.VCF_VERSIONS)) +DEFAULT_VCF_VERSION = "auto" -XSD_DATE_URI = "http://www.w3.org/2001/XMLSchema#date" -#: ##fileDate has no mandated format. These are the two forms seen in practice -#: that map unambiguously onto xsd:date, which the SHACL shape requires. -FILE_DATE_PATTERNS = ( - (re.compile(r"^(\d{4})(\d{2})(\d{2})$"), "{0}-{1}-{2}"), - (re.compile(r"^(\d{4})-(\d{2})-(\d{2})$"), "{0}-{1}-{2}"), -) - - -def file_date_object(value: str) -> str | None: - """Serialize ##fileDate as xsd:date when its form allows, else lexically. - Returns None for an absent value so no triple is emitted, matching RML's - behaviour for an empty reference. +def _emit_field_definition_attributes( + emit, + line_uri: str, + *, + field_id: str, + number: str, + value_type: str | None, + description: str | None, +) -> None: + """Emit the ID/Number/Type/Description quartet shared by field definitions. + + ``vcfc:fieldNumber`` keeps the source token losslessly. ``vcfc:fieldArity`` + is added for a symbolic token and ``vcfc:fieldNumberInteger`` for a fixed + count; the SPARQL profile requires the arity individual's code to equal the + fieldNumber token, so both are derived from the same string. """ - value = (value or "").strip() - if not value or value == ".": - return None - for pattern, template in FILE_DATE_PATTERNS: - match = pattern.match(value) - if match: - return f'"{template.format(*match.groups())}"^^<{XSD_DATE_URI}>' - # An unrecognized form is preserved verbatim rather than dropped; the SHACL - # layer reports it as non-conformant. - return _ntriples_string_literal(value) + emit( + f"<{line_uri}> <{_vocab('fieldId')}> " + f"{_ntriples_string_literal(field_id)} .\n" + ) + emit( + f"<{line_uri}> <{_vocab('fieldNumber')}> " + f"{_ntriples_string_literal(number)} .\n" + ) + arity = vocab.arity_individual(number) + if arity is not None: + emit(f"<{line_uri}> <{_vocab('fieldArity')}> <{_vocab(arity)}> .\n") + fixed_count = vocab.number_as_integer(number) + if fixed_count is not None: + emit( + f"<{line_uri}> <{_vocab('fieldNumberInteger')}> " + f"{_ordinal_literal(fixed_count)} .\n" + ) + if value_type: + individual = VCF_VALUE_TYPE_INDIVIDUALS.get(value_type, "StringType") + emit(f"<{line_uri}> <{_vocab('fieldType')}> <{_vocab(individual)}> .\n") + if description is not None: + emit( + f"<{line_uri}> <{_vocab('fieldDescription')}> " + f"{_ntriples_string_literal(description)} .\n" + ) def append_header_representation_rdf( header_lines_tsv: Path, rdf_path: Path, ) -> dict: - """Append typed header lines and structured FILTER/ALT/contig declarations. + """Append the structured VCF meta-information representation. - The default mapping emits every '##' line as an untyped ``vcfr:HeaderLine`` - with a raw key and value. The vocabulary already defines a subclass per line - type and dedicated properties for the FILTER, ALT and contig attributes; - this emits them, which is what makes the meta-information block queryable. + The RML mapping emits every '##' line as an untyped ``vcfc:HeaderLine`` with + its raw key, value and line index. Everything that lives inside the + angle-bracketed value is added here: the line's vocabulary subclass, one + ordered ``vcfc:HeaderAttribute`` per source attribute, and the dedicated + properties for INFO, FORMAT, FILTER, ALT, contig, META, SAMPLE and PEDIGREE + declarations. Emitted directly rather than through RML because the attributes live inside - a single ```` value that RML cannot decompose. + a single ```` cell that RML cannot decompose, and because + several of them (contig length, contig URL, assembly URL) need a datatype + that the raw string does not carry. """ stats = { "representation": "header", "header_lines": 0, "typed_lines": 0, + "attributes": 0, + "field_definitions": 0, "filter_definitions": 0, "alt_definitions": 0, "contigs": 0, + "sample_declarations": 0, + "pedigree_relations": 0, "file_dates": 0, "triples": 0, "appended_bytes": 0, @@ -2465,6 +3052,17 @@ def produce(emit): source_file = rows[0].get("SOURCE_FILE", "") if rows else "" file_uri = f"file://{_rml_uri_component(source_file)}" contig_count = 0 + # ##SAMPLE declares a subject that ##PEDIGREE lines refer to by ID, so + # both mint the same IRI for one declared sample. + declaration_uris: dict[str, str] = {} + + def declaration_uri(identifier: str) -> str: + existing = declaration_uris.get(identifier) + if existing is None: + existing = f"{file_uri}#header/sample/{_rml_uri_component(identifier)}" + declaration_uris[identifier] = existing + return existing + for row in rows: key = (row.get("HEADER_KEY") or "").strip() value = row.get("HEADER_VALUE") or "" @@ -2472,95 +3070,663 @@ def produce(emit): line_uri = f"file://{_rml_uri_component(row.get('SOURCE_FILE') or '')}" \ f"#header/line/{index_component}" stats["header_lines"] += 1 + key_lower = key.lower() - line_class = HEADER_LINE_CLASSES.get(key.lower()) + line_class = HEADER_LINE_CLASSES.get(key_lower) + structured = vocab.is_structured_header_value(value) if line_class is None: - # An unrecognized '##' key keeps only the base HeaderLine type - # the mapping already emitted; inventing a subclass for it would - # put a term in the graph that the vocabulary does not define. - continue - emit(f"<{line_uri}> <{RDF_TYPE_URI}> <{VCFR_NAMESPACE}{line_class}> .\n") + # An unrecognized '##' key still belongs to one of the two VCF + # meta-information forms. Typing it keeps it queryable without + # inventing a subclass the vocabulary does not define. An + # unstructured line is only typed when it has a value, because + # vcfc:UnstructuredHeaderLineShape requires one. + if structured: + line_class = "StructuredHeaderLine" + elif value: + line_class = "UnstructuredHeaderLine" + else: + continue + emit(f"<{line_uri}> <{RDF_TYPE_URI}> <{_vocab(line_class)}> .\n") stats["typed_lines"] += 1 - if key.lower() not in {"filter", "alt", "contig"}: + if not structured: + if key_lower == "assembly" and value: + emit( + f"<{line_uri}> <{_vocab('assemblyUrl')}> " + f"{_ntriples_typed_literal(value, XSD_ANY_URI)} .\n" + ) + elif key_lower == "pedigreedb" and value: + emit( + f"<{line_uri}> <{_vocab('pedigreeDbUrl')}> " + f"{_ntriples_typed_literal(value, XSD_ANY_URI)} .\n" + ) continue - fields = _parse_structured_header_fields(value) + + # vcfc:StructuredHeaderLineShape requires at least one attribute on + # every structured line, so these are emitted for all of them, not + # only the ones with dedicated properties below. + attributes = _parse_structured_header_attributes(value) + for position, (attribute_key, attribute_value) in enumerate(attributes, start=1): + attribute_uri = f"{line_uri}/attribute/{position}" + emit(f"<{line_uri}> <{_vocab('hasAttribute')}> <{attribute_uri}> .\n") + emit( + f"<{attribute_uri}> <{RDF_TYPE_URI}> " + f"<{_vocab('HeaderAttribute')}> .\n" + ) + emit( + f"<{attribute_uri}> <{_vocab('attributeKey')}> " + f"{_ntriples_string_literal(attribute_key)} .\n" + ) + emit( + f"<{attribute_uri}> <{_vocab('attributeValue')}> " + f"{_ntriples_string_literal(attribute_value)} .\n" + ) + emit( + f"<{attribute_uri}> <{_vocab('attributeIndex')}> " + f"{_ordinal_literal(position)} .\n" + ) + stats["attributes"] += 1 + + fields = dict(attributes) identifier = (fields.get("ID") or "").strip() - if not identifier: - continue description = fields.get("Description") - if key.lower() == "filter": - emit(f"<{line_uri}> <{RDF_TYPE_URI}> <{VCFR_NAMESPACE}FilterDefinition> .\n") + if key_lower in {"info", "format"}: + if not identifier: + continue + _emit_field_definition_attributes( + emit, + line_uri, + field_id=identifier, + number=fields.get("Number") or ".", + value_type=fields.get("Type") or "String", + description=( + description + or f"{key.upper()} field {identifier} " + "(source declaration has no Description)" + ), + ) + if key_lower == "info": + for attribute, predicate in vocab.INFO_EXTRA_ATTRIBUTES.items(): + attribute_value = fields.get(attribute) + if attribute_value: + emit( + f"<{line_uri}> <{_vocab(predicate)}> " + f"{_ntriples_string_literal(attribute_value)} .\n" + ) + stats["field_definitions"] += 1 + elif key_lower == "filter": + if not identifier: + continue emit( - f"<{line_uri}> <{VCFR_NAMESPACE}filterId> " + f"<{line_uri}> <{_vocab('filterId')}> " f"{_ntriples_string_literal(identifier)} .\n" ) + emit( + f"<{line_uri}> <{_vocab('fieldDescription')}> " + f"{_ntriples_string_literal(description or f'FILTER {identifier}')} .\n" + ) stats["filter_definitions"] += 1 - elif key.lower() == "alt": - emit(f"<{line_uri}> <{RDF_TYPE_URI}> <{VCFR_NAMESPACE}AltDefinition> .\n") + elif key_lower == "alt": + if not identifier: + continue emit( - f"<{line_uri}> <{VCFR_NAMESPACE}altId> " + f"<{line_uri}> <{_vocab('altId')}> " f"{_ntriples_string_literal(identifier)} .\n" ) + emit( + f"<{line_uri}> <{_vocab('fieldDescription')}> " + f"{_ntriples_string_literal(description or f'ALT {identifier}')} .\n" + ) stats["alt_definitions"] += 1 - else: + elif key_lower == "contig": + if not identifier: + continue contig_count += 1 stats["contigs"] += 1 emit( - f"<{line_uri}> <{VCFR_NAMESPACE}contigId> " + f"<{line_uri}> <{_vocab('contigId')}> " f"{_ntriples_string_literal(identifier)} .\n" ) - for attribute, predicate in CONTIG_ATTRIBUTES.items(): + for attribute, (predicate, datatype) in CONTIG_ATTRIBUTES.items(): attribute_value = fields.get(attribute) - if attribute_value: + if not attribute_value: + continue + if datatype == XSD_INTEGER_URI and not attribute_value.lstrip("-").isdigit(): + # A non-numeric length is preserved lexically rather + # than dropped; the SHACL layer reports it. emit( - f"<{line_uri}> <{VCFR_NAMESPACE}{predicate}> " + f"<{line_uri}> <{_vocab(predicate)}> " f"{_ntriples_string_literal(attribute_value)} .\n" ) + continue + emit( + f"<{line_uri}> <{_vocab(predicate)}> " + f"{_ntriples_typed_literal(attribute_value, datatype)} .\n" + ) + elif key_lower == "meta": + if not identifier: + continue + _emit_field_definition_attributes( + emit, + line_uri, + field_id=identifier, + number=fields.get("Number") or ".", + # vcfc:MetaDefinitionShape excludes Flag, which VCF also + # forbids for a ##META attribute. + value_type=fields.get("Type") or "String", + description=description, + ) + for allowed in _parse_meta_values(fields.get("Values")): + emit( + f"<{line_uri}> <{_vocab('metaAllowedValue')}> " + f"{_ntriples_string_literal(allowed)} .\n" + ) + stats["field_definitions"] += 1 + elif key_lower == "sample": + if not identifier: + continue + subject_uri = declaration_uri(identifier) + emit(f"<{line_uri}> <{_vocab('declaresSample')}> <{subject_uri}> .\n") + emit( + f"<{subject_uri}> <{RDF_TYPE_URI}> " + f"<{_vocab('SampleDeclaration')}> .\n" + ) + stats["sample_declarations"] += 1 + elif key_lower == "pedigree": + emitted_relation = False + for attribute_key, attribute_value in attributes: + if attribute_key == "ID" or not attribute_value: + continue + ancestor_uri = declaration_uri(attribute_value) + predicate = vocab.PEDIGREE_ROLE_PROPERTIES.get( + attribute_key, "pedigreeAncestor" + ) + emit(f"<{line_uri}> <{_vocab(predicate)}> <{ancestor_uri}> .\n") + emit( + f"<{ancestor_uri}> <{RDF_TYPE_URI}> " + f"<{_vocab('SampleDeclaration')}> .\n" + ) + emit( + f"<{line_uri}> <{_vocab('ancestorRole')}> " + f"{_ntriples_string_literal(attribute_key)} .\n" + ) + emitted_relation = True + if emitted_relation: + stats["pedigree_relations"] += 1 + + for row in rows: + if (row.get("HEADER_KEY") or "").strip().lower() != "filedate": continue + date_object = file_date_object(row.get("HEADER_VALUE") or "") + if date_object is not None: + emit(f"<{file_uri}> <{_vocab('fileDate')}> {date_object} .\n") + stats["file_dates"] += 1 + break + + if contig_count: + emit( + f"<{file_uri}> <{_vocab('contigCount')}> " + f"{_ordinal_literal(contig_count)} .\n" + ) + + return _append_rdf_atomically(rdf_path, stats, produce) + + +def _parse_meta_values(raw: str | None) -> list[str]: + """Split a ##META ``Values=[a, b, c]`` list into its members.""" + if not raw: + return [] + inner = raw.strip() + if inner.startswith("[") and inner.endswith("]"): + inner = inner[1:-1] + return [item.strip() for item in inner.split(",") if item.strip()] + + +_qual_object = vocab.qual_object + + +def _emit_allele_layer( + emit, + *, + record_uri: str, + alleles: list, + alt_definitions: dict[str, str], + stats: dict, +) -> dict[int, str]: + """Emit the record's ordered REF/ALT allele resources. + + Returns allele index -> allele IRI so the INFO value items, the SV carriers + and the per-sample genotype calls can all join the same resources. The index + is record-global: 0 for REF, then 1..n in source ALT order, which is exactly + the join key VCF Number=A and Number=R define. + """ + allele_uris: dict[int, str] = {} + for allele in alleles: + allele_uri = f"{record_uri}/allele/{allele.index}" + allele_uris[allele.index] = allele_uri + if allele.index == 0: + emit(f"<{record_uri}> <{_vocab('hasReferenceAllele')}> <{allele_uri}> .\n") + allele_class = "ReferenceAllele" + else: + emit(f"<{record_uri}> <{_vocab('hasAltAllele')}> <{allele_uri}> .\n") + allele_class = "AltAllele" + emit(f"<{allele_uri}> <{RDF_TYPE_URI}> <{_vocab(allele_class)}> .\n") + emit( + f"<{allele_uri}> <{_vocab('alleleIndex')}> " + f"{_ordinal_literal(allele.index)} .\n" + ) + emit( + f"<{allele_uri}> <{_vocab('alleleValue')}> " + f"{_ntriples_literal(allele.value)} .\n" + ) + emit(f"<{allele_uri}> <{_vocab('alleleKind')}> <{_vocab(allele.kind)}> .\n") + stats["alleles"] += 1 + + if allele.symbolic_id: + definition_uri = alt_definitions.get(allele.symbolic_id) + if definition_uri is not None: + emit( + f"<{allele_uri}> <{_vocab('declaredByAlt')}> " + f"<{definition_uri}> .\n" + ) + if allele.symbolic_type: + emit( + f"<{allele_uri}> <{_vocab('svType')}> " + f"<{_vocab(allele.symbolic_type)}> .\n" + ) + # vcfc:TandemRepeatAllele and vcfc:ReferenceBlock are deliberately not + # assigned here. Both shapes require a property that only the record's + # INFO can supply -- repeatSequenceCount from RN, referenceBlockLength + # from END -- so _emit_sv_layer types those alleles once it has them. + if allele.breakend is not None: + _emit_breakend(emit, allele_uri, allele.breakend) + stats["breakends"] += 1 + return allele_uris + + +def _emit_breakend(emit, allele_uri: str, breakend) -> None: + """Emit the parsed components of one breakend ALT expression.""" + emit(f"<{allele_uri}> <{RDF_TYPE_URI}> <{_vocab('Breakend')}> .\n") + if breakend.orientation is not None: + emit( + f"<{allele_uri}> <{_vocab('breakendOrientation')}> " + f"<{_vocab(breakend.orientation)}> .\n" + ) + if breakend.replacement: + emit( + f"<{allele_uri}> <{_vocab('breakendReplacementString')}> " + f"{_ntriples_typed_literal(breakend.replacement, vocab.BREAKEND_STRING_DATATYPE_URI)} .\n" + ) + if breakend.is_single: + emit( + f"<{allele_uri}> <{_vocab('isSingleBreakend')}> " + f'"true"^^<{XSD_BOOLEAN_URI}> .\n' + ) + + +def _emit_value_items( + emit, + *, + parent_uri: str, + field_key: str, + value: str, + number: str, + allele_uris: dict[int, str], + version: "vocab.VCFVersion", + stats: dict, +) -> None: + """Decompose a comma-separated payload into indexed vcfc:FieldValueItems. + + A field is decomposed when its items have a meaning per position. That is + usually decided by the declared Number -- A, R, LA, LR (one per allele), G, + LG (one per genotype), P (one per GT allele) -- but the version's flattened + tuple keys are positional too, even though the specification declares them + ``Number=.``: the version overlay checks their item count against the ALT + count, so their items have to be materialized. + + Which keys those are, and whether the tuple repeats per ALT allele, is + version-dependent. CIPOS is one pair for the whole record in VCF 4.1-4.3 and + one pair per ALT from 4.4; CILEN and CICN only become tuple keys in 4.4. + + Once any item of a field is materialized, all of them must be, contiguously + indexed from zero -- vcfc:ValueItemCountShape checks exactly that. + """ + if not version.is_positional(field_key, number): + return + items = vocab.split_value_items(value) + if not items: + return + tuple_arity = version.tuple_arity(field_key) + for item_index, item in enumerate(items): + item_uri = f"{parent_uri}/value/{item_index}" + emit(f"<{parent_uri}> <{_vocab('hasValueItem')}> <{item_uri}> .\n") + emit(f"<{item_uri}> <{RDF_TYPE_URI}> <{_vocab('FieldValueItem')}> .\n") + emit( + f"<{item_uri}> <{_vocab('valueIndex')}> " + f"{_ordinal_literal(item_index)} .\n" + ) + emit(f"<{item_uri}> <{_vocab('itemValue')}> {_ntriples_literal(item)} .\n") + if tuple_arity is not None: + emit( + f"<{item_uri}> <{_vocab('tupleArity')}> " + f"{_ordinal_literal(tuple_arity)} .\n" + ) + link = version.value_item_link(field_key, number, item_index) + if link.allele_index is not None: + allele_uri = allele_uris.get(link.allele_index) + if allele_uri is not None: + emit(f"<{item_uri}> <{_vocab('forAllele')}> <{allele_uri}> .\n") + elif link.genotype_index is not None: + emit( + f"<{item_uri}> <{_vocab('forGenotypeIndex')}> " + f"{_ordinal_literal(link.genotype_index)} .\n" + ) + elif link.gt_allele_index is not None: + emit( + f"<{item_uri}> <{_vocab('forGTAlleleIndex')}> " + f"{_ordinal_literal(link.gt_allele_index)} .\n" + ) + stats["value_items"] += 1 + + +def _emit_sv_layer( + emit, + *, + record_uri: str, + pos: str, + info_map: dict[str, str | None], + allele_uris: dict[int, str], + alleles_by_index: dict[int, object], + version: "vocab.VCFVersion", + stats: dict, +) -> None: + """Attach the reserved SV INFO keys to the alleles they describe. + + Called once per record with the whole INFO map rather than per key, because + several of these carriers need more than one key to be well formed: a + vcfc:VariantEvent needs both EVENT and EVENTTYPE, a reference block needs + both END and POS, and a tandem repeat needs RN before its RUS/RUL/RUC/RB + values mean anything. + + That coupling matters because the vocabulary's properties carry rdfs:domain + axioms. Emitting vcfc:endPosition on an allele infers vcfc:ReferenceBlock + for it under RDFS, which then requires vcfc:referenceBlockLength -- so a + carrier is only emitted when the record supplies everything its shape needs. + """ + alt_uris = {index: uri for index, uri in allele_uris.items() if index >= 1} + if not alt_uris: + return + + for key, predicate in vocab.SV_FLAG_INFO_KEYS.items(): + if key in info_map: + for allele_uri in alt_uris.values(): + emit( + f"<{allele_uri}> <{_vocab(predicate)}> " + f'"true"^^<{XSD_BOOLEAN_URI}> .\n' + ) + + for key, predicate in vocab.SV_ALLELE_INFO_KEYS.items(): + value = info_map.get(key) + if value is None: + continue + for offset, item in enumerate(vocab.split_value_items(value), start=1): + allele_uri = alt_uris.get(offset) + if allele_uri is not None: + emit( + f"<{allele_uri}> <{_vocab(predicate)}> " + f"{_ntriples_literal(item)} .\n" + ) + + claims = info_map.get("SVCLAIM") + if claims is not None: + for offset, item in enumerate(vocab.split_value_items(claims), start=1): + allele_uri = alt_uris.get(offset) + claim = vocab.SV_CLAIMS.get(item.strip().upper()) + if allele_uri is not None and claim is not None: + emit(f"<{allele_uri}> <{_vocab('svClaim')}> <{_vocab(claim)}> .\n") + + _emit_reference_block( + emit, + pos=pos, + end=info_map.get("END"), + alt_uris=alt_uris, + alleles_by_index=alleles_by_index, + stats=stats, + ) + _emit_variant_event( + emit, + record_uri=record_uri, + event=info_map.get("EVENT"), + event_type=info_map.get("EVENTTYPE"), + alt_uris=alt_uris, + version=version, + stats=stats, + ) + _emit_tandem_repeats( + emit, info_map=info_map, alt_uris=alt_uris, + alleles_by_index=alleles_by_index, stats=stats, + ) + + for key in (*vocab.FALDO_INTERVAL_INFO_KEYS, *vocab.CONFIDENCE_INTERVAL_INFO_KEYS): + value = info_map.get(key) + if value is None: + continue + if version.tuple_arity(key) is None: + # CILEN and CICN do not exist before VCF 4.4. A file that uses the + # name anyway keeps it as an ordinary INFO value rather than being + # given a carrier this version does not define. + continue + _emit_confidence_intervals( + emit, key=key, value=value, alt_uris=alt_uris, + version=version, stats=stats, + ) + - if description: - emit( - f"<{line_uri}> <{VCFR_NAMESPACE}fieldDescription> " - f"{_ntriples_string_literal(description)} .\n" - ) +def _emit_reference_block( + emit, *, pos: str, end: str | None, alt_uris: dict, alleles_by_index: dict, stats: dict +) -> None: + """Close a gVCF reference block with its END position and derived length. - for row in rows: - if (row.get("HEADER_KEY") or "").strip().lower() != "filedate": - continue - date_object = file_date_object(row.get("HEADER_VALUE") or "") - if date_object is not None: - emit(f"<{file_uri}> <{VCFR_NAMESPACE}fileDate> {date_object} .\n") - stats["file_dates"] += 1 - break + Only the ``<*>``/```` allele is a reference block. VCF also uses + END on symbolic SV records, but the vocabulary has no generic END property + there: the extent of such an allele is carried by SVLEN and the CI keys, so + END stays in its INFO value alone rather than being forced onto a class it + does not belong to. + """ + if end is None or not end.lstrip("-").isdigit(): + return + for index, allele_uri in alt_uris.items(): + allele = alleles_by_index.get(index) + if allele is None or allele.kind != "UnspecifiedAllele": + continue + emit(f"<{allele_uri}> <{RDF_TYPE_URI}> <{_vocab('ReferenceBlock')}> .\n") + emit( + f"<{allele_uri}> <{_vocab('endPosition')}> " + f"{_ntriples_typed_literal(end, XSD_INTEGER_URI)} .\n" + ) + if pos.isdigit(): + # VCF 4.5 reference blocks are inclusive of both endpoints. + length = max(int(end) - int(pos) + 1, 0) + emit( + f"<{allele_uri}> <{_vocab('referenceBlockLength')}> " + f"{_ordinal_literal(length)} .\n" + ) + emit( + f"<{allele_uri}> <{_vocab('isReferenceBlockStart')}> " + f'"true"^^<{XSD_BOOLEAN_URI}> .\n' + ) + stats["reference_blocks"] += 1 - if contig_count: + +def _emit_variant_event( + emit, + *, + record_uri: str, + event: str | None, + event_type: str | None, + alt_uris: dict, + version: "vocab.VCFVersion", + stats: dict, +) -> None: + """Group a record, or each of its ALT alleles, into a named EVENT. + + EVENT changed arity in VCF 4.4. Before that it is ``Number=1``: one event + for the whole record, and EVENTTYPE does not exist. From 4.4 both are + ``Number=A``: one event per ALT allele, in source order, so the link hangs + off the allele rather than the record. + + ``vcfc:VariantEventShape`` requires exactly one ``vcfc:eventType``, so an + event resource is only minted when a reserved EVENTTYPE code is available to + give it one. In a pre-4.4 file there is none, so EVENT stays an ordinary + INFO value rather than becoming a resource that fails its shape. + """ + if not event: + return + if not version.event_types or not event_type: + return + + events = vocab.split_value_items(event) + types = vocab.split_value_items(event_type) + if not version.events_per_alt: + events, types = events[:1], types[:1] + + for offset, name in enumerate(events): + if vocab.is_missing(name): + continue + code = types[offset] if offset < len(types) else None + individual = ( + vocab.EVENT_TYPES.get(code.strip().upper()) if code else None + ) + if individual is None: + continue + event_uri = f"{record_uri}/event/{_rml_uri_component(name)}" + subject = record_uri + if version.events_per_alt: + allele_uri = alt_uris.get(offset + 1) + if allele_uri is None: + continue + subject = allele_uri + emit(f"<{subject}> <{_vocab('inEvent')}> <{event_uri}> .\n") + emit(f"<{event_uri}> <{RDF_TYPE_URI}> <{_vocab('VariantEvent')}> .\n") + emit(f"<{event_uri}> <{_vocab('eventType')}> <{_vocab(individual)}> .\n") + stats["variant_events"] += 1 + + +def _emit_tandem_repeats( + emit, *, info_map: dict, alt_uris: dict, alleles_by_index: dict, stats: dict +) -> None: + """Emit the RN/RUS/RUL/RUC/RB repeat structure of a allele. + + RN gives the number of repeat sequences per ALT allele and is what makes an + allele a ``vcfc:TandemRepeatAllele``: without it the remaining keys have no + grouping, and the class's required ``vcfc:repeatSequenceCount`` would be + missing. The repeat-sequence values are flat comma lists across all alleles, + so RN also supplies the stride into them. + """ + counts = vocab.split_value_items(info_map.get("RN") or "") + if not counts: + return + consumed = 0 + for offset, count_token in enumerate(counts, start=1): + allele_uri = alt_uris.get(offset) + allele = alleles_by_index.get(offset) + if allele_uri is None or not count_token.isdigit(): + continue + count = int(count_token) + if allele is not None and allele.symbolic_type != "SymbolicTandemRepeat": + consumed += count + continue + emit( + f"<{allele_uri}> <{RDF_TYPE_URI}> " + f"<{_vocab('TandemRepeatAllele')}> .\n" + ) + emit( + f"<{allele_uri}> <{_vocab('repeatSequenceCount')}> " + f"{_ordinal_literal(count)} .\n" + ) + for position in range(count): + sequence_uri = f"{allele_uri}/repeat/{position + 1}" + emit( + f"<{allele_uri}> <{_vocab('hasRepeatSequence')}> " + f"<{sequence_uri}> .\n" + ) emit( - f"<{file_uri}> <{VCFR_NAMESPACE}contigCount> " - f'"{contig_count}"^^<{XSD_INTEGER_URI}> .\n' + f"<{sequence_uri}> <{RDF_TYPE_URI}> " + f"<{_vocab('RepeatSequence')}> .\n" ) + emit( + f"<{sequence_uri}> <{_vocab('repeatSequenceIndex')}> " + f"{_ordinal_literal(position + 1)} .\n" + ) + for key, predicate in vocab.REPEAT_SEQUENCE_INFO_KEYS.items(): + items = vocab.split_value_items(info_map.get(key) or "") + flat_index = consumed + position + if flat_index < len(items): + emit( + f"<{sequence_uri}> <{_vocab(predicate)}> " + f"{_ntriples_literal(items[flat_index])} .\n" + ) + stats["repeat_sequences"] += 1 + consumed += count + stats["tandem_repeats"] += 1 - return _append_rdf_atomically(rdf_path, stats, produce) +def _emit_confidence_intervals( + emit, *, key: str, value: str, alt_uris: dict, version: "vocab.VCFVersion", stats: dict +) -> None: + """Emit the (lower, upper) pairs of a CI* INFO key. -def _qual_object(value: str) -> str: - """Serialize QUAL as the SHACL shape requires: xsd:decimal, or vcfr:Null. + CIPOS and CIEND use a FALDO InRangePosition, as the SV module specifies; the + remaining CI keys use a vcfc:ConfidenceInterval. - The shape is ``sh:or([sh:datatype xsd:decimal] [sh:datatype vcfr:Null])``, - which RML cannot satisfy because the datatype depends on the row. A value - that is neither numeric nor the missing token is kept as a plain literal - rather than dropped: a non-conformant graph is more useful than a lossy one, - and the SHACL layer reports it. + Before VCF 4.4 the list holds exactly one pair describing the record, so + every ALT allele of a multi-allelic record shares it. From 4.4 the list + holds one pair per ALT allele in source order, and each allele gets its own. """ - if value in ("", "."): - return f'"."^^<{VCFR_NAMESPACE}Null>' - try: - float(value) - except ValueError: - return _ntriples_string_literal(value) - # Serialize the source lexical form so no precision is gained or lost. - return f'"{value}"^^<{XSD_DECIMAL_URI}>' + items = vocab.split_value_items(value) + for offset, allele_index in enumerate(sorted(alt_uris)): + # A pre-4.4 record has a single pair, so every allele reads items 0-1. + pair = offset if version.tuples_per_alt else 0 + lower = items[pair * 2] if pair * 2 < len(items) else None + upper = items[pair * 2 + 1] if pair * 2 + 1 < len(items) else None + if lower is None or upper is None: + break + allele_uri = alt_uris[allele_index] + interval_uri = f"{allele_uri}/ci/{key}" + if key in vocab.FALDO_INTERVAL_INFO_KEYS: + predicate = vocab.FALDO_INTERVAL_INFO_KEYS[key] + emit(f"<{allele_uri}> <{_vocab(predicate)}> <{interval_uri}> .\n") + emit( + f"<{interval_uri}> <{RDF_TYPE_URI}> " + f"<{vocab.FALDO_NAMESPACE}InRangePosition> .\n" + ) + emit( + f"<{interval_uri}> <{vocab.FALDO_NAMESPACE}begin> " + f"{vocab.numeric_literal(lower)} .\n" + ) + emit( + f"<{interval_uri}> <{vocab.FALDO_NAMESPACE}end> " + f"{vocab.numeric_literal(upper)} .\n" + ) + else: + predicate = vocab.CONFIDENCE_INTERVAL_INFO_KEYS[key] + emit(f"<{allele_uri}> <{_vocab(predicate)}> <{interval_uri}> .\n") + emit( + f"<{interval_uri}> <{RDF_TYPE_URI}> " + f"<{_vocab('ConfidenceInterval')}> .\n" + ) + # vcfc:ConfidenceIntervalShape puts both bounds through + # vcfc:NumericLiteralShape, so a plain string will not do. + emit( + f"<{interval_uri}> <{_vocab('ciLower')}> " + f"{vocab.numeric_literal(lower)} .\n" + ) + emit( + f"<{interval_uri}> <{_vocab('ciUpper')}> " + f"{vocab.numeric_literal(upper)} .\n" + ) + stats["confidence_intervals"] += 1 def append_record_detail_rdf( @@ -2570,24 +3736,25 @@ def append_record_detail_rdf( *, emit_qual: bool = True, emit_info: bool = True, + emit_alleles: bool = True, + version: "vocab.VCFVersion | None" = None, progress_interval_records: int = 10_000, ) -> dict: """Append per-record detail the RML mapping cannot express. - Covers QUAL (whose datatype depends on the value) and the structured INFO - representation (which would otherwise need a materialized helper table of - variants x INFO keys). Both are per-record, so they share one pass over - ``records.tsv``. - - The default mapping emits INFO only as an opaque ``vcfr:infoRaw`` string. - This adds the structured form the vocabulary already defines - one - ``vcfr:InfoFieldValue`` per record and key, at the IRI template the - vocabulary declares - so INFO becomes queryable rather than just present. - - Emitted directly rather than through RML for the same reason the genotype - representations are: RML would need a materialized helper table of - variants x INFO keys, and it cannot switch value datatype on a declared - ``Type``. + Three groups of things land here, all of them per-record, so they share one + pass over ``records.tsv``: + + * The fixed fields whose datatype depends on the row -- ID, ALT, QUAL and + FILTER. Each may be the VCF missing token, which the vocabulary requires + as ``"."^^vcfc:Null``, and RML cannot switch datatype per row. + * The allele layer: ``vcfc:ReferenceAllele`` and ``vcfc:AltAllele`` + resources parsed out of REF and ALT, with their index, kind, symbolic + type and breakend components, plus the ``vcfc:chromosome`` link to the + contig declaration named by CHROM. + * The structured INFO representation: one ``vcfc:InfoFieldValue`` per record + and key, its ``vcfc:FieldValueItem`` decomposition for Number=A/R/G/P + fields, and the SV carriers the reserved INFO keys imply. """ stats = { "representation": "record-detail", @@ -2595,6 +3762,17 @@ def append_record_detail_rdf( "qual_values": 0, "info_values": 0, "info_definitions": 0, + "alleles": 0, + "breakends": 0, + "value_items": 0, + "variant_events": 0, + "confidence_intervals": 0, + "reference_blocks": 0, + "tandem_repeats": 0, + "repeat_sequences": 0, + "contig_links": 0, + "assembly_contigs": 0, + "assembly_contig_links": 0, "triples": 0, "appended_bytes": 0, } @@ -2603,7 +3781,16 @@ def append_record_detail_rdf( if not rdf_path.is_file(): raise FileNotFoundError(f"RDF aggregate not found for INFO streaming: {rdf_path}") + version = version or vocab.FALLBACK_VERSION definitions = _load_field_definitions(header_lines_tsv, "INFO") + contig_line_uris = _load_contig_line_uris(header_lines_tsv) if emit_alleles else {} + assembly_line_uri = _load_assembly_line_uri(header_lines_tsv) if emit_alleles else None + alt_definition_uris = { + identifier: definition.uri + for identifier, definition in _load_field_definitions( + header_lines_tsv, "ALT" + ).items() + } with SampleRecordStream(records_tsv) as record_stream: if not record_stream.source_file: @@ -2613,6 +3800,7 @@ def produce(emit): source_component = _rml_uri_component(record_stream.source_file) file_uri = f"file://{source_component}" emitted_definitions: set[str] = set() + emitted_assembly_contigs: set[str] = set() key_components: dict[str, str] = {} value_count = 0 definition_count = 0 @@ -2620,14 +3808,90 @@ def produce(emit): qual_count = 0 for record in record_stream: row_component = _rml_uri_component(record.row_id) + record_uri = f"{file_uri}#record/{row_component}" call_uri = f"{file_uri}#call/{row_component}" + + # ID and ALT are record-level; FILTER and QUAL are call-level. + # All four are emitted here so the missing token is typed. + emit( + f"<{record_uri}> <{_vocab('recordId')}> " + f"{_ntriples_literal(record.record_id or '.')} .\n" + ) + emit( + f"<{record_uri}> <{_vocab('alt')}> " + f"{_ntriples_literal(record.alt or '.')} .\n" + ) + emit( + f"<{call_uri}> <{_vocab('filter')}> " + f"{_ntriples_literal(record.filter_value or '.')} .\n" + ) + emit( + f"<{call_uri}> <{_vocab('infoRaw')}> " + f"{_ntriples_literal(record.info or '.')} .\n" + ) if emit_qual: emit( - f"<{call_uri}> <{VCFR_NAMESPACE}qual> " + f"<{call_uri}> <{_vocab('qual')}> " f"{_qual_object(record.qual)} .\n" ) qual_count += 1 - for key, value in (parse_info_entries(record.info) if emit_info else ()): + + allele_uris: dict[int, str] = {} + alleles_by_index: dict[int, object] = {} + if emit_alleles: + # VCF 4.5 lets CHROM name a breakpoint-assembly contig in + # angle brackets instead of a declared reference sequence. + # The two are mutually exclusive: the vocabulary's + # consistency profile rejects a bracketed CHROM that also + # resolves to a ##contig declaration. + assembly_id = vocab.parse_bracketed_chrom(record.chrom) + if assembly_id is not None: + assembly_uri = ( + f"{file_uri}#assembly/contig/" + f"{_rml_uri_component(assembly_id)}" + ) + emit( + f"<{record_uri}> <{_vocab('chromAssemblyContig')}> " + f"<{assembly_uri}> .\n" + ) + if assembly_uri not in emitted_assembly_contigs: + emitted_assembly_contigs.add(assembly_uri) + emit( + f"<{assembly_uri}> <{RDF_TYPE_URI}> " + f"<{_vocab('AssemblyContig')}> .\n" + ) + emit( + f"<{assembly_uri}> <{_vocab('assemblyContigId')}> " + f"{_ntriples_string_literal(assembly_id)} .\n" + ) + if assembly_line_uri is not None: + emit( + f"<{assembly_uri}> " + f"<{_vocab('declaredInAssembly')}> " + f"<{assembly_line_uri}> .\n" + ) + stats["assembly_contigs"] += 1 + stats["assembly_contig_links"] += 1 + else: + contig_uri = contig_line_uris.get(record.chrom) + if contig_uri is not None: + emit( + f"<{record_uri}> <{_vocab('chromosome')}> " + f"<{contig_uri}> .\n" + ) + stats["contig_links"] += 1 + parsed_alleles = vocab.parse_alt_alleles(record.ref, record.alt) + alleles_by_index = {a.index: a for a in parsed_alleles} + allele_uris = _emit_allele_layer( + emit, + record_uri=record_uri, + alleles=parsed_alleles, + alt_definitions=alt_definition_uris, + stats=stats, + ) + + info_entries = parse_info_entries(record.info) if emit_info else [] + for key, value in info_entries: key_component = key_components.get(key) if key_component is None: key_component = _rml_uri_component(key) @@ -2638,67 +3902,84 @@ def produce(emit): if definition is None: definition = FormatDefinition( uri=f"{file_uri}#header/info/{key_component}", - field_number=".", + field_number="0" if value is None else ".", description=( f"Synthesized definition for undeclared INFO key {key}" ), value_type="Flag" if value is None else "String", + synthesized=True, ) if definition.uri not in emitted_definitions: emitted_definitions.add(definition.uri) definition_count += 1 - type_class = VCF_VALUE_TYPE_CLASSES.get( - definition.value_type, "StringType" - ) emit( f"<{definition.uri}> <{RDF_TYPE_URI}> " - f"<{VCFR_NAMESPACE}InfoFieldDefinition> .\n" - ) - emit( - f"<{definition.uri}> <{VCFR_NAMESPACE}fieldId> " - f"{_ntriples_string_literal(key)} .\n" - ) - emit( - f"<{definition.uri}> <{VCFR_NAMESPACE}fieldNumber> " - f"{_ntriples_string_literal(definition.field_number)} .\n" - ) - emit( - f"<{definition.uri}> <{VCFR_NAMESPACE}fieldDescription> " - f"{_ntriples_string_literal(definition.description)} .\n" - ) - emit( - f"<{definition.uri}> <{VCFR_NAMESPACE}fieldType> " - f"<{VCFR_NAMESPACE}{type_class}> .\n" + f"<{_vocab('InfoFieldDefinition')}> .\n" ) + # A key declared by a '##INFO' line already has its + # ID/Number/Type/Description from the header emitter, at + # this very IRI. Only an invented definition needs them. + if definition.synthesized: + _emit_field_definition_attributes( + emit, + definition.uri, + field_id=key, + number=definition.field_number, + value_type=definition.value_type, + description=definition.description, + ) - emit(f"<{call_uri}> <{VCFR_NAMESPACE}hasInfoValue> <{info_uri}> .\n") + emit(f"<{call_uri}> <{_vocab('hasInfoValue')}> <{info_uri}> .\n") emit( f"<{info_uri}> <{RDF_TYPE_URI}> " - f"<{VCFR_NAMESPACE}InfoFieldValue> .\n" + f"<{_vocab('InfoFieldValue')}> .\n" ) emit( - f"<{info_uri}> <{VCFR_NAMESPACE}declaredBy> " + f"<{info_uri}> <{_vocab('declaredBy')}> " f"<{definition.uri}> .\n" ) if value is None: emit( - f"<{info_uri}> <{VCFR_NAMESPACE}fieldValueBoolean> " + f"<{info_uri}> <{_vocab('fieldValueBoolean')}> " f'"true"^^<{XSD_BOOLEAN_URI}> .\n' ) else: emit( - f"<{info_uri}> <{VCFR_NAMESPACE}fieldValue> " + f"<{info_uri}> <{_vocab('fieldValue')}> " f"{_ntriples_literal(value)} .\n" ) typed = _typed_info_object(value, definition.value_type) if typed is not None: predicate, literal = typed emit( - f"<{info_uri}> <{VCFR_NAMESPACE}{predicate}> " + f"<{info_uri}> <{_vocab(predicate)}> " f"{literal} .\n" ) + if allele_uris: + _emit_value_items( + emit, + parent_uri=info_uri, + field_key=key, + value=value, + number=definition.field_number, + allele_uris=allele_uris, + version=version, + stats=stats, + ) value_count += 1 + if allele_uris and info_entries: + _emit_sv_layer( + emit, + record_uri=record_uri, + pos=record.pos, + info_map=dict(info_entries), + allele_uris=allele_uris, + alleles_by_index=alleles_by_index, + version=version, + stats=stats, + ) + stats["records"] += 1 stats["qual_values"] = qual_count stats["info_values"] = value_count @@ -2721,9 +4002,24 @@ def append_condensed_sample_rdf( header_lines_tsv: Path, rdf_path: Path, *, + emit_sample_data_raw: bool = False, progress_interval_records: int = 10_000, ) -> dict: - """Append sample-ordered cohort matrices and FORMAT value vectors.""" + """Append sample-ordered cohort matrices and FORMAT value vectors. + + This profile deliberately stops at the vector. The vocabulary defines + ``vcfc:encodedValues`` as a compact payload rather than a set of + independently asserted values, so the genotype, phase-set, local-allele and + base-modification resources that the expanded emitter derives per sample are + not produced here: doing so would reintroduce the per-sample growth the + condensed profile exists to avoid. Every value remains recoverable by + decoding a vector against its FORMAT definition and the matrix SampleSet. + + ``emit_sample_data_raw`` additionally stores each record's genotype columns + verbatim on the matrix. It is off by default because the FORMAT vectors + already preserve the semantic values, and the raw copy roughly doubles the + condensed payload. + """ stats = { "representation": "condensed", "records": 0, @@ -2741,38 +4037,23 @@ def append_condensed_sample_rdf( definitions = _load_format_definitions(header_lines_tsv) with SampleRecordStream(records_tsv) as record_stream: - if not record_stream.columns or not record_stream.source_file: + if not record_stream.source_file: return stats def produce(emit): source_component = _rml_uri_component(record_stream.source_file) file_uri = f"file://{source_component}" - sample_set_uri = f"{file_uri}#samples" emitted_definitions: set[str] = set() # FORMAT keys repeat on every record; encode each distinct key once. format_components: dict[str, str] = {} emit( - f"<{file_uri}> <{VCFR_NAMESPACE}representationProfile> " - f"<{VCFR_NAMESPACE}CondensedRepresentation> .\n" + f"<{file_uri}> <{_vocab('representationProfile')}> " + f"<{_vocab('CondensedRepresentation')}> .\n" + ) + sample_set_uri = _emit_sample_set( + emit, file_uri=file_uri, columns=record_stream.columns, stats=stats ) - emit(f"<{file_uri}> <{VCFR_NAMESPACE}hasSampleSet> <{sample_set_uri}> .\n") - emit(f"<{sample_set_uri}> <{RDF_TYPE_URI}> <{VCFR_NAMESPACE}SampleSet> .\n") - - for sample_column in record_stream.columns: - sample_component = _rml_uri_component(sample_column.uri_id) - sample_uri = f"{sample_set_uri}/{sample_component}" - emit(f"<{sample_set_uri}> <{VCFR_NAMESPACE}hasSample> <{sample_uri}> .\n") - emit(f"<{sample_uri}> <{RDF_TYPE_URI}> <{VCFR_NAMESPACE}VCFSample> .\n") - emit( - f"<{sample_uri}> <{VCFR_NAMESPACE}sampleName> " - f"{_ntriples_string_literal(sample_column.sample_id)} .\n" - ) - emit( - f"<{sample_uri}> <{VCFR_NAMESPACE}sampleIndex> " - f'"{sample_column.index}"^^<{XSD_POSITIVE_INTEGER_URI}> .\n' - ) - stats["samples"] += 1 for record in record_stream: if not record.format_keys: @@ -2780,12 +4061,20 @@ def produce(emit): row_component = _rml_uri_component(record.row_id) call_uri = f"{file_uri}#call/{row_component}" matrix_uri = f"{call_uri}/matrix" - emit(f"<{call_uri}> <{VCFR_NAMESPACE}hasCallMatrix> <{matrix_uri}> .\n") - emit(f"<{matrix_uri}> <{RDF_TYPE_URI}> <{VCFR_NAMESPACE}CohortCallMatrix> .\n") + emit(f"<{call_uri}> <{_vocab('hasCallMatrix')}> <{matrix_uri}> .\n") + emit( + f"<{matrix_uri}> <{RDF_TYPE_URI}> " + f"<{_vocab('CohortCallMatrix')}> .\n" + ) emit( - f"<{matrix_uri}> <{VCFR_NAMESPACE}appliesToSampleSet> " + f"<{matrix_uri}> <{_vocab('appliesToSampleSet')}> " f"<{sample_set_uri}> .\n" ) + if emit_sample_data_raw: + emit( + f"<{matrix_uri}> <{_vocab('sampleDataRaw')}> " + f"{_ntriples_string_literal(chr(9).join(record.sample_payloads))} .\n" + ) stats["matrices"] += 1 for format_index, format_key in enumerate(record.format_keys): @@ -2796,52 +4085,39 @@ def produce(emit): vector_uri = f"{matrix_uri}/fmt/{format_component}" definition = definitions.get(format_key) if definition is None: - definition = FormatDefinition( - uri=f"{file_uri}#header/format/{format_component}", - field_number=".", - description=( - f"Synthesized definition for undeclared FORMAT key {format_key}" - ), - ) - definition_uri = definition.uri - if definition_uri not in emitted_definitions: - emit( - f"<{definition_uri}> <{RDF_TYPE_URI}> " - f"<{VCFR_NAMESPACE}FormatFieldDefinition> .\n" - ) - emit( - f"<{definition_uri}> <{VCFR_NAMESPACE}fieldId> " - f"{_ntriples_string_literal(format_key)} .\n" - ) - emit( - f"<{definition_uri}> <{VCFR_NAMESPACE}fieldNumber> " - f"{_ntriples_string_literal(definition.field_number)} .\n" - ) - emit( - f"<{definition_uri}> <{VCFR_NAMESPACE}fieldDescription> " - f"{_ntriples_string_literal(definition.description)} .\n" + definition = _synthesized_format_definition( + file_uri, format_key, format_component ) - emitted_definitions.add(definition_uri) - stats["format_definitions"] += 1 + definitions[format_key] = definition + definition_uri = _emit_format_field_definition( + emit, + definition=definition, + field_id=format_key, + emitted=emitted_definitions, + stats=stats, + ) encoded_values = "\t".join( values[format_index] or "." for values in record.sample_values ) emit( - f"<{matrix_uri}> <{VCFR_NAMESPACE}hasFormatValueVector> " + f"<{matrix_uri}> <{_vocab('hasFormatValueVector')}> " f"<{vector_uri}> .\n" ) - emit(f"<{vector_uri}> <{RDF_TYPE_URI}> <{VCFR_NAMESPACE}FormatValueVector> .\n") emit( - f"<{vector_uri}> <{VCFR_NAMESPACE}declaredBy> " + f"<{vector_uri}> <{RDF_TYPE_URI}> " + f"<{_vocab('FormatValueVector')}> .\n" + ) + emit( + f"<{vector_uri}> <{_vocab('declaredBy')}> " f"<{definition_uri}> .\n" ) emit( - f"<{vector_uri}> <{VCFR_NAMESPACE}valueEncoding> " - f"<{VCFR_NAMESPACE}VCFTextVector> .\n" + f"<{vector_uri}> <{_vocab('valueEncoding')}> " + f"<{_vocab('VCFTextVector')}> .\n" ) emit( - f"<{vector_uri}> <{VCFR_NAMESPACE}encodedValues> " + f"<{vector_uri}> <{_vocab('encodedValues')}> " f"{_ntriples_string_literal(encoded_values)} .\n" ) stats["format_vectors"] += 1 @@ -2863,17 +4139,23 @@ def emit_record_detail( records_tsv: Path, header_lines_tsv: Path, rdf_path: Path, + version: "vocab.VCFVersion | None" = None, ) -> dict | None: - """Append QUAL and, when selected, the structured INFO representation. + """Append the record detail, and when selected the structured INFO form. - QUAL is always emitted: the RML mapping cannot type it per row, so this is - the only place it can come from. + ID, ALT, QUAL, FILTER and INFO raw are always emitted: the RML mapping + cannot type the missing token per row, so this is the only place they can + come from. The allele layer travels with the structured INFO representation, + because the Number=A/R/G value items are joined to the allele resources it + mints. """ if info_representation not in INFO_REPRESENTATION_CHOICES: raise ValueError(f"unknown INFO representation: {info_representation}") + structured = info_representation == "structured" return append_record_detail_rdf( records_tsv, header_lines_tsv, rdf_path, - emit_qual=True, emit_info=info_representation == "structured", + emit_qual=True, emit_info=structured, emit_alleles=structured, + version=version, ) @@ -2883,12 +4165,15 @@ def emit_sample_representation( records_tsv: Path, header_lines_tsv: Path, rdf_path: Path, + version: "vocab.VCFVersion | None" = None, ) -> dict | None: """Execute the workflow's sole direct RDF emitter, if it has one.""" if workflow.emitter is None: return None if workflow.emitter == "expanded": - return append_expanded_sample_rdf(records_tsv, rdf_path) + return append_expanded_sample_rdf( + records_tsv, rdf_path, header_lines_tsv, version=version + ) if workflow.emitter == "condensed": return append_condensed_sample_rdf(records_tsv, header_lines_tsv, rdf_path) raise RuntimeError(f"unknown sample RDF emitter: {workflow.emitter}") @@ -2902,6 +4187,8 @@ def update_conversion_metrics_after_sample_stream( rdf_path: Path, total_triples: int, sample_stats: dict, + vcf_version: str | None = None, + vcf_version_source: str | None = None, ): """Bring conversion JSON/CSV metrics in sync after direct sample emission.""" safe_name = safe_metrics_name(output_name) @@ -2922,6 +4209,9 @@ def update_conversion_metrics_after_sample_stream( payload["sample_representation"] = sample_stats # Retained for consumers of pre-condensed conversion metrics. payload["sample_streaming"] = sample_stats + if vcf_version is not None: + payload["vcf_version"] = vcf_version + payload["vcf_version_source"] = vcf_version_source metrics_json.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") except (OSError, json.JSONDecodeError): pass @@ -2939,6 +4229,9 @@ def update_conversion_metrics_after_sample_stream( if row.get("run_id") == run_id and row.get("output_name") == output_name: row["output_triples"] = str(int(total_triples)) row["output_dir_size_bytes"] = str(output_size) + if vcf_version is not None and "vcf_version" in fieldnames: + row["vcf_version"] = vcf_version + row["vcf_version_source"] = vcf_version_source or "" changed = True if changed: with metrics_csv.open("w", newline="", encoding="utf-8") as handle: @@ -2998,6 +4291,12 @@ def build_sample_support_tsvs(records_tsv: Path, sample_calls_tsv: Path, sample_ # --------------------------------------------------------------------------- # RML mapping rendering and Docker image resolution # --------------------------------------------------------------------------- +#: The mapping's placeholder for the optional per-version file subclass. It is +#: rewritten per input, the same way the TSV paths are, so one mapping stays the +#: single source of truth instead of five near-identical per-version copies. +VCF_VERSION_CLASS_SENTINEL = "vcfc:VCFVersionFile" + + def render_rules_for_triplet( template_rules: Path, output_rules: Path, @@ -3006,17 +4305,56 @@ def render_rules_for_triplet( metadata_name: str, sample_calls_name: str, sample_format_name: str, + vcf_version: "vocab.VCFVersion | None" = None, ): - """Render per-input mapping rules by substituting TSV placeholders.""" + """Render per-input mapping rules by substituting TSV placeholders. + + ``vcf_version`` additionally resolves the version sentinel to that version's + optional ``vcfc:VCF4xFile`` subclass, which activates its SHACL overlay + gate. Passing None resolves the sentinel to ``vcfc:VCFFile``, which the + subject map already asserts: an unrecognized version therefore produces the + version-neutral graph rather than a claim the vocabulary cannot check. + + A custom mapping without the sentinel is unaffected; the substitution simply + finds nothing. + """ text = template_rules.read_text() text = text.replace('/data/tsv/records.tsv', f'/data/tsv/{records_name}') text = text.replace('/data/tsv/header_lines.tsv', f'/data/tsv/{headers_name}') text = text.replace('/data/tsv/file_metadata.tsv', f'/data/tsv/{metadata_name}') text = text.replace('/data/tsv/sample_calls.tsv', f'/data/tsv/{sample_calls_name}') text = text.replace('/data/tsv/sample_format_values.tsv', f'/data/tsv/{sample_format_name}') + version_class = ( + f"vcfc:{vcf_version.file_class}" if vcf_version is not None else "vcfc:VCFFile" + ) + text = text.replace(VCF_VERSION_CLASS_SENTINEL, version_class) output_rules.write_text(text) +def read_declared_vcf_version(metadata_tsv: Path) -> tuple["vocab.VCFVersion", bool, str]: + """Detect the VCF version from an input's ``##fileformat`` line. + + VCF requires ``##fileformat`` to be the first line of the file, and + ``src/vcf_as_tsv.sh`` already lifts it into ``file_metadata.tsv``, so + detection costs one small read and never needs the user to supply anything. + + Returns the version to convert with, whether it was recognized, and the raw + declared token for reporting. An unrecognized version -- absent, malformed, + or VCFv4.0, for which VCF Core claims no conformance overlay -- falls back to + the newest supported version so no representable content is dropped, and is + reported rather than silently assumed. + """ + declared = "" + if metadata_tsv.is_file(): + _set_max_csv_field_size() + with metadata_tsv.open(newline="", encoding="utf-8") as handle: + for row in csv.DictReader(handle, delimiter="\t"): + declared = (row.get("FILE_FORMAT") or "").strip() + break + version, recognized = vocab.resolve_vcf_version(declared) + return version, recognized, declared + + def resolve_default_rules_path(repo_root: Path) -> Path: """Resolve default rules path for both source checkout and installed package. @@ -3208,6 +4546,46 @@ def should_use_partitioned_hdt( return mode == "full" and rdf_storage_mode in RDF_STORAGE_MODES +def hdt_strategy_rejection( + *, + hdt_strategy: str, + methods: list[str], + gzip_aggregate: bool, + gzip_remedy: str, +) -> str | None: + """Return why `--hdt-strategy single` cannot be honoured here, or None. + + `single` is a verification path rather than a performance alternative: it + exists so one `rdf2hdt` pass can be compared against the chunked `hdtc` + merge on the same graph. It applies only to an HDT-only run over an + uncompressed aggregate. + + Two requests cannot be honoured, and both are refused rather than silently + downgraded to the partitioned path under a `single` label: + + * a gzip aggregate, which one pass could only read by materializing a + second full uncompressed copy; and + * a run that also selects COTTAS, which always needs bounded chunks, so the + partitioned path would run for both representations regardless. + """ + if hdt_strategy != "single" or not compression_uses_hdt(methods): + return None + if gzip_aggregate: + return ( + "--hdt-strategy single cannot read a gzip aggregate: one rdf2hdt pass " + "would have to materialize a second full uncompressed copy. " + + gzip_remedy + ) + if any(method in COTTAS_COMPRESSION_METHODS for method in methods): + return ( + "--hdt-strategy single applies only to an HDT-only run. COTTAS always " + "requires chunked generation, so the partitioned path would run for " + "both representations and the strategy would have no effect. Drop " + "cottas from --representations, or use --hdt-strategy partitioned." + ) + return None + + # --------------------------------------------------------------------------- # Run metrics layout: naming, manifest, and summary # --------------------------------------------------------------------------- @@ -3316,6 +4694,55 @@ def update_run_manifest(metrics_dir: Path, **updates) -> None: pass +def _validation_index(metrics_dir: Path) -> list[dict]: + """Point at every artifact one validation target produced. + + The reports are spread across three trees by design -- the stage JSON with + the other stages, the detail beside the other reports, the raw engine output + under it -- which makes them hard to find when a run is being diagnosed + rather than analysed. This collects them per target. + """ + stage_dir = metrics_dir / "stages" / "validation" + if not stage_dir.is_dir(): + return [] + entries: list[dict] = [] + for stage_path in sorted(stage_dir.glob("*.json")): + target_id = stage_path.stem + results_dir = metrics_dir / "reports" / "validation" / target_id + entry = { + "validation_id": target_id, + "stage_report": stage_path.relative_to(metrics_dir).as_posix(), + "results_dir": ( + results_dir.relative_to(metrics_dir).as_posix() + if results_dir.is_dir() else None + ), + } + try: + stage = json.loads(stage_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + stage = {} + entry["status"] = stage.get("status") + entry["engine"] = stage.get("engine") + entry["validation_target"] = stage.get("validation_target") + entry["wall_seconds"] = (stage.get("timing") or {}).get("wall_seconds") + for label, name in ( + ("summary", "summary.json"), + ("benchmark", "benchmark.json"), + ("benchmark_csv", "benchmark.csv"), + ): + candidate = results_dir / name + entry[label] = ( + candidate.relative_to(metrics_dir).as_posix() + if candidate.is_file() else None + ) + engines_dir = results_dir / "engines" + entry["engines"] = sorted( + child.name for child in engines_dir.iterdir() if child.is_dir() + ) if engines_dir.is_dir() else [] + entries.append(entry) + return entries + + def write_run_summary( *, metrics_dir: Path, @@ -3363,6 +4790,10 @@ def write_run_summary( }, "stage_reports": stage_reports, "reports": reports, + # Validation writes a per-stage JSON, a detailed report directory and a + # benchmark CSV per target, which is a lot of places to look. Name them + # explicitly rather than leaving them to be found among the globs above. + "validation": _validation_index(metrics_dir), "logs": [ path.relative_to(metrics_dir).as_posix() for path in sorted((metrics_dir / "logs").rglob("*")) @@ -4957,6 +6388,9 @@ def run_containerized_partitioned_representation_methods( eprint(f"Error: unable to create temporary Docker volume '{volume_name}'. See log: {wrapper_log_path}") return False, {} volume_created = True + # Track it from the moment it exists, so an interrupt can still find it + # if this function's own cleanup cannot remove it. + register_temp_volume(volume_name) # A named volume is normally initialized by Docker as root. Make its # workspace writable for the mapped host user before running COTTAS. @@ -5113,13 +6547,13 @@ def run_containerized_partitioned_representation_methods( result_path.unlink() except OSError: pass - if volume_created: - cleanup_exit_code = run([*docker_cmd_prefix(), "volume", "rm", "-f", volume_name]) - if cleanup_exit_code != 0: - eprint( - f"Warning: failed to remove temporary Docker volume '{volume_name}'. " - f"See log: {wrapper_log_path}" - ) + if volume_created and not remove_temp_volume(volume_name): + # Still registered, so the interrupt sweep retries after the + # containers holding it are stopped. + eprint( + f"Warning: failed to remove temporary Docker volume '{volume_name}' " + f"for now; it will be retried during cleanup. See log: {wrapper_log_path}" + ) def run_partitioned_representation_methods_for_rdf_files( @@ -5188,6 +6622,7 @@ def run_full_mode( sample_workflow: SampleWorkflow, info_representation: str = DEFAULT_INFO_REPRESENTATION, header_representation: str = DEFAULT_HEADER_REPRESENTATION, + vcf_version: str = DEFAULT_VCF_VERSION, run_validation: bool = False, validation_artifacts: list[str] | None = None, validation_engine: str | list[str] = DEFAULT_VALIDATION_ENGINE, @@ -5209,8 +6644,14 @@ def run_full_mode( timestamp: str, wrapper_log_path: Path, run_tracker: RunTracker | None = None, + linking_manifests: list | None = None, + linking_options: dict | None = None, ): """Execute full pipeline: per-input TSV -> RDF -> compression -> validation.""" + linking_options = dict(linking_options or {}) + if linking_manifests: + from vcf_rdfizer_linking.session import NetworkPolicy + linking_options["policy"] = NetworkPolicy(linking_manifests) pipeline_label = "TSV -> RDF -> compression" if run_validation: pipeline_label += " -> validation" @@ -5392,6 +6833,41 @@ def fail_current(stage: str, message: str): triplet["sample_calls"] = sample_calls_tsv triplet["sample_format_values"] = sample_format_tsv safe_prefix = slugify(prefix) + + # The VCF version is detected per input, not per run: a directory may + # hold files of different versions, and each one's mapping and emitters + # have to follow its own ##fileformat line. + detected_version, version_recognized, declared_version = ( + read_declared_vcf_version(triplet["metadata"]) + ) + if vcf_version == "auto": + active_version = detected_version if version_recognized else None + else: + active_version = vocab.VCF_VERSIONS[vcf_version] + effective_version = active_version or vocab.FALLBACK_VERSION + + if vcf_version != "auto": + if version_recognized and detected_version.short != vcf_version: + print( + f" ! VCF version overridden: file declares " + f"{detected_version.token}, converting as " + f"{effective_version.token}" + ) + else: + print(f" * VCF version: {effective_version.token} (forced)") + elif version_recognized: + print(f" * VCF version: {effective_version.token} (detected)") + else: + # VCFv4.0 and anything malformed land here. VCF Core claims no + # conformance overlay for those, so no version class is emitted and + # the newest supported rules are used, which is stated rather than + # assumed silently. + shown = declared_version or "no ##fileformat line" + print( + f" ! VCF version not recognized ({shown}); converting with " + f"{effective_version.token} rules and no version class" + ) + generated_rules = generated_rules_dir / f"{safe_prefix}.rules.ttl" render_rules_for_triplet( rules_path, @@ -5401,6 +6877,7 @@ def fail_current(stage: str, message: str): triplet["metadata"].name, triplet["sample_calls"].name, triplet["sample_format_values"].name, + vcf_version=active_version, ) output_name = safe_prefix or slugify(out_name) @@ -5557,6 +7034,7 @@ def fail_current(stage: str, message: str): records_tsv=triplet["records"], header_lines_tsv=triplet["headers"], rdf_path=raw_rdf_files[0], + version=effective_version, ) except Exception as exc: fail_current( @@ -5612,6 +7090,7 @@ def fail_current(stage: str, message: str): records_tsv=triplet["records"], header_lines_tsv=triplet["headers"], rdf_path=raw_rdf_files[0], + version=effective_version, ) except Exception as exc: fail_current( @@ -5629,6 +7108,19 @@ def fail_current(stage: str, message: str): f"declarations: {info_stats['info_definitions']:,}" ) + if linking_manifests: + from vcf_rdfizer_linking.runner import run_stage + print(" * Linking: " + ", ".join(m.id for m in linking_manifests)) + try: + run_stage( + raw_rdf_files[0], linking_manifests, + out_dir / output_name / f"{output_name}.links.nt", + metrics_dir=metrics_dir, tracker=run_tracker, **linking_options, + ) + except Exception as exc: + fail_current("data-linking", str(exc)) + continue + if triples_produced is None: triples_produced = count_triples_in_nt_files(raw_rdf_files) if header_stats is not None: @@ -5643,6 +7135,12 @@ def fail_current(stage: str, message: str): rdf_path=raw_rdf_files[0], total_triples=triples_produced, sample_stats=sample_stats, + vcf_version=effective_version.token, + vcf_version_source=( + "declared" if vcf_version == "auto" and version_recognized + else "forced" if vcf_version != "auto" + else "fallback" + ), ) if triples_produced is not None: saw_triple_counts = True @@ -6149,11 +7647,17 @@ def run_compress_mode( print("No compression methods selected (`none`). Nothing to do.") return 0 - if rdf_path.name.endswith(".gz") and compression_uses_hdt(methods) and hdt_strategy == "single": - eprint( - "Error: --hdt-strategy single cannot read a gzip aggregate without materializing " - "an uncompressed RDF file; use --hdt-strategy partitioned." - ) + hdt_strategy_error = hdt_strategy_rejection( + hdt_strategy=hdt_strategy, + methods=methods, + gzip_aggregate=rdf_path.name.endswith(".gz"), + gzip_remedy=( + "Use --hdt-strategy partitioned, or supply an uncompressed .nt if you " + "specifically need a single-pass HDT for verification." + ), + ) + if hdt_strategy_error is not None: + eprint(f"Error: {hdt_strategy_error}") return 2 if any(method in HDT_COMPRESSION_METHODS for method in methods): @@ -6699,13 +8203,21 @@ def run_validation_mode( engine_args: list[str] = ["--engine", ",".join(engine_list)] for flag, key in ( ("--query-timeout", "query_timeout"), + ("--validation-time-budget", "validation_time_budget"), ("--qlever-memory-gb", "qlever_memory_gb"), ("--qlever-port", "qlever_port"), ("--qlever-startup-timeout", "qlever_startup_timeout"), + ("--comunica-port", "comunica_port"), + ("--hdt-port", "hdt_port"), + ("--comunica-warmup-timeout", "comunica_warmup_timeout"), ): value = options.get(key) if value is not None: engine_args.extend([flag, str(value)]) + # Running every query is the default; the runner still accepts the old + # continue flag, and stopping early is now the thing you opt into. + if options.get("stop_after_query_timeout"): + engine_args.append("--stop-after-query-timeout") for flag, key in ( ("--qlever-index-arg", "qlever_index_args"), ("--qlever-server-arg", "qlever_server_args"), @@ -6729,7 +8241,10 @@ def run_validation_mode( "-v", f"{str(rdf_path.parent)}:/data/rdf:ro", "-v", - f"{str(results_dir)}:/data/validation", + # Resolved, like every other mount here: Docker reads a relative + # source as a *named volume*, so an unresolved path would silently send + # the reports into a volume instead of onto the host. + f"{str(results_dir.resolve())}:/data/validation", "-v", f"{str(metrics_dir.resolve())}:/data/metrics", *shacl_mount, @@ -6880,29 +8395,32 @@ def main(): formatter_class=argparse.RawTextHelpFormatter, epilog=( "Examples:\n" - " Full pipeline (plain aggregate):\n" + " Full pipeline (space-optimized aggregate is the default):\n" + " vcf_rdfizer.py -m full -i ./vcf_files --representations hdt,cottas " + "--rdf-compression none -o ./results\n" + " Full pipeline keeping one uncompressed .nt aggregate:\n" " vcf_rdfizer.py -m full -i ./vcf_files --rdf-storage-mode plain " "--representations hdt -o ./results\n" - " Full pipeline (space-optimized aggregate):\n" - " vcf_rdfizer.py -m full -i ./vcf_files --rdf-storage-mode space-optimized " - "--representations hdt,cottas --rdf-compression none -o ./results\n" " Full pipeline with semantic validation:\n" - " vcf_rdfizer.py -m full -i ./cohort.vcf.gz --rdf-storage-mode plain " + " vcf_rdfizer.py -m full -i ./cohort.vcf.gz " "--representations none --rdf-compression none --validate -o ./results\n" " Condensed multi-sample representation:\n" " vcf_rdfizer.py -m full -i ./cohort.vcf.gz --sample-representation condensed " - "--rdf-storage-mode space-optimized --representations hdt -o ./results\n" - " Space-optimized full pipeline with shared HDT/COTTAS chunks:\n" - " vcf_rdfizer.py -m full -i ./vcf_files --rdf-storage-mode space-optimized " + "--representations hdt -o ./results\n" + " Full pipeline with shared HDT/COTTAS chunks:\n" + " vcf_rdfizer.py -m full -i ./vcf_files " "--representations hdt,cottas --rdf-compression none " "--chunk-target-bytes 536870912 -o ./results\n" " Ultra-small full pipeline (remove aggregate RDF after compression):\n" - " vcf_rdfizer.py -m full -i ./vcf_files --rdf-storage-mode space-optimized " + " vcf_rdfizer.py -m full -i ./vcf_files " "--representations hdt --rdf-compression none --hdt-strategy partitioned " "--remove-rdf-storage-output -o ./results\n" " Queryable HDT plus gzip-packaged HDT:\n" - " vcf_rdfizer.py -m full -i ./vcf_files --rdf-storage-mode space-optimized " + " vcf_rdfizer.py -m full -i ./vcf_files " "--rdf-compression none --representations hdt --artifact-compression gzip -o ./results\n" + " Single-pass HDT for verification against the partitioned merge:\n" + " vcf_rdfizer.py -m full -i ./vcf_files --rdf-storage-mode plain " + "--representations hdt --rdf-compression none --hdt-strategy single -o ./results\n" " TSV-only benchmark:\n" " vcf_rdfizer.py -m tsv -i ./vcf_files -o ./results\n" " Compression-only:\n" @@ -6925,9 +8443,9 @@ def main(): parser.add_argument( "-m", "--mode", - choices=["full", "compress", "decompress", "tsv", "index", "validation"], + choices=["full", "compress", "decompress", "tsv", "index", "validation", "link"], default="full", - help="Run mode: full pipeline, TSV benchmark, compression, decompression, validation, or index-only regeneration", + help="Run mode: full pipeline, TSV benchmark, compression, decompression, validation, data linking, or index-only regeneration", ) parser.add_argument( "-i", @@ -6990,14 +8508,31 @@ def main(): f"(default: {DEFAULT_HEADER_REPRESENTATION})" ), ) + parser.add_argument( + "--vcf-version", + choices=VCF_VERSION_CHOICES, + default=DEFAULT_VCF_VERSION, + help=( + "VCF specification version to convert against. auto (the default) " + "reads each input's ##fileformat line, so nothing has to be " + "supplied; an explicit version overrides that for a file whose " + "declaration is missing or wrong. The version selects the " + "vcfc:VCF4xFile subclass in the mapping and the version-dependent " + "emitter behaviour (which INFO keys flatten a tuple, whether that " + "tuple repeats per ALT allele, and whether the local-allele and " + "base-modification FORMAT families exist). VCF 4.0 has no " + "conformance overlay and converts with no version class" + ), + ) parser.add_argument( "--info-representation", choices=INFO_REPRESENTATION_CHOICES, default=DEFAULT_INFO_REPRESENTATION, help=( "INFO column representation: structured also emits one " - "vcfr:InfoFieldValue per record and key alongside the raw string, " - "making INFO queryable; raw keeps only vcfr:infoRaw " + "vcfc:InfoFieldValue per record and key alongside the raw string, " + "along with the allele layer and per-allele value items, making " + "INFO queryable; raw keeps only vcfc:infoRaw " f"(default: {DEFAULT_INFO_REPRESENTATION})" ), ) @@ -7014,10 +8549,12 @@ def main(): parser.add_argument( "--rdf-storage-mode", choices=sorted(RDF_STORAGE_MODES), - default=None, + default=DEFAULT_RDF_STORAGE_MODE, help=( "Full-mode aggregate storage: plain keeps one .nt; space-optimized builds one .nt.gz " - "from RMLStreamer parts before record-safe HDT/COTTAS chunking" + "from RMLStreamer parts before record-safe HDT/COTTAS chunking " + f"(default: {DEFAULT_RDF_STORAGE_MODE}). plain is required by " + "--hdt-strategy single, which cannot read a gzip aggregate" ), ) parser.add_argument( @@ -7096,8 +8633,12 @@ def main(): choices=sorted(HDT_STRATEGY_CHOICES), default=DEFAULT_HDT_STRATEGY, help=( - "HDT generation strategy: auto uses partitioned HDT+hdtc merge for full-mode aggregate storage, " - "single uses one rdf2hdt run, partitioned forces chunked HDT generation" + "HDT generation strategy. auto uses partitioned HDT+hdtc merge for full-mode " + "aggregate storage; partitioned forces chunked generation and is the setting to " + "pin for reproducible runs. single uses one rdf2hdt run and is a verification " + "path, not a faster alternative: it exists so a one-shot conversion can be " + "compared against the chunked merge, and it requires an HDT-only run over an " + "uncompressed aggregate (--rdf-storage-mode plain, no cottas)" ), ) parser.add_argument( @@ -7161,7 +8702,7 @@ def main(): action="store_true", help=( "Fail validation when a missing token is serialized as a plain '.' " - "literal instead of '.'^^vcfr:Null (reported but non-fatal by default)" + "literal instead of '.'^^vcfc:Null (reported but non-fatal by default)" ), ) parser.add_argument( @@ -7193,6 +8734,32 @@ def main(): default=None, help="Per-query timeout in seconds for validation (default: engine default)", ) + parser.add_argument( + "--validation-time-budget", + default=None, + help=( + "Wall-clock ceiling in seconds for one engine's whole validation " + "query set (0 = no ceiling, the default). A backstop for queries " + "that are slow but never individually time out" + ), + ) + parser.add_argument( + "--validation-continue-after-query-timeout", + action="store_true", + help=( + "Run every validation query even after one times out. This is now " + "the default; the flag is kept so existing callers keep working" + ), + ) + parser.add_argument( + "--validation-stop-after-query-timeout", + action="store_true", + help=( + "Abandon an engine's remaining validation queries once one exceeds " + "the per-query timeout. Off by default: every query gets a " + "recorded verdict, which is what makes a validation run reportable" + ), + ) parser.add_argument( "--qlever-memory-gb", default=None, @@ -7235,7 +8802,12 @@ def main(): action="store_true", help="Explicitly remove the aggregate .nt/.nt.gz output after successful compression", ) + from vcf_rdfizer_link import add_link_arguments, run_options, run_posthoc, selected_linkers + add_link_arguments(parser) args = parser.parse_args() + if args.mode not in {"full", "link"} and (args.link or args.linker_path or args.offline or args.links_cache_only or args.assembly or args.links_contact_email): + eprint("Error: linking options require --mode full or --mode link") + return 2 global _PROGRESS_ALLOWED, _PROGRESS_EVENTS_ALLOWED, _QUIET _QUIET = bool(args.quiet) @@ -7245,6 +8817,9 @@ def main(): _PROGRESS_ALLOWED = not (args.no_progress or args.quiet) _PROGRESS_EVENTS_ALLOWED = not args.no_progress + if args.mode == "link": + return run_posthoc(args) + if args.build and args.no_build: eprint("Error: --build and --no-build are mutually exclusive.") return 2 @@ -7269,7 +8844,10 @@ def main(): validation_engines: list[str] = [DEFAULT_VALIDATION_ENGINE] validation_engine_options: dict = {} shacl_shapes_path: Path | None = None + linking_manifests = [] try: + if args.link: + linking_manifests = selected_linkers(args) for option_name, key in ( ("--validation-query-timeout", "query_timeout"), ("--qlever-memory-gb", "qlever_memory_gb"), @@ -7281,6 +8859,21 @@ def main(): validation_engine_options[key] = parse_positive_int( raw_value, name=option_name ) + if args.validation_time_budget is not None: + # 0 is meaningful: no ceiling. + budget = int(args.validation_time_budget) + if budget < 0: + raise ValueError( + "--validation-time-budget must be zero or a positive integer" + ) + validation_engine_options["validation_time_budget"] = budget + if args.validation_stop_after_query_timeout: + if args.validation_continue_after_query_timeout: + raise ValueError( + "--validation-continue-after-query-timeout and " + "--validation-stop-after-query-timeout are mutually exclusive" + ) + validation_engine_options["stop_after_query_timeout"] = True if args.qlever_index_arg: validation_engine_options["qlever_index_args"] = list(args.qlever_index_arg) if args.qlever_server_arg: @@ -7312,8 +8905,6 @@ def main(): if mode == "full": if args.input is None: raise ValueError("--input is required in --mode full") - if args.rdf_storage_mode is None: - raise ValueError("--rdf-storage-mode is required in --mode full") input_path = Path(args.input).expanduser().resolve() ( input_mount_dir, @@ -7356,15 +8947,17 @@ def main(): representations=args.representations, artifact_compression=args.artifact_compression, ) - if ( - args.rdf_storage_mode == "space-optimized" - and args.hdt_strategy == "single" - and compression_uses_hdt(full_methods) - ): - raise ValueError( - "--hdt-strategy single cannot be used with space-optimized HDT input; " - "use --hdt-strategy partitioned" - ) + hdt_strategy_error = hdt_strategy_rejection( + hdt_strategy=args.hdt_strategy, + methods=full_methods, + gzip_aggregate=args.rdf_storage_mode == "space-optimized", + gzip_remedy=( + "Use --hdt-strategy partitioned, or --rdf-storage-mode plain if you " + "specifically need a single-pass HDT for verification." + ), + ) + if hdt_strategy_error is not None: + raise ValueError(hdt_strategy_error) if args.spark_partitions is not None: spark_partitions = parse_positive_int( args.spark_partitions, name="--spark-partitions" @@ -7391,6 +8984,10 @@ def main(): methods=full_methods, partitioned=full_uses_partitioning, ) + if linking_manifests: + output_plans[f"input {index} ({prefix})"].add( + out_dir / output_name / f"{output_name}.links.nt" + ) validate_no_output_collisions(output_plans) elif args.run_validation: raise ValueError("--validate/--run-validation is only valid in --mode full") @@ -7462,15 +9059,17 @@ def main(): representations=args.representations, artifact_compression=args.artifact_compression, ) - if ( - rdf_path.name.endswith(".gz") - and args.hdt_strategy == "single" - and compression_uses_hdt(methods) - ): - raise ValueError( - "--hdt-strategy single cannot be used with a .nt.gz HDT input; " - "use --hdt-strategy partitioned" - ) + hdt_strategy_error = hdt_strategy_rejection( + hdt_strategy=args.hdt_strategy, + methods=methods, + gzip_aggregate=rdf_path.name.endswith(".gz"), + gzip_remedy=( + "Use --hdt-strategy partitioned, or supply an uncompressed .nt if you " + "specifically need a single-pass HDT for verification." + ), + ) + if hdt_strategy_error is not None: + raise ValueError(hdt_strategy_error) validate_mode_dirs([out_root, out_dir, metrics_root]) compression_uses_partitioning_for_input = compression_uses_partitioning(methods) and ( any(method in COTTAS_COMPRESSION_METHODS for method in methods) @@ -7592,11 +9191,14 @@ def main(): ) return 2 manifest_options = { + "link": [m.id for m in linking_manifests], + "linking_options": {k: str(v) if isinstance(v, Path) else v for k, v in run_options(args).items()} if linking_manifests else None, "requested_image": args.image, "requested_image_version": args.image_version, "sample_representation": args.sample_representation if mode == "full" else None, "info_representation": args.info_representation if mode == "full" else None, "header_representation": args.header_representation if mode == "full" else None, + "vcf_version": args.vcf_version if mode == "full" else None, "rdf_storage_mode": args.rdf_storage_mode if mode == "full" else None, "compression_methods": ( full_methods if mode == "full" else methods if mode == "compress" else [] @@ -7666,6 +9268,9 @@ def main(): wrapper_log_path = metrics_dir / "logs" / "wrapper.log" progress_log_path = metrics_dir / "logs" / "progress.log" execution_started = time.perf_counter() + # Stamp containers with this run before any of them start, so an interrupt + # can identify exactly the ones it owns. + set_docker_run_label(run_id) global _COMMAND_LOGGER _COMMAND_LOGGER = CommandLogger(wrapper_log_path) run_tracker = RunTracker(progress_log_path) @@ -7801,6 +9406,9 @@ def execute_mode(): sample_workflow=sample_workflow, info_representation=args.info_representation, header_representation=args.header_representation, + vcf_version=args.vcf_version, + linking_manifests=linking_manifests, + linking_options=run_options(args), run_validation=args.run_validation, validation_artifacts=validation_artifacts, validation_engine=validation_engines, @@ -7908,23 +9516,97 @@ def _interrupt_handler(_signum, _frame): result_code = 130 eprint("Run interrupted by user signal; starting best-effort cleanup.") run_tracker.mark("Run interrupted by user signal") - if mode == "full": - removed, failed = cleanup_interrupted_full_run( - run_tracker=run_tracker, - out_root=out_root, - image_ref=resolved_image_ref, - keep_rmlstreamer_rdf_output=args.keep_rmlstreamer_rdf_output, - wrapper_log_path=wrapper_log_path, + + # Ignore a second interrupt for the duration of cleanup. Ctrl-C twice + # is exactly what an impatient operator does, and it used to abort the + # cleanup halfway -- leaving behind the containers this is here to + # stop. Cleanup is bounded and short; let it finish. + previous_sigint = signal.getsignal(signal.SIGINT) + try: + signal.signal(signal.SIGINT, signal.SIG_IGN) + except (OSError, ValueError): # not the main thread, or unsupported + previous_sigint = None + + removed = failed = None + containers_killed = containers_failed = 0 + volumes_removed = volumes_failed = 0 + try: + # Containers first, and for every mode -- validation, compress and + # index all start containers too, and every one of them outlives + # the wrapper unless it is killed here. This must precede the file + # removal below: deleting inputs from under a live container is + # how an orphaned run spent three days reading files that no + # longer existed. + containers_killed, containers_failed = kill_run_containers( + run_tracker=run_tracker ) - eprint( - "Interrupt cleanup summary: " - "removed=" - f"{removed}, failed={failed}, " - "keep_rmlstreamer_rdf_output=" - f"{str(args.keep_rmlstreamer_rdf_output).lower()}" + if containers_killed or containers_failed: + eprint( + "Interrupt cleanup: stopped containers " + f"killed={containers_killed}, failed={containers_failed}" + ) + + # Only now can a temporary volume be removed: Docker refuses while + # a container still holds it, which is why each volume's own + # `finally` could not do this during an interrupt. + volumes_removed, volumes_failed = remove_tracked_volumes( + run_tracker=run_tracker + ) + if volumes_removed or volumes_failed: + eprint( + "Interrupt cleanup: temporary volumes " + f"removed={volumes_removed}, failed={volumes_failed}" + ) + + if mode == "full": + removed, failed = cleanup_interrupted_full_run( + run_tracker=run_tracker, + out_root=out_root, + image_ref=resolved_image_ref, + keep_rmlstreamer_rdf_output=args.keep_rmlstreamer_rdf_output, + wrapper_log_path=wrapper_log_path, + ) + eprint( + "Interrupt cleanup summary: " + "removed=" + f"{removed}, failed={failed}, " + "keep_rmlstreamer_rdf_output=" + f"{str(args.keep_rmlstreamer_rdf_output).lower()}" + ) + + checkpoint_path = write_interrupt_checkpoint( + run_tracker=run_tracker, + metrics_dir=metrics_dir, + run_id=run_id, + mode=mode, + containers_killed=containers_killed, + containers_failed=containers_failed, + removed=removed, + failed=failed, + volumes_removed=volumes_removed, + volumes_failed=volumes_failed, ) + if checkpoint_path is not None: + eprint(f"Interrupt checkpoint: {checkpoint_path}") + except Exception as cleanup_error: # noqa: BLE001 - never mask the interrupt + eprint(f"Interrupt cleanup did not complete: {cleanup_error}") + run_tracker.mark(f"Interrupt cleanup error: {cleanup_error}") + finally: + if previous_sigint is not None: + try: + signal.signal(signal.SIGINT, previous_sigint) + except (OSError, ValueError): + pass + eprint(f"Progress log: {progress_log_path}") finally: + # Last chance for any temporary volume still outstanding. On the normal + # path each is already gone; this catches the one whose own removal + # failed and which would otherwise be left behind on a clean exit too. + try: + remove_tracked_volumes(run_tracker=run_tracker) + except Exception: # noqa: BLE001 - teardown must not mask a result + pass if hasattr(signal, "SIGTERM") and original_sigterm is not None: signal.signal(signal.SIGTERM, original_sigterm) # Always report/record wrapper runtime, even on failure paths. diff --git a/vcf_rdfizer_data/linkers/__init__.py b/vcf_rdfizer_data/linkers/__init__.py new file mode 100644 index 0000000..bc54638 --- /dev/null +++ b/vcf_rdfizer_data/linkers/__init__.py @@ -0,0 +1 @@ +"""Packaged example linkers; copy with vcf-rdfizer-link init --example ID.""" diff --git a/vcf_rdfizer_data/linkers/gene-demo/README.md b/vcf_rdfizer_data/linkers/gene-demo/README.md new file mode 100644 index 0000000..2eaf602 --- /dev/null +++ b/vcf_rdfizer_data/linkers/gene-demo/README.md @@ -0,0 +1,25 @@ +# Tier 2 — synthetic gene intervals + +This plug-in exercises an interval join without Python. `linker.ttl` declares +the local `genes.gff3`, its SHA-256 and an assembly guard. The genes and their +coordinates are **synthetic**, licensed with the example under MIT. The GRCh38 +label exists to test assembly matching; these are not biological annotations. + +```bash +vcf-rdfizer-link init --example gene-demo -o my-gene-linker +vcf-rdfizer-link check my-gene-linker --offline +vcf-rdfizer-link dry-run my-gene-linker -i sample.vcf --limit 100 +``` + +Intervals are 1-based closed; keys cover POS through POS + len(REF) - 1. The +example contains overlapping genes A/B and a gene C on chromosome 2. Exons are +ignored. Chromosome names match exactly. Symbolic alleles are skipped. + +For a real reference, change the plug-in ID, reference URL, actual SHA-256, +assembly and object template. Point `idAttribute` at the GFF3 attribute your +template needs; do not assume every publisher's `ID` is an Ensembl gene ID. +HTTPS and local file URLs are supported. Gzip digests cover compressed bytes. +Keep the reference's licensing and terms visible in the manifest. + +The source repository's `docs/datalinking.md` explains caching and assembly +refusal, including the limited role of `--assembly`. diff --git a/vcf_rdfizer_data/linkers/gene-demo/genes.gff3 b/vcf_rdfizer_data/linkers/gene-demo/genes.gff3 new file mode 100644 index 0000000..8aedec7 --- /dev/null +++ b/vcf_rdfizer_data/linkers/gene-demo/genes.gff3 @@ -0,0 +1,7 @@ +##gff-version 3 +##sequence-region 1 1 1000 +# Synthetic intervals for testing; these are not biological gene annotations. +1 VCF-RDFizer-demo gene 100 200 . + . ID=demo-gene-A +1 VCF-RDFizer-demo gene 180 250 . - . ID=demo-gene-B +1 VCF-RDFizer-demo exon 190 195 . + . ID=ignored-exon;Parent=demo-gene-A +2 VCF-RDFizer-demo gene 10 20 . + . ID=demo-gene-C diff --git a/vcf_rdfizer_data/linkers/gene-demo/linker.ttl b/vcf_rdfizer_data/linkers/gene-demo/linker.ttl new file mode 100644 index 0000000..7ef2ddd --- /dev/null +++ b/vcf_rdfizer_data/linkers/gene-demo/linker.ttl @@ -0,0 +1,22 @@ +@prefix vcfl: . + +# Tier 2: a deliberately tiny SYNTHETIC bundle, not a gene annotation release. +# Relative IRIs resolve beside this manifest, including after `init`/pip install. +# To adapt: declare your GFF3 HTTPS URL, its actual SHA-256 and assembly, then +# change featureType/idAttribute/objectTemplate to match that bundle. +<#demoGenes> a vcfl:Linker ; + vcfl:id "gene-demo" ; + vcfl:version "1.0.0" ; + vcfl:title "Synthetic GFF3 gene overlap example" ; + vcfl:license "MIT (manifest and synthetic reference data)" ; + vcfl:termsOfUse ; + vcfl:join [ a vcfl:IntervalJoin ; + vcfl:featureType "gene" ; + vcfl:idAttribute "ID" ] ; + vcfl:reference [ vcfl:url ; + vcfl:sha256 "c343c83fbb27ca6a22079fccab7e385fc3e9c89967b521fd77bd9b0930b87aa3" ; + vcfl:assembly "GRCh38" ; + vcfl:format vcfl:GFF3 ] ; + vcfl:emit [ vcfl:subject vcfl:VariantCall ; + vcfl:predicate vcfl:overlapsGene ; + vcfl:objectTemplate "https://example.org/genes/{ID}" ] . diff --git a/vcf_rdfizer_data/linkers/rsid-dbsnp/README.md b/vcf_rdfizer_data/linkers/rsid-dbsnp/README.md new file mode 100644 index 0000000..ffb0215 --- /dev/null +++ b/vcf_rdfizer_data/linkers/rsid-dbsnp/README.md @@ -0,0 +1,19 @@ +# Tier 1 — rsID to dbSNP + +This entire plug-in is `linker.ttl`: split VCF ID on semicolons, accept complete +`rs` + digits tokens, and emit a `vcfl:sameVariantAs` link to the dbSNP IRI. +There is no resolver, reference or network. This constructs identifier links; +it does not check whether an identifier is current or retired. + +```bash +vcf-rdfizer-link init --example rsid-dbsnp -o my-rsid-linker +vcf-rdfizer-link check my-rsid-linker +vcf-rdfizer-link dry-run my-rsid-linker -i sample.vcf --limit 100 +``` + +To read INFO tokens, change `field` to `INFO/` and select the delimiter. +To change the target, edit `objectTemplate` while retaining `{TOKEN}`. Change +`id` before installing a copy beside this example. The runner escapes the +token, owns the subject and removes duplicate triples. + +The source repository's `docs/datalinking.md` documents the full contract. diff --git a/vcf_rdfizer_data/linkers/rsid-dbsnp/linker.ttl b/vcf_rdfizer_data/linkers/rsid-dbsnp/linker.ttl new file mode 100644 index 0000000..027434a --- /dev/null +++ b/vcf_rdfizer_data/linkers/rsid-dbsnp/linker.ttl @@ -0,0 +1,17 @@ +@prefix vcfl: . + +# Tier 1: no code, reference download, or API request. +# ID is a semicolon-separated VCF column. Matching is case-sensitive. +<#rsIDdbSNP> a vcfl:Linker ; + vcfl:id "rsid-dbsnp" ; + vcfl:version "1.0.0" ; + vcfl:title "rsID to dbSNP" ; + vcfl:license "MIT (plug-in code and manifest)" ; + vcfl:termsOfUse ; + vcfl:join [ a vcfl:TokenJoin ; + vcfl:field "ID" ; + vcfl:splitOn ";" ; + vcfl:accept "^rs[0-9]+$" ] ; + vcfl:emit [ vcfl:subject vcfl:VariantCall ; + vcfl:predicate vcfl:sameVariantAs ; + vcfl:objectTemplate "https://identifiers.org/dbsnp:{TOKEN}" ] . diff --git a/vcf_rdfizer_data/linkers/rsid-ensembl/README.md b/vcf_rdfizer_data/linkers/rsid-ensembl/README.md new file mode 100644 index 0000000..87596bc --- /dev/null +++ b/vcf_rdfizer_data/linkers/rsid-ensembl/README.md @@ -0,0 +1,29 @@ +# Tier 3 — Ensembl variation service + +`resolver.py` sends one POST per deduplicated batch to Ensembl's +[variation endpoint](https://rest.ensembl.org/documentation/info/variation_post). +It links returned identifiers to Ensembl variation pages; it does not add +clinical significance, frequencies, or allele-level equivalence assertions. + +```bash +vcf-rdfizer-link init --example rsid-ensembl -o my-api-linker +vcf-rdfizer-link check my-api-linker +vcf-rdfizer-link dry-run my-api-linker -i sample.vcf --limit 100 + +vcf-rdfizer-link run -i sample.vcf --link rsid-ensembl \ + --links-contact-email you@your-institution.org \ + --links-cache ./api-cache -o sample.ensembl.links.nt +``` + +Set the contact address before a live cache miss. The shipped budget is 2 +requests/second, 100 total HTTP attempts per invocation, 100 rsIDs per batch; +retries consume the ceiling. Use filtered inputs. The runner's session owns +HTTPS access, caching, host pacing and retries; the resolver imports no HTTP +client. The session is a policy API, not a sandbox for untrusted Python. + +`dry-run` reports batches without calling the service or executing the resolver. +An offline replay needs an existing matching cache. A missing ID is skipped; +malformed responses, service failures and exhausted budgets abort the linker. +The example's code/manifest are MIT; service terms are linked in the manifest. + +The source repository's `docs/datalinking.md` documents the protocol and limits. diff --git a/vcf_rdfizer_data/linkers/rsid-ensembl/linker.ttl b/vcf_rdfizer_data/linkers/rsid-ensembl/linker.ttl new file mode 100644 index 0000000..842e75d --- /dev/null +++ b/vcf_rdfizer_data/linkers/rsid-ensembl/linker.ttl @@ -0,0 +1,21 @@ +@prefix vcfl: . + +# Tier 3: POST one batch of rsIDs to Ensembl, linking only returned variants. +# Copy this example and set contactEmail to your own address before live use. +<#rsIDEnsembl> a vcfl:Linker ; + vcfl:id "rsid-ensembl" ; + vcfl:version "1.0.0" ; + vcfl:title "Service-confirmed rsID to Ensembl" ; + vcfl:license "MIT (plug-in code and manifest)" ; + vcfl:termsOfUse ; + vcfl:endpoint ; + vcfl:maxRequestsPerSecond 2 ; + vcfl:maxRequestsPerRun 100 ; + vcfl:batchSize 100 ; + vcfl:contactEmail "replace-me@example.org" ; + vcfl:join [ a vcfl:TokenJoin ; + vcfl:field "ID" ; + vcfl:splitOn ";" ; + vcfl:accept "^rs[0-9]+$" ] ; + vcfl:emit [ vcfl:subject vcfl:VariantCall ; + vcfl:predicate vcfl:sameVariantAs ] . diff --git a/vcf_rdfizer_data/linkers/rsid-ensembl/resolver.py b/vcf_rdfizer_data/linkers/rsid-ensembl/resolver.py new file mode 100644 index 0000000..9779156 --- /dev/null +++ b/vcf_rdfizer_data/linkers/rsid-ensembl/resolver.py @@ -0,0 +1,34 @@ +"""Ensembl variation POST example: one request per deduplicated batch. + +API contract: https://rest.ensembl.org/documentation/info/variation_post +Only confirm returned identifiers. Do not infer clinical significance, allele +equivalence, frequencies, or current identifiers for merged/retired rsIDs. +""" + +from collections.abc import Iterable, Sequence +from urllib.parse import quote + +from vcf_rdfizer_linking import Link, LinkKey, LinkerContext + + +def resolve(batch: Sequence[LinkKey], ctx: LinkerContext) -> Iterable[Link]: + response = ctx.session.post( + "https://rest.ensembl.org/variation/homo_sapiens", + json={"ids": [key.token for key in batch]}, + ) + response.raise_for_status() + payload = response.json() + if not isinstance(payload, dict): + raise ValueError("Expected an Ensembl object keyed by requested identifiers") + for key in batch: + variant = payload.get(key.token) + if variant is None: + continue + if not isinstance(variant, dict): + raise ValueError(f"Invalid Ensembl variation object for {key.token}") + if variant.get("error"): + continue + if not isinstance(variant.get("name"), str): + raise ValueError(f"Ensembl response for {key.token} has no variant name") + # Preserve the submitted identifier; the website handles redirects. + yield Link(key, "https://www.ensembl.org/Homo_sapiens/Variation/Explore?v=" + quote(key.token, safe="")) diff --git a/vcf_rdfizer_data/rules/default_rules.ttl b/vcf_rdfizer_data/rules/default_rules.ttl index 7332230..a4e0107 100644 --- a/vcf_rdfizer_data/rules/default_rules.ttl +++ b/vcf_rdfizer_data/rules/default_rules.ttl @@ -3,9 +3,17 @@ @prefix ql: . @prefix csvw: . @prefix xsd: . -@prefix vcfr: . +@prefix vcfc: . -# Default VCF-RDFizer RML rules. +# Default VCF-RDFizer RML rules, targeting the VCF Core vocabulary. +# +# Namespace +# --------- +# The target is https://w3id.org/vcf-core/vocab# (prefix vcfc:), which replaces +# the retired https://w3id.org/vcf-rdfizer/vocab# (prefix vcfr:). The retired +# namespace is not redirected: it serves a deprecation document in which every +# former term is linked to its successor. No version IRI is pinned; these rules +# target the namespace, whose release cadence is independent of the converter's. # # Expected TSV inputs produced by src/vcf_as_tsv.sh: # - /data/tsv/.file_metadata.tsv @@ -14,12 +22,60 @@ # - /data/tsv/.sample_calls.tsv (header-only compatibility source) # - /data/tsv/.sample_format_values.tsv (header-only compatibility source) # The wrapper rewrites the template paths below for each input sample. -# The wrapper selects exactly one direct sample emitter: expanded SampleCall / -# FormatFieldValue triples, or condensed SampleSet / CohortCallMatrix / -# FormatValueVector triples. The helper tables stay header-only in either mode. +# +# What is here and what is not +# ---------------------------- +# The division is a single rule: this mapping carries every field whose RDF +# datatype is the same for every row, and the wrapper's direct N-Triples +# emitters carry everything else. RML cannot choose an object datatype per row, +# and it cannot decompose one source cell into several resources without a +# materialized helper table whose size is the product of its dimensions. +# +# Emitted by the wrapper, not here: +# - ID, ALT, QUAL, FILTER and INFO. Each may be the VCF missing token, which the +# vocabulary requires as "."^^vcfc:Null rather than a plain string, so the +# datatype depends on the row. +# - fileDate. The published shape requires xsd:date, but ##fileDate has no +# mandated format in the VCF specification, so the value must be inspected. +# - The structured header representation: header-line subclasses, the +# vcfc:HeaderAttribute resources that vcfc:StructuredHeaderLineShape requires +# on every ##INFO/##FORMAT/##FILTER/##ALT/##contig/##META/##SAMPLE/##PEDIGREE +# line, and the INFO/FORMAT/FILTER/ALT/contig/META declaration properties. +# These all live inside one angle-bracketed cell. +# - The allele layer: vcfc:ReferenceAllele and vcfc:AltAllele resources with +# their index, kind, symbolic type and breakend components, parsed out of the +# REF and ALT columns. +# - The structured INFO representation, its vcfc:FieldValueItem decomposition, +# and the SV carriers derived from reserved INFO keys. +# - Genotype data. The wrapper selects exactly one direct sample emitter: +# expanded SampleCall / FormatFieldValue / Genotype triples, or condensed +# SampleSet / CohortCallMatrix / FormatValueVector triples. The helper tables +# stay header-only in either mode. +# +# Ordinals +# -------- +# vcfc:lineIndex and vcfc:recordIndex are typed xsd:integer, not +# xsd:positiveInteger. The SHACL profiles constrain every ordinal with +# sh:datatype xsd:integer plus a sh:minInclusive bound, and a SHACL datatype +# constraint compares the datatype IRI exactly. An xsd:integer literal +# satisfies both the shape and the ontology's xsd:positiveInteger range, since +# the value 1 belongs to that value space however it is written. +# +# VCF versions +# ------------ +# VCF Core supplies one SHACL overlay per VCF version, each scoped by the file's +# vcfc:fileFormat. Most of this mapping is version-neutral, because the fixed +# columns and the header-line structure do not change between 4.1 and 4.5. The +# one version-dependent thing RML can carry is the optional vcfc:VCF4xFile +# subclass on the file subject, which is the sentinel in <#VCFFileMap> below. +# +# The version-dependent *semantics* -- which INFO keys flatten a tuple, whether +# that tuple repeats per ALT allele, and whether the local-allele and +# base-modification FORMAT families exist -- are decompositions of one source +# cell, so they live in the wrapper's emitters like every other decomposition. # # Output format: -# - The default conversion now targets triples (N-Triples) without named graph terms. +# - The default conversion targets triples (N-Triples) without named graph terms. <#VCFFileMap> a rr:TriplesMap ; @@ -37,22 +93,30 @@ ] ; rr:subjectMap [ rr:template "file://{SOURCE_FILE}" ; - rr:class vcfr:VCFFile + rr:class vcfc:VCFFile ; + # Version overlay. The wrapper rewrites this sentinel per input to the + # optional subclass for the version the file's ##fileformat line declares + # -- vcfc:VCF41File through vcfc:VCF45File -- which activates that version's + # SHACL gate. When the version has no conformance overlay (VCFv4.0, or a + # missing or malformed line) the sentinel is rewritten to vcfc:VCFFile, + # which is already asserted above, so the graph never claims a version gate + # it cannot support and never gains an undefined term. + rr:class vcfc:VCFVersionFile ] ; rr:predicateObjectMap [ - rr:predicate vcfr:hasHeader ; + rr:predicate vcfc:hasHeader ; rr:objectMap [ rr:template "file://{SOURCE_FILE}#header" ] ] ; rr:predicateObjectMap [ - rr:predicate vcfr:fileFormat ; + rr:predicate vcfc:fileFormat ; rr:objectMap [ rml:reference "FILE_FORMAT" ] ] ; rr:predicateObjectMap [ - rr:predicate vcfr:sourceSoftware ; + rr:predicate vcfc:sourceSoftware ; rr:objectMap [ rml:reference "SOURCE_SOFTWARE" ] ] ; rr:predicateObjectMap [ - rr:predicate vcfr:referenceGenome ; + rr:predicate vcfc:referenceGenome ; rr:objectMap [ rml:reference "REFERENCE_GENOME" ] ] . @@ -72,9 +136,34 @@ ] ; rr:subjectMap [ rr:template "file://{SOURCE_FILE}#header" ; - rr:class vcfr:VCFHeader + rr:class vcfc:VCFHeader + ] ; + rr:predicateObjectMap [ + rr:predicate vcfc:hasColumnHeader ; + rr:objectMap [ rr:template "file://{SOURCE_FILE}#header/columns" ] + ] . + +# The mandatory #CHROM line. Its ordered sample columns are attached by the +# wrapper with vcfc:hasGenotypeColumns, because they live in the records TSV +# header rather than in a row of any source table. +<#ColumnHeaderLineMap> + a rr:TriplesMap ; + rml:logicalSource [ + rml:source [ + a csvw:Table ; + csvw:url "/data/tsv/file_metadata.tsv" ; + csvw:dialect [ + a csvw:Dialect ; + csvw:delimiter "\t" ; + csvw:headerRowCount 1 + ] + ] ; + rml:referenceFormulation ql:CSV ] ; - . + rr:subjectMap [ + rr:template "file://{SOURCE_FILE}#header/columns" ; + rr:class vcfc:ColumnHeaderLine + ] . # NOTE: # Use explicit link maps (template-based) instead of parentTriplesMap joins. @@ -97,7 +186,7 @@ rr:template "file://{SOURCE_FILE}" ] ; rr:predicateObjectMap [ - rr:predicate vcfr:hasRecord ; + rr:predicate vcfc:hasRecord ; rr:objectMap [ rr:template "file://{SOURCE_FILE}#record/{ROW_ID}" ] @@ -121,12 +210,16 @@ rr:template "file://{SOURCE_FILE}#header" ] ; rr:predicateObjectMap [ - rr:predicate vcfr:hasHeaderLine ; + rr:predicate vcfc:hasHeaderLine ; rr:objectMap [ rr:template "file://{SOURCE_FILE}#header/line/{HEADER_INDEX}" ] ] . +# Every '##' line as a base vcfc:HeaderLine. The wrapper adds the subclass, the +# ordered vcfc:HeaderAttribute resources and the declaration properties. +# vcfc:lineIndex is what the SPARQL SHACL profile uses to check that header +# indices are unique and that ##fileformat is line 1. <#HeaderLineMap> a rr:TriplesMap ; rml:logicalSource [ @@ -143,17 +236,24 @@ ] ; rr:subjectMap [ rr:template "file://{SOURCE_FILE}#header/line/{HEADER_INDEX}" ; - rr:class vcfr:HeaderLine + rr:class vcfc:HeaderLine ] ; rr:predicateObjectMap [ - rr:predicate vcfr:headerKey ; + rr:predicate vcfc:headerKey ; rr:objectMap [ rml:reference "HEADER_KEY" ] ] ; rr:predicateObjectMap [ - rr:predicate vcfr:headerValue ; + rr:predicate vcfc:headerValue ; rr:objectMap [ rml:reference "HEADER_VALUE" ] + ] ; + rr:predicateObjectMap [ + rr:predicate vcfc:lineIndex ; + rr:objectMap [ rml:reference "HEADER_INDEX" ; rr:datatype xsd:integer ] ] . +# CHROM, POS and REF are the three fixed fields VCF 4.5 forbids from being the +# missing token, so their datatype is fixed for every row and they belong here. +# ID and ALT may be missing and are emitted by the wrapper. <#VCFRecordMap> a rr:TriplesMap ; rml:logicalSource [ @@ -170,33 +270,32 @@ ] ; rr:subjectMap [ rr:template "file://{SOURCE_FILE}#record/{ROW_ID}" ; - rr:class vcfr:VCFRecord + rr:class vcfc:VCFRecord ] ; rr:predicateObjectMap [ - rr:predicate vcfr:chrom ; + rr:predicate vcfc:chrom ; rr:objectMap [ rml:reference "CHROM" ] ] ; rr:predicateObjectMap [ - rr:predicate vcfr:pos ; + rr:predicate vcfc:pos ; rr:objectMap [ rml:reference "POS" ; rr:datatype xsd:integer ] ] ; rr:predicateObjectMap [ - rr:predicate vcfr:recordId ; - rr:objectMap [ rml:reference "ID" ] - ] ; - rr:predicateObjectMap [ - rr:predicate vcfr:ref ; + rr:predicate vcfc:ref ; rr:objectMap [ rml:reference "REF" ] ] ; rr:predicateObjectMap [ - rr:predicate vcfr:alt ; - rr:objectMap [ rml:reference "ALT" ] + rr:predicate vcfc:recordIndex ; + rr:objectMap [ rml:reference "ROW_ID" ; rr:datatype xsd:integer ] ] ; rr:predicateObjectMap [ - rr:predicate vcfr:hasCall ; + rr:predicate vcfc:hasCall ; rr:objectMap [ rr:template "file://{SOURCE_FILE}#call/{ROW_ID}" ] ] . +# FORMAT is the only call-level column whose datatype is fixed: VCF 4.5 defines +# it as a colon-separated key list with no missing form. QUAL, FILTER and INFO +# are emitted by the wrapper. <#VariantCallMap> a rr:TriplesMap ; rml:logicalSource [ @@ -213,18 +312,10 @@ ] ; rr:subjectMap [ rr:template "file://{SOURCE_FILE}#call/{ROW_ID}" ; - rr:class vcfr:VariantCall - ] ; - rr:predicateObjectMap [ - rr:predicate vcfr:filter ; - rr:objectMap [ rml:reference "FILTER" ] - ] ; - rr:predicateObjectMap [ - rr:predicate vcfr:infoRaw ; - rr:objectMap [ rml:reference "INFO" ] + rr:class vcfc:VariantCall ] ; rr:predicateObjectMap [ - rr:predicate vcfr:formatRaw ; + rr:predicate vcfc:formatRaw ; rr:objectMap [ rml:reference "FORMAT" ] ] . @@ -246,7 +337,7 @@ rr:template "file://{SOURCE_FILE}#call/{ROW_ID}" ] ; rr:predicateObjectMap [ - rr:predicate vcfr:hasSampleCall ; + rr:predicate vcfc:hasSampleCall ; rr:objectMap [ rr:template "file://{SOURCE_FILE}#sample/{ROW_ID}/{SAMPLE_URI_ID}" ] @@ -268,11 +359,17 @@ ] ; rr:subjectMap [ rr:template "file://{SOURCE_FILE}#sample/{ROW_ID}/{SAMPLE_URI_ID}" ; - rr:class vcfr:SampleCall + rr:class vcfc:SampleCall ] ; rr:predicateObjectMap [ - rr:predicate vcfr:sampleId ; + rr:predicate vcfc:sampleId ; rr:objectMap [ rml:reference "SAMPLE_ID" ] + ] ; + rr:predicateObjectMap [ + rr:predicate vcfc:forSample ; + rr:objectMap [ + rr:template "file://{SOURCE_FILE}#samples/{SAMPLE_URI_ID}" + ] ] . <#SampleCallToFormatValueLinkMap> @@ -293,7 +390,7 @@ rr:template "file://{SOURCE_FILE}#sample/{ROW_ID}/{SAMPLE_URI_ID}" ] ; rr:predicateObjectMap [ - rr:predicate vcfr:hasFormatValue ; + rr:predicate vcfc:hasFormatValue ; rr:objectMap [ rr:template "file://{SOURCE_FILE}#sample/{ROW_ID}/{SAMPLE_URI_ID}/fmt/{FORMAT_KEY}" ] @@ -315,9 +412,9 @@ ] ; rr:subjectMap [ rr:template "file://{SOURCE_FILE}#sample/{ROW_ID}/{SAMPLE_URI_ID}/fmt/{FORMAT_KEY}" ; - rr:class vcfr:FormatFieldValue + rr:class vcfc:FormatFieldValue ] ; rr:predicateObjectMap [ - rr:predicate vcfr:fieldValue ; + rr:predicate vcfc:fieldValue ; rr:objectMap [ rml:reference "FORMAT_VALUE" ] ] . diff --git a/vcf_rdfizer_link.py b/vcf_rdfizer_link.py new file mode 100644 index 0000000..1fd1901 --- /dev/null +++ b/vcf_rdfizer_link.py @@ -0,0 +1,199 @@ +#!/usr/bin/env python3 +"""Inspect, scaffold, preview and run data-linking plug-ins without Docker.""" + +import argparse +from dataclasses import asdict, replace +from datetime import datetime +import json +from pathlib import Path +import shutil +import sys +import time + +from vcf_rdfizer_linking.inputs import read_rdf, read_vcf +from vcf_rdfizer_linking.manifest import discover, load_manifest, select +from vcf_rdfizer_linking.reference import acquire_reference, check_assembly, IntervalIndex +from vcf_rdfizer_linking.runner import DEFAULT_CACHE, LinkRunError, run_linkers, run_stage + + +def _require_rdflib(): + """Fail with an actionable message when the linking dependency is absent. + + rdflib is a declared runtime dependency, but the tool is routinely run from + a checkout, where nothing installs it. Every other mode works without it; + this turns the ModuleNotFoundError that only linking would hit into an + instruction. + """ + try: + import rdflib # noqa: F401 + except ModuleNotFoundError: + raise ValueError( + "data linking requires the 'rdflib' package, which is not " + "installed. Install the project's dependencies with " + "'python -m pip install -e .' from the repository, or " + "'python -m pip install rdflib'. Every other mode runs without it." + ) from None + + +def add_link_arguments(parser): + parser.add_argument("--link", help="Comma-separated linker IDs (full/link modes)") + parser.add_argument("--linker-path", action="append", default=[], help="Additional linker search directory (repeatable)") + parser.add_argument("--links-cache", default=str(DEFAULT_CACHE), help="Reference and HTTP response cache directory") + parser.add_argument("--offline", action="store_true", help="Disable linker network access; local references and cached responses only") + parser.add_argument("--links-cache-only", action="store_true", help="Alias for --offline for linking") + parser.add_argument("--assembly", help="Input assembly when reference metadata is absent/unrecognized; cannot override a mismatch") + parser.add_argument("--links-contact-email", help="Contact address sent in live linker User-Agent headers") + + +def selected_linkers(args): + import re + _require_rdflib() + manifests = select(args.link or "", args.linker_path) + if args.links_contact_email: + if not re.fullmatch(r"[^\s<>@]+@[^\s<>@]+\.[^\s<>@]+", args.links_contact_email): + raise ValueError("--links-contact-email must be an email address") + manifests = [replace(m, contact_email=args.links_contact_email) if m.tier == 3 else m for m in manifests] + return manifests + + +def run_options(args): + return {"cache_dir": Path(args.links_cache).expanduser(), + "offline": args.offline or args.links_cache_only, "assembly": args.assembly} + + +def run_posthoc(args): + """Host-only --mode link, with the standard run_metrics tree.""" + from vcf_rdfizer import (RunTracker, metrics_run_directory, metrics_run_label, + write_run_manifest, write_run_summary, rdf_output_basename) + try: + # Inside the guard so a missing dependency is reported the same way as + # any other bad invocation, rather than as a traceback. + _require_rdflib() + if not args.rdf or not args.link: + raise ValueError("--mode link requires --rdf and --link") + rdf = Path(args.rdf).expanduser().resolve() + if not rdf.is_file() or not rdf.name.endswith((".nt", ".nt.gz")): + raise ValueError("--rdf must be an existing .nt or .nt.gz file") + manifests = selected_linkers(args) + root = Path(args.out).expanduser().resolve() + output = root / (rdf_output_basename(rdf) + ".links.nt") + if output.exists(): + raise ValueError(f"Refusing to overwrite existing linkset: {output}") + except (ValueError, OSError) as exc: + print(f"error: {exc}", file=sys.stderr) + return 2 + run_id = datetime.now().strftime("%Y%m%dT%H%M%S%f") + timestamp = datetime.now().isoformat(timespec="seconds") + metrics = metrics_run_directory(root / "run_metrics", metrics_run_label([rdf], rdf), run_id) + started, tracker, report, code = time.monotonic(), None, None, 1 + try: + write_run_manifest(metrics_dir=metrics, run_id=run_id, timestamp=timestamp, + mode="link", source_label=rdf.name, source_paths=[rdf], out_root=root, + options={"link": [m.id for m in manifests], **{k: str(v) if isinstance(v, Path) else v for k, v in run_options(args).items()}}) + tracker = RunTracker(metrics / "logs" / "progress.log") + report = run_stage(rdf, manifests, output, metrics_dir=metrics, tracker=tracker, **run_options(args)) + print(f"Wrote {output} ({report['link_triples']} links); metrics: {metrics}") + code = 0 + except (ValueError, OSError) as exc: + print(f"error: {exc}", file=sys.stderr) + except KeyboardInterrupt: + print("Linking interrupted; no partial side-graph published", file=sys.stderr) + code = 130 + finally: + if tracker: + tracker.close() + write_run_summary(metrics_dir=metrics, run_id=run_id, timestamp=timestamp, mode="link", + exit_code=code, elapsed_seconds=time.monotonic() - started, + total_triples=report["triples"] if report else None) + return code + + +def build_parser(): + parser = argparse.ArgumentParser(prog="vcf-rdfizer-link", description=__doc__) + commands = parser.add_subparsers(dest="command", required=True) + listing = commands.add_parser("list", help="List installed IDs, versions, tiers, references and licensing") + listing.add_argument("--linker-path", action="append", default=[]) + listing.add_argument("--json", action="store_true") + commands.add_parser("keys", help="Describe the supported join-key contract") + init = commands.add_parser("init", help="Copy an annotated example into a new linker directory") + init.add_argument("-o", "--output", required=True) + init.add_argument("--example", default="rsid-dbsnp", choices=["rsid-dbsnp", "gene-demo", "rsid-ensembl"]) + check = commands.add_parser("check", help="Parse a manifest and verify its reference (no resolver execution)") + check.add_argument("directory") + check.add_argument("--json", action="store_true") + add_link_arguments(check) + dry = commands.add_parser("dry-run", help="Preview first N VCF records; no network or persistent writes") + dry.add_argument("directory") + dry.add_argument("-i", "--input", required=True) + dry.add_argument("--limit", type=int, default=100) + add_link_arguments(dry) + run = commands.add_parser("run", help="Write a side-graph from VCF or an existing RDF aggregate") + inputs = run.add_mutually_exclusive_group(required=True) + inputs.add_argument("-i", "--input", help="VCF to link using the default subject templates") + inputs.add_argument("--rdf", help="Existing .nt/.nt.gz; subjects are read from the graph") + run.add_argument("-o", "--output", required=True, help="New .links.nt output file") + add_link_arguments(run) + return parser + + +def main(argv=None): + args = build_parser().parse_args(argv) + try: + if args.command == "keys": + print("TokenJoin: field ID (splitOn ';') or INFO/ (splitOn ','); accept uses full regex matches; {TOKEN} is percent-encoded.") + print("IntervalJoin: CHROM exact match; [POS, POS + len(REF) - 1], 1-based closed; explicit DNA alleles only. GFF3 attribute -> {ID}.") + print("Subjects: vcfl:VariantCall or vcfl:VCFRecord. AlleleJoin is not implemented.") + elif args.command == "list": + manifests = discover(args.linker_path) + if args.json: + print(json.dumps([asdict(m) for m in manifests.values()], default=str, indent=2)) + else: + for m in manifests.values(): + print(f"{m.id} {m.version} — tier {m.tier}: {m.title}") + print(f" License: {m.license}; terms: {m.terms_of_use or 'unspecified'}") + if m.reference: + print(f" Reference: {m.reference.url}; {m.reference.assembly}; sha256:{m.reference.sha256}") + elif args.command == "init": + output = Path(args.output).expanduser() + if output.exists(): + raise ValueError(f"Refusing to overwrite {output}") + source = discover()[args.example].directory + shutil.copytree(source, output, ignore=shutil.ignore_patterns("__pycache__", "*.pyc")) + print(f"Wrote {output}. Change its id before adding it to a search path, then run check.") + elif args.command == "check": + m = load_manifest(Path(args.directory)) + if m.reference: + if args.assembly: + check_assembly(args.assembly, m.reference.assembly) + path = acquire_reference(m.reference, Path(args.links_cache), offline=args.offline or args.links_cache_only) + IntervalIndex(path, m.reference) + print(json.dumps({"ok": True, "id": m.id, "tier": m.tier, "assembly": m.reference.assembly if m.reference else None, + "note": "Input assembly is checked at run time; resolver code was not executed"}, indent=2)) + elif args.command == "dry-run": + if args.limit < 1: + raise ValueError("--limit must be positive") + report = run_linkers(read_vcf(Path(args.input).expanduser(), args.limit), [load_manifest(Path(args.directory))], None, dry_run=True, **run_options(args)) + print(json.dumps(report, indent=2)) + else: + manifests = selected_linkers(args) + output = Path(args.output).expanduser() + if not output.name.endswith(".nt"): + raise ValueError("Link output must end in .nt") + if output.with_suffix(".json").exists(): + raise ValueError(f"Refusing to overwrite existing report: {output.with_suffix('.json')}") + records = read_rdf(Path(args.rdf).expanduser()) if args.rdf else read_vcf(Path(args.input).expanduser()) + report = run_linkers(records, manifests, output, **run_options(args)) + from vcf_rdfizer_linking.session import atomic_bytes + atomic_bytes(output.with_suffix(".json"), (json.dumps(report, indent=2) + "\n").encode()) + print(f"Wrote {output} ({report['link_triples']} links)") + return 0 + except (ValueError, OSError) as exc: + if getattr(args, "json", False): + print(json.dumps({"ok": False, "errors": [str(exc)]})) + else: + print(f"error: {exc}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/vcf_rdfizer_linking/__init__.py b/vcf_rdfizer_linking/__init__.py new file mode 100644 index 0000000..7689715 --- /dev/null +++ b/vcf_rdfizer_linking/__init__.py @@ -0,0 +1,30 @@ +"""Public, deliberately narrow protocol for data-linking plug-ins. + +Resolvers receive deduplicated keys and return objects; the runner owns subject +identity, predicates, serialization, batching, and network policy. +""" + +from dataclasses import dataclass +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from .session import CachedSession + + +@dataclass(frozen=True, order=True) +class LinkKey: + token: str = "" + chrom: str = "" + start: int = 0 + end: int = 0 + + +@dataclass(frozen=True) +class Link: + key: LinkKey + object: str + + +@dataclass(frozen=True) +class LinkerContext: + session: "CachedSession" diff --git a/vcf_rdfizer_linking/inputs.py b/vcf_rdfizer_linking/inputs.py new file mode 100644 index 0000000..0ea1540 --- /dev/null +++ b/vcf_rdfizer_linking/inputs.py @@ -0,0 +1,153 @@ +"""Join-key inputs, preserving the wrapper's file/record/call identities.""" + +import csv +from dataclasses import dataclass +import gzip +from pathlib import Path +import sqlite3 +import tempfile + + +from .manifest import VCFR + + +@dataclass(frozen=True) +class Source: + source: str + reference: str + + +@dataclass(frozen=True) +class Record: + source: str + record: str + call: str + reference: str + chrom: str + pos: str + ref: str + alt: str + id: str + info: str + + +def open_text(path): + return gzip.open(path, "rt", encoding="utf-8") if path.name.endswith(".gz") else path.open(encoding="utf-8") + + +def make_record(source_file, row_id, reference, fields): + from vcf_rdfizer import _rml_uri_component + + source = "file://" + _rml_uri_component(source_file) + row = _rml_uri_component(str(row_id)) + return Record(source, f"{source}#record/{row}", f"{source}#call/{row}", reference, + *(fields.get(k, ".") for k in ("CHROM", "POS", "REF", "ALT", "ID", "INFO"))) + + +def read_vcf(path: Path, limit=None): + reference, row, header = "", 0, False + with open_text(path) as handle: + for line in handle: + if line.startswith("##reference="): + value = line.strip().split("=", 1)[1] + if reference and value != reference: + raise ValueError("VCF declares conflicting ##reference values") + reference = value + if line.startswith("#CHROM\t"): + header = True + if line.startswith("#") or not line.strip(): + continue + if not header: + raise ValueError("VCF is missing the #CHROM header") + if limit is not None and row >= limit: + break + if row == 0: + from vcf_rdfizer import _rml_uri_component + yield Source("file://" + _rml_uri_component(path.name), reference) + fields = line.rstrip("\r\n").split("\t") + row += 1 + if len(fields) < 8: + raise ValueError(f"VCF record {row} has fewer than eight columns") + yield make_record(path.name, row, reference, + dict(zip(("CHROM", "POS", "ID", "REF", "ALT", "QUAL", "FILTER", "INFO"), fields))) + if not header: + raise ValueError("VCF is missing the #CHROM header") + if row == 0: + from vcf_rdfizer import _rml_uri_component + yield Source("file://" + _rml_uri_component(path.name), reference) + + +def read_tsv(records: Path, metadata: Path): + with metadata.open(encoding="utf-8", newline="") as handle: + references = {r["SOURCE_FILE"]: r["REFERENCE_GENOME"] for r in csv.DictReader(handle, delimiter="\t")} + from vcf_rdfizer import _rml_uri_component + for source, reference in references.items(): + yield Source("file://" + _rml_uri_component(source), reference) + with records.open(encoding="utf-8", newline="") as handle: + for row in csv.DictReader(handle, delimiter="\t"): + yield make_record(row["SOURCE_FILE"], row["ROW_ID"], references[row["SOURCE_FILE"]], row) + + +def read_rdf(path: Path): + """Stream unordered N-Triples into a disk-backed index of just join fields. + + Follow hasRecord/hasCall edges rather than guessing a subject from its IRI. + Genotype triples are parsed but never retained. RDFLib owns term decoding. + """ + # Deferred so the module imports without rdflib: the CLI loads it while + # building its argument parser, long before any linking runs. + from rdflib import Literal, RDF, URIRef + from rdflib.plugins.parsers.ntriples import W3CNTriplesParser + if not path.is_file() or not path.name.endswith((".nt", ".nt.gz")): + raise ValueError("Link input must be an existing .nt or .nt.gz file") + keep = {str(VCFR[name]) for name in ("hasRecord", "hasCall", "referenceGenome", "chrom", "pos", "ref", "alt", "recordId", "infoRaw")} + types = {VCFR.VCFFile, VCFR.VCFRecord, VCFR.VariantCall} + with tempfile.TemporaryDirectory(prefix="vcfr-link-input-") as work: + db = sqlite3.connect(str(Path(work) / "input.sqlite")) + try: + db.execute("CREATE TABLE terms (s TEXT, p TEXT, o TEXT, iri INTEGER, PRIMARY KEY(s,p,o,iri))") + db.execute("CREATE INDEX predicates ON terms(p,s)") + + class Sink: + def triple(self, s, p, o): + if str(p) not in keep and not (p == RDF.type and o in types): + return + if not isinstance(s, URIRef) or not isinstance(o, (URIRef, Literal)): + raise ValueError("Linking requires IRI subjects and IRI/literal join fields") + db.execute("INSERT OR IGNORE INTO terms VALUES (?,?,?,?)", (str(s), str(p), str(o), isinstance(o, URIRef))) + + with open_text(path) as handle: + W3CNTriplesParser(sink=Sink()).parse(handle) + db.commit() + + def one(subject, predicate, *, iri=False, required=True): + rows = db.execute("SELECT o,iri FROM terms WHERE s=? AND p=?", (subject, str(predicate))).fetchall() + if not rows and not required: + return "." + if len(rows) != 1 or bool(rows[0][1]) != iri: + raise ValueError(f"Expected one {'IRI' if iri else 'literal'} {predicate} on {subject}") + return rows[0][0] + + sources = db.execute("SELECT DISTINCT s FROM terms WHERE p=?", (str(VCFR.hasRecord),)).fetchall() + if not sources: + # A typed empty VCF graph is a valid zero-link input. + sources = db.execute("SELECT s FROM terms WHERE p=? AND o=?", (str(RDF.type), str(VCFR.VCFFile))).fetchall() + if not sources: + raise ValueError("No VCFFile/hasRecord edges found; the RDF must retain the VCF-RDFizer vocabulary") + duplicate_owner = db.execute("SELECT o FROM terms WHERE p=? GROUP BY o HAVING COUNT(*) > 1 LIMIT 1", (str(VCFR.hasRecord),)).fetchone() + if duplicate_owner: + raise ValueError("Each hasRecord object must belong to one source file") + for (source,) in sources: + reference = one(source, VCFR.referenceGenome, required=False) + yield Source(source, reference) + for record, iri in db.execute("SELECT o,iri FROM terms WHERE s=? AND p=?", (source, str(VCFR.hasRecord))): + if not iri: + raise ValueError("Each hasRecord object must be an IRI belonging to one source file") + call = one(record, VCFR.hasCall, iri=True) + if not db.execute("SELECT 1 FROM terms WHERE s=? LIMIT 1", (call,)).fetchone(): + raise ValueError(f"Call subject does not exist in the base graph: {call}") + yield Record(source, record, call, reference, + *(one(record, VCFR[k], required=False) for k in ("chrom", "pos", "ref", "alt", "recordId")), + one(call, VCFR.infoRaw, required=False)) + finally: + db.close() diff --git a/vcf_rdfizer_linking/manifest.py b/vcf_rdfizer_linking/manifest.py new file mode 100644 index 0000000..1ee64c3 --- /dev/null +++ b/vcf_rdfizer_linking/manifest.py @@ -0,0 +1,282 @@ +"""Turtle manifests and deterministic directory/entry-point discovery.""" + +import importlib.metadata +import math +import os +from dataclasses import dataclass +from pathlib import Path +import re +from string import Formatter +from urllib.parse import quote, urlsplit + + +class _LazyNamespace: + """An rdflib Namespace that is not built until a term is asked for. + + rdflib is a required dependency of the data-linking feature, but the CLI + imports this module while merely building its argument parser -- so a + checkout without the dependency installed could not print ``--help``, let + alone run a conversion that does no linking at all. Deferring the import to + the first term lookup keeps every non-linking code path working, and makes + the failure land where the feature is actually used. + + Term access behaves exactly as ``rdflib.Namespace``: attribute and item + lookup both return a ``URIRef``, and ``str()`` gives the namespace IRI. + """ + + __slots__ = ("_iri", "_namespace") + + def __init__(self, iri: str): + self._iri = iri + self._namespace = None + + def _resolve(self): + if self._namespace is None: + from rdflib import Namespace + + self._namespace = Namespace(self._iri) + return self._namespace + + def __getattr__(self, name): + return getattr(self._resolve(), name) + + def __getitem__(self, name): + return self._resolve()[name] + + def __str__(self): + return self._iri + + def __repr__(self): + return f"_LazyNamespace({self._iri!r})" + + +VCFL = _LazyNamespace("https://w3id.org/vcf-rdfizer/linking#") +# The converter's own linking vocabulary keeps its namespace; only the +# conversion target moved to VCF Core. +VCFC = _LazyNamespace("https://w3id.org/vcf-core/vocab#") +#: Retained alias: the linking stage reads back converted RDF, so it must +#: use whatever namespace the emitters wrote. +VCFR = VCFC +SAFE_NAME = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]*\Z") + + +def absolute_iri(value: str) -> str: + """Reject terms that cannot safely appear inside N-Triples angle brackets.""" + if (not value or not re.match(r"^[A-Za-z][A-Za-z0-9+.-]*:", value) + or re.search(r'[\x00-\x20<>"{}|^`\\\x7f]', value)): + raise ValueError(f"Not an absolute, N-Triples-safe IRI: {value!r}") + parts = urlsplit(value) + if parts.scheme in {"http", "https"} and not parts.hostname: + raise ValueError(f"IRI has no host: {value!r}") + return value + + +@dataclass(frozen=True) +class Reference: + url: str + sha256: str + assembly: str + feature_type: str + id_attribute: str + + +@dataclass(frozen=True) +class Manifest: + directory: Path + id: str + version: str + title: str + license: str + terms_of_use: str + tier: int + strategy: str + field: str + split_on: str + accept: str + subject: str + predicate: str + object_template: str + reference: Reference | None + endpoint: str + requests_per_second: float + max_requests: int + batch_size: int + contact_email: str + + +def load_manifest(directory: Path) -> Manifest: + directory = directory.expanduser().resolve() + if directory.is_file(): + directory = directory.parent + # rdflib is needed only to read a manifest, which is the point at which the + # data-linking feature is genuinely in use. + from rdflib import Graph, Literal, RDF, URIRef # noqa: F401 + + graph = Graph() + try: + graph.parse(directory / "linker.ttl", format="turtle") + except Exception as exc: + raise ValueError(f"Cannot parse {directory / 'linker.ttl'}: {exc}") from exc + roots = list(graph.subjects(RDF.type, VCFL.Linker)) + if len(roots) != 1: + raise ValueError("Manifest must declare exactly one vcfl:Linker") + root = roots[0] + + allowed = {"id", "version", "title", "license", "termsOfUse", "join", "emit", "reference", + "field", "splitOn", "accept", "subject", "predicate", "objectTemplate", "format", + "url", "sha256", "assembly", "featureType", "idAttribute", "endpoint", + "maxRequestsPerSecond", "maxRequestsPerRun", "batchSize", "contactEmail"} + for predicate in graph.predicates(): + if str(predicate).startswith(str(VCFL)) and str(predicate)[len(str(VCFL)):] not in allowed: + raise ValueError(f"Unsupported manifest property: {predicate}") + + def one(node, name, default=None, iri=False, resource=False, url=False): + values = list(graph.objects(node, VCFL[name])) + if len(values) > 1 or (not values and default is None): + raise ValueError(f"Expected one vcfl:{name}") + if not values: + return default + value = values[0] + if url: + if not isinstance(value, (URIRef, Literal)): + raise ValueError(f"vcfl:{name} must be a URL IRI or string") + return absolute_iri(str(value)) + if resource: + if isinstance(value, Literal): + raise ValueError(f"vcfl:{name} must be a resource") + return value + if not isinstance(value, URIRef if iri else Literal): + raise ValueError(f"vcfl:{name} must be {'an IRI' if iri else 'a literal'}") + return absolute_iri(str(value)) if iri else str(value) + + linker_id = one(root, "id") + version = one(root, "version") + if not SAFE_NAME.fullmatch(linker_id) or not SAFE_NAME.fullmatch(version): + raise ValueError("Linker id/version must be safe names (letters, digits, . _ -)") + join = one(root, "join", resource=True) + types = set(graph.objects(join, RDF.type)) + if types == {VCFL.TokenJoin}: + strategy = "token" + elif types == {VCFL.IntervalJoin}: + strategy = "interval" + else: + raise ValueError("Supported joins: TokenJoin and IntervalJoin; allele normalization is not implemented") + emit = one(root, "emit", resource=True) + subject = one(emit, "subject", iri=True) + if subject not in {str(VCFL.VariantCall), str(VCFL.VCFRecord)}: + raise ValueError("vcfl:subject must be vcfl:VariantCall or vcfl:VCFRecord") + field = one(join, "field", "ID") + if field != "ID" and not re.fullmatch(r"INFO/[A-Za-z_][A-Za-z0-9_.]*", field): + raise ValueError("Token field must be ID or INFO/") + split_on = one(join, "splitOn", ";" if field == "ID" else ",") + if not split_on: + raise ValueError("vcfl:splitOn cannot be empty") + accept = one(join, "accept", ".+") + try: + re.compile(accept) + except re.error as exc: + raise ValueError(f"Invalid vcfl:accept regular expression: {exc}") from exc + + reference_node = one(root, "reference", "", resource=True) + reference = None + if reference_node: + if one(reference_node, "format", iri=True) != str(VCFL.GFF3): + raise ValueError("Only vcfl:GFF3 reference bundles are supported") + digest = one(reference_node, "sha256") + if not re.fullmatch(r"[0-9a-f]{64}", digest): + raise ValueError("vcfl:sha256 must contain 64 lowercase hexadecimal digits") + assembly = one(reference_node, "assembly") + if not assembly: + raise ValueError("A reference must declare a nonempty assembly") + reference = Reference(one(reference_node, "url", url=True), digest, assembly, + one(join, "featureType", "gene"), one(join, "idAttribute", "ID")) + resolver = directory / "resolver.py" + tier = 3 if resolver.is_file() else 2 if reference else 1 + if strategy == "interval" and (reference is None or tier == 3): + raise ValueError("IntervalJoin requires a GFF3 reference and no resolver.py") + if reference and strategy != "interval": + raise ValueError("Reference bundles currently require IntervalJoin") + template = one(emit, "objectTemplate", "") + if tier != 3: + allowed = "TOKEN" if strategy == "token" else "ID" + try: + fields = list(Formatter().parse(template)) + variables = [name for _, name, _, _ in fields if name is not None] + if variables != [allowed] or any(spec or conv for _, _, spec, conv in fields): + raise ValueError(f"Template must contain exactly one {{{allowed}}} placeholder") + absolute_iri(template.format(**{allowed: "example"})) + except (KeyError, IndexError, ValueError) as exc: + raise ValueError(f"Invalid vcfl:objectTemplate: {exc}") from exc + elif template: + raise ValueError("Tier 3 objects come from the resolver; omit objectTemplate") + + endpoint, rps, max_requests, batch_size, contact = "", 0.0, 0, 1000, "" + if tier == 3: + endpoint = one(root, "endpoint", iri=True) + parts = urlsplit(endpoint) + if parts.scheme != "https" or parts.username or parts.password or parts.fragment: + raise ValueError("vcfl:endpoint must be an HTTPS URL without credentials or fragment") + try: + rps = float(one(root, "maxRequestsPerSecond")) + max_requests = int(one(root, "maxRequestsPerRun")) + batch_size = int(one(root, "batchSize")) + except ValueError as exc: + raise ValueError("Invalid network budget: expected positive numbers") from exc + if not math.isfinite(rps) or rps <= 0 or max_requests <= 0 or not 1 <= batch_size <= 1000: + raise ValueError("Network budgets must be positive; batchSize must be 1..1000") + contact = one(root, "contactEmail") + if not re.fullmatch(r"[^\s<>@]+@[^\s<>@]+\.[^\s<>@]+", contact): + raise ValueError("vcfl:contactEmail must be an email address") + return Manifest(directory, linker_id, version, one(root, "title"), + one(root, "license", "unspecified"), one(root, "termsOfUse", "", iri=True), + tier, strategy, field, split_on, accept, subject, + one(emit, "predicate", iri=True), template, reference, endpoint, + rps, max_requests, batch_size, contact) + + +def template_object(manifest: Manifest, value: str) -> str: + name = "TOKEN" if manifest.strategy == "token" else "ID" + return absolute_iri(manifest.object_template.format(**{name: quote(value, safe="")})) + + +def discover(search_paths=()) -> dict[str, Manifest]: + """Reject ambiguous IDs; never silently replace an installed linker.""" + import vcf_rdfizer_data.linkers + + roots = [Path(vcf_rdfizer_data.linkers.__file__).parent] + roots.extend(Path(p).expanduser() for p in search_paths) + roots.extend(Path(p).expanduser() for p in os.environ.get("VCF_RDFIZER_LINKER_PATH", "").split(os.pathsep) if p) + directories = set() + for root in roots: + if not root.is_dir(): + raise ValueError(f"Linker search path is not a directory: {root}") + if (root / "linker.ttl").is_file(): + directories.add(root.resolve()) + else: + directories.update(p.parent.resolve() for p in root.glob("*/linker.ttl")) + for entry in importlib.metadata.entry_points(group="vcf_rdfizer.linkers"): + # An entry point exports a zero-argument function returning a directory. + directory = Path(entry.load()()).resolve() + if not (directory / "linker.ttl").is_file(): + raise ValueError(f"Entry point {entry.name} did not return a linker directory") + directories.add(directory) + found = {} + for directory in sorted(directories): + manifest = load_manifest(directory) + if manifest.id in found: + raise ValueError(f"Ambiguous linker id {manifest.id!r}: {found[manifest.id].directory} and {directory}") + found[manifest.id] = manifest + return found + + +def select(raw: str, search_paths=()) -> list[Manifest]: + found = discover(search_paths) + names = raw.split(",") + if not raw or any(not n.strip() for n in names): + raise ValueError("--link requires comma-separated linker IDs") + selected = [] + for name in dict.fromkeys(n.strip() for n in names): + if name not in found: + raise ValueError(f"Unknown linker {name!r}; run vcf-rdfizer-link list") + selected.append(found[name]) + return selected diff --git a/vcf_rdfizer_linking/reference.py b/vcf_rdfizer_linking/reference.py new file mode 100644 index 0000000..796b3ea --- /dev/null +++ b/vcf_rdfizer_linking/reference.py @@ -0,0 +1,153 @@ +"""Digest-pinned GFF3 acquisition and interval lookup.""" + +from bisect import bisect_left +from collections import defaultdict +import gzip +import hashlib +from pathlib import Path +import re +import tempfile +from urllib.parse import unquote, urlsplit +from urllib.request import build_opener, Request, url2pathname + +from .session import NoRedirects + + +#: Where linker reference downloads and cached HTTP responses live. It sits in +#: this module rather than in ``runner`` because the argument parser needs it as +#: a default, and ``runner`` imports rdflib -- which would make merely printing +#: ``--help`` require the data-linking dependency. +DEFAULT_CACHE = Path.home() / ".cache" / "vcf-rdfizer" / "linkers" + + +def digest_file(path): + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def check_assembly(reference, declared, override=None): + """An explicit assembly can fill missing metadata, never override a mismatch.""" + aliases = {"hg19": "GRCh37", "hg38": "GRCh38"} + + def identify(value): + value = value.strip().strip('"') + if value == declared: + return declared + names = set(re.findall(r"(? 1: + raise ValueError(f"Ambiguous reference assembly: {value}") + return next(iter(names), None) + + expected = aliases.get(declared, declared) + observed = identify(reference) + explicit = identify(override) if override else None + if observed and aliases.get(observed, observed) != expected: + raise ValueError(f"Assembly mismatch: input is {observed}; reference bundle is {declared}") + if override and explicit != expected: + raise ValueError(f"Assembly mismatch: --assembly {override}; reference bundle is {declared}") + if not observed and not explicit: + raise ValueError(f"Cannot establish assembly from {reference!r}; supply --assembly {declared} after checking the input") + + +def acquire_reference(reference, cache_dir, *, offline=False, dry_run=False, stats=None): + """Hash the stored bytes (compressed bytes for .gz), including on cache hits.""" + parts = urlsplit(reference.url) + target = cache_dir / "references" / reference.sha256 + if target.is_file(): + if digest_file(target) != reference.sha256: + raise ValueError(f"Reference cache digest mismatch: {target}") + if stats is not None: + stats["cache_hits"] += 1 + return target + if parts.scheme == "file": + if parts.netloc not in {"", "localhost"}: + raise ValueError("Reference file URL must be local") + source = Path(url2pathname(parts.path)) + if digest_file(source) != reference.sha256: + raise ValueError(f"Reference digest mismatch: {source}") + # Local bundles already are offline references. No copy on dry-run. + if dry_run: + return source + elif parts.scheme != "https": + raise ValueError("References must use file: or https: URLs") + elif offline or dry_run: + raise ValueError(f"Reference cache miss (network disabled): {reference.sha256}") + target.parent.mkdir(parents=True, exist_ok=True) + with tempfile.NamedTemporaryFile(dir=target.parent, delete=False) as output: + temporary = Path(output.name) + try: + if parts.scheme == "file": + input_handle = source.open("rb") + else: + if stats is not None: + stats["requests"] += 1 + input_handle = build_opener(NoRedirects()).open(Request(reference.url, headers={"User-Agent": "VCF-RDFizer/reference-fetch"}), timeout=60) + if stats is not None: + stats["final_service_status"] = input_handle.code + with input_handle: + for chunk in iter(lambda: input_handle.read(1024 * 1024), b""): + output.write(chunk) + if parts.scheme != "file" and stats is not None: + stats["bytes_transferred"] += len(chunk) + output.close() + if digest_file(temporary) != reference.sha256: + raise ValueError(f"Reference digest mismatch: {reference.url}") + temporary.replace(target) + finally: + temporary.unlink(missing_ok=True) + return target + + +class IntervalIndex: + """Sorted starts plus prefix-max ends: handles nested/overlapping features.""" + + def __init__(self, path, reference): + features = defaultdict(list) + with path.open("rb") as raw: + compressed = raw.read(2) == b"\x1f\x8b" + opener = gzip.open if compressed else open + with opener(path, "rt", encoding="utf-8") as handle: + for number, line in enumerate(handle, 1): + if line.startswith("##FASTA"): + break + if line.startswith("#") or not line.strip(): + continue + columns = line.rstrip("\r\n").split("\t") + if len(columns) != 9: + raise ValueError(f"GFF3 line {number}: expected nine columns") + if columns[2] != reference.feature_type: + continue + try: + start, end = int(columns[3]), int(columns[4]) + except ValueError as exc: + raise ValueError(f"GFF3 line {number}: invalid coordinates") from exc + if start < 1 or end < start: + raise ValueError(f"GFF3 line {number}: invalid interval") + attributes = dict(item.split("=", 1) for item in columns[8].split(";") if "=" in item) + identifier = unquote(attributes.get(reference.id_attribute, "")) + if not identifier: + raise ValueError(f"GFF3 line {number}: missing {reference.id_attribute}") + features[columns[0]].append((start, end, identifier)) + self.chromosomes = {} + for chrom, rows in features.items(): + rows.sort() + maximum, ends = 0, [] + for _, end, _ in rows: + maximum = max(maximum, end) + ends.append(maximum) + self.chromosomes[chrom] = (rows, [r[0] for r in rows], ends) + if not features: + raise ValueError(f"GFF3 contains no {reference.feature_type!r} features") + + def overlaps(self, key): + rows, starts, max_ends = self.chromosomes.get(key.chrom, ([], [], [])) + index = bisect_left(starts, key.end + 1) - 1 + while index >= 0 and max_ends[index] >= key.start: + start, end, identifier = rows[index] + if end >= key.start: + yield identifier + index -= 1 diff --git a/vcf_rdfizer_linking/runner.py b/vcf_rdfizer_linking/runner.py new file mode 100644 index 0000000..21dc9b0 --- /dev/null +++ b/vcf_rdfizer_linking/runner.py @@ -0,0 +1,275 @@ +"""One linking implementation for previews, full conversion, and existing RDF.""" + +from dataclasses import asdict +from datetime import datetime, timezone +import importlib.util +import hashlib +from itertools import islice +import json +import os +from pathlib import Path +import re +import sqlite3 +import tempfile +import time + + +from . import Link, LinkKey, LinkerContext +from .inputs import Source +from .manifest import VCFL, absolute_iri, template_object +from .reference import IntervalIndex, acquire_reference, check_assembly +from .session import CachedSession, NetworkPolicy, atomic_bytes + +# Re-exported from .reference, which has no rdflib dependency, so the CLI can +# offer it as an argument default without importing this module. +from .reference import DEFAULT_CACHE # noqa: F401 +PROV_TIME = "http://www.w3.org/ns/prov#generatedAtTime" + + +class LinkRunError(ValueError): + def __init__(self, message, report): + super().__init__(message) + self.report = report + + +def keys_for(record, manifest): + if manifest.strategy == "token": + value = record.id + if manifest.field.startswith("INFO/"): + wanted = manifest.field.split("/", 1)[1] + values = [part.split("=", 1)[1] for part in record.info.split(";") + if "=" in part and part.split("=", 1)[0] == wanted] + if len(values) > 1: + raise ValueError(f"Duplicate INFO key {wanted} on {record.record}") + value = values[0] if values else "." + return [LinkKey(token=token) for token in dict.fromkeys(value.split(manifest.split_on)) + if token and token != "." and re.fullmatch(manifest.accept, token)] + # POS and GFF3 intervals are both 1-based closed. Explicit REF spans only; + # END, symbolic alleles and breakends require a different coordinate policy. + if not re.fullmatch(r"[ACGTNacgtn]+", record.ref) or any( + not re.fullmatch(r"[ACGTNacgtn]+|\.", alt) for alt in record.alt.split(",")): + return [] + try: + start = int(record.pos) + except ValueError as exc: + raise ValueError(f"Invalid POS on {record.record}: {record.pos!r}") from exc + if start < 1 or record.chrom in {"", "."}: + raise ValueError(f"Invalid interval key on {record.record}") + return [LinkKey(chrom=record.chrom, start=start, end=start + len(record.ref) - 1)] + + +def load_resolver(manifest): + path = manifest.directory / "resolver.py" + spec = importlib.util.spec_from_file_location(f"vcfr_linker_{manifest.id.replace('-', '_')}", path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + if not callable(getattr(module, "resolve", None)): + raise ValueError(f"{path} must define resolve(batch, ctx)") + return module.resolve + + +def triple(subject, predicate, obj): + return f"<{absolute_iri(subject)}> <{absolute_iri(str(predicate))}> {obj} .\n" + + +def literal(value): + from vcf_rdfizer import _ntriples_literal + return _ntriples_literal(str(value)) + + +def run_linkers(records, manifests, output: Path | None, *, cache_dir=DEFAULT_CACHE, + offline=False, assembly=None, dry_run=False, policy=None, session_factory=CachedSession, + progress=None): + """Deduplicate all keys on disk before any resolution; publish only on success. + + Dry-run writes no persistent artifacts, imports no resolver code, and never + fetches a reference. It resolves local tiers and reports Tier 3 batch plans. + """ + # Deferred so the module imports without rdflib: the CLI loads it while + # building its argument parser, long before any linking runs. + from rdflib import Literal, RDF, URIRef, XSD + started = time.monotonic() + report = {"status": "running", "records": 0, "triples": 0, "appended_bytes": 0, + "output": str(output) if output else None, "dry_run": dry_run, "linkers": [], "preview": []} + if not manifests: + raise ValueError("Select at least one linker") + if not dry_run and output is None: + raise ValueError("An output path is required") + if output and not output.name.endswith(".nt"): + raise ValueError("Link output must end in .nt") + if output and output.exists(): + raise ValueError(f"Refusing to overwrite existing linkset: {output}") + policy = policy or NetworkPolicy(manifests) + cache_dir = Path(cache_dir).expanduser() + sources, stats_by_id = {}, {} + for manifest in manifests: + stats = {"id": manifest.id, "version": manifest.version, "tier": manifest.tier, + "manifest_sha256": hashlib.sha256((manifest.directory / "linker.ttl").read_bytes()).hexdigest(), + "reference_digest": manifest.reference.sha256 if manifest.reference else None, + "assembly": manifest.reference.assembly if manifest.reference else None, + "unique_keys": 0, "skipped_records": 0, "links": 0, "requests": 0, + "cache_hits": 0, "bytes_transferred": 0, "final_service_status": None, + "status": "pending", "wall_seconds": 0} + if manifest.tier == 3: + stats["resolver_sha256"] = hashlib.sha256((manifest.directory / "resolver.py").read_bytes()).hexdigest() + stats_by_id[manifest.id] = stats + report["linkers"].append(stats) + current = None + try: + with tempfile.TemporaryDirectory(prefix="vcfr-link-keys-") as work: + db = sqlite3.connect(str(Path(work) / "joins.sqlite")) + try: + db.execute("CREATE TABLE keys (linker TEXT, key TEXT, subject TEXT, source TEXT, PRIMARY KEY(linker,key,subject,source))") + db.execute("CREATE TABLE links (linker TEXT, source TEXT, subject TEXT, predicate TEXT, object TEXT, PRIMARY KEY(linker,source,subject,predicate,object))") + # Finish key extraction and assembly checks for every linker first. + for record in records: + absolute_iri(record.source) + if record.source not in sources: + for manifest in manifests: + if manifest.reference: + check_assembly(record.reference, manifest.reference.assembly, assembly) + sources[record.source] = record.reference + elif sources[record.source] != record.reference: + raise ValueError(f"Conflicting reference metadata for {record.source}") + if isinstance(record, Source): + continue + report["records"] += 1 + if progress and report["records"] % 10_000 == 0: + progress(f"Indexed {report['records']} records for linking") + for manifest in manifests: + keys = keys_for(record, manifest) + if not keys: + stats_by_id[manifest.id]["skipped_records"] += 1 + subject = record.call if manifest.subject == str(VCFL.VariantCall) else record.record + absolute_iri(subject) + db.executemany("INSERT OR IGNORE INTO keys VALUES (?,?,?,?)", [ + (manifest.id, json.dumps(asdict(key), sort_keys=True), subject, record.source) for key in keys]) + db.commit() + for manifest in manifests: + current = stats_by_id[manifest.id] + linker_start = time.monotonic() + current["status"] = "running" + current["unique_keys"] = db.execute("SELECT COUNT(DISTINCT key) FROM keys WHERE linker=?", (manifest.id,)).fetchone()[0] + if progress: + progress(f"Linking {manifest.id}: {current['unique_keys']} unique keys") + reference_index, session, resolver = None, None, None + try: + if manifest.tier == 2: + path = acquire_reference(manifest.reference, cache_dir, offline=offline, dry_run=dry_run, stats=current) + reference_index = IntervalIndex(path, manifest.reference) + if manifest.tier == 3: + current["batches"] = (current["unique_keys"] + manifest.batch_size - 1) // manifest.batch_size + current["max_requests_per_run"] = manifest.max_requests + if dry_run: + current["status"] = "planned" + continue + session = session_factory(manifest, cache_dir, policy, offline=offline) + resolver = load_resolver(manifest) + rows = db.execute("SELECT DISTINCT key FROM keys WHERE linker=? ORDER BY key", (manifest.id,)) + while True: + raw_keys = [r[0] for r in islice(rows, manifest.batch_size)] + if not raw_keys: + break + batch = [LinkKey(**json.loads(key)) for key in raw_keys] + encoded = dict(zip(batch, raw_keys)) + if manifest.tier == 1: + links = (Link(key, template_object(manifest, key.token)) for key in batch) + elif manifest.tier == 2: + links = (Link(key, template_object(manifest, identifier)) for key in batch for identifier in reference_index.overlaps(key)) + else: + links = resolver(tuple(batch), LinkerContext(session)) + for link in links: + if not isinstance(link, Link) or link.key not in encoded: + raise ValueError(f"{manifest.id}: resolver returned a link for an undispatched key") + obj = absolute_iri(link.object) + db.execute("INSERT OR IGNORE INTO links SELECT linker,source,subject,?,? FROM keys WHERE linker=? AND key=?", + (manifest.predicate, obj, manifest.id, encoded[link.key])) + current["links"] = db.execute("SELECT COUNT(*) FROM links WHERE linker=?", (manifest.id,)).fetchone()[0] + current["status"] = "success" + except BaseException: + current["status"] = "failed" + raise + finally: + if session: + current.update(session.stats) + current["wall_seconds"] = round(time.monotonic() - linker_start, 6) + db.commit() + report["preview"] = [triple(s, p, f"<{o}>").rstrip() for s, p, o in db.execute("SELECT DISTINCT subject,predicate,object FROM links ORDER BY subject,predicate,object LIMIT 20")] + report["link_triples"] = db.execute("SELECT COUNT(*) FROM (SELECT DISTINCT subject,predicate,object FROM links)").fetchone()[0] + if not dry_run: + output.parent.mkdir(parents=True, exist_ok=True) + with tempfile.NamedTemporaryFile(dir=output.parent, suffix=".nt", delete=False) as handle: + temporary = Path(handle.name) + try: + from vcf_rdfizer import _append_rdf_atomically + + def produce(emit): + for s, p, o in db.execute("SELECT DISTINCT subject,predicate,object FROM links ORDER BY subject,predicate,object"): + emit(triple(s, p, f"<{o}>")) + for source in sorted(sources): + for manifest in manifests: + node = source + "#linkset/" + manifest.id + count = db.execute("SELECT COUNT(*) FROM links WHERE linker=? AND source=?", (manifest.id, source)).fetchone()[0] + emit(triple(node, RDF.type, URIRef(VCFL.Linkset).n3())) + emit(triple(node, VCFL.producedBy, URIRef(f"https://w3id.org/vcf-rdfizer/linker/{manifest.id}/{manifest.version}").n3())) + emit(triple(node, VCFL.linkCount, Literal(count, datatype=XSD.integer).n3())) + emit(triple(node, VCFL.source, URIRef(source).n3())) + emit(triple(node, VCFL.manifestDigest, literal("sha256:" + stats_by_id[manifest.id]["manifest_sha256"]))) + if manifest.tier == 3: + emit(triple(node, VCFL.resolverDigest, literal("sha256:" + stats_by_id[manifest.id]["resolver_sha256"]))) + for digest in sorted({r["sha256"] for r in stats_by_id[manifest.id].get("responses", [])}): + emit(triple(node, VCFL.responseDigest, literal("sha256:" + digest))) + emit(triple(node, PROV_TIME, Literal(datetime.now(timezone.utc).isoformat(), datatype=XSD.dateTime).n3())) + if manifest.reference: + emit(triple(node, VCFL.referenceDigest, literal("sha256:" + manifest.reference.sha256))) + emit(triple(node, VCFL.assembly, literal(manifest.reference.assembly))) + _append_rdf_atomically(temporary, report, produce) + # Hard-link publication is atomic and fails if another run + # claimed this output after preflight; never overwrite it. + os.link(temporary, output) + finally: + temporary.unlink(missing_ok=True) + finally: + db.close() + report["status"] = "success" + return report + except Exception as exc: + report["status"] = "failed" + report["error"] = str(exc) + if current and current["status"] == "running": + current["status"] = "failed" + raise LinkRunError(str(exc), report) from exc + finally: + report["wall_seconds"] = round(time.monotonic() - started, 6) + + +def run_stage(rdf_path, manifests, output, *, metrics_dir, tracker=None, **options): + """Record success and failure using the wrapper's existing run manifest.""" + from vcf_rdfizer import (update_run_manifest, ProgressSession, + progress_event_path, progress_events_enabled) + from .inputs import read_rdf + + stage = metrics_dir / "stages" / "linking" / (output.stem + ".json") + event_path = progress_event_path(metrics_dir, "linking", output.stem) if progress_events_enabled() else None + try: + with ProgressSession(event_path, f"Linking: {output.stem}") as session: + def progress(message): + if tracker: + tracker.mark(message) + if event_path: + with event_path.open("a", encoding="utf-8") as handle: + handle.write(json.dumps({"stage": "linking", "phase": "working", "detail": message}) + "\n") + session.poll_events() + report = run_linkers(read_rdf(rdf_path), manifests, output, progress=progress, **options) + except LinkRunError as exc: + report = exc.report + raise + finally: + if "report" in locals(): + atomic_bytes(stage, (json.dumps(report, indent=2) + "\n").encode()) + # Preserve other inputs' linking reports in a directory conversion. + reports = {p.stem: json.loads(p.read_text()) for p in stage.parent.glob("*.json")} + update_run_manifest(metrics_dir, linking=reports) + if tracker: + tracker.mark(f"Linking {report['status']}: {output.name}") + return report diff --git a/vcf_rdfizer_linking/session.py b/vcf_rdfizer_linking/session.py new file mode 100644 index 0000000..4bc0285 --- /dev/null +++ b/vcf_rdfizer_linking/session.py @@ -0,0 +1,193 @@ +"""Runner-owned HTTP access: cache, budgets, host pacing, retries and accounting. + +Python resolvers are trusted code, not sandboxed code. These guarantees cover +requests made through this session, not arbitrary sockets opened by a plug-in. +""" + +import base64 +from datetime import datetime, timezone +from email.utils import parsedate_to_datetime +import hashlib +import importlib.metadata +import json +import json as json_module +import math +from pathlib import Path +import tempfile +import threading +import time +from urllib.error import HTTPError, URLError +from urllib.parse import urlencode, urlsplit +from urllib.request import HTTPRedirectHandler, Request, build_opener + +from .manifest import Manifest, absolute_iri + +MAX_RESPONSE_BYTES = 16 * 1024 * 1024 + + +def atomic_bytes(path: Path, data: bytes): + path.parent.mkdir(parents=True, exist_ok=True) + with tempfile.NamedTemporaryFile(dir=path.parent, delete=False) as handle: + temporary = Path(handle.name) + try: + handle.write(data) + handle.close() + temporary.replace(path) + finally: + temporary.unlink(missing_ok=True) + + +class NoRedirects(HTTPRedirectHandler): + def redirect_request(self, req, fp, code, msg, headers, newurl): + # Redirects otherwise evade host declarations and request accounting. + return None + + +class NetworkPolicy: + """A capacity-one bucket per host, and one in-flight request per run. + + All selected linkers are registered up front; a host gets the strictest + declared rate. Retry attempts consume the same per-linker run budget. + """ + + def __init__(self, manifests, *, clock=time.monotonic, sleep=time.sleep): + self.clock, self.sleep = clock, sleep + self.lock = threading.Lock() + self.rates, self.next_request, self.requests = {}, {}, {} + for manifest in manifests: + if manifest.tier == 3: + host = urlsplit(manifest.endpoint).hostname + self.rates[host] = min(self.rates.get(host, math.inf), manifest.requests_per_second) + + def issue(self, manifest, operation): + with self.lock: + count = self.requests.get(manifest.id, 0) + if count >= manifest.max_requests: + raise ValueError(f"{manifest.id}: maxRequestsPerRun={manifest.max_requests} exhausted") + host = urlsplit(manifest.endpoint).hostname + delay = self.next_request.get(host, 0) - self.clock() + if delay > 0: + self.sleep(delay) + self.next_request[host] = self.clock() + 1 / self.rates[host] + self.requests[manifest.id] = count + 1 + return operation() + + def defer(self, host, seconds): + with self.lock: + self.next_request[host] = max(self.next_request.get(host, 0), self.clock() + seconds) + + +class Response: + def __init__(self, status_code, headers, content): + self.status_code, self.headers, self.content = status_code, headers, content + + def json(self): + return json.loads(self.content) + + def raise_for_status(self): + if not 200 <= self.status_code < 300: + raise ValueError(f"Linker service returned HTTP {self.status_code}") + + +class CachedSession: + def __init__(self, manifest: Manifest, cache_dir: Path, policy: NetworkPolicy, + *, offline=False, transport=None): + self.manifest, self.cache_dir, self.policy = manifest, cache_dir, policy + self.offline = offline + self.transport = transport or build_opener(NoRedirects()).open + self.stats = {"requests": 0, "cache_hits": 0, "bytes_transferred": 0, + "final_service_status": None, "responses": []} + + def get(self, url, *, params=None): + if params: + url += ("&" if "?" in url else "?") + urlencode(sorted(params.items())) + return self._request("GET", url, None) + + def post(self, url, *, json): + body = json_module.dumps(json, sort_keys=True, separators=(",", ":")).encode() + return self._request("POST", url, body) + + def _request(self, method, url, body): + absolute_iri(url) + declared, requested = urlsplit(self.manifest.endpoint), urlsplit(url) + if (requested.scheme, requested.netloc) != (declared.scheme, declared.netloc) or requested.fragment: + raise ValueError(f"{self.manifest.id}: request outside declared endpoint origin") + if requested.path != declared.path and not requested.path.startswith(declared.path.rstrip("/") + "/"): + raise ValueError(f"{self.manifest.id}: request outside declared endpoint path") + signature = method.encode() + b"\n" + url.encode() + b"\n" + (body or b"") + request_hash = hashlib.sha256(signature).hexdigest() + path = self.cache_dir / "responses" / self.manifest.id / self.manifest.version / (request_hash + ".json") + if path.is_file(): + try: + cached = json.loads(path.read_text(encoding="utf-8")) + content = base64.b64decode(cached["body"], validate=True) + if cached["sha256"] != hashlib.sha256(content).hexdigest() or cached["request_hash"] != request_hash: + raise ValueError("digest mismatch") + response = Response(cached["status"], cached["headers"], content) + response.raise_for_status() + except (ValueError, KeyError, TypeError) as exc: + raise ValueError(f"Corrupt response cache: {path}: {exc}") from exc + self.stats["cache_hits"] += 1 + self._account_response(request_hash, cached) + return response + if self.offline: + raise ValueError(f"{self.manifest.id}: offline/cache-only response miss: {request_hash}") + if self.manifest.contact_email == "replace-me@example.org": + raise ValueError("Set --links-contact-email (or vcfl:contactEmail in a copied manifest) before live requests") + try: + version = importlib.metadata.version("vcf-rdfizer") + except importlib.metadata.PackageNotFoundError: + version = "development" + request = Request(url, data=body, method=method, headers={ + "Accept": "application/json", "Content-Type": "application/json", + "User-Agent": f"VCF-RDFizer/{version} ({self.manifest.contact_email})", + }) + for attempt in range(4): + def fetch(): + self.stats["requests"] += 1 + try: + raw = self.transport(request, timeout=30) + except HTTPError as exc: + raw = exc + with raw: + content = raw.read(MAX_RESPONSE_BYTES + 1) + self.stats["bytes_transferred"] += len(content) + self.stats["final_service_status"] = raw.code + if len(content) > MAX_RESPONSE_BYTES: + raise ValueError("Linker service response exceeds 16 MiB") + return Response(raw.code, dict(raw.headers.items()), content) + try: + response = self.policy.issue(self.manifest, fetch) + except (URLError, TimeoutError, OSError) as exc: + self.stats["final_service_status"] = "network-error" + raise ValueError(f"{self.manifest.id}: network request failed: {exc}") from exc + if response.status_code == 429 or 500 <= response.status_code <= 599: + retry = next((v for k, v in response.headers.items() if k.lower() == "retry-after"), "") + delay = float(2 ** attempt) + if retry: + try: + retry_seconds = float(retry) + except ValueError: + try: + retry_seconds = (parsedate_to_datetime(retry) - datetime.now(timezone.utc)).total_seconds() + except (ValueError, TypeError, OverflowError): + retry_seconds = 0 + if math.isfinite(retry_seconds): + delay = max(delay, retry_seconds) + self.policy.defer(requested.hostname, delay) + if attempt < 3: + continue + response.raise_for_status() + cached = {"request_hash": request_hash, "status": response.status_code, + "headers": response.headers, "body": base64.b64encode(response.content).decode(), + "sha256": hashlib.sha256(response.content).hexdigest(), + "fetched_at": datetime.now(timezone.utc).isoformat()} + atomic_bytes(path, (json.dumps(cached, sort_keys=True) + "\n").encode()) + self._account_response(request_hash, cached) + return response + raise AssertionError("unreachable") + + def _account_response(self, request_hash, cached): + self.stats["final_service_status"] = cached["status"] + self.stats["responses"].append({"request_hash": request_hash, + "sha256": cached["sha256"], "fetched_at": cached["fetched_at"]}) diff --git a/vcf_rdfizer_rules.py b/vcf_rdfizer_rules.py index ea7b5ef..d2fa03d 100644 --- a/vcf_rdfizer_rules.py +++ b/vcf_rdfizer_rules.py @@ -200,6 +200,11 @@ def referenced_columns(text: str) -> set[str]: return names +#: Placeholder the wrapper rewrites to the per-input vcfc:VCF4xFile subclass. +#: Kept in step with vcf_rdfizer.VCF_VERSION_CLASS_SENTINEL. +VCF_VERSION_CLASS_SENTINEL = "vcfc:VCFVersionFile" + + def check_rules(rules_path: Path) -> dict: """Validate one mapping against the wrapper contract. @@ -214,9 +219,30 @@ def check_rules(rules_path: Path) -> dict: "sources": [], "columns": [], "sample_representations": {}, + "vcf_version_sentinel": False, } text = rules_path.read_text(encoding="utf-8") + # The wrapper rewrites this sentinel per input to the vcfc:VCF4xFile + # subclass for the version that input's ##fileformat line declares. A + # mapping without it still converts; it just never claims a version gate. + report["vcf_version_sentinel"] = VCF_VERSION_CLASS_SENTINEL in text + if report["vcf_version_sentinel"]: + report["info"].append( + "Version overlay: the file subject carries the " + f"{VCF_VERSION_CLASS_SENTINEL} sentinel, so each input is typed with " + "the vcfc:VCF4xFile subclass for its own declared VCF version." + ) + else: + report["warnings"].append( + f"No {VCF_VERSION_CLASS_SENTINEL} sentinel in the file subject map. The " + "conversion still works and the version-dependent emitter behaviour " + "still follows each input's ##fileformat line, but the graph will not " + "carry a vcfc:VCF4xFile class, so the version-specific SHACL overlay " + "gate stays inactive. Copy the rr:class line from the shipped default " + "mapping to enable it." + ) + if not TRIPLES_MAP_RE.search(text): report["errors"].append( "No rml:logicalSource or rr:subjectMap found; this does not look like an " diff --git a/vcf_rdfizer_vocab.py b/vcf_rdfizer_vocab.py new file mode 100644 index 0000000..20c25b3 --- /dev/null +++ b/vcf_rdfizer_vocab.py @@ -0,0 +1,1156 @@ +#!/usr/bin/env python3 +"""VCF Core vocabulary terms and the VCF 4.5 parsers the RDF emitters need. + +The conversion splits in two. The RML mapping in ``rules/default_rules.ttl`` +carries every field whose RDF datatype is the same for every row of the TSV. +Everything else lives here and in the ``append_*_rdf`` emitters of +``vcf_rdfizer.py``: a value whose datatype depends on the row (QUAL, ALT, ID, +FILTER, INFO), and any value that has to be decomposed before it can be +represented (a structured header line, an ALT list, a GT string, a +comma-separated ``Number=A``/``R``/``G`` payload, a breakend expression). + +This module holds only pure functions and constants, so the parsers can be +tested without producing RDF. + +Target vocabulary: VCF Core (``https://w3id.org/vcf-core/vocab#``), which +replaces the retired ``https://w3id.org/vcf-rdfizer/vocab#`` namespace. No +version IRI is pinned here: the converter targets the namespace, and the +vocabulary's own release cadence is independent of this converter's. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass +from urllib.parse import quote_plus + +#: The VCF Core vocabulary namespace. Formerly +#: ``https://w3id.org/vcf-rdfizer/vocab#``, which is retired and serves a +#: deprecation document rather than redirecting here. +VCFC_NAMESPACE = "https://w3id.org/vcf-core/vocab#" + +RDF_TYPE_URI = "http://www.w3.org/1999/02/22-rdf-syntax-ns#type" +FALDO_NAMESPACE = "http://biohackathon.org/resource/faldo#" + +XSD_NAMESPACE = "http://www.w3.org/2001/XMLSchema#" +XSD_STRING_URI = f"{XSD_NAMESPACE}string" +XSD_INTEGER_URI = f"{XSD_NAMESPACE}integer" +XSD_DECIMAL_URI = f"{XSD_NAMESPACE}decimal" +XSD_BOOLEAN_URI = f"{XSD_NAMESPACE}boolean" +XSD_DATE_URI = f"{XSD_NAMESPACE}date" +XSD_ANY_URI = f"{XSD_NAMESPACE}anyURI" + +#: Every ordinal in the published SHACL profiles is constrained with +#: ``sh:datatype xsd:integer`` and a ``sh:minInclusive`` bound, even where the +#: ontology declares ``rdfs:range xsd:positiveInteger``. A SHACL datatype +#: constraint compares the literal's datatype IRI exactly, so an +#: ``xsd:positiveInteger`` literal fails the shape while an ``xsd:integer`` +#: literal satisfies both the shape and the range (the value 1 is in the value +#: space of xsd:positiveInteger however it is written). Ordinals are therefore +#: always serialized as xsd:integer. +XSD_ORDINAL_URI = XSD_INTEGER_URI + + +def term(local_name: str) -> str: + """Return the absolute IRI of one VCF Core term.""" + return f"{VCFC_NAMESPACE}{local_name}" + + +NULL_DATATYPE_URI = term("Null") +VCF_FLOAT_DATATYPE_URI = term("VCFFloat") +GENOTYPE_STRING_DATATYPE_URI = term("GenotypeString") +BREAKEND_STRING_DATATYPE_URI = term("BreakendString") + + +# --------------------------------------------------------------------------- +# Lexical serialization +# --------------------------------------------------------------------------- + +# Characters that ``quote_plus(..., safe="*-._")`` leaves untouched. Sample ids, +# FORMAT keys and numeric row ids are almost always drawn from this set, so the +# fast path below skips percent-encoding entirely for them. +_URI_COMPONENT_PASSTHROUGH_RE = re.compile(r"[A-Za-z0-9*\-._]*\Z") +# Characters that require N-Triples escaping. VCF genotype/FORMAT payloads are +# overwhelmingly free of them, so testing before substituting is worthwhile. +_NTRIPLES_ESCAPE_RE = re.compile(r'[\\"\n\r\t]') +_NTRIPLES_ESCAPES = { + "\\": "\\\\", + '"': '\\"', + "\n": "\\n", + "\r": "\\r", + "\t": "\\t", +} + + +def rml_uri_component(value: str) -> str: + """Match RMLStreamer's Java URLEncoder-based template substitution. + + The wrapper's directly emitted IRIs have to be byte-identical to the ones + RMLStreamer mints from the same value, or the two halves of the graph would + describe different resources. + """ + if _URI_COMPONENT_PASSTHROUGH_RE.match(value) is not None: + return value + encoded = quote_plus(value, safe="*-._", encoding="utf-8", errors="strict") + # urllib follows current RFC rules and always leaves '~' unescaped, whereas + # java.net.URLEncoder (used by RMLStreamer 2.5.0) encodes it. + return encoded.replace("+", "%20").replace("~", "%7E") + + +def ntriples_string_literal(value: str) -> str: + """Serialize ``value`` as a plain (xsd:string) N-Triples literal.""" + if _NTRIPLES_ESCAPE_RE.search(value): + value = _NTRIPLES_ESCAPE_RE.sub(lambda m: _NTRIPLES_ESCAPES[m.group(0)], value) + return f'"{value}"' + + +def ntriples_typed_literal(value: str, datatype_uri: str) -> str: + """Serialize ``value`` as an N-Triples literal with an explicit datatype.""" + return f"{ntriples_string_literal(value)}^^<{datatype_uri}>" + + +def ntriples_literal(value: str) -> str: + """Serialize a VCF value, typing the missing token as ``vcfc:Null``. + + The vocabulary's missing-value policy is that the VCF token ``.`` is + serialized as ``"."^^vcfc:Null`` rather than as a plain string, so that + missingness stays distinguishable from a literal period. + """ + if value == ".": + return f'"."^^<{NULL_DATATYPE_URI}>' + return ntriples_string_literal(value) + + +def ordinal_literal(value: int | str) -> str: + """Serialize a one-based or zero-based ordinal as xsd:integer.""" + return f'"{value}"^^<{XSD_ORDINAL_URI}>' + + +def is_missing(value: str | None) -> bool: + """Return True for an absent value or the VCF missing token.""" + return value is None or value == "" or value == "." + + +# --------------------------------------------------------------------------- +# VCF lexical spaces +# --------------------------------------------------------------------------- + +#: VCF 4.5 Float: a decimal/scientific form, or INF/INFINITY/NAN in any case, +#: optionally signed. The non-finite spellings have no XSD numeric datatype, so +#: they are serialized with vcfc:VCFFloat instead of being coerced. +_VCF_FLOAT_FINITE_RE = re.compile(r"^[-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)?$") +_VCF_FLOAT_SPECIAL_RE = re.compile(r"^[-+]?(INF|INFINITY|NAN)$", re.IGNORECASE) + +#: VCF 4.5 reserves Integer values -2147483648..-2147483641 because BCF cannot +#: encode them; a value in that range is kept lexically rather than typed. +_VCF_INTEGER_RESERVED_LOW = -2147483648 +_VCF_INTEGER_RESERVED_HIGH = -2147483641 + +_GENOTYPE_RE = re.compile(r"^[|/]?[0-9.]+([|/][0-9.]+)*$") + + +def is_vcf_float(value: str) -> bool: + """Return True when ``value`` is in the VCF Float lexical space.""" + return bool(_VCF_FLOAT_FINITE_RE.match(value) or _VCF_FLOAT_SPECIAL_RE.match(value)) + + +def is_finite_vcf_float(value: str) -> bool: + """Return True for a VCF Float that xsd:decimal can carry without loss.""" + return bool(_VCF_FLOAT_FINITE_RE.match(value)) + + +def is_vcf_integer(value: str) -> bool: + """Return True for a VCF Integer outside the BCF-reserved range.""" + try: + parsed = int(value) + except ValueError: + return False + return not (_VCF_INTEGER_RESERVED_LOW <= parsed <= _VCF_INTEGER_RESERVED_HIGH) + + +def is_genotype_string(value: str) -> bool: + """Return True when ``value`` is in the vcfc:GenotypeString lexical space.""" + return bool(_GENOTYPE_RE.match(value)) + + +# --------------------------------------------------------------------------- +# VCF Number arities +# --------------------------------------------------------------------------- + +#: Number token -> the vcfc:VCFNumberArity individual carrying that arityCode. +#: The SPARQL SHACL profile requires fieldArity and fieldNumber to agree, so the +#: two are always derived from the same source token. +ARITY_INDIVIDUALS = { + "A": "ArityPerAlt", + "R": "ArityPerAllele", + "G": "ArityPerGenotype", + ".": "ArityVariable", + "LA": "ArityPerLocalAlt", + "LR": "ArityPerLocalAllele", + "LG": "ArityPerLocalGenotype", + "P": "ArityPerGTAllele", + "M": "ArityPerBaseModification", +} + +#: Number tokens whose values are one-per-ALT-allele, so value item i joins ALT +#: allele i+1; and those that are one-per-allele including REF, where item i +#: joins allele i. +_ARITY_PER_ALT = {"A", "LA"} +_ARITY_PER_ALLELE = {"R", "LR"} +_ARITY_PER_GENOTYPE = {"G", "LG"} + + +def arity_individual(number: str) -> str | None: + """Return the vcfc arity individual for a Number token, if it is symbolic.""" + return ARITY_INDIVIDUALS.get((number or "").strip()) + + +def number_as_integer(number: str) -> int | None: + """Return the fixed count declared by a Number token, if it is an integer.""" + token = (number or "").strip() + if not token.isdigit(): + return None + return int(token) + + +#: Keys that flatten fixed-width tuples into one comma list, and the width of +#: each tuple. Recording the width with vcfc:tupleArity lets a consumer regroup +#: them without knowing the key. Which keys are in this set, and whether the +#: list repeats per ALT allele, depends on the VCF version -- see VCFVersion. +TUPLE_ARITIES = { + "CIPOS": 2, + "CIEND": 2, + "CILEN": 2, + "CICN": 2, + "CIRUC": 2, + "CIRB": 2, + "MEINFO": 4, + "METRANS": 4, +} + + +@dataclass(frozen=True) +class ValueItemLink: + """How one parsed value item joins back to the record's alleles. + + ``allele_index`` is the record-global allele index (0 for REF), which is + what vcfc:forAllele points at. ``genotype_index`` and ``gt_allele_index`` + carry the Number=G and Number=P orderings, which are ordinals rather than + allele references. All three being None means the item has a position but no + resource to join it to, which is correct for a version whose tuple keys + describe the record rather than one allele. + """ + + allele_index: int | None = None + genotype_index: int | None = None + gt_allele_index: int | None = None + + @property + def is_empty(self) -> bool: + return ( + self.allele_index is None + and self.genotype_index is None + and self.gt_allele_index is None + ) + + +def value_item_link(number: str, item_index: int, *, tuple_arity: int | None) -> ValueItemLink: + """Resolve which allele or ordinal one comma-list item belongs to. + + ``item_index`` is zero-based within the comma list. When the field flattens + fixed-width tuples, the allele stride is the tuple width, so items 0 and 1 + of a per-ALT CIPOS list both belong to ALT allele 1. + + This is the version-neutral form, driven only by the declared Number token. + Prefer :meth:`VCFVersion.value_item_link`, which also knows which keys are + positional in a given version despite declaring ``Number=.``. + """ + token = (number or "").strip() + stride = tuple_arity or 1 + group = item_index // stride + if token in _ARITY_PER_ALT: + return ValueItemLink(allele_index=group + 1) + if token in _ARITY_PER_ALLELE: + return ValueItemLink(allele_index=group) + if token in _ARITY_PER_GENOTYPE: + return ValueItemLink(genotype_index=group) + if token == "P": + return ValueItemLink(gt_allele_index=group) + return ValueItemLink() + + +# --------------------------------------------------------------------------- +# VCF versions +# --------------------------------------------------------------------------- +# +# VCF Core ships one SHACL overlay per VCF version, each scoped by the file's +# vcfc:fileFormat, plus an optional vcfc:VCF4xFile subclass that turns the +# version gate on. Three things the converter emits genuinely differ between +# versions, so the emitters take a VCFVersion rather than assuming 4.5: +# +# * which Number codes exist at all -- R arrived in 4.2, P in 4.4, and +# LA/LR/LG/M in 4.5; +# * which INFO keys carry flattened tuples, and whether the tuple repeats per +# ALT allele. CIPOS is two values for the whole record in 4.1-4.3 and two +# values per ALT from 4.4; CILEN and CICN only become tuple keys in 4.4; +# * whether the local-allele and base-modification FORMAT families exist. + + +@dataclass(frozen=True) +class VCFVersion: + """One VCF specification version and the conversion behaviour it implies.""" + + #: The exact ##fileformat token, e.g. "VCFv4.5". + token: str + #: The bare version, e.g. "4.5". This is what --vcf-version accepts. + short: str + #: The optional vcfc subclass that activates this version's SHACL gate. + file_class: str + #: Number codes this version permits in an INFO / FORMAT declaration. + info_numbers: frozenset[str] + format_numbers: frozenset[str] + #: INFO keys whose value is a flattened tuple, and that tuple's width. + tuple_keys: dict[str, int] + #: True when a tuple key's list repeats once per ALT allele (4.4 onward); + #: False when it carries exactly one tuple for the whole record. + tuples_per_alt: bool + #: Whether the LAA/LA/LR/LG local-allele family exists. + local_alleles: bool + #: Whether the M/DPM/ADM base-modification families exist. + base_modifications: bool + #: Whether GT may carry a leading phase indicator (4.4 onward). + leading_phase_indicator: bool + #: Whether EVENT names one event per ALT allele (Number=A, 4.4 onward) + #: rather than one event for the whole record (Number=1, 4.1-4.3). + events_per_alt: bool + #: Whether the EVENTTYPE key exists at all (4.4 onward). + event_types: bool + + def allows_number(self, number: str, *, is_format: bool) -> bool: + """Whether a Number token is legal for this version and field kind.""" + token = (number or "").strip() + if token.isdigit(): + return True + allowed = self.format_numbers if is_format else self.info_numbers + return token in allowed + + def tuple_arity(self, field_id: str) -> int | None: + """The flattened-tuple width of an INFO key in this version, if any.""" + return self.tuple_keys.get(field_id) + + def is_positional(self, field_id: str, number: str) -> bool: + """Whether a field's comma list has one meaning per position. + + A tuple key is positional even though the specification declares it + ``Number=.``: the version overlay checks its item count against the ALT + count, so its items must be materialized. Everything else is positional + only when its declared Number says so. + """ + if self.tuple_arity(field_id) is not None: + return True + return not value_item_link(number, 0, tuple_arity=None).is_empty + + def value_item_link(self, field_id: str, number: str, item_index: int) -> ValueItemLink: + """Resolve one comma-list item's allele or ordinal, for this version.""" + arity = self.tuple_arity(field_id) + if arity is not None: + if not self.tuples_per_alt: + # The tuple describes the record, not one allele, so the item + # has an index but nothing to point vcfc:forAllele at. + return ValueItemLink() + return ValueItemLink(allele_index=(item_index // arity) + 1) + return value_item_link(number, item_index, tuple_arity=None) + + +def _version( + short: str, + *, + info_numbers: str, + format_numbers: str, + tuple_keys: dict[str, int], + tuples_per_alt: bool, + local_alleles: bool = False, + base_modifications: bool = False, + leading_phase_indicator: bool = False, + events_per_alt: bool = False, + event_types: bool = False, +) -> VCFVersion: + return VCFVersion( + token=f"VCFv{short}", + short=short, + file_class=f"VCF{short.replace('.', '')}File", + info_numbers=frozenset(info_numbers.split()), + format_numbers=frozenset(format_numbers.split()), + tuple_keys=tuple_keys, + tuples_per_alt=tuples_per_alt, + local_alleles=local_alleles, + base_modifications=base_modifications, + leading_phase_indicator=leading_phase_indicator, + events_per_alt=events_per_alt, + event_types=event_types, + ) + + +#: Tuple keys before 4.4: one (lower, upper) pair per record, and MEINFO / +#: METRANS as one four-value tuple. CILEN and CICN are not yet defined. +_EARLY_TUPLE_KEYS = {"CIPOS": 2, "CIEND": 2, "MEINFO": 4, "METRANS": 4} +#: From 4.4 the confidence-interval family grew and every tuple repeats per ALT. +_LATE_TUPLE_KEYS = { + "CIPOS": 2, + "CIEND": 2, + "CILEN": 2, + "CICN": 2, + "MEINFO": 4, + "METRANS": 4, +} + +#: The VCF versions VCF Core supplies a conformance overlay for. VCF 4.0 file +#: labels remain syntactically valid but no 4.0 overlay is claimed, so this +#: converter treats 4.0 as unrecognized: it emits the version-neutral graph and +#: says so, rather than validating a 4.0 file against 4.1 rules. +VCF_VERSIONS: dict[str, VCFVersion] = { + version.short: version + for version in ( + _version( + "4.1", + info_numbers="A G .", + format_numbers="A G .", + tuple_keys=_EARLY_TUPLE_KEYS, + tuples_per_alt=False, + ), + _version( + "4.2", + info_numbers="A R G .", + format_numbers="A R G .", + tuple_keys=_EARLY_TUPLE_KEYS, + tuples_per_alt=False, + ), + _version( + "4.3", + info_numbers="A R G .", + format_numbers="A R G .", + tuple_keys=_EARLY_TUPLE_KEYS, + tuples_per_alt=False, + ), + _version( + "4.4", + info_numbers="A R G .", + format_numbers="A R G . P", + tuple_keys=_LATE_TUPLE_KEYS, + tuples_per_alt=True, + leading_phase_indicator=True, + events_per_alt=True, + event_types=True, + ), + _version( + "4.5", + info_numbers="A R G .", + format_numbers="A R G . P LA LR LG M", + tuple_keys=_LATE_TUPLE_KEYS, + tuples_per_alt=True, + local_alleles=True, + base_modifications=True, + leading_phase_indicator=True, + events_per_alt=True, + event_types=True, + ), + ) +} + +#: Used when the ##fileformat line is absent, malformed, or names a version with +#: no conformance overlay. It behaves as the newest supported version so no +#: representable content is silently dropped, but no vcfc:VCF4xFile class is +#: emitted, so no version gate is claimed for the graph. +FALLBACK_VERSION = VCF_VERSIONS["4.5"] + +_FILEFORMAT_RE = re.compile(r"^\s*VCFv(?P[0-9]+\.[0-9]+)\s*$", re.IGNORECASE) + + +def parse_vcf_version(file_format: str | None) -> VCFVersion | None: + """Resolve a ##fileformat value to a supported version, or None. + + Returns None for an absent or malformed value, and for a syntactically valid + version that has no conformance overlay (notably VCFv4.0). The caller + decides what to do about it; nothing here guesses. + """ + if not file_format: + return None + match = _FILEFORMAT_RE.match(file_format) + if not match: + return None + return VCF_VERSIONS.get(match.group("short")) + + +def resolve_vcf_version(file_format: str | None) -> tuple[VCFVersion, bool]: + """Return the version to convert with, and whether it was recognized.""" + detected = parse_vcf_version(file_format) + if detected is not None: + return detected, True + return FALLBACK_VERSION, False + + +# --------------------------------------------------------------------------- +# Alleles +# --------------------------------------------------------------------------- + +#: The reserved symbolic ALT identifiers VCF 4.5 defines, mapped to the +#: vcfc:SymbolicAlleleType individual for each. Subtypes are matched before +#: their first-level type so ```` does not resolve to ``DUP``. +SYMBOLIC_ALLELE_TYPES = { + "CNV:TR": "SymbolicTandemRepeat", + "DUP:TANDEM": "SymbolicTandemDuplication", + "DEL:ME": "SymbolicMobileElementDeletion", + "INS:ME": "SymbolicMobileElementInsertion", + "DEL": "SymbolicDeletion", + "INS": "SymbolicInsertion", + "DUP": "SymbolicDuplication", + "INV": "SymbolicInversion", + "CNV": "SymbolicCopyNumberVariation", +} + +#: The reserved EVENTTYPE codes, mapped to their vcfc:EventType individual. +EVENT_TYPES = { + "DEL": "EventDeletion", + "DEL:ME": "EventMobileElementDeletion", + "INS": "EventInsertion", + "INS:ME": "EventMobileElementInsertion", + "DUP": "EventDuplication", + "DUP:TANDEM": "EventTandemDuplication", + "DUP:DISPERSED": "EventDispersedDuplication", + "INV": "EventInversion", + "TRA": "EventTranslocation", + "TRA:BALANCED": "EventBalancedTranslocation", + "TRA:UNBALANCED": "EventUnbalancedTranslocation", + "CHROMOTHRIPSIS": "EventChromothripsis", + "CHROMOPLEXY": "EventChromoplexy", + "BFB": "EventBreakageFusionBridge", + "DOUBLEMINUTE": "EventDoubleMinute", +} + +#: The per-ALT SVCLAIM codes. +SV_CLAIMS = { + "D": "AbundanceClaim", + "J": "AdjacencyClaim", + "DJ": "AbundanceAndAdjacencyClaim", + "JD": "AbundanceAndAdjacencyClaim", +} + +#: ALT forms that assert an unspecified alternate allele rather than a symbolic +#: variant type. gVCF uses <*>; GATK's older spelling is . +_UNSPECIFIED_ALT_IDS = {"*", "NON_REF"} + +_BREAKEND_RE = re.compile( + r"^(?:" + r"(?P[ACGTNacgtn]*)(?P[\[\]])(?P[^\[\]]+)(?P=open)" + r"|" + r"(?P[\[\]])(?P[^\[\]]+)(?P=open2)(?P[ACGTNacgtn]*)" + r")$" +) +_SINGLE_BREAKEND_RE = re.compile(r"^(?:\.[ACGTNacgtn]+|[ACGTNacgtn]+\.)$") + + +@dataclass(frozen=True) +class ParsedAllele: + """One allele of a record, classified by its VCF lexical form.""" + + index: int + value: str + #: The vcfc:AlleleKind individual local name. + kind: str + #: The vcfc:SymbolicAlleleType individual local name, for a symbolic ALT + #: whose identifier is one of the reserved codes. + symbolic_type: str | None = None + #: The bare symbolic identifier without angle brackets, used to join an ALT + #: allele to the ##ALT declaration that defines it. + symbolic_id: str | None = None + #: Set for a bracketed or single breakend expression. + breakend: "ParsedBreakend | None" = None + + +@dataclass(frozen=True) +class ParsedBreakend: + """The queryable components of a VCF breakend ALT expression.""" + + #: The vcfc:BreakendOrientation individual local name, or None for a single + #: breakend, which has no bracket and therefore no orientation. + orientation: str | None + #: The replacement sequence on the record's own side of the adjacency. + replacement: str + #: The mate position, as written: ``chrom:pos``. + mate: str | None + is_single: bool = False + + +_ORIENTATIONS = { + ("before", "["): "SequenceBeforeLeftBracket", + ("before", "]"): "SequenceBeforeRightBracket", + ("after", "["): "SequenceAfterLeftBracket", + ("after", "]"): "SequenceAfterRightBracket", +} + + +def parse_breakend(value: str) -> ParsedBreakend | None: + """Parse a VCF breakend ALT expression into its structural components. + + VCF 4.5 writes an adjacency in one of four bracketed forms -- ``t[p[``, + ``t]p]``, ``[p[t`` and ``]p]t`` -- where ``t`` is the replacement sequence + and ``p`` the mate position. A single breakend, whose partner is unknown, + is written ``.t`` or ``t.`` and has no bracket. + """ + match = _BREAKEND_RE.match(value) + if match: + if match.group("open"): + bracket = match.group("open") + return ParsedBreakend( + orientation=_ORIENTATIONS[("before", bracket)], + replacement=match.group("before"), + mate=match.group("mate"), + ) + bracket = match.group("open2") + return ParsedBreakend( + orientation=_ORIENTATIONS[("after", bracket)], + replacement=match.group("after"), + mate=match.group("mate2"), + ) + if _SINGLE_BREAKEND_RE.match(value): + return ParsedBreakend( + orientation=None, + replacement=value.replace(".", ""), + mate=None, + is_single=True, + ) + return None + + +def classify_allele(value: str, index: int) -> ParsedAllele: + """Classify one REF or ALT lexical form into a vcfc:AlleleKind. + + ``index`` is the record-global allele index: 0 for REF, then 1..n in source + ALT order, which is the join key for every Number=A and Number=R value. + """ + if value == ".": + return ParsedAllele(index, value, "MissingAllele") + if value == "*": + return ParsedAllele(index, value, "OverlappingDeletionAllele") + if value.startswith("<") and value.endswith(">"): + identifier = value[1:-1] + if identifier in _UNSPECIFIED_ALT_IDS: + return ParsedAllele( + index, value, "UnspecifiedAllele", symbolic_id=identifier + ) + return ParsedAllele( + index, + value, + "SymbolicAllele", + symbolic_type=_symbolic_type_for(identifier), + symbolic_id=identifier, + ) + breakend = parse_breakend(value) + if breakend is not None: + return ParsedAllele(index, value, "BreakendAllele", breakend=breakend) + return ParsedAllele(index, value, "BaseSequenceAllele") + + +def _symbolic_type_for(identifier: str) -> str | None: + """Resolve a symbolic ALT identifier to its reserved type, subtype first. + + An unreserved identifier such as ```` has no vcfc type; + it stays a plain SymbolicAllele joined to its ##ALT declaration. + """ + upper = identifier.upper() + if upper in SYMBOLIC_ALLELE_TYPES: + return SYMBOLIC_ALLELE_TYPES[upper] + # A subtype the vocabulary does not enumerate still resolves to its + # first-level type, which VCF 4.5 requires to be a reserved code. + head = upper.split(":", 1)[0] + return SYMBOLIC_ALLELE_TYPES.get(head) + + +#: VCF 4.5 allows CHROM to name a breakpoint assembly contig instead of a +#: reference sequence, written in angle brackets. The identifier may not contain +#: brackets, commas or whitespace. +_BRACKETED_CHROM_RE = re.compile(r"^<(?P[^<>,\s]+)>$") + + +def parse_bracketed_chrom(chrom: str) -> str | None: + """Return the assembly-contig identifier of a bracketed CHROM, else None. + + A bracketed CHROM names a contig from the ##assembly FASTA rather than a + declared ##contig, so it links to a vcfc:AssemblyContig and must *not* also + resolve to a contig declaration. + """ + match = _BRACKETED_CHROM_RE.match(chrom or "") + return match.group("identifier") if match else None + + +def parse_alt_alleles(ref: str, alt: str) -> list[ParsedAllele]: + """Split a record's REF and ALT columns into indexed allele resources. + + Returns REF at index 0 followed by each ALT item in source order. An ALT of + ``.`` means the record asserts no alternate allele, so only REF is returned. + """ + alleles = [classify_allele(ref, 0)] if ref else [] + if is_missing(alt): + return alleles + for offset, item in enumerate(alt.split(","), start=1): + alleles.append(classify_allele(item, offset)) + return alleles + + +# --------------------------------------------------------------------------- +# Genotypes +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class ParsedGenotypeCall: + """One allele position within a GT value.""" + + call_index: int + #: The record-global allele index, or None when the position is a no-call. + allele_index: int | None + + @property + def is_no_call(self) -> bool: + return self.allele_index is None + + +@dataclass(frozen=True) +class ParsedGenotype: + """A parsed GT value: its calls, ploidy, and phasing.""" + + genotype_string: str + calls: tuple[ParsedGenotypeCall, ...] + #: True when every separator is ``|``. VCF permits a leading separator for + #: partial phasing, which is treated as phased for the purposes of the + #: vcfc:PhasingStatus individual. + is_phased: bool + + @property + def ploidy(self) -> int: + return len(self.calls) + + +def parse_genotype(value: str) -> ParsedGenotype | None: + """Parse a GT value into ordered allele calls and a phasing status. + + Returns None when the value is absent or not in the GT lexical space, so a + malformed field is preserved as a plain FORMAT value rather than producing + a Genotype resource that would fail vcfc:GenotypeShape. + """ + if is_missing(value): + return None + if not is_genotype_string(value): + return None + body = value + leading_phased = False + if body[0] in "|/": + leading_phased = body[0] == "|" + body = body[1:] + if not body: + return None + + tokens = re.split(r"([|/])", body) + allele_tokens = tokens[0::2] + separators = tokens[1::2] + + calls = [] + for call_index, token in enumerate(allele_tokens): + if token == "." or token == "": + calls.append(ParsedGenotypeCall(call_index, None)) + else: + try: + calls.append(ParsedGenotypeCall(call_index, int(token))) + except ValueError: + return None + + is_phased = bool(separators) and all(sep == "|" for sep in separators) + if not separators: + # A haploid call carries phasing only through the optional leading + # separator; without one there is nothing to phase against. + is_phased = leading_phased + return ParsedGenotype(value, tuple(calls), is_phased) + + +def genotype_index(allele_indices: list[int]) -> int: + """VCF Number=G ordering: Index(k1..kP) = sum over m of C(k_m + m - 1, m). + + Used to place a Number=G value item against the genotype it scores. + """ + from math import comb + + total = 0 + for position, allele in enumerate(sorted(allele_indices), start=1): + total += comb(allele + position - 1, position) + return total + + +# --------------------------------------------------------------------------- +# Structural variation, repeats and base modifications +# --------------------------------------------------------------------------- + +#: INFO keys whose value is a per-ALT confidence interval expressed through +#: FALDO rather than a vcfc:ConfidenceInterval, per the SV module's design. +FALDO_INTERVAL_INFO_KEYS = {"CIPOS": "posConfidenceInterval", "CIEND": "endConfidenceInterval"} + +#: INFO keys whose value is a per-ALT vcfc:ConfidenceInterval, and the property +#: that attaches it. +CONFIDENCE_INTERVAL_INFO_KEYS = { + "CILEN": "lenConfidenceInterval", + "CICN": "copyNumberConfidenceInterval", +} + +#: INFO Flag keys that set a boolean directly on the ALT allele. +SV_FLAG_INFO_KEYS = {"IMPRECISE": "isImprecise", "NOVEL": "isNovel"} + +#: Per-ALT INFO keys that become a datatype property on the allele. +SV_ALLELE_INFO_KEYS = {"SVLEN": "svLength"} + +#: Tandem-repeat INFO keys, in the RN/RUS/RUL/RUC/RB order VCF 4.5 defines, and +#: the vcfc:RepeatSequence property each populates. +REPEAT_SEQUENCE_INFO_KEYS = { + "RUS": "repeatUnitSequence", + "RUL": "repeatUnitLength", + "RUC": "repeatUnitCount", + "RB": "repeatBases", +} + +#: FORMAT keys that carry copy-number information on a sample call. +COPY_NUMBER_FORMAT_KEYS = { + "CN": "copyNumber", + "CNQ": "copyNumberQuality", + "CNL": "copyNumberLikelihood", + "CNP": "copyNumberPosterior", +} + +#: FORMAT keys that carry haplotype identity on a sample call. +HAPLOTYPE_FORMAT_KEYS = {"HAP": "haplotypeId", "AHAP": "ancestralHaplotypeId"} + +#: FORMAT keys that carry phase-set information. +PHASE_SET_FORMAT_KEYS = ("PS", "PSL", "PSO", "PSQ") + +#: VCF 4.5 defines the base-modification FORMAT families by pattern rather than +#: by a single key: M, DPM and ADM followed by a ChEBI id and the modified base. +BASE_MODIFICATION_KEY_RE = re.compile(r"^(?PM|DPM|ADM)(?P[0-9]+)(?P[ACGTUN])$") + +#: The ChEBI namespace the SV module's vcfc:modifiedResidue points into. +CHEBI_NAMESPACE = "http://purl.obolibrary.org/obo/CHEBI_" + +_BASE_MODIFICATION_PROPERTIES = { + "M": "modificationFraction", + "DPM": "modificationDepth", + "ADM": "modificationAlleleDepth", +} + + +@dataclass(frozen=True) +class ParsedBaseModification: + """A FORMAT key from the M/DPM/ADM base-modification families.""" + + family: str + chebi_id: str + base: str + #: The vcfc property that carries this family's value. + value_property: str + + @property + def residue_uri(self) -> str: + return f"{CHEBI_NAMESPACE}{self.chebi_id}" + + @property + def key(self) -> str: + return f"{self.family}{self.chebi_id}{self.base}" + + +def parse_base_modification_key(key: str) -> ParsedBaseModification | None: + """Recognize a base-modification FORMAT key and resolve its ChEBI residue.""" + match = BASE_MODIFICATION_KEY_RE.match(key) + if not match: + return None + family = match.group("family") + return ParsedBaseModification( + family=family, + chebi_id=match.group("chebi"), + base=match.group("base"), + value_property=_BASE_MODIFICATION_PROPERTIES[family], + ) + + +def split_value_items(value: str) -> list[str]: + """Split a comma-separated VCF value payload into its ordered items.""" + if is_missing(value): + return [] + return value.split(",") + + +def parse_local_allele_indices(value: str) -> list[int]: + """Parse an LAA value into the one-based ALT indices it declares.""" + indices = [] + for item in split_value_items(value): + if item.isdigit(): + parsed = int(item) + if parsed >= 1: + indices.append(parsed) + return indices + + +# --------------------------------------------------------------------------- +# Header lines +# --------------------------------------------------------------------------- + +#: '##' key (lower-cased) -> the vocabulary subclass for that line. Every entry +#: is a subclass of either vcfc:StructuredHeaderLine or +#: vcfc:UnstructuredHeaderLine, which decides whether the line needs +#: vcfc:HeaderAttribute resources. +HEADER_LINE_CLASSES = { + "fileformat": "FileFormatHeaderLine", + "filedate": "FileDateHeaderLine", + "source": "SourceHeaderLine", + "reference": "ReferenceHeaderLine", + "assembly": "AssemblyHeaderLine", + "pedigreedb": "PedigreeDBHeaderLine", + "info": "INFOHeaderLine", + "format": "FORMATHeaderLine", + "filter": "FILTERHeaderLine", + "alt": "ALTHeaderLine", + "contig": "ContigHeaderLine", + "meta": "MetaHeaderLine", + "sample": "SampleHeaderLine", + "pedigree": "PedigreeHeaderLine", +} + +#: The subset of HEADER_LINE_CLASSES whose value is an angle-bracketed +#: attribute list. vcfc:StructuredHeaderLineShape requires at least one +#: vcfc:hasAttribute on each of these. +STRUCTURED_HEADER_KEYS = { + "info", + "format", + "filter", + "alt", + "contig", + "meta", + "sample", + "pedigree", +} + +#: contig attribute -> vocabulary predicate, with the datatype each needs. +CONTIG_ATTRIBUTES = { + "length": ("contigLength", XSD_INTEGER_URI), + "md5": ("contigMd5", XSD_STRING_URI), + "assembly": ("contigAssembly", XSD_STRING_URI), + "URL": ("contigUrl", XSD_ANY_URI), +} + +#: ##INFO attributes beyond ID/Number/Type/Description that the vocabulary +#: represents with a dedicated property. +INFO_EXTRA_ATTRIBUTES = {"Source": "fieldSource", "Version": "fieldVersion"} + +#: ##PEDIGREE role attribute -> the vcfc property for that relation. Any other +#: role falls back to vcfc:pedigreeAncestor with a vcfc:ancestorRole literal. +PEDIGREE_ROLE_PROPERTIES = { + "Mother": "pedigreeMother", + "Father": "pedigreeFather", + "Original": "pedigreeOriginal", +} + +#: Maps a declared VCF INFO/FORMAT Type to its vocabulary value-type +#: individual. Since vocabulary 3.0.0 these are owl:NamedIndividual members of +#: vcfc:VCFValueType, not classes, so they are only ever used as the object of +#: vcfc:fieldType. +VCF_VALUE_TYPE_INDIVIDUALS = { + "Integer": "IntegerType", + "Float": "FloatType", + "Flag": "FlagType", + "Character": "CharacterType", + "String": "StringType", +} + + +def is_structured_header_value(value: str) -> bool: + """Return True when a header value is an angle-bracketed attribute list.""" + stripped = (value or "").strip() + return stripped.startswith("<") and stripped.endswith(">") + + +def parse_structured_header_fields(value: str) -> dict[str, str]: + """Parse comma-delimited VCF header attributes while respecting quotes.""" + return dict(parse_structured_header_attributes(value)) + + +def _split_header_attributes(inner: str, *, bracket_aware: bool) -> tuple[list[str], int]: + """Split a structured header body on its attribute-separating commas. + + A comma separates attributes only outside quotes and, when *bracket_aware*, + outside a bracketed list. VCF 4.5 writes a ``##META`` allowed-value set as + ``Values=[a, b, c]``, whose internal commas are part of one attribute; + splitting on them drops every member after the first, because the fragments + carry no ``=`` and are discarded as non-attributes. + + Returns the tokens and the final bracket depth, which is non-zero only when + the input's brackets are unbalanced. + """ + tokens: list[str] = [] + token: list[str] = [] + in_quotes = False + escaped = False + depth = 0 + for character in inner: + if escaped: + token.append(character) + escaped = False + elif character == "\\" and in_quotes: + token.append(character) + escaped = True + elif character == '"': + token.append(character) + in_quotes = not in_quotes + elif bracket_aware and character == "[" and not in_quotes: + depth += 1 + token.append(character) + elif bracket_aware and character == "]" and not in_quotes: + # Clamped, so a stray ']' cannot make the depth negative and turn a + # later, genuine '[' into a no-op. + depth = max(0, depth - 1) + token.append(character) + elif character == "," and not in_quotes and depth == 0: + tokens.append("".join(token)) + token = [] + else: + token.append(character) + tokens.append("".join(token)) + return tokens, depth + + +def parse_structured_header_attributes(value: str) -> list[tuple[str, str]]: + """Parse a structured header value into ordered (key, value) attributes. + + Order is preserved because vcfc:HeaderAttribute carries a one-based + vcfc:attributeIndex; VCF 4.5 forbids relying on source attribute order for + meaning, but the vocabulary keeps it recoverable. + """ + inner = (value or "").strip() + if inner.startswith("<") and inner.endswith(">"): + inner = inner[1:-1] + + tokens, depth = _split_header_attributes(inner, bracket_aware=True) + if depth: + # Unbalanced brackets are malformed. Bracket-aware splitting would treat + # the rest of the line as one token and silently drop every attribute + # after the stray '[', so fall back to splitting on commas alone. + tokens, _ = _split_header_attributes(inner, bracket_aware=False) + + attributes: list[tuple[str, str]] = [] + for item in tokens: + if "=" not in item: + continue + key, raw_value = item.split("=", 1) + parsed_value = raw_value.strip() + if len(parsed_value) >= 2 and parsed_value[0] == parsed_value[-1] == '"': + parsed_value = parsed_value[1:-1] + parsed_value = parsed_value.replace('\\"', '"').replace("\\\\", "\\") + attributes.append((key.strip(), parsed_value)) + return attributes + + +def parse_info_entries(info: str) -> list[tuple[str, str | None]]: + """Split an INFO column into ``(key, value)`` pairs. + + An entry without ``=`` is a Flag: a presence assertion with no value, which + the vocabulary models as ``vcfc:fieldValueBoolean true``. + """ + if info in ("", "."): + return [] + entries: list[tuple[str, str | None]] = [] + for item in info.split(";"): + if not item: + continue + key, separator, value = item.partition("=") + entries.append((key, value if separator else None)) + return entries + + +#: ##fileDate has no mandated format. These are the two forms seen in practice +#: that map unambiguously onto xsd:date, which the SHACL shape requires. +FILE_DATE_PATTERNS = ( + (re.compile(r"^(\d{4})(\d{2})(\d{2})$"), "{0}-{1}-{2}"), + (re.compile(r"^(\d{4})-(\d{2})-(\d{2})$"), "{0}-{1}-{2}"), +) + + +def file_date_object(value: str) -> str | None: + """Serialize ##fileDate as xsd:date when its form allows, else lexically. + + Returns None for an absent value so no triple is emitted, matching RML's + behaviour for an empty reference. + """ + value = (value or "").strip() + if not value or value == ".": + return None + for pattern, template in FILE_DATE_PATTERNS: + match = pattern.match(value) + if match: + return ntriples_typed_literal(template.format(*match.groups()), XSD_DATE_URI) + # An unrecognized form is preserved verbatim rather than dropped; the SHACL + # layer reports it as non-conformant. + return ntriples_string_literal(value) + + +def qual_object(value: str) -> str: + """Serialize QUAL the way vcfc:VariantCallShape accepts it. + + The shape is a disjunction over vcfc:VCFFloat, the XSD numeric datatypes + and vcfc:Null, which RML cannot satisfy because the branch depends on the + row. INF/INFINITY/NAN are in the VCF Float lexical space but in no XSD + numeric one, so they take vcfc:VCFFloat; a finite value takes xsd:decimal + in its source lexical form, so no precision is gained or lost. + """ + if is_missing(value): + return ntriples_typed_literal(".", NULL_DATATYPE_URI) + if is_finite_vcf_float(value): + return ntriples_typed_literal(value, XSD_DECIMAL_URI) + if is_vcf_float(value): + return ntriples_typed_literal(value, VCF_FLOAT_DATATYPE_URI) + # A value outside the VCF Float lexical space is kept as a plain literal + # rather than dropped: a non-conformant graph is more useful than a lossy + # one, and the SHACL layer reports it. + return ntriples_string_literal(value) + + +def numeric_literal(value: str) -> str: + """Serialize a VCF numeric value with a datatype a numeric shape accepts. + + vcfc:NumericLiteralShape requires an integer- or decimal-derived datatype, + so a confidence-interval bound cannot be a plain string. A value outside + both lexical spaces is kept verbatim rather than dropped, and the SHACL + layer reports it. + """ + if is_missing(value): + return ntriples_typed_literal(".", NULL_DATATYPE_URI) + stripped = value.strip() + if is_vcf_integer(stripped) and stripped.lstrip("+-").isdigit(): + return ntriples_typed_literal(stripped, XSD_INTEGER_URI) + if is_finite_vcf_float(stripped): + return ntriples_typed_literal(stripped, XSD_DECIMAL_URI) + return ntriples_string_literal(value) + + +def typed_field_object(value: str, declared_type: str) -> tuple[str, str] | None: + """Return ``(property_local_name, literal)`` for a typed single value. + + Only a single, non-missing value gets a typed companion property; a comma + list is represented by vcfc:FieldValueItem resources instead. + """ + if "," in value or is_missing(value): + return None + if declared_type == "Integer": + if is_vcf_integer(value): + return "fieldValueInteger", ntriples_typed_literal(value, XSD_INTEGER_URI) + return None + if declared_type == "Float": + # Serialize the source lexical form rather than a reparsed float, so + # the graph never gains or loses precision relative to the VCF. A + # non-finite Float has no xsd:decimal form and keeps only fieldValue. + if is_finite_vcf_float(value): + return "fieldValueDecimal", ntriples_typed_literal(value, XSD_DECIMAL_URI) + return None + return None