diff --git a/.github/workflows/publish-docker.yml b/.github/workflows/publish-docker.yml index 3b2e6fd..4759375 100644 --- a/.github/workflows/publish-docker.yml +++ b/.github/workflows/publish-docker.yml @@ -9,10 +9,13 @@ on: workflow_dispatch: inputs: docker_tag: - description: "Optional explicit Docker tag (example: v1.1.1)" + description: "Optional explicit semantic Docker tag (example: 1.2.4 or v1.2.4)" required: false type: string +env: + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true + jobs: docker: name: build-and-push-docker @@ -71,8 +74,8 @@ jobs: ref_name = os.environ.get("GITHUB_REF_NAME", "") head_message = os.environ.get("HEAD_COMMIT_MESSAGE", "") manual_tag = os.environ.get("INPUT_DOCKER_TAG", "").strip() - semver = re.compile(r"^v(\d+)\.(\d+)\.(\d+)$") - explicit_from_commit = re.search(r"\[(?:publish-docker|docker)=([A-Za-z0-9_.-]+)\]", head_message) + semver = re.compile(r"^v?(\d+)\.(\d+)\.(\d+)$") + explicit_from_commit = re.search(r"\[(?:publish-docker|docker)=(v?\d+\.\d+\.\d+)\]", head_message) def parse_max(tags): best = (-1, -1, -1) @@ -105,16 +108,22 @@ jobs: best = max(best, parse_max(tags)) major, minor, patch = best if major < 0: - resolved_tag = "v1.0.0" + resolved_tag = "1.0.0" else: - resolved_tag = f"v{major}.{minor}.{patch + 1}" + resolved_tag = f"{major}.{minor}.{patch + 1}" - if not re.match(r"^[A-Za-z0-9_.-]+$", resolved_tag): - raise SystemExit(f"Invalid Docker tag: {resolved_tag}") - print(f"Resolved Docker tag: {resolved_tag}") + match = semver.fullmatch(resolved_tag) + if not match: + raise SystemExit( + f"Docker release tag must match MAJOR.MINOR.PATCH or vMAJOR.MINOR.PATCH: {resolved_tag}" + ) + version_tag = ".".join(match.groups()) + version_tag_with_v = f"v{version_tag}" + print(f"Resolved Docker tags: {version_tag}, {version_tag_with_v}, latest") with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as f: - f.write(f"next_version={resolved_tag}\n") + f.write(f"version_tag={version_tag}\n") + f.write(f"version_tag_with_v={version_tag_with_v}\n") PY - name: Docker metadata @@ -123,7 +132,8 @@ jobs: with: images: ecrum19/vcf-rdfizer tags: | - type=raw,value=${{ steps.version.outputs.next_version }} + type=raw,value=${{ steps.version.outputs.version_tag }} + type=raw,value=${{ steps.version.outputs.version_tag_with_v }} type=raw,value=latest - name: Build and push diff --git a/.github/workflows/publish-python.yml b/.github/workflows/publish-python.yml index acbab8f..0001d09 100644 --- a/.github/workflows/publish-python.yml +++ b/.github/workflows/publish-python.yml @@ -3,32 +3,36 @@ name: publish-python on: push: tags: - - "v*" + - "v*.*.*" workflow_dispatch: inputs: release_tag: - description: "Optional Git tag to sync from, for example v1.2.3" - required: false + description: "Existing Git tag to validate, for example v1.2.4. Manual runs never publish to PyPI." + required: true type: string +env: + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true + jobs: build: name: build-sdist-wheel runs-on: ubuntu-latest + env: + RELEASE_TAG: ${{ github.event.inputs.release_tag || github.ref_name }} steps: - name: Checkout uses: actions/checkout@v4 + with: + ref: ${{ github.event.inputs.release_tag || github.ref }} - name: Setup Python uses: actions/setup-python@v5 with: python-version: "3.11" - - name: Sync release metadata from tag - env: - RELEASE_TAG: ${{ github.event.inputs.release_tag || github.ref_name }} - run: | - python scripts/release.py --tag "${RELEASE_TAG}" --workflow + - name: Validate committed release metadata + run: python scripts/release.py --check-tag "${RELEASE_TAG}" - name: Build package run: | @@ -38,8 +42,6 @@ jobs: python -m twine check dist/* - name: Verify built artifact versions - env: - RELEASE_TAG: ${{ github.event.inputs.release_tag || github.ref_name }} run: | python - <<'PY' from pathlib import Path @@ -85,6 +87,27 @@ jobs: print(f" {name}") PY + - name: Smoke-test installed wheel + run: | + python -m venv /tmp/vcf-rdfizer-wheel-test + /tmp/vcf-rdfizer-wheel-test/bin/python -m pip install --no-deps dist/*.whl + ( + cd /tmp + /tmp/vcf-rdfizer-wheel-test/bin/vcf-rdfizer --help + /tmp/vcf-rdfizer-wheel-test/bin/python - <<'PY' + import importlib.resources + from pathlib import Path + + import vcf_rdfizer + + rules = importlib.resources.files("vcf_rdfizer_data").joinpath("rules/default_rules.ttl") + if not rules.is_file(): + raise SystemExit("Packaged default rules are missing") + if not vcf_rdfizer.resolve_default_rules_path(Path(vcf_rdfizer.__file__).resolve().parent).is_file(): + raise SystemExit("Installed wrapper cannot resolve packaged default rules") + PY + ) + - name: Upload artifacts uses: actions/upload-artifact@v4 with: diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index f0dfddf..05c8481 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -6,6 +6,9 @@ on: - "**" pull_request: +env: + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true + jobs: wrapper-cross-platform: name: wrapper-unit (${{ matrix.os }}) diff --git a/.gitignore b/.gitignore index d8ae6e8..0b1216f 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,8 @@ test/test_vcf_files/* !test/test_vcf_files/test-100.vcf !test/test_vcf_files/test-1k.vcf !test/test_vcf_files/test-10k.vcf +!test/test_vcf_files/test-larger.vcf.gz + RMLStreamer-v2.5.0-standalone.jar out/ tsv/ diff --git a/Dockerfile b/Dockerfile index 9c685c2..8b3eb98 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,5 @@ ARG RMLSTREAMER_VERSION=2.5.0 +ARG HDT_JAVA_PACKAGE_VERSION=3.0.10 FROM eclipse-temurin:11-jre AS build-hdt-cpp @@ -37,6 +38,7 @@ RUN mkdir -p /opt/third_party_licenses \ FROM eclipse-temurin:11-jre ARG RMLSTREAMER_VERSION +ARG HDT_JAVA_PACKAGE_VERSION RUN apt-get update \ && apt-get install -y --no-install-recommends \ @@ -50,14 +52,26 @@ RUN apt-get update \ gzip \ libserd-0-0 \ nodejs \ + python3 \ + python3-venv \ time \ && rm -rf /var/lib/apt/lists/* +RUN python3 -m venv /opt/pycottas-venv \ + && /opt/pycottas-venv/bin/pip install --no-cache-dir pycottas + RUN mkdir -p /opt/rmlstreamer \ && curl -fsSL \ -o /opt/rmlstreamer/RMLStreamer-v${RMLSTREAMER_VERSION}-standalone.jar \ https://github.com/RMLio/RMLStreamer/releases/download/v${RMLSTREAMER_VERSION}/RMLStreamer-v${RMLSTREAMER_VERSION}-standalone.jar +RUN mkdir -p /opt/hdt-java \ + && curl -fsSL \ + -o /tmp/hdt-java-package.tar.gz \ + https://repo1.maven.org/maven2/org/rdfhdt/hdt-java-package/${HDT_JAVA_PACKAGE_VERSION}/hdt-java-package-${HDT_JAVA_PACKAGE_VERSION}-distribution.tar.gz \ + && tar -xzf /tmp/hdt-java-package.tar.gz -C /opt/hdt-java --strip-components=1 \ + && rm -f /tmp/hdt-java-package.tar.gz + COPY --from=build-hdt-cpp /usr/local/bin/rdf2hdt /usr/local/bin/rdf2hdt COPY --from=build-hdt-cpp /usr/local/bin/hdt2rdf /usr/local/bin/hdt2rdf COPY --from=build-hdt-cpp /usr/local/lib/libcds* /usr/local/lib/ @@ -65,15 +79,23 @@ COPY --from=build-hdt-cpp /usr/local/lib/libhdt* /usr/local/lib/ COPY --from=build-hdt-cpp /opt/third_party_licenses/ /usr/share/licenses/vcf-rdfizer/ COPY THIRD_PARTY_NOTICES.md /usr/share/licenses/vcf-rdfizer/THIRD_PARTY_NOTICES.md COPY src/*.sh /opt/vcf-rdfizer/ +COPY src/*.py /opt/vcf-rdfizer/ RUN chmod +x /opt/vcf-rdfizer/*.sh \ + && find /opt/hdt-java/bin -type f -exec chmod +x {} \; \ && chmod +x /usr/local/bin/rdf2hdt \ && chmod +x /usr/local/bin/hdt2rdf ENV RMLSTREAMER_JAR=/opt/rmlstreamer/RMLStreamer-v${RMLSTREAMER_VERSION}-standalone.jar ENV JAR=/opt/rmlstreamer/RMLStreamer-v${RMLSTREAMER_VERSION}-standalone.jar +ENV HDT_JAVA_HOME=/opt/hdt-java ENV RDF2HDT_BIN=/usr/local/bin/rdf2hdt ENV HDT2RDF_BIN=/usr/local/bin/hdt2rdf +ENV COTTAS_PYTHON_BIN=/opt/pycottas-venv/bin/python ENV LD_LIBRARY_PATH=/usr/local/lib +# COTTAS creates a temporary DuckDB database in the container working +# directory. The wrapper runs containers as the host UID/GID, so this path +# must be writable without requiring root or creating root-owned host files. +RUN mkdir -p /work && chmod 1777 /work WORKDIR /work diff --git a/README.md b/README.md index 9c142d8..097a589 100644 --- a/README.md +++ b/README.md @@ -47,6 +47,9 @@ or pull the prebuilt Docker image directly: docker pull ecrum19/vcf-rdfizer:latest ``` +Release maintainers: see [`RELEASING.md`](RELEASING.md) for the PyPI, +Docker Hub, and conda-forge release procedure. + ## Important CLI Rule `--out` is required for all modes. @@ -62,8 +65,9 @@ inside this directory. - `full`: VCF -> TSV -> RDF -> compression - `tsv`: VCF -> TSV only (benchmarking) -- `compress`: compress an existing `.nt` +- `compress`: compress an existing `.nt` or `.nt.gz` - `decompress`: decompress `.nt.gz`, `.nt.br`, or `.hdt` +- `index`: eagerly initialize HDT Java's versioned `.hdt.index.*` sidecar for an existing `.hdt` In `full` mode with multiple VCF inputs, failures are isolated per input: - the run continues with remaining files @@ -71,25 +75,74 @@ In `full` mode with multiple VCF inputs, failures are isolated per input: ## Main Flags (Most Used) -- `-m, --mode {full,compress,decompress,tsv}` +- `-m, --mode {full,compress,decompress,tsv,index}` - `-o, --out` required output root directory -- `-c, --compression` methods: `gzip,brotli,hdt,hdt_gzip,hdt_brotli,none` +- `--rdf-compression` final raw RDF codecs: `gzip`, `brotli`, or `none` +- `--representations` queryable RDF outputs: `hdt`, `cottas`, or `none` +- `--artifact-compression` packaging codecs for selected representations: `gzip`, `brotli`, or `none` +- `--hdt-strategy {auto,partitioned,single}` HDT generation policy +- `--chunk-target-bytes`, `--chunk-min-bytes`, `--chunk-max-bytes` shared record-safe chunk sizing - `-I, --image` Docker image repo (default `ecrum19/vcf-rdfizer`) - `-v, --image-version` Docker tag/version - `-b, --build` force Docker build - `-B, --no-build` fail if image not found - `-h, --help` show full usage +## Compression Plan + +Compression is configured as three independent decisions: + +1. **RDF staging**: `--rdf-storage-mode` controls how RMLStreamer output is + assembled before chunking. `plain` creates one `.nt` aggregate; + `space-optimized` streams the parts into one `.nt.gz` aggregate and removes + each source part immediately. +2. **Raw RDF artifacts**: `--rdf-compression gzip,brotli` creates compressed + copies of the RDF aggregate. Use `--rdf-compression none` when RDF is only a + temporary input to HDT/COTTAS. +3. **Queryable representations and packaging**: `--representations hdt,cottas` + creates the selected indexed formats. `--artifact-compression gzip,brotli` + packages each selected representation, producing `.hdt.gz`, `.hdt.br`, + `.cottas.gz`, or `.cottas.br` in addition to the queryable base artifact. + +Each selector accepts comma-separated values. Use `none` by itself to disable +that stage; do not combine `none` with another value. `--artifact-compression` +requires at least one selected value in `--representations`. + +The gzip used by `--rdf-storage-mode space-optimized` is staging storage; it is +not automatically a final raw RDF artifact. The default plan preserves the +historical outputs: `--rdf-compression gzip,brotli` and +`--representations hdt`. For the smallest final output, select +`--rdf-compression none`, one representation, and +`--remove-rdf-storage-output`. + +Packaged `.hdt.gz`, `.hdt.br`, `.cottas.gz`, and `.cottas.br` files are archives, +not directly queryable indexed files. Keep the unwrapped `.hdt`/`.cottas` file +when queries must run without a decompression step. + ## Full Mode Flags - `-i, --input` required VCF file or directory - `-r, --rules` mapping rules file (`.ttl`) - default: `rules/default_rules.ttl` -- `-l, --rdf-layout {aggregate,batch}` required in full mode +- `--rdf-storage-mode {plain,space-optimized}` required full-mode aggregate storage policy + - `plain`: merge RMLStreamer parts into one uncompressed `.nt` + - `space-optimized`: gzip each part into one `.nt.gz` aggregate and delete the source part immediately +- `--rdf-compression {gzip,brotli,none}` raw RDF artifacts to retain +- `--representations {hdt,cottas,none}` queryable primary representations +- `--artifact-compression {gzip,brotli,none}` optional packaging applied to each selected representation +- `--hdt-strategy {auto,partitioned,single}` + - `auto`: in full mode, build smaller HDT chunks and merge them with `HDTCat` + - `partitioned`: always use chunked HDT generation for HDT-based methods + - `single`: always use one `rdf2hdt` run per RDF input + - with `space-optimized`, use `auto` or `partitioned`; `single` cannot consume the gzip stream without expanding it +- `--chunk-target-bytes` target uncompressed bytes per HDT/COTTAS chunk +- `--chunk-min-bytes` minimum uncompressed bytes before flushing a chunk group +- `--chunk-max-bytes` maximum uncompressed bytes in a chunk; boundaries remain on complete NT lines - `-P, --spark-partitions` optional Spark partition hint (positive integer) - - low-cost way to reduce output part count by setting `spark.default.parallelism` and `spark.sql.shuffle.partitions` + - low-cost way to tune RMLStreamer parallelism while the wrapper still produces one aggregate RDF output - `-k, --keep-tsv` keep hidden TSV intermediates -- `-R, --keep-rdf` keep raw `.nt` after compression +- `-R, --keep-rmlstreamer-rdf-output` keep the aggregate RDF output produced by RMLStreamer +- `--remove-rdf-storage-output` explicitly remove the aggregate `.nt`/`.nt.gz` after successful compression - `-e, --estimate-size` preflight size estimate ## TSV Mode Flags @@ -100,13 +153,19 @@ In `full` mode with multiple VCF inputs, failures are isolated per input: ## Compression Mode Flags -- `-q, --rdf, --nt` required input `.nt` file +- `--rdf` required input `.nt` or `.nt.gz` file ## Decompression Mode Flags - `-C, --compressed-input` required `.nt.gz`, `.nt.br`, or `.hdt` - `-d, --decompress-out` optional explicit output `.nt` path (must be inside `--out`) +## HDT Index Mode Flags + +- `-H, --hdt` required existing `.hdt` file +- HDT Java 3.0.10 generates an HDT v1-1 index named `.hdt.index.v1-1`. +- The operation is also run automatically after each partitioned HDT merge. + ## Quick Start Show help: @@ -115,49 +174,100 @@ Show help: vcf-rdfizer --help ``` -Full pipeline (aggregate RDF): +Full pipeline (plain aggregate RDF): + +```bash +vcf-rdfizer \ + --mode full \ + --input ./vcf_files \ + --rdf-storage-mode plain \ + --rdf-compression none \ + --representations none \ + --out ./results +``` + +Full pipeline (plain aggregate, chunked HDT + HDTCat merge): ```bash vcf-rdfizer \ --mode full \ --input ./vcf_files \ - --rdf-layout aggregate \ + --rdf-storage-mode plain \ + --rdf-compression none \ + --representations hdt \ + --hdt-strategy partitioned \ + --chunk-target-bytes 536870912 \ + --chunk-min-bytes 134217728 \ + --chunk-max-bytes 1073741824 \ --out ./results ``` -Full pipeline (batch RDF parts): +Full pipeline (space-optimized aggregate with shared HDT and COTTAS chunks): ```bash vcf-rdfizer \ --mode full \ --input ./vcf_files \ - --rdf-layout batch \ - --compression hdt \ + --rdf-storage-mode space-optimized \ + --rdf-compression none \ + --representations hdt,cottas \ + --chunk-target-bytes 536870912 \ + --chunk-min-bytes 134217728 \ + --chunk-max-bytes 1073741824 \ --out ./results ``` -Full pipeline with low-cost partition cap (helps avoid too many tiny batch files): +Full pipeline with a Spark partition hint: ```bash vcf-rdfizer \ --mode full \ --input ./vcf_files \ - --rdf-layout batch \ + --rdf-storage-mode space-optimized \ --spark-partitions 8 \ - --compression hdt \ + --rdf-compression none \ + --representations hdt \ --out ./results ``` -Full pipeline with custom rules + keep RDF: +Full pipeline with custom rules + keep RMLStreamer RDF output: ```bash vcf-rdfizer \ --mode full \ --input ./vcf_files \ --rules ./rules/my_rules.ttl \ - --rdf-layout aggregate \ - --compression hdt,brotli \ - --keep-rdf \ + --rdf-storage-mode plain \ + --rdf-compression brotli \ + --representations hdt \ + --keep-rmlstreamer-rdf-output \ + --out ./results +``` + +Ultra-small full pipeline: + +```bash +vcf-rdfizer \ + --mode full \ + --input ./vcf_files \ + --rdf-storage-mode space-optimized \ + --rdf-compression none \ + --representations hdt \ + --hdt-strategy partitioned \ + --remove-rdf-storage-output \ + --out ./results +``` + +Queryable HDT and COTTAS plus gzip/Brotli packages: + +```bash +vcf-rdfizer \ + --mode full \ + --input ./vcf_files \ + --rdf-storage-mode space-optimized \ + --rdf-compression none \ + --representations hdt,cottas \ + --artifact-compression gzip,brotli \ --out ./results ``` @@ -176,7 +286,21 @@ Compression-only: vcf-rdfizer \ --mode compress \ --rdf ./results/sample/sample.nt \ - --compression hdt_gzip \ + --rdf-compression none \ + --representations hdt \ + --artifact-compression gzip \ + --out ./results +``` + +Compression-only from a space-optimized aggregate: + +```bash +vcf-rdfizer \ + --mode compress \ + --rdf ./results/sample/sample.nt.gz \ + --rdf-compression none \ + --representations hdt,cottas \ + --chunk-target-bytes 536870912 \ --out ./results ``` @@ -189,6 +313,15 @@ vcf-rdfizer \ --out ./results ``` +Initialize an index for an existing HDT: + +```bash +vcf-rdfizer \ + --mode index \ + --hdt ./results/sample/sample.hdt \ + --out ./results +``` + ## Output Layout Given `--out ./results`: @@ -200,8 +333,27 @@ Given `--out ./results`: - hidden intermediates: - `./results/.intermediate/tsv/` +For an existing RDF input named `test-larger.nt` or `test-larger.nt.gz`, +`--mode compress --out ./results` always uses one directory: + +```text +./results/test-larger/test-larger.hdt +./results/test-larger/test-larger.hdt.index.v1-1 +./results/test-larger/test-larger.cottas +./results/test-larger/test-larger.nt.gz +``` + +The same basename rule applies in full mode. VCF-RDFizer performs an output +collision check before Docker or conversion starts and never overwrites a +planned artifact. Choose a new `--out` directory, or rename/remove the +conflicting output before rerunning. + Intermediates are hidden by default. -Raw `.nt` files are removed after compression unless `--keep-rdf` is provided. +Raw RDF files are removed after successful compression by default. Use +`--remove-rdf-storage-output` to make that cleanup explicit, or use +`--keep-rmlstreamer-rdf-output` to retain the aggregate RDF output instead. +The space-optimized mode retains the `.nt.gz` aggregate when `gzip` is selected +because that file is the gzip artifact itself. ## Metrics @@ -210,6 +362,7 @@ For each run, VCF-RDFizer writes: - `run_metrics//metrics.csv` - `run_metrics//wrapper_execution_times.csv` - `run_metrics//progress.log` +- `run_metrics//hdt_index_metrics.json` for standalone HDT index mode Compression metrics now include per-method: @@ -218,6 +371,69 @@ Compression metrics now include per-method: - `sys_seconds_*` - `max_rss_kb_*` +When HDT or COTTAS is selected, compression also validates the final base +artifact before packaging or RDF cleanup. The validator reads the source +triple count, streams the artifact back through the native decoder, and +requires equal counts. HDT validation also initializes the versioned `.hdt.index.*` +sidecar. Validation results and `source_triples`/`decoded_triples` are stored +in the per-run compression JSON and in the HDT/COTTAS columns of `metrics.csv`. +Compression fails closed if the artifact cannot be decoded or the counts do +not match. In compression-only mode, the source count is obtained by a +streaming fallback when no upstream conversion metrics are available. + +For partitioned HDT/COTTAS runs, the final method metric reports one +sample-level result, while raw metrics also include a sample-scoped +`__partitioned_compression__` artifact describing chunk conversion, merge +rounds, and the generated chunk guide. + +Metrics may use internal stage names such as `hdt_gzip` and `cottas_brotli`. +These correspond to the public combination of `--representations` and +`--artifact-compression`; users do not need to pass those compound names. + +## Chunked Compression + +Full mode always uses one of the two aggregate storage modes. Both modes create +one logical N-Triples aggregate. The space-optimized mode streams each +RMLStreamer part through gzip and deletes that part before processing the next +one, so it avoids retaining both the part files and a full uncompressed +aggregate. + +When HDT or COTTAS is selected, the aggregate is read sequentially and split +into complete N-Triples records. The same temporary chunks are consumed by +both converters before cleanup. HDT chunks are merged with `HDTCat`, the final +HDT index is generated after merging through HDT Java's indexed search loader, +and COTTAS chunks are merged with `pycottas.cat`, which rebuilds the query +indexes for the merged representation. + +After each final HDT/COTTAS base artifact is produced, VCF-RDFizer performs a +streaming decode/count check. This verifies both readability and that the +decoded artifact contains exactly the number of source triples. The check is +performed before `.hdt.gz`, `.hdt.br`, `.cottas.gz`, or `.cottas.br` packaging, +and before raw RDF cleanup. + +Partitioned HDT/COTTAS compression runs in an ephemeral Docker-managed +workspace. Temporary RDF chunks, COTTAS/DuckDB scratch data, intermediate +representations, and merge files are not written to the output directory. +After a successful or failed run, the temporary workspace is removed; only +the selected final artifacts and normal run metrics remain on the host. +Each COTTAS conversion and merge also receives a fresh container-local DuckDB +workspace, which is removed as soon as that operation completes. This prevents +state from one chunk being reused by another and requires no user configuration. + +HDT Java 3.0.10 does not provide a standalone `hdtGenerateIndex` executable. +VCF-RDFizer sends an `exit` command to the supported `hdtSearch.sh` launcher; +this opens the HDT through `mapIndexedHDT()` without executing a data query and +creates the sibling `.hdt.index` sidecar before the run is marked successful. +For the pinned HDT Java 3.0.10 distribution, this is the HDT v1-1 sidecar +`.hdt.index.v1-1`; VCF-RDFizer reports the actual path in its metrics. + +The record-safe chunk plan and per-stage timings are retained in the raw +partitioned-compression metrics JSON for diagnostics. The temporary chunk +files and guide are not retained as host files. + +See [`COMPRESSION_CHANGELOG.md`](COMPRESSION_CHANGELOG.md) for the detailed +implementation approach and operational constraints. + ## Rules - default rules file: `rules/default_rules.ttl` @@ -227,15 +443,18 @@ Compression metrics now include per-method: If Docker permission issues occur, rerun with a Docker-allowed user (or configure Docker group/sudo access on your system). -If HDT compression fails on very large `.nt` files, use batch layout and/or non-HDT compression methods. +If HDT compression fails on very large RDF files, use +`--rdf-storage-mode space-optimized` or `--rdf-storage-mode plain` with +`--hdt-strategy partitioned`, then lower `--chunk-target-bytes` and +`--chunk-max-bytes` to reduce each converter's working set. Safe termination: - Press `Ctrl+C` to interrupt a run. - The wrapper exits with code `130`, writes progress to `run_metrics//progress.log`, and performs best-effort cleanup of tracked intermediates. -- Raw RDF cleanup on interrupt follows `--keep-rdf`: - - with `--keep-rdf`, raw `.nt` files are preserved - - without `--keep-rdf`, tracked raw `.nt` files are removed during interrupt cleanup +- Raw RDF cleanup on interrupt follows `--keep-rmlstreamer-rdf-output`: + - with `--keep-rmlstreamer-rdf-output`, raw RDF files are preserved + - without it, tracked raw RDF files are removed during interrupt cleanup ## Citation diff --git a/RELEASING.md b/RELEASING.md new file mode 100644 index 0000000..18d559a --- /dev/null +++ b/RELEASING.md @@ -0,0 +1,73 @@ +# Releasing VCF-RDFizer + +This repository releases the Python wrapper to PyPI, a multi-architecture +Docker image to Docker Hub, and the wrapper package to conda-forge. + +## Preconditions + +- The target branch has passed `.github/workflows/tests.yml`. +- The GitHub `pypi` environment is configured as a trusted publisher for the + `vcf-rdfizer` PyPI project. +- `DOCKERHUB_USERNAME` and `DOCKERHUB_TOKEN` are configured as repository + secrets. +- Release from the branch intended to become `main`; do not tag an unmerged + development branch. + +## 1. Prepare And Validate The Source Release + +Choose a new semantic version, for example `1.2.4`, then run: + +```bash +python3 scripts/release.py 1.2.4 +git diff --check +python3 -m unittest discover -s test -p "test_*_unit.py" -q +git add pyproject.toml CITATION.cff README.md conda-recipe/meta.yaml conda-recipe/README.md +git commit -m "Release v1.2.4" +git push origin main +git tag -a v1.2.4 -m "VCF-RDFizer v1.2.4" +git push origin v1.2.4 +``` + +The version must be committed before creating the tag. The PyPI workflow +checks that the tag and all version markers agree; it deliberately does not +modify metadata in CI. + +The pushed tag runs both publishing workflows. The Python workflow builds, +checks, installs, and publishes only on a `vMAJOR.MINOR.PATCH` tag. The Docker +workflow publishes `X.Y.Z`, `vX.Y.Z`, and `latest` as a single +`linux/amd64,linux/arm64` manifest. + +To exercise the PyPI build checks without uploading, manually dispatch the +`publish-python` workflow and provide an existing release tag. It checks out +that tag and never runs the publish job for a manual dispatch. + +## 2. Update Conda-Forge + +The conda-forge source archive does not exist until the tag is pushed. After +the tag is available, populate this repository's reference recipe with the +real checksum: + +```bash +python3 scripts/release.py 1.2.4 --fetch-conda-sha256 +``` + +Copy the resulting `conda-recipe/meta.yaml` values into a branch in your fork +of [`conda-forge/vcf-rdfizer-feedstock`](https://github.com/conda-forge/vcf-rdfizer-feedstock), +edit `recipe/meta.yaml`, and open a PR. For a version bump, keep +`build/number: 0`. The feedstock CI builds and publishes the conda package +after the PR merges. + +## 3. Verify Published Artifacts + +```bash +python3 -m pip install --upgrade vcf-rdfizer==1.2.4 +vcf-rdfizer --help + +conda install -c conda-forge vcf-rdfizer=1.2.4 +vcf-rdfizer --help + +docker pull ecrum19/vcf-rdfizer:1.2.4 +docker buildx imagetools inspect ecrum19/vcf-rdfizer:1.2.4 +``` + +The Docker inspection must list both `linux/amd64` and `linux/arm64`. diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md index cb7c6ab..f8bffb5 100644 --- a/THIRD_PARTY_NOTICES.md +++ b/THIRD_PARTY_NOTICES.md @@ -21,6 +21,12 @@ their respective authors and apply to those components. - License files copied into image: - `/usr/share/licenses/vcf-rdfizer/RMLStreamer.LICENSE` +3. `pycottas` +- Repository: +- Usage in this project: RDF chunk compression and indexed COTTAS merging +- Upstream license: Apache License 2.0 +- Installed in the image's `/opt/pycottas-venv` environment + ## Notes for package users - The `pip` and `conda` packages install the Python wrapper CLI only. diff --git a/conda-recipe/README.md b/conda-recipe/README.md index f142a10..ff048f6 100644 --- a/conda-recipe/README.md +++ b/conda-recipe/README.md @@ -1,26 +1,32 @@ # Conda Recipe Notes -This directory contains a starter `meta.yaml` for publishing `vcf-rdfizer` on conda-forge. +This directory contains the release-reference recipe for `vcf-rdfizer`. +The published recipe is maintained in the existing +[`conda-forge/vcf-rdfizer-feedstock`](https://github.com/conda-forge/vcf-rdfizer-feedstock); +do not submit this package to `staged-recipes`. ## Before submitting to conda-forge -1. Create a GitHub release/tag (for example `v1.2.3`). +1. Commit the version bump, then create and push a Git tag (for example `v1.2.3`). 2. Download the source tarball and compute sha256: ```bash curl -L -o vcf-rdfizer.tar.gz \ https://github.com/ecrum19/VCF-RDFizer/archive/refs/tags/v1.2.3.tar.gz shasum -a 256 vcf-rdfizer.tar.gz ``` -3. Replace `version` and `sha256` in `meta.yaml`. -4. Submit to [`conda-forge/staged-recipes`](https://github.com/conda-forge/staged-recipes). +3. Replace `version` and `sha256` in the feedstock's `recipe/meta.yaml`. +4. Open a PR against [`conda-forge/vcf-rdfizer-feedstock`](https://github.com/conda-forge/vcf-rdfizer-feedstock). ## Ongoing releases After the initial feedstock is created: -1. Open PR in your feedstock bumping `version` and `sha256`. -2. Merge once CI passes. -3. Verify install: +1. Fork `conda-forge/vcf-rdfizer-feedstock` and create a branch in your fork. +2. Bump `version`, set the SHA256 of the corresponding pushed Git tag archive, + and reset `build/number` to `0` in `recipe/meta.yaml`. +3. Open a PR from that branch to the feedstock. +4. Merge once CI passes. +5. Verify install: ```bash conda install -c conda-forge vcf-rdfizer vcf-rdfizer --help diff --git a/conda-recipe/meta.yaml b/conda-recipe/meta.yaml index dec394f..8776e43 100644 --- a/conda-recipe/meta.yaml +++ b/conda-recipe/meta.yaml @@ -7,7 +7,7 @@ package: source: url: https://github.com/ecrum19/VCF-RDFizer/archive/refs/tags/v{{ version }}.tar.gz - sha256: REPLACE_WITH_GITHUB_TARBALL_SHA256 + sha256: 1587e0b51e4394eff884b503cff64f7283f2ea5bf13039ed262be8763f8ff691 build: noarch: python @@ -20,7 +20,7 @@ requirements: host: - python {{ python_min }} - pip - - setuptools >=69 + - setuptools >=77 - wheel run: - python >={{ python_min }} diff --git a/pyproject.toml b/pyproject.toml index 80621e6..b6cfc47 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["setuptools>=69", "wheel"] +requires = ["setuptools>=77", "wheel"] build-backend = "setuptools.build_meta" [project] @@ -8,7 +8,8 @@ version = "1.2.3" description = "Docker-first VCF to RDF conversion wrapper with compression/decompression modes" readme = "README.md" requires-python = ">=3.10" -license = { file = "LICENSE" } +license = "MIT" +license-files = ["LICENSE"] authors = [ { name = "VCF-RDFizer maintainers" } ] @@ -17,7 +18,6 @@ classifiers = [ "Development Status :: 4 - Beta", "Environment :: Console", "Intended Audience :: Science/Research", - "License :: OSI Approved :: MIT License", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", @@ -41,8 +41,8 @@ dev = [ [tool.setuptools] py-modules = ["vcf_rdfizer"] -packages = ["vcf_rdfizer_data"] +packages = ["vcf_rdfizer_data", "vcf_rdfizer_data.rules"] include-package-data = true [tool.setuptools.package-data] -vcf_rdfizer_data = ["rules/default_rules.ttl"] +"vcf_rdfizer_data.rules" = ["default_rules.ttl"] diff --git a/scripts/release.py b/scripts/release.py index 5c2d156..c6b39ae 100644 --- a/scripts/release.py +++ b/scripts/release.py @@ -1,17 +1,17 @@ #!/usr/bin/env python3 -"""Bump repository release metadata from a single semantic version. +"""Prepare or validate VCF-RDFizer release metadata. -Usage: - python3 scripts/release.py 1.2.3 +Before creating a Git tag, update the committed release metadata: -The script updates the project version markers, README citation text, the conda -recipe version, and a few release-oriented examples. It can also fetch the -GitHub source tarball SHA256 for the conda recipe when network access is -available. + python3 scripts/release.py 1.2.4 -In workflow mode, pass a Git tag via ``--tag v1.2.3`` together with -``--workflow`` to sync release metadata in CI without touching the conda recipe -SHA256. +After pushing ``v1.2.4``, fetch the immutable source-archive checksum needed +for a conda-forge feedstock update: + + python3 scripts/release.py 1.2.4 --fetch-conda-sha256 + +CI uses ``--check-tag``. It never rewrites metadata: a release tag must point +at source whose committed metadata already matches that tag. """ from __future__ import annotations @@ -19,12 +19,17 @@ import argparse import hashlib import re +import shutil +import subprocess import sys +import tempfile import urllib.request from pathlib import Path ROOT = Path(__file__).resolve().parents[1] +VERSION_PATTERN = r"\d+\.\d+\.\d+" +PLACEHOLDER_CONDA_SHA256 = "REPLACE_WITH_GITHUB_TARBALL_SHA256" def read_text(path: Path) -> str: @@ -38,140 +43,219 @@ def write_text(path: Path, text: str) -> None: def replace_once(text: str, pattern: str, replacement: str, *, file: Path) -> str: new_text, count = re.subn(pattern, replacement, text, count=1, flags=re.MULTILINE) if count != 1: - raise SystemExit(f"Expected exactly one match for {pattern!r} in {file}") + raise ValueError(f"Expected exactly one match for {pattern!r} in {file}") return new_text -def sha256_for_url(url: str) -> str: - with urllib.request.urlopen(url, timeout=60) as response: - digest = hashlib.sha256() - for chunk in iter(lambda: response.read(1024 * 1024), b""): - digest.update(chunk) - return digest.hexdigest() - - def version_from_tag(tag: str) -> str: - match = re.fullmatch(r"v(\d+\.\d+\.\d+)", tag.strip()) + match = re.fullmatch(rf"v({VERSION_PATTERN})", tag.strip()) if not match: - raise SystemExit("tag must match vMAJOR.MINOR.PATCH, for example v1.2.3") + raise ValueError("tag must match vMAJOR.MINOR.PATCH, for example v1.2.4") return match.group(1) -def main() -> int: - parser = argparse.ArgumentParser(description="Bump VCF-RDFizer release metadata.") - parser.add_argument("version", nargs="?", help="Target semantic version, for example 1.2.3") - parser.add_argument( - "--tag", - default=None, - help="Git tag to use as the version source, for example v1.2.3", - ) - parser.add_argument( - "--workflow", - action="store_true", - help="Sync build metadata from a tag in CI and skip conda recipe sha256 updates.", - ) - parser.add_argument( - "--conda-sha256", - default=None, - help="Optional sha256 for the GitHub release tarball. If omitted, the script will try to fetch it.", - ) - args = parser.parse_args() - - version_arg = args.version.strip() if args.version else None - tag_arg = args.tag.strip() if args.tag else None - if tag_arg: - version = version_from_tag(tag_arg) - if version_arg and version_arg != version: - raise SystemExit( - f"version argument {version_arg!r} does not match tag-derived version {version!r}" +def validate_version(version: str) -> str: + value = version.strip() + if not re.fullmatch(VERSION_PATTERN, value): + raise ValueError("version must match MAJOR.MINOR.PATCH, for example 1.2.4") + return value + + +def validate_sha256(value: str) -> str: + digest = value.strip().lower() + if not re.fullmatch(r"[0-9a-f]{64}", digest): + raise ValueError("--conda-sha256 must be a 64-character lowercase hexadecimal SHA256") + return digest + + +def source_archive_url(version: str) -> str: + return f"https://github.com/ecrum19/VCF-RDFizer/archive/refs/tags/v{version}.tar.gz" + + +def sha256_for_url(url: str) -> str: + """Download *url* and return its SHA256, with a system-curl TLS fallback.""" + try: + with urllib.request.urlopen(url, timeout=60) as response: + digest = hashlib.sha256() + for chunk in iter(lambda: response.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + except urllib.error.URLError as url_error: + curl = shutil.which("curl") + if curl is None: + raise url_error + + # Some macOS Python installations do not inherit the system CA bundle. + # Curl uses the system trust store and keeps the release command usable. + with tempfile.TemporaryDirectory(prefix="vcf-rdfizer-release-") as directory: + archive = Path(directory) / "source.tar.gz" + result = subprocess.run( + [curl, "--fail", "--location", "--retry", "3", "--output", str(archive), url], + capture_output=True, + text=True, + check=False, ) - elif version_arg: - version = version_arg - if not re.fullmatch(r"\d+\.\d+\.\d+", version): - raise SystemExit("version must match MAJOR.MINOR.PATCH, for example 1.2.3") - else: - raise SystemExit("provide either a semantic version argument or --tag vMAJOR.MINOR.PATCH") + if result.returncode != 0: + details = result.stderr.strip() or result.stdout.strip() or "unknown curl error" + raise ValueError(f"could not download {url}: {details}") from url_error + return hashlib.sha256(archive.read_bytes()).hexdigest() - tag = f"v{version}" +def release_metadata_errors(version: str) -> list[str]: + """Return every committed release marker that disagrees with *version*.""" + expected_tag = f"v{version}" + checks = ( + (ROOT / "pyproject.toml", rf'^version\s*=\s*"{re.escape(version)}"$'), + (ROOT / "CITATION.cff", rf'^version:\s*"{re.escape(version)}"$'), + (ROOT / "README.md", rf"Version\s+{re.escape(version)}"), + (ROOT / "README.md", rf"version\s*=\s*\{{{re.escape(version)}\}}"), + ( + ROOT / "conda-recipe" / "meta.yaml", + rf'^\{{% set version = "{re.escape(version)}" %\}}$', + ), + ) + errors = [] + for path, pattern in checks: + if not re.search(pattern, read_text(path), flags=re.MULTILINE): + errors.append(f"{path.relative_to(ROOT)} does not declare version {version}") + + conda_readme = read_text(ROOT / "conda-recipe" / "README.md") + if conda_readme.count(expected_tag) < 2: + errors.append( + f"conda-recipe/README.md must contain at least two {expected_tag} release examples" + ) + return errors + + +def update_release_metadata(version: str, conda_sha256: str | None) -> None: + """Synchronize local source metadata before a release tag is created.""" files = { ROOT / "pyproject.toml": [ - (r'^(version\s*=\s*)"\d+\.\d+\.\d+"$', rf'\g<1>"{version}"'), + (rf'^(version\s*=\s*)"{VERSION_PATTERN}"$', rf'\g<1>"{version}"'), ], ROOT / "CITATION.cff": [ - (r'^(version:\s*)"\d+\.\d+\.\d+"$', rf'\g<1>"{version}"'), + (rf'^(version:\s*)"{VERSION_PATTERN}"$', rf'\g<1>"{version}"'), ], ROOT / "README.md": [ - (r'(Version\s+)\d+\.\d+\.\d+', rf"\g<1>{version}"), - (r'(version\s*=\s*\{)\d+\.\d+\.\d+(\})', rf"\g<1>{version}\2"), - ], - ROOT / "conda-recipe" / "README.md": [ - (r'v\d+\.\d+\.\d+', tag), - ], - ROOT / "test" / "test_vcf_rdfizer_cross_platform_unit.py": [ - (r'("ecrum19/vcf-rdfizer",\s*")\d+\.\d+\.\d+(")', rf"\g<1>{version}\2"), - (r'("ecrum19/vcf-rdfizer:)\d+\.\d+\.\d+(")', rf"\g<1>{version}\2"), + (rf'(Version\s+){VERSION_PATTERN}', rf"\g<1>{version}"), + (rf'(version\s*=\s*\{{){VERSION_PATTERN}(\}})', rf"\g<1>{version}\2"), ], } - for path, replacements in files.items(): text = read_text(path) - if path == ROOT / "conda-recipe" / "README.md": - text, count = re.subn(r"v\d+\.\d+\.\d+", tag, text) - if count < 2: - raise SystemExit(f"Expected to update at least two version references in {path}") - else: - for pattern, replacement in replacements: - text = replace_once(text, pattern, replacement, file=path) + for pattern, replacement in replacements: + text = replace_once(text, pattern, replacement, file=path) write_text(path, text) - if not args.workflow: - meta_path = ROOT / "conda-recipe" / "meta.yaml" - meta_text = read_text(meta_path) + conda_readme_path = ROOT / "conda-recipe" / "README.md" + conda_readme, count = re.subn(rf"v{VERSION_PATTERN}", f"v{version}", read_text(conda_readme_path)) + if count < 2: + raise ValueError(f"Expected to update at least two version references in {conda_readme_path}") + write_text(conda_readme_path, conda_readme) + + meta_path = ROOT / "conda-recipe" / "meta.yaml" + meta_text = read_text(meta_path) + previous_match = re.search(rf'^\{{% set version = "({VERSION_PATTERN})" %\}}$', meta_text, re.MULTILINE) + if previous_match is None: + raise ValueError(f"Could not find the conda recipe version in {meta_path}") + previous_version = previous_match.group(1) + meta_text = replace_once( + meta_text, + rf'^\{{% set version = "{VERSION_PATTERN}" %\}}$', + f'{{% set version = "{version}" %}}', + file=meta_path, + ) + if conda_sha256 is not None: meta_text = replace_once( meta_text, - r'^\{% set version = "\d+\.\d+\.\d+" %\}$', - f'{{% set version = "{version}" %}}', + r"^\s*sha256:\s*.*$", + f" sha256: {conda_sha256}", file=meta_path, ) - - sha256 = args.conda_sha256 - if sha256 is None: - url = f"https://github.com/ecrum19/VCF-RDFizer/archive/refs/tags/{tag}.tar.gz" - try: - sha256 = sha256_for_url(url) - except Exception as exc: - sha256 = "REPLACE_WITH_GITHUB_TARBALL_SHA256" - print( - f"Warning: could not fetch {url} to compute sha256 automatically ({exc}).", - file=sys.stderr, - ) - print( - "Use the tarball URL above to compute the sha256 before opening the conda-forge PR.", - file=sys.stderr, - ) - + elif previous_version != version: + # The archive for the new tag does not exist yet. Never retain a + # checksum from the previous release, because it would be invalid. meta_text = replace_once( meta_text, - r'^\s*sha256:\s*.*$', - f" sha256: {sha256}", + r"^\s*sha256:\s*.*$", + f" sha256: {PLACEHOLDER_CONDA_SHA256}", file=meta_path, ) - write_text(meta_path, meta_text) + write_text(meta_path, meta_text) + +def print_next_steps(version: str, conda_sha256: str | None) -> None: + tag = f"v{version}" print(f"Updated release metadata to {version}") - print(f"Next steps:") - print(f" git add pyproject.toml CITATION.cff README.md conda-recipe/meta.yaml conda-recipe/README.md test/test_vcf_rdfizer_cross_platform_unit.py") - print(f" git commit -m \"Release {tag}\"") - print(f" git tag -a {tag} -m \"VCF-RDFizer {tag}\"") + if conda_sha256 is not None: + print("The conda source-archive checksum is now recorded.") + print("Next steps:") + print(" git add conda-recipe/meta.yaml") + print(f' git commit -m "Record conda checksum for {tag}"') + print(" git push origin main") + print(" # Then update conda-forge/vcf-rdfizer-feedstock with conda-recipe/meta.yaml") + return + + print("Next steps:") + print(" git add pyproject.toml CITATION.cff README.md conda-recipe/meta.yaml conda-recipe/README.md") + print(f' git commit -m "Release {tag}"') print(" git push origin main") + print(f' git tag -a {tag} -m "VCF-RDFizer {tag}"') print(f" git push origin {tag}") - print(" # Before building locally for upload, clean old artifacts:") - print(" rm -rf dist build *.egg-info") - print(" python -m build") - print(" python -m twine check dist/*") + if conda_sha256 is None: + print(f" python3 scripts/release.py {version} --fetch-conda-sha256") + print(" # Then update conda-forge/vcf-rdfizer-feedstock with conda-recipe/meta.yaml") + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description="Prepare or validate VCF-RDFizer release metadata.") + parser.add_argument("version", nargs="?", help="Target semantic version, for example 1.2.4") + parser.add_argument( + "--check-tag", + help="Validate that committed metadata matches vMAJOR.MINOR.PATCH without editing files.", + ) + checksum_group = parser.add_mutually_exclusive_group() + checksum_group.add_argument( + "--conda-sha256", + help="Set the SHA256 for the GitHub tag archive after that tag has been pushed.", + ) + checksum_group.add_argument( + "--fetch-conda-sha256", + action="store_true", + help="Fetch and set the SHA256 for the already-pushed GitHub tag archive.", + ) + args = parser.parse_args(argv) + + try: + if args.check_tag: + if args.version or args.conda_sha256 or args.fetch_conda_sha256: + raise ValueError("--check-tag cannot be combined with a version or conda checksum option") + version = version_from_tag(args.check_tag) + errors = release_metadata_errors(version) + if errors: + raise ValueError("Release metadata validation failed:\n- " + "\n- ".join(errors)) + print(f"Release metadata matches {args.check_tag.strip()}") + return 0 + + if not args.version: + raise ValueError("provide a semantic version or --check-tag vMAJOR.MINOR.PATCH") + version = validate_version(args.version) + conda_sha256 = ( + validate_sha256(args.conda_sha256) if args.conda_sha256 is not None else None + ) + if args.fetch_conda_sha256: + conda_sha256 = sha256_for_url(source_archive_url(version)) + print(f"Fetched SHA256 for v{version}: {conda_sha256}") - return 0 + update_release_metadata(version, conda_sha256) + errors = release_metadata_errors(version) + if errors: + raise ValueError("Updated metadata did not validate:\n- " + "\n- ".join(errors)) + print_next_steps(version, conda_sha256) + return 0 + except (OSError, ValueError, urllib.error.URLError) as exc: + print(f"Error: {exc}", file=sys.stderr) + return 2 if __name__ == "__main__": diff --git a/src/cottas_tool.py b/src/cottas_tool.py new file mode 100644 index 0000000..6dbca5f --- /dev/null +++ b/src/cottas_tool.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python3 +"""Small Docker-side adapter for the pycottas conversion and merge API.""" + +import argparse +import os +import sys +import tempfile +from contextlib import contextmanager +from pathlib import Path + + +@contextmanager +def cottas_scratch_workspace(): + """Run one pycottas operation with an isolated DuckDB working directory.""" + scratch_root = Path(os.environ.get("COTTAS_SCRATCH_DIR", "/work")).resolve() + scratch_root.mkdir(parents=True, exist_ok=True) + original_working_directory = Path.cwd() + # pycottas defaults to ``pycottas.duckdb`` in the current directory. + # A fresh directory prevents one chunk from reusing another chunk's + # database, whose ``quads`` table already exists. + with tempfile.TemporaryDirectory(prefix="vcf-rdfizer-cottas-", dir=scratch_root) as directory: + try: + os.chdir(directory) + yield + finally: + # Leave the directory before TemporaryDirectory removes it. + os.chdir(original_working_directory) + + +def main() -> int: + parser = argparse.ArgumentParser(description="VCF-RDFizer COTTAS adapter") + subparsers = parser.add_subparsers(dest="command", required=True) + + convert = subparsers.add_parser("convert", help="convert one RDF file to COTTAS") + convert.add_argument("rdf_path") + convert.add_argument("cottas_path") + convert.add_argument("index", nargs="?", default="spo") + + merge = subparsers.add_parser("merge", help="merge two COTTAS files") + merge.add_argument("left_path") + merge.add_argument("right_path") + merge.add_argument("cottas_path") + merge.add_argument("index", nargs="?", default="spo") + + args = parser.parse_args() + try: + import pycottas + except ImportError as exc: + print(f"COTTAS dependency is unavailable: {exc}", file=sys.stderr) + return 127 + + if args.command == "convert": + rdf_path = str(Path(args.rdf_path).resolve()) + cottas_path = str(Path(args.cottas_path).resolve()) + # disk=True keeps parser/index construction from requiring the whole + # RDF chunk in Python memory. + with cottas_scratch_workspace(): + pycottas.rdf2cottas( + rdf_path, + cottas_path, + index=args.index, + disk=True, + ) + return 0 + + left_path = str(Path(args.left_path).resolve()) + right_path = str(Path(args.right_path).resolve()) + cottas_path = str(Path(args.cottas_path).resolve()) + with cottas_scratch_workspace(): + pycottas.cat( + [left_path, right_path], + cottas_path, + index=args.index, + remove_input_files=True, + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/ensure_hdt_index.sh b/src/ensure_hdt_index.sh new file mode 100644 index 0000000..93a875a --- /dev/null +++ b/src/ensure_hdt_index.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ $# -ne 1 ]]; then + echo "Usage: ensure_hdt_index.sh " >&2 + exit 2 +fi + +HDT_PATH=$1 +if [[ ! -f "$HDT_PATH" ]]; then + echo "HDT file not found: $HDT_PATH" >&2 + exit 2 +fi + +HDT_JAVA_HOME=${HDT_JAVA_HOME:-/opt/hdt-java} +HDT_SEARCH_BIN=${HDT_SEARCH_BIN:-$HDT_JAVA_HOME/bin/hdtSearch.sh} +if [[ ! -x "$HDT_SEARCH_BIN" ]]; then + echo "Missing HDT Java search launcher: $HDT_SEARCH_BIN" >&2 + exit 127 +fi + +# hdtSearch uses HDT Java's mapIndexedHDT(), which creates the sibling index +# lazily. Sending only `exit` initializes the index without running a query or +# materializing an unbounded result set. +printf 'exit\n' | "$HDT_SEARCH_BIN" "$HDT_PATH" >/dev/null + +shopt -s nullglob +# HDT Java 3.0.10 writes the v1-1 index as ``.hdt.index.v1-1``. +INDEX_CANDIDATES=("${HDT_PATH}".index.*) +INDEX_PATH="" +for candidate in "${INDEX_CANDIDATES[@]}"; do + if [[ -f "$candidate" && -s "$candidate" ]]; then + INDEX_PATH=$candidate + break + fi +done + +if [[ -z "$INDEX_PATH" ]]; then + echo "HDT Java did not create a non-empty index beside: $HDT_PATH" >&2 + exit 1 +fi + +echo "HDT index ready: $INDEX_PATH" diff --git a/src/partitioned_compression.py b/src/partitioned_compression.py new file mode 100644 index 0000000..ff7079a --- /dev/null +++ b/src/partitioned_compression.py @@ -0,0 +1,541 @@ +#!/usr/bin/env python3 +"""Run the partitioned HDT/COTTAS pipeline inside one Docker container. + +The wrapper deliberately keeps this runner independent from ``vcf_rdfizer.py``. +The latter is the host-side CLI, while this file is copied into the Docker +image and owns the ephemeral workspace mounted at ``/work``. Keeping chunk +and merge files here prevents large intermediate artifacts, including COTTAS's +DuckDB scratch database, from consuming the user's output filesystem. +""" + +from __future__ import annotations + +import argparse +import gzip +import json +import os +import shutil +import subprocess +import sys +import time +from pathlib import Path + + +HDT_METHODS = {"hdt", "hdt_gzip", "hdt_brotli"} +COTTAS_METHODS = {"cottas", "cottas_gzip", "cottas_brotli"} +HDT_CAT_CANDIDATES = ( + "hdtCat", + "hdtCat.sh", + "/opt/hdt-java/bin/hdtCat.sh", + "/opt/hdt-java/bin/hdtCat", +) + + +def is_triple_line(line: bytes) -> bool: + """Identify an N-Triples record without parsing or copying its terms.""" + stripped = line.strip() + return bool(stripped) and not stripped.startswith(b"#") and stripped.endswith(b".") + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="VCF-RDFizer partitioned compression runner") + parser.add_argument("--source", required=True, help="plain or gzip-compressed N-Triples input") + parser.add_argument("--output-dir", required=True) + parser.add_argument("--output-name", required=True) + parser.add_argument("--methods", required=True, help="comma-separated internal method names") + parser.add_argument("--target-chunk-bytes", required=True, type=int) + parser.add_argument("--min-chunk-bytes", required=True, type=int) + parser.add_argument("--max-chunk-bytes", required=True, type=int) + parser.add_argument("--expected-triples", type=int) + parser.add_argument("--result-path", required=True) + return parser.parse_args() + + +def iter_rdf_lines(path: Path): + """Read plain or gzip RDF incrementally without materializing the source.""" + opener = gzip.open if path.name.endswith(".gz") else Path.open + with opener(path, "rb") as handle: + yield from handle + + +def plan_chunks( + source: Path, + chunk_dir: Path, + *, + target_bytes: int, + min_bytes: int, + max_bytes: int, +) -> tuple[list[Path], dict]: + """Create chunks on complete N-Triples records in one sequential pass.""" + if target_bytes <= 0 or min_bytes <= 0 or max_bytes <= 0: + raise ValueError("RDF chunk sizes must be positive") + if min_bytes > target_bytes or target_bytes > max_bytes: + raise ValueError("RDF chunk sizes must satisfy min <= target <= max") + + chunk_dir.mkdir(parents=True, exist_ok=True) + chunk_paths: list[Path] = [] + chunk_metadata: list[dict] = [] + handle = None + chunk_path = None + chunk_size = 0 + chunk_start_offset = 0 + chunk_start_record = 0 + logical_offset = 0 + record_count = 0 + chunk_index = 0 + + def close_chunk(): + nonlocal handle, chunk_path, chunk_size + if handle is None or chunk_path is None: + return + handle.close() + chunk_paths.append(chunk_path) + chunk_metadata.append( + { + "chunk_id": len(chunk_metadata), + "path": str(chunk_path), + "start_record": chunk_start_record, + "end_record": record_count, + "start_uncompressed_byte": chunk_start_offset, + "end_uncompressed_byte": logical_offset, + "record_count": record_count - chunk_start_record, + "payload_bytes": chunk_size, + } + ) + handle = None + chunk_path = None + chunk_size = 0 + + try: + for line in iter_rdf_lines(source): + if not line.endswith(b"\n"): + raise ValueError(f"RDF source contains a non-line-terminated record: {source}") + line_size = len(line) + if handle is None: + chunk_path = chunk_dir / f"chunk-{chunk_index:05d}.nt" + chunk_index += 1 + handle = chunk_path.open("wb") + chunk_start_offset = logical_offset + chunk_start_record = record_count + elif chunk_size > 0 and ( + (chunk_size >= target_bytes and chunk_size >= min_bytes) + or chunk_size + line_size > max_bytes + ): + close_chunk() + chunk_path = chunk_dir / f"chunk-{chunk_index:05d}.nt" + chunk_index += 1 + handle = chunk_path.open("wb") + chunk_start_offset = logical_offset + chunk_start_record = record_count + + handle.write(line) + chunk_size += line_size + logical_offset += line_size + if is_triple_line(line): + record_count += 1 + finally: + close_chunk() + + return chunk_paths, { + "source_file_count": 1, + "source_paths": [str(source)], + "chunk_count": len(chunk_paths), + "chunk_input_bytes": logical_offset, + "record_count": record_count, + "target_chunk_bytes": target_bytes, + "min_chunk_bytes": min_bytes, + "max_chunk_bytes": max_bytes, + "chunks": chunk_metadata, + } + + +def resolve_executable(candidates: tuple[str, ...], label: str) -> str: + for candidate in candidates: + resolved = shutil.which(candidate) if "/" not in candidate else candidate + if resolved and Path(resolved).is_file() and os.access(resolved, os.X_OK): + return resolved + raise RuntimeError(f"Missing {label} in container") + + +def find_hdt_index_sidecar(hdt_path: Path) -> Path | None: + """Locate HDT Java's non-empty versioned index sidecar.""" + for candidate in sorted(hdt_path.parent.glob(f"{hdt_path.name}.index.*")): + if candidate.is_file() and candidate.stat().st_size > 0: + return candidate + return None + + +def parse_time_log(path: Path) -> dict: + if not path.exists(): + return {"user_seconds": None, "sys_seconds": None, "max_rss_kb": None} + text = path.read_text(encoding="utf-8", errors="replace") + + def number(pattern: str, integer: bool = False): + import re + + match = re.search(pattern, text, flags=re.MULTILINE) + if not match: + return None + try: + return int(float(match.group(1))) if integer else float(match.group(1)) + except ValueError: + return None + + return { + "user_seconds": number(r"User time \(seconds\):\s*([0-9]+(?:\.[0-9]+)?)"), + "sys_seconds": number(r"System time \(seconds\):\s*([0-9]+(?:\.[0-9]+)?)"), + "max_rss_kb": number(r"Maximum resident set size.*:\s*([0-9]+)", integer=True), + } + + +class StageRunner: + """Execute stages and accumulate both detailed and method-level metrics.""" + + def __init__(self, work_dir: Path): + self.work_dir = work_dir + self.stages: list[dict] = [] + + def run( + self, + name: str, + command: list[str], + output_path: Path | None = None, + stdout_path: Path | None = None, + ) -> dict: + time_path = self.work_dir / f".{name}.time" + if time_path.exists(): + time_path.unlink() + started = time.perf_counter() + time_bin = "/usr/bin/time" if Path("/usr/bin/time").exists() else None + if time_bin: + timed_command = [time_bin, "-v", "-o", str(time_path), *command] + else: + timed_command = command + stdout_handle = None + try: + if stdout_path is not None: + stdout_path.parent.mkdir(parents=True, exist_ok=True) + stdout_handle = stdout_path.open("wb") + completed = subprocess.run(timed_command, stdout=stdout_handle, check=False) + finally: + if stdout_handle is not None: + stdout_handle.close() + result = { + "exit_code": completed.returncode, + "wall_seconds": time.perf_counter() - started, + "output_path": "" if output_path is None else str(output_path), + "output_size_bytes": output_path.stat().st_size + if output_path is not None and output_path.is_file() + else 0, + } + result.update(parse_time_log(time_path)) + self.stages.append({"name": name, **result}) + time_path.unlink(missing_ok=True) + return result + + +def add_totals(total: dict, stage: dict): + total["exit_code"] = max(total["exit_code"], int(stage.get("exit_code") or 0)) + total["wall_seconds"] += float(stage.get("wall_seconds") or 0.0) + if stage.get("user_seconds") is not None: + total["user_seconds"] += float(stage["user_seconds"]) + total["has_user"] = True + if stage.get("sys_seconds") is not None: + total["sys_seconds"] += float(stage["sys_seconds"]) + total["has_sys"] = True + if stage.get("max_rss_kb") is not None: + total["max_rss_kb"] = max(total["max_rss_kb"], int(stage["max_rss_kb"])) + total["has_rss"] = True + + +def finalize_totals(total: dict) -> dict: + return { + "exit_code": total["exit_code"], + "wall_seconds": total["wall_seconds"], + "user_seconds": total["user_seconds"] if total["has_user"] else None, + "sys_seconds": total["sys_seconds"] if total["has_sys"] else None, + "max_rss_kb": total["max_rss_kb"] if total["has_rss"] else None, + } + + +def merge_pairwise( + paths: list[Path], + *, + prefix: str, + runner: StageRunner, + merge_command, + total: dict, +) -> tuple[Path | None, int]: + rounds = 0 + current = list(paths) + while len(current) > 1: + rounds += 1 + next_paths: list[Path] = [] + for pair_index in range(0, len(current), 2): + left = current[pair_index] + if pair_index + 1 >= len(current): + next_paths.append(left) + continue + right = current[pair_index + 1] + merged = left.parent / f"{prefix}-merge-r{rounds:02d}-{pair_index // 2:05d}{left.suffix}" + stage = runner.run( + f"{prefix}-merge-r{rounds:02d}-{pair_index // 2:05d}", + merge_command(left, right, merged), + merged, + ) + add_totals(total, stage) + if stage["exit_code"] != 0: + return None, rounds + left.unlink(missing_ok=True) + right.unlink(missing_ok=True) + next_paths.append(merged) + current = next_paths + return (current[0] if current else None), rounds + + +def main() -> int: + args = parse_args() + source = Path(args.source) + output_dir = Path(args.output_dir) + result_path = Path(args.result_path) + methods = [method.strip() for method in args.methods.split(",") if method.strip()] + work_dir = Path("/work") + chunk_dir = work_dir / "rdf_chunks" + runner = StageRunner(work_dir) + results: dict[str, dict] = {} + + hdt_total = {"exit_code": 0, "wall_seconds": 0.0, "user_seconds": 0.0, "sys_seconds": 0.0, "max_rss_kb": 0, "has_user": False, "has_sys": False, "has_rss": False} + cottas_total = {"exit_code": 0, "wall_seconds": 0.0, "user_seconds": 0.0, "sys_seconds": 0.0, "max_rss_kb": 0, "has_user": False, "has_sys": False, "has_rss": False} + + try: + if not source.is_file(): + raise FileNotFoundError(f"RDF source not found: {source}") + if not methods or not all(method in HDT_METHODS | COTTAS_METHODS for method in methods): + raise ValueError(f"Unsupported partitioned method list: {methods}") + + chunks, plan = plan_chunks( + source, + chunk_dir, + target_bytes=args.target_chunk_bytes, + min_bytes=args.min_chunk_bytes, + max_bytes=args.max_chunk_bytes, + ) + if not chunks: + raise ValueError("RDF source contains no complete records") + source_triples = int(plan["record_count"]) + if args.expected_triples is not None and source_triples != args.expected_triples: + raise ValueError( + "source triple count does not match the upstream conversion count: " + f"source={source_triples}, expected={args.expected_triples}" + ) + hdt_paths: list[Path] = [] + cottas_paths: list[Path] = [] + needs_hdt = any(method in HDT_METHODS for method in methods) + hdt_bin = ( + resolve_executable( + ( + os.environ.get("RDF2HDT_BIN", ""), + "/usr/local/bin/rdf2hdt", + "/opt/hdt-cpp/bin/rdf2hdt", + ), + "rdf2hdt", + ) + if needs_hdt + else None + ) + hdt_cat = resolve_executable(HDT_CAT_CANDIDATES, "HDTCat") if needs_hdt else None + cottas_python = os.environ.get("COTTAS_PYTHON_BIN") or shutil.which("python3") + if any(method in COTTAS_METHODS for method in methods) and not cottas_python: + raise RuntimeError("Missing Python runtime for COTTAS") + + def validate_artifact( + *, + name: str, + artifact: Path, + artifact_format: str, + python_bin: str, + skip_index_check: bool = False, + ) -> dict: + """Decode/count one final artifact and return its validation report.""" + validation_path = work_dir / f".{name}.validation.json" + command = [ + python_bin, + "/opt/vcf-rdfizer/validate_compression.py", + "--source", + str(source), + "--artifact", + str(artifact), + "--format", + artifact_format, + "--source-triples", + str(source_triples), + "--result-path", + str(validation_path), + ] + if args.expected_triples is not None: + command.extend(["--expected-triples", str(args.expected_triples)]) + if skip_index_check: + command.append("--skip-index-check") + stage = runner.run(name, command, validation_path) + try: + report = json.loads(validation_path.read_text(encoding="utf-8")) + except (FileNotFoundError, json.JSONDecodeError) as exc: + report = { + "valid": False, + "count_match": False, + "error": f"validator did not produce a valid report: {exc}", + } + finally: + validation_path.unlink(missing_ok=True) + report.setdefault( + "timing", + { + "wall_seconds": stage.get("wall_seconds"), + "user_seconds": stage.get("user_seconds"), + "sys_seconds": stage.get("sys_seconds"), + "max_rss_kb": stage.get("max_rss_kb"), + }, + ) + if ( + stage["exit_code"] != 0 + or not report.get("valid") + or not report.get("count_match") + ): + raise RuntimeError( + f"{artifact_format.upper()} validation failed for {artifact.name}: " + f"{report.get('error', 'decoded triple count mismatch')}" + ) + return report + + for index, chunk in enumerate(chunks): + if needs_hdt: + chunk_hdt = work_dir / f"chunk-{index:05d}.hdt" + stage = runner.run(f"hdt-build-{index:05d}", [hdt_bin, str(chunk), str(chunk_hdt)], chunk_hdt) + add_totals(hdt_total, stage) + if stage["exit_code"] != 0: + raise RuntimeError("partitioned HDT chunk conversion failed") + hdt_paths.append(chunk_hdt) + if any(method in COTTAS_METHODS for method in methods): + chunk_cottas = work_dir / f"chunk-{index:05d}.cottas" + stage = runner.run( + f"cottas-build-{index:05d}", + [cottas_python, "/opt/vcf-rdfizer/cottas_tool.py", "convert", str(chunk), str(chunk_cottas), "spo"], + chunk_cottas, + ) + add_totals(cottas_total, stage) + if stage["exit_code"] != 0: + raise RuntimeError("partitioned COTTAS chunk conversion failed") + cottas_paths.append(chunk_cottas) + chunk.unlink(missing_ok=True) + + output_hdt = output_dir / f"{args.output_name}.hdt" + if hdt_paths: + final_hdt, hdt_rounds = merge_pairwise( + hdt_paths, + prefix="hdt", + runner=runner, + merge_command=lambda left, right, merged: [hdt_cat, str(left), str(right), str(merged)], + total=hdt_total, + ) + if final_hdt is None: + raise RuntimeError("HDTCat merge failed") + shutil.copyfile(final_hdt, output_hdt) + final_hdt.unlink(missing_ok=True) + index_stage = runner.run( + "hdt-index", + ["/opt/vcf-rdfizer/ensure_hdt_index.sh", str(output_hdt)], + ) + add_totals(hdt_total, index_stage) + output_index = find_hdt_index_sidecar(output_hdt) + if output_index is not None: + # The filename is versioned by HDT Java, so record the actual + # sidecar after the helper completes instead of assuming .index. + index_stage["output_path"] = str(output_index) + index_stage["output_size_bytes"] = output_index.stat().st_size + runner.stages[-1].update(index_stage) + if index_stage["exit_code"] != 0 or output_index is None: + raise RuntimeError("final HDT index initialization failed") + hdt_validation = validate_artifact( + name="hdt-validate", + artifact=output_hdt, + artifact_format="hdt", + python_bin=sys.executable, + skip_index_check=True, + ) + results["hdt"] = { + **finalize_totals(hdt_total), + "output_path": str(output_hdt), + "output_size_bytes": output_hdt.stat().st_size, + "source": "partitioned_generated", + "details": { + **plan, + "merge_rounds": hdt_rounds, + "index_path": str(output_index), + "index_size_bytes": output_index.stat().st_size, + "validation": hdt_validation, + }, + } + + output_cottas = output_dir / f"{args.output_name}.cottas" + if cottas_paths: + final_cottas, cottas_rounds = merge_pairwise( + cottas_paths, + prefix="cottas", + runner=runner, + merge_command=lambda left, right, merged: [cottas_python, "/opt/vcf-rdfizer/cottas_tool.py", "merge", str(left), str(right), str(merged), "spo"], + total=cottas_total, + ) + if final_cottas is None: + raise RuntimeError("COTTAS merge failed") + shutil.copyfile(final_cottas, output_cottas) + final_cottas.unlink(missing_ok=True) + cottas_validation = validate_artifact( + name="cottas-validate", + artifact=output_cottas, + artifact_format="cottas", + python_bin=cottas_python, + ) + results["cottas"] = { + **finalize_totals(cottas_total), + "output_path": str(output_cottas), + "output_size_bytes": output_cottas.stat().st_size, + "source": "partitioned_generated", + "details": { + **plan, + "merge_rounds": cottas_rounds, + "index": "spo", + "validation": cottas_validation, + }, + } + + for method in methods: + if method == "hdt_gzip": + artifact = output_dir / f"{args.output_name}.hdt.gz" + stage = runner.run("hdt-gzip", ["gzip", "-c", str(output_hdt)], artifact, artifact) + elif method == "hdt_brotli": + artifact = output_dir / f"{args.output_name}.hdt.br" + stage = runner.run("hdt-brotli", ["brotli", "-q", "7", "-c", str(output_hdt)], artifact, artifact) + elif method == "cottas_gzip": + artifact = output_dir / f"{args.output_name}.cottas.gz" + stage = runner.run("cottas-gzip", ["gzip", "-c", str(output_cottas)], artifact, artifact) + elif method == "cottas_brotli": + artifact = output_dir / f"{args.output_name}.cottas.br" + stage = runner.run("cottas-brotli", ["brotli", "-q", "7", "-c", str(output_cottas)], artifact, artifact) + else: + continue + if stage["exit_code"] != 0: + raise RuntimeError(f"{method} packaging failed") + results[method] = stage + + result_path.parent.mkdir(parents=True, exist_ok=True) + result_path.write_text(json.dumps({"exit_code": 0, "methods": results, "stages": runner.stages}, indent=2) + "\n", encoding="utf-8") + return 0 + except Exception as exc: + result_path.parent.mkdir(parents=True, exist_ok=True) + result_path.write_text(json.dumps({"exit_code": 1, "methods": results, "stages": runner.stages, "error": str(exc)}, indent=2) + "\n", encoding="utf-8") + print(f"partitioned compression failed: {exc}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/run_conversion.sh b/src/run_conversion.sh index 5f38606..0fdb0ff 100644 --- a/src/run_conversion.sh +++ b/src/run_conversion.sh @@ -7,7 +7,7 @@ set -euo pipefail # Responsibilities: # 1) run RMLStreamer with stable output naming # 2) normalize Spark part outputs to .nt -# 3) optionally aggregate part files into a single .nt +# 3) aggregate part files into a single .nt or .nt.gz # 4) collect conversion timing + output metrics # 5) upsert conversion row in run_metrics/metrics.csv # ------------------------------------------------------------------------------ @@ -19,9 +19,16 @@ IN_VCF=${IN_VCF:-input.vcf} OUT_NAME=${OUT_NAME:-rdf} OUT_DIR=${OUT_DIR:-run_output} OUT="$OUT_DIR/$OUT_NAME" -AGGREGATE_RDF=${AGGREGATE_RDF:-1} +# Full-mode storage policy. `plain` keeps a normal aggregate N-Triples file; +# `space-optimized` appends gzip members and removes each source part after it +# has been compressed. Plain storage is the default for direct script use. +RDF_STORAGE_MODE=${RDF_STORAGE_MODE:-plain} +if [[ "$RDF_STORAGE_MODE" != "plain" && "$RDF_STORAGE_MODE" != "space-optimized" ]]; then + echo "ERROR: invalid RDF_STORAGE_MODE='$RDF_STORAGE_MODE' (expected plain or space-optimized)." >&2 + exit 2 +fi LOGDIR=${LOGDIR:-run_metrics} -mkdir -p "$LOGDIR" "$OUT_DIR" +mkdir -p "$LOGDIR" "$OUT_DIR" "$OUT" RUN_ID=${RUN_ID:-$(date +%Y%m%dT%H%M%S)} TIMESTAMP=${TIMESTAMP:-$(date +"%Y-%m-%dT%H:%M:%S")} @@ -29,6 +36,27 @@ SAFE_OUT_NAME=$(printf "%s" "$OUT_NAME" | tr -cs 'A-Za-z0-9._-' '_') if [[ -z "$SAFE_OUT_NAME" ]]; then SAFE_OUT_NAME="rdf" fi +MERGED_NT="$OUT/$OUT_NAME.nt" +if [[ "$RDF_STORAGE_MODE" == "space-optimized" ]]; then + FINAL_RDF="${MERGED_NT}.gz" +else + FINAL_RDF="$MERGED_NT" +fi +if [[ -e "$FINAL_RDF" ]]; then + echo "ERROR: refusing to overwrite existing RDF output '$FINAL_RDF'." >&2 + exit 2 +fi +# Keep RMLStreamer parts isolated from final artifacts. This lets the wrapper +# preserve unrelated files in an existing sample directory. +PARTS_DIR="$OUT/.rmlstreamer-parts-${SAFE_OUT_NAME}-$$" +if [[ -e "$PARTS_DIR" ]]; then + echo "ERROR: temporary RMLStreamer directory already exists: '$PARTS_DIR'." >&2 + exit 2 +fi +cleanup_parts_dir() { + rm -rf "$PARTS_DIR" +} +trap cleanup_parts_dir EXIT TIME_LOG_DIR="$LOGDIR/conversion_time/${SAFE_OUT_NAME}" METRICS_JSON_DIR="$LOGDIR/conversion_metrics/${SAFE_OUT_NAME}" mkdir -p "$TIME_LOG_DIR" "$METRICS_JSON_DIR" @@ -141,7 +169,11 @@ count_triples_json() { if [[ -f "$path" ]]; then local count - count=$( (grep -E '^[[:space:]]*[^#].*\.[[:space:]]*$' "$path" || true) | wc -l | tr -d ' ' ) + if [[ "$path" == *.gz ]]; then + count=$(gzip -dc "$path" | awk '/^[[:space:]]*[^#].*\.[[:space:]]*$/ { count++ } END { print count + 0 }') + else + count=$( (grep -E '^[[:space:]]*[^#].*\.[[:space:]]*$' "$path" || true) | wc -l | tr -d ' ' ) + fi echo "{" printf " \"%s\": %s,\n" "$path" "$count" printf " \"TOTAL\": %s\n" "$count" @@ -211,14 +243,9 @@ fi # Build Java command in a macOS Bash-3-compatible way: # with `set -u`, expanding an empty array (`"${arr[@]}"`) raises "unbound variable". -JAVA_CMD=(java -jar "$JAR" toFile -m "$IN" -o "$OUT_DIR/$OUT_NAME") +JAVA_CMD=(java -jar "$JAR" toFile -m "$IN" -o "$PARTS_DIR") if (( ${#JAVA_SPARK_OPTS[@]} > 0 )); then - JAVA_CMD=(java "${JAVA_SPARK_OPTS[@]}" -jar "$JAR" toFile -m "$IN" -o "$OUT_DIR/$OUT_NAME") -fi - -# Ensure repeated runs with the same OUT_NAME do not accumulate old artifacts. -if [[ -d "$OUT_DIR/$OUT_NAME" ]]; then - rm -rf "$OUT_DIR/$OUT_NAME" + JAVA_CMD=(java "${JAVA_SPARK_OPTS[@]}" -jar "$JAR" toFile -m "$IN" -o "$PARTS_DIR") fi # ---------- Pre-run ---------- @@ -233,11 +260,10 @@ else { time -p "${JAVA_CMD[@]}"; } >"$TIME_LOG" 2>&1 || EXIT_CODE=$? fi -# ---------- Post-run normalization ---------- -mkdir -p "$OUT_DIR/$OUT_NAME" - -# Normalize output files to .nt for downstream compression/HDT conversion. -for RDF_FILE in "$OUT_DIR/$OUT_NAME"/*; do +# Normalize output files to .nt for downstream line-oriented processing. The +# wrapper does not infer or rewrite RDF syntax beyond preserving the part as a +# line-oriented file; callers must validate whether the stream is NT or NQ. +for RDF_FILE in "$PARTS_DIR"/*; do if [[ ! -f "$RDF_FILE" ]]; then continue fi @@ -247,20 +273,49 @@ for RDF_FILE in "$OUT_DIR/$OUT_NAME"/*; do mv "$RDF_FILE" "${RDF_FILE}.nt" done -# Merge all RMLStreamer output parts into one N-Triples file named after output -# basename when AGGREGATE_RDF=1. Stream merge + delete each part immediately to -# avoid temporary 2x disk spikes. -if [[ "$AGGREGATE_RDF" == "1" ]]; then - MERGED_NT="$OUT_DIR/$OUT_NAME/$OUT_NAME.nt" - shopt -s nullglob - PART_FILES=("$OUT_DIR/$OUT_NAME"/*.nt) - if (( ${#PART_FILES[@]} > 0 )); then +# Merge all RMLStreamer output parts into one aggregate named after the output +# basename. Stream merge + delete each part immediately to avoid temporary 2x +# disk spikes. +shopt -s nullglob +PART_FILES=("$PARTS_DIR"/*.nt) +if [[ "$RDF_STORAGE_MODE" == "space-optimized" ]]; then + MERGED_RDF="${MERGED_NT}.gz" + MERGED_TMP="${MERGED_RDF}.partial.$$" + # Keep duplicate-part protection in the compressed path as well. Hashing + # each part is linear in the bytes already being read and avoids retaining + # an uncompressed duplicate just to compare it later. + SEEN_HASH_FILE="$PARTS_DIR/.seen_part_hashes" + SEEN_MAP_FILE="$PARTS_DIR/.seen_part_hash_map" + : > "$SEEN_HASH_FILE" + : > "$SEEN_MAP_FILE" + # Feed an empty stream on stdin for GNU and BSD gzip compatibility. + gzip -c < /dev/null > "$MERGED_TMP" + for PART_NT in "${PART_FILES[@]}"; do + PART_HASH=$(hash_file_sha256 "$PART_NT") + if grep -Fqx "$PART_HASH" "$SEEN_HASH_FILE"; then + FIRST_SEEN=$(awk -F'\t' -v hash="$PART_HASH" '$1 == hash { print $2; exit }' "$SEEN_MAP_FILE") + echo "WARNING: skipping duplicate RDF part '$PART_NT' (same content as '$FIRST_SEEN')." >&2 + rm -f "$PART_NT" + continue + fi + printf "%s\n" "$PART_HASH" >> "$SEEN_HASH_FILE" + printf "%s\t%s\n" "$PART_HASH" "$PART_NT" >> "$SEEN_MAP_FILE" + annotate_null_literals_nt "$PART_NT" + # Concatenated gzip members form one valid sequential gzip stream while + # allowing each completed RMLStreamer part to be deleted immediately. + gzip -c "$PART_NT" >> "$MERGED_TMP" + rm -f "$PART_NT" + done + rm -f "$SEEN_HASH_FILE" "$SEEN_MAP_FILE" + mv "$MERGED_TMP" "$MERGED_RDF" + OUTPUT_PATH="$MERGED_RDF" + elif (( ${#PART_FILES[@]} > 0 )); then : > "$MERGED_NT" # Defensive dedupe: some Spark/RMLStreamer runs can emit identical part # files for the same dataset. Skip exact duplicate part payloads to avoid # doubling every triple in the merged output. - SEEN_HASH_FILE="$OUT_DIR/$OUT_NAME/.seen_part_hashes.$$" - SEEN_MAP_FILE="$OUT_DIR/$OUT_NAME/.seen_part_hash_map.$$" + SEEN_HASH_FILE="$PARTS_DIR/.seen_part_hashes" + SEEN_MAP_FILE="$PARTS_DIR/.seen_part_hash_map" : > "$SEEN_HASH_FILE" : > "$SEEN_MAP_FILE" for PART_NT in "${PART_FILES[@]}"; do @@ -282,15 +337,18 @@ if [[ "$AGGREGATE_RDF" == "1" ]]; then rm -f "$SEEN_HASH_FILE" "$SEEN_MAP_FILE" else : > "$MERGED_NT" - fi - shopt -u nullglob - OUTPUT_PATH="$MERGED_NT" -else - OUTPUT_PATH="$OUT_DIR/$OUT_NAME" + OUTPUT_PATH="$MERGED_NT" fi + shopt -u nullglob + if [[ "$RDF_STORAGE_MODE" != "space-optimized" ]]; then + OUTPUT_PATH="$MERGED_NT" + fi + +rm -rf "$PARTS_DIR" +trap - EXIT # Apply ontology-compliant null datatype annotation to produced RDF files. -if [[ -f "$OUTPUT_PATH" ]]; then +if [[ -f "$OUTPUT_PATH" && "$OUTPUT_PATH" != *.gz ]]; then annotate_null_literals_nt "$OUTPUT_PATH" elif [[ -d "$OUTPUT_PATH" ]]; then shopt -s nullglob @@ -344,6 +402,11 @@ cat > "$METRICS_JSON" < bool: + """Return whether a serialized line represents an N-Triples statement.""" + stripped = line.strip() + return bool(stripped) and not stripped.startswith(b"#") and stripped.endswith(b".") + + +def count_nt(path: Path) -> int: + """Count N-Triples records while transparently reading ``.nt.gz``.""" + opener = gzip.open if path.name.endswith(".gz") else Path.open + with opener(path, "rb") as handle: + return sum(1 for line in handle if is_triple_line(line)) + + +def count_decoded(command: list[str]) -> int: + """Run an RDF exporter and count its streamed N-Triples output.""" + process = subprocess.Popen( + command, + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + ) + assert process.stdout is not None + count = sum(1 for line in process.stdout if is_triple_line(line)) + return_code = process.wait() + if return_code != 0: + raise RuntimeError(f"decoder exited with status {return_code}: {' '.join(command)}") + return count + + +def resolve_hdt2rdf() -> str: + candidates = ( + os.environ.get("HDT2RDF_BIN", ""), + "/usr/local/bin/hdt2rdf", + "/opt/hdt-cpp/bin/hdt2rdf", + ) + for candidate in candidates: + if candidate and Path(candidate).is_file() and os.access(candidate, os.X_OK): + return candidate + raise RuntimeError("Missing hdt2rdf binary in container") + + +def validate(args: argparse.Namespace) -> dict: + source = Path(args.source) + artifact = Path(args.artifact) + if not source.is_file(): + raise FileNotFoundError(f"source RDF file not found: {source}") + if not artifact.is_file() or artifact.stat().st_size == 0: + raise FileNotFoundError(f"compression artifact is missing or empty: {artifact}") + + source_triples = ( + args.source_triples if args.source_triples is not None else count_nt(source) + ) + if args.expected_triples is not None and source_triples != args.expected_triples: + return { + "valid": False, + "source_triples": source_triples, + "decoded_triples": None, + "expected_triples": args.expected_triples, + "count_match": False, + "error": ( + "source triple count does not match the upstream conversion count: " + f"source={source_triples}, expected={args.expected_triples}" + ), + } + + if args.format == "hdt": + # Loading through the bundled Java launcher also verifies that the HDT + # structure is readable and eagerly creates the query index. + if not args.skip_index_check: + index_helper = Path("/opt/vcf-rdfizer/ensure_hdt_index.sh") + if not index_helper.is_file(): + raise RuntimeError(f"Missing HDT index helper: {index_helper}") + index_check = subprocess.run( + [str(index_helper), str(artifact)], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + check=False, + ) + if index_check.returncode != 0: + raise RuntimeError( + "HDT index/readability check failed with status " + f"{index_check.returncode}" + ) + # hdt2rdf uses "-" (not /dev/stdout) as its stdout sentinel. This + # keeps the decoded RDF in the pipe and avoids another large file. + decoded_triples = count_decoded([resolve_hdt2rdf(), str(artifact), "-"]) + validator = "hdt2rdf" + else: + try: + import pycottas + except ImportError as exc: + raise RuntimeError(f"COTTAS dependency is unavailable: {exc}") from exc + + # pycottas exposes cottas2rdf; its output path can be /dev/stdout, so + # the COTTAS reader is exercised without materializing decoded RDF. + decoded_triples = count_decoded( + [ + sys.executable, + "-c", + ( + "import pycottas, sys; " + "pycottas.cottas2rdf(sys.argv[1], '/dev/stdout')" + ), + str(artifact), + ] + ) + validator = "pycottas.cottas2rdf" + + return { + "valid": True, + "source_triples": source_triples, + "decoded_triples": decoded_triples, + "expected_triples": args.expected_triples, + "count_match": source_triples == decoded_triples, + "validator": validator, + } | ({ + "error": ( + "decoded triple count does not match the source: " + f"source={source_triples}, decoded={decoded_triples}" + ) + } if source_triples != decoded_triples else {}) + + +def main() -> int: + parser = argparse.ArgumentParser(description="Validate an HDT or COTTAS artifact") + parser.add_argument("--source", required=True, help="plain or gzip-compressed N-Triples") + parser.add_argument("--artifact", required=True, help="HDT or COTTAS artifact") + parser.add_argument("--format", required=True, choices=("hdt", "cottas")) + parser.add_argument("--expected-triples", type=int) + parser.add_argument( + "--source-triples", + type=int, + help="source count already collected by an upstream streaming pass", + ) + parser.add_argument( + "--skip-index-check", + action="store_true", + help="skip HDT index initialization when the caller already performed it", + ) + parser.add_argument("--result-path", required=True) + args = parser.parse_args() + + result_path = Path(args.result_path) + try: + result = validate(args) + exit_code = 0 if result.get("valid") and result.get("count_match") else 1 + except Exception as exc: + result = { + "valid": False, + "source_triples": None, + "decoded_triples": None, + "expected_triples": args.expected_triples, + "count_match": False, + "error": str(exc), + } + exit_code = 1 + + result_path.parent.mkdir(parents=True, exist_ok=True) + result_path.write_text(json.dumps(result, indent=2) + "\n", encoding="utf-8") + if exit_code: + print(result.get("error", "compression validation failed"), file=sys.stderr) + return exit_code + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/test/README.md b/test/README.md index 773e06c..108d4e2 100644 --- a/test/README.md +++ b/test/README.md @@ -7,7 +7,7 @@ This repository uses `unittest` (Python standard library) to isolate orchestrati - `test/test_vcf_rdfizer_unit.py` - Verifies wrapper control flow for the 5-step pipeline. - Verifies image/version resolution behavior and error handling. - - Verifies CLI compression option propagation into `compression.sh`. + - Verifies the separate RDF-compression, representation, and artifact-packaging plan. - `test/test_vcf_rdfizer_cross_platform_unit.py` - Runs a Windows/macOS/Linux-safe subset of wrapper tests. @@ -25,9 +25,9 @@ This repository uses `unittest` (Python standard library) to isolate orchestrati - Verifies unified metrics CSV row creation and schema consistency. - `test/test_compression_unit.py` - - Replaces `gzip`, `brotli`, and `rdf2hdt` with fake executables. - - Verifies compression artifact generation and metrics row update. - - Verifies `-m none` behavior (no compression outputs, metrics still updated). + - Replaces `gzip`, `brotli`, and `rdf2hdt` with fake executables for the standalone helper. + - Verifies helper compression artifact generation and metrics row update. + - Verifies no-op behavior (no compression outputs, metrics still updated). ## CI matrix behavior diff --git a/test/test_cottas_tool.py b/test/test_cottas_tool.py new file mode 100644 index 0000000..8da86f6 --- /dev/null +++ b/test/test_cottas_tool.py @@ -0,0 +1,104 @@ +import importlib.util +import os +import sys +import tempfile +import types +import unittest +from pathlib import Path +from unittest import mock + + +COTTAS_TOOL_PATH = Path(__file__).resolve().parents[1] / "src" / "cottas_tool.py" + + +def load_cottas_tool(): + """Load the Docker-side adapter without requiring pycottas locally.""" + spec = importlib.util.spec_from_file_location("cottas_tool_for_test", COTTAS_TOOL_PATH) + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + spec.loader.exec_module(module) + return module + + +class CottasToolTests(unittest.TestCase): + def test_convert_uses_a_fresh_duckdb_workspace_for_each_invocation(self): + """Sequential chunk builds cannot reuse pycottas.duckdb or its quads table.""" + module = load_cottas_tool() + observed_workspaces = [] + + def fake_rdf2cottas(rdf_path, cottas_path, *, index, disk): + self.assertTrue(Path(rdf_path).is_absolute()) + self.assertTrue(Path(cottas_path).is_absolute()) + self.assertEqual(index, "spo") + self.assertTrue(disk) + database_path = Path.cwd() / "pycottas.duckdb" + if database_path.exists(): + raise RuntimeError("Table with name quads already exists") + database_path.write_text("temporary DuckDB state\n") + Path(cottas_path).write_text("COTTAS output\n") + observed_workspaces.append(Path.cwd()) + + with tempfile.TemporaryDirectory() as temporary_directory: + root = Path(temporary_directory) + scratch_root = (root / "scratch").resolve() + source = root / "input.nt" + source.write_text("

.\n") + original_working_directory = Path.cwd() + + with mock.patch.dict(sys.modules, {"pycottas": types.SimpleNamespace(rdf2cottas=fake_rdf2cottas)}), mock.patch.dict( + os.environ, {"COTTAS_SCRATCH_DIR": str(scratch_root)}, clear=False + ): + for chunk_number in range(2): + output = root / f"chunk-{chunk_number}.cottas" + with mock.patch.object( + sys, + "argv", + ["cottas_tool.py", "convert", str(source), str(output)], + ): + self.assertEqual(module.main(), 0) + self.assertTrue(output.is_file()) + + self.assertEqual(Path.cwd(), original_working_directory) + self.assertEqual(len(observed_workspaces), 2) + self.assertNotEqual(observed_workspaces[0], observed_workspaces[1]) + self.assertTrue(all(path.parent == scratch_root for path in observed_workspaces)) + self.assertFalse(any(scratch_root.iterdir())) + + def test_merge_uses_the_same_isolated_scratch_policy(self): + """Pairwise merges do not inherit a DuckDB database from chunk conversion.""" + module = load_cottas_tool() + observed_workspaces = [] + + def fake_cat(paths, cottas_path, *, index, remove_input_files): + self.assertTrue(all(Path(path).is_absolute() for path in paths)) + self.assertTrue(Path(cottas_path).is_absolute()) + self.assertEqual(index, "spo") + self.assertTrue(remove_input_files) + database_path = Path.cwd() / "pycottas.duckdb" + if database_path.exists(): + raise RuntimeError("Table with name quads already exists") + database_path.write_text("temporary DuckDB state\n") + Path(cottas_path).write_text("merged COTTAS output\n") + observed_workspaces.append(Path.cwd()) + + with tempfile.TemporaryDirectory() as temporary_directory: + root = Path(temporary_directory) + scratch_root = (root / "scratch").resolve() + left = root / "left.cottas" + right = root / "right.cottas" + output = root / "merged.cottas" + left.write_text("left\n") + right.write_text("right\n") + + with mock.patch.dict(sys.modules, {"pycottas": types.SimpleNamespace(cat=fake_cat)}), mock.patch.dict( + os.environ, {"COTTAS_SCRATCH_DIR": str(scratch_root)}, clear=False + ), mock.patch.object( + sys, + "argv", + ["cottas_tool.py", "merge", str(left), str(right), str(output)], + ): + self.assertEqual(module.main(), 0) + + self.assertTrue(output.is_file()) + self.assertEqual(len(observed_workspaces), 1) + self.assertFalse(any(scratch_root.iterdir())) diff --git a/test/test_release_unit.py b/test/test_release_unit.py new file mode 100644 index 0000000..dd38298 --- /dev/null +++ b/test/test_release_unit.py @@ -0,0 +1,68 @@ +"""Unit tests for the release metadata helper.""" + +from __future__ import annotations + +import importlib.util +import re +import tempfile +import unittest +from pathlib import Path +from unittest import mock + + +REPO_ROOT = Path(__file__).resolve().parents[1] +RELEASE_SCRIPT = REPO_ROOT / "scripts" / "release.py" + + +def load_release_module(): + spec = importlib.util.spec_from_file_location("release_script", RELEASE_SCRIPT) + if spec is None or spec.loader is None: + raise RuntimeError("could not load release script") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +class ReleaseScriptTests(unittest.TestCase): + def setUp(self): + self.release = load_release_module() + + def test_current_metadata_matches_current_tag(self): + pyproject = (REPO_ROOT / "pyproject.toml").read_text(encoding="utf-8") + version_match = re.search(r'^version = "(\d+\.\d+\.\d+)"$', pyproject, re.MULTILINE) + self.assertIsNotNone(version_match) + version = version_match.group(1) + self.assertEqual(self.release.release_metadata_errors(version), []) + self.assertEqual(self.release.main(["--check-tag", f"v{version}"]), 0) + + def test_version_and_tag_validation_reject_invalid_values(self): + with self.assertRaises(ValueError): + self.release.validate_version("1.2") + with self.assertRaises(ValueError): + self.release.version_from_tag("1.2.3") + with self.assertRaises(ValueError): + self.release.validate_sha256("not-a-sha") + + def test_update_resets_stale_conda_checksum_until_tag_checksum_is_known(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / "conda-recipe").mkdir() + (root / "pyproject.toml").write_text('version = "1.2.3"\n', encoding="utf-8") + (root / "CITATION.cff").write_text('version: "1.2.3"\n', encoding="utf-8") + (root / "README.md").write_text( + "Version 1.2.3\nversion = {1.2.3}\n", encoding="utf-8" + ) + (root / "conda-recipe" / "README.md").write_text( + "v1.2.3\nv1.2.3\n", encoding="utf-8" + ) + (root / "conda-recipe" / "meta.yaml").write_text( + '{% set version = "1.2.3" %}\nsource:\n sha256: old-checksum\n', + encoding="utf-8", + ) + + with mock.patch.object(self.release, "ROOT", root): + self.release.update_release_metadata("1.2.4", None) + meta_text = (root / "conda-recipe" / "meta.yaml").read_text(encoding="utf-8") + self.assertIn('{% set version = "1.2.4" %}', meta_text) + self.assertIn(self.release.PLACEHOLDER_CONDA_SHA256, meta_text) + self.assertEqual(self.release.release_metadata_errors("1.2.4"), []) diff --git a/test/test_run_conversion_unit.py b/test/test_run_conversion_unit.py index be26f24..79b1ab3 100644 --- a/test/test_run_conversion_unit.py +++ b/test/test_run_conversion_unit.py @@ -1,5 +1,6 @@ import csv import gzip +import json import subprocess import tempfile import unittest @@ -82,6 +83,62 @@ def test_run_conversion_writes_nt_and_metrics_without_real_java(self): self.assertEqual(row["output_name"], "rdf") self.assertEqual(row["exit_code_java"], "0") + def test_run_conversion_preserves_unrelated_files_in_existing_output_directory(self): + """RMLStreamer parts do not require deleting the sample output directory.""" + with tempfile.TemporaryDirectory() as td: + tmp_path = Path(td) + fake_bin = tmp_path / "bin" + fake_bin.mkdir() + make_executable( + fake_bin / "java", + """#!/usr/bin/env bash +set -euo pipefail +if [[ "${1:-}" == "-version" ]]; then + echo 'openjdk version "11.0.0"' >&2 + exit 0 +fi +out="" +while [[ $# -gt 0 ]]; do + if [[ "$1" == "-o" ]]; then + out="$2" + shift 2 + continue + fi + shift +done +mkdir -p "$out" +printf '

.\\n' > "$out/part-000" +""", + ) + out_dir = tmp_path / "out" + sample_dir = out_dir / "rdf" + sample_dir.mkdir(parents=True) + note = sample_dir / "notes.txt" + note.write_text("preserve me\n", encoding="utf-8") + rules = tmp_path / "rules.ttl" + rules.write_text("@prefix ex: .\n", encoding="utf-8") + vcf = tmp_path / "input.vcf" + vcf.write_text("##fileformat=VCFv4.2\n#CHROM\tPOS\n1\t5\n", encoding="utf-8") + + env = env_with_path(fake_bin) + env.update( + { + "JAR": "fake.jar", + "IN": str(rules), + "IN_VCF": str(vcf), + "OUT_DIR": str(out_dir), + "OUT_NAME": "rdf", + "LOGDIR": str(tmp_path / "metrics"), + "RUN_ID": "run-preserve", + } + ) + result = subprocess.run(["bash", str(SCRIPT)], env=env, capture_output=True, text=True) + + self.assertEqual(result.returncode, 0, msg=result.stderr) + self.assertEqual(note.read_text(encoding="utf-8"), "preserve me\n") + self.assertTrue((sample_dir / "rdf.nt").is_file()) + self.assertEqual(list(sample_dir.glob(".rmlstreamer-parts-*")), []) + def test_run_conversion_records_normalized_size_for_gz_input_vcf(self): """input_vcf_size_bytes uses decompressed size for .vcf.gz inputs.""" with tempfile.TemporaryDirectory() as td: @@ -356,7 +413,7 @@ def test_run_conversion_applies_spark_partition_hint_when_set(self): self.assertIn("-Dspark.default.parallelism=3", java_args) self.assertIn("-Dspark.sql.shuffle.partitions=3", java_args) - def test_run_conversion_normalizes_legacy_and_writes_merged_nt(self): + def test_run_conversion_normalizes_parts_and_writes_merged_nt(self): """Normalization + merge writes rdf.nt with combined converted content.""" with tempfile.TemporaryDirectory() as td: tmp_path = Path(td) @@ -414,8 +471,8 @@ def test_run_conversion_normalizes_legacy_and_writes_merged_nt(self): self.assertIn("

.", text) self.assertIn(" .", text) - def test_run_conversion_batch_mode_keeps_individual_nt_parts(self): - """AGGREGATE_RDF=0 preserves individual normalized .nt part files.""" + def test_run_conversion_aggregate_skips_duplicate_part_payloads(self): + """Aggregate merge skips exact duplicate part files to avoid duplicated triples.""" with tempfile.TemporaryDirectory() as td: tmp_path = Path(td) fake_bin = tmp_path / "bin" @@ -438,8 +495,9 @@ def test_run_conversion_batch_mode_keeps_individual_nt_parts(self): shift done mkdir -p "$out" -printf '

.\\n' > "$out/part-00000" -printf '

.\\n' > "$out/part-00001" +printf '

.\\n' > "$out/part-00000" +printf '

.\\n' > "$out/part-00001" +printf '

.\\n' > "$out/part-00002" """, ) @@ -459,20 +517,23 @@ def test_run_conversion_batch_mode_keeps_individual_nt_parts(self): "OUT_DIR": str(out_dir), "OUT_NAME": "rdf", "LOGDIR": str(metrics_dir), - "RUN_ID": "run-batch", + "RUN_ID": "run-dedupe", "TIMESTAMP": "2026-01-01T00:00:00", - "AGGREGATE_RDF": "0", } ) result = subprocess.run(["bash", str(SCRIPT)], env=env, capture_output=True, text=True) self.assertEqual(result.returncode, 0, msg=result.stderr) - self.assertTrue((out_dir / "rdf" / "part-00000.nt").exists()) - self.assertTrue((out_dir / "rdf" / "part-00001.nt").exists()) - self.assertFalse((out_dir / "rdf" / "rdf.nt").exists()) + self.assertIn("skipping duplicate RDF part", result.stderr) - def test_run_conversion_aggregate_skips_duplicate_part_payloads(self): - """Aggregate merge skips exact duplicate part files to avoid duplicated triples.""" + merged_nt = out_dir / "rdf" / "rdf.nt" + self.assertTrue(merged_nt.exists()) + lines = [line.strip() for line in merged_nt.read_text().splitlines() if line.strip()] + self.assertEqual(lines.count("

."), 1) + self.assertEqual(lines.count("

."), 1) + + def test_run_conversion_space_optimized_streams_gzip_members_and_deletes_parts(self): + """Space-optimized aggregation keeps one readable gzip stream and no source parts.""" with tempfile.TemporaryDirectory() as td: tmp_path = Path(td) fake_bin = tmp_path / "bin" @@ -495,9 +556,8 @@ def test_run_conversion_aggregate_skips_duplicate_part_payloads(self): shift done mkdir -p "$out" -printf '

.\\n' > "$out/part-00000" -printf '

.\\n' > "$out/part-00001" -printf '

.\\n' > "$out/part-00002" +printf '

.\\n' > "$out/part-00000" +printf '

.\\n' > "$out/part-00001" """, ) @@ -517,20 +577,25 @@ def test_run_conversion_aggregate_skips_duplicate_part_payloads(self): "OUT_DIR": str(out_dir), "OUT_NAME": "rdf", "LOGDIR": str(metrics_dir), - "RUN_ID": "run-dedupe", + "RUN_ID": "run-space", "TIMESTAMP": "2026-01-01T00:00:00", + "RDF_STORAGE_MODE": "space-optimized", } ) result = subprocess.run(["bash", str(SCRIPT)], env=env, capture_output=True, text=True) self.assertEqual(result.returncode, 0, msg=result.stderr) - self.assertIn("skipping duplicate RDF part", result.stderr) - - merged_nt = out_dir / "rdf" / "rdf.nt" - self.assertTrue(merged_nt.exists()) - lines = [line.strip() for line in merged_nt.read_text().splitlines() if line.strip()] - self.assertEqual(lines.count("

."), 1) - self.assertEqual(lines.count("

."), 1) + aggregate = out_dir / "rdf" / "rdf.nt.gz" + self.assertTrue(aggregate.exists()) + with gzip.open(aggregate, "rt", encoding="utf-8") as handle: + aggregate_text = handle.read() + self.assertIn("

.", aggregate_text) + self.assertIn("

.", aggregate_text) + self.assertEqual(list((out_dir / "rdf").glob("part-*.nt")), []) + metrics = metrics_dir / "conversion_metrics" / "rdf" / "run-space.json" + payload = json.loads(metrics.read_text()) + self.assertEqual(payload["rdf_storage"]["mode"], "space-optimized") + self.assertTrue(payload["rdf_storage"]["compressed"]) def test_run_conversion_rewrites_dot_literal_to_typed_null(self): """Plain literal dot is normalized to ontology null typed literal in RDF output.""" diff --git a/test/test_vcf_files/test-larger.vcf.gz b/test/test_vcf_files/test-larger.vcf.gz new file mode 100644 index 0000000..be43cab Binary files /dev/null and b/test/test_vcf_files/test-larger.vcf.gz differ diff --git a/test/test_vcf_rdfizer_cross_platform_unit.py b/test/test_vcf_rdfizer_cross_platform_unit.py index 1fece0f..7985bf6 100644 --- a/test/test_vcf_rdfizer_cross_platform_unit.py +++ b/test/test_vcf_rdfizer_cross_platform_unit.py @@ -1,4 +1,6 @@ +import json import os +import re import sys import tempfile import unittest @@ -33,6 +35,18 @@ def invoke_main(argv): return vcf_rdfizer.main() +def host_path_for_bind_mount(command, container_path: str) -> Path: + """Return a Docker bind-mount source without splitting Windows drive letters.""" + suffix = f":{container_path}" + for index, token in enumerate(command[:-1]): + if token != "-v": + continue + mount = str(command[index + 1]) + if mount.endswith(suffix): + return Path(mount[: -len(suffix)]) + raise AssertionError(f"missing bind mount for {container_path}: {command}") + + class WrapperCrossPlatformUnitTests(VerboseTestCase): def test_help_flag_prints_usage(self): """CLI help succeeds and prints usage examples.""" @@ -45,8 +59,8 @@ def test_help_flag_prints_usage(self): def test_resolve_image_ref_with_version(self): """Image repository + explicit version resolves to a tagged image.""" - image, requested = vcf_rdfizer.resolve_image_ref("ecrum19/vcf-rdfizer", "1.2.3") - self.assertEqual(image, "ecrum19/vcf-rdfizer:1.2.3") + image, requested = vcf_rdfizer.resolve_image_ref("ecrum19/vcf-rdfizer", "9.9.9") + self.assertEqual(image, "ecrum19/vcf-rdfizer:9.9.9") self.assertTrue(requested) def test_success_symbol_falls_back_for_cp1252_console(self): @@ -64,12 +78,47 @@ def test_parse_compression_methods_rejects_unknown(self): with self.assertRaises(ValueError): vcf_rdfizer.parse_compression_methods("gzip,unknown") + def test_build_compression_methods_separates_representation_packaging(self): + """Public plan options expand into shared base and packaging stages.""" + self.assertEqual( + vcf_rdfizer.build_compression_methods( + rdf_compression="none", + representations="hdt,cottas", + artifact_compression="gzip,brotli", + ), + [ + "hdt", + "hdt_gzip", + "hdt_brotli", + "cottas", + "cottas_gzip", + "cottas_brotli", + ], + ) + + def test_build_compression_methods_rejects_packaging_without_representation(self): + """Packaging cannot be requested without a base representation.""" + with self.assertRaises(ValueError): + vcf_rdfizer.build_compression_methods( + rdf_compression="none", + representations="none", + artifact_compression="gzip", + ) + def test_detect_compressed_format(self): """Compressed format detection works by extension.""" self.assertEqual(vcf_rdfizer.detect_compressed_format(Path("sample.nt.gz")), "gzip") self.assertEqual(vcf_rdfizer.detect_compressed_format(Path("sample.nt.br")), "brotli") self.assertEqual(vcf_rdfizer.detect_compressed_format(Path("sample.hdt")), "hdt") + def test_bind_mount_parser_preserves_windows_drive_letter(self): + """Validation mocks preserve Windows bind-mount source paths.""" + mount = r"C:\Users\runneradmin\AppData\Local\Temp\output:/data/out" + self.assertEqual( + str(host_path_for_bind_mount(["docker", "run", "-v", mount], "/data/out")), + r"C:\Users\runneradmin\AppData\Local\Temp\output", + ) + def test_compress_mode_runs_with_mocks(self): """Compress mode succeeds with mocked Docker execution across OSes.""" with tempfile.TemporaryDirectory() as td: @@ -82,6 +131,27 @@ def test_compress_mode_runs_with_mocks(self): def fake_run(cmd, cwd=None, env=None): commands.append(cmd) + rendered = str(cmd[-1]) if cmd else "" + if "validate_compression.py" in rendered: + out_mount = host_path_for_bind_mount(cmd, "/data/out") + result_match = re.search( + r"--result-path\s+['\"]?(/data/out/[^\s'\";]+)", + rendered, + ) + result_path = out_mount / result_match.group(1).replace( + "/data/out/", "", 1 + ) + result_path.parent.mkdir(parents=True, exist_ok=True) + result_path.write_text( + json.dumps( + { + "valid": True, + "source_triples": 1, + "decoded_triples": 1, + "count_match": True, + } + ) + ) return 0 old_cwd = os.getcwd() @@ -98,7 +168,7 @@ def fake_run(cmd, cwd=None, env=None): "compress", "--rdf", str(nt_path), - "--compression", + "--rdf-compression", "gzip", "--out", str(out_dir), diff --git a/test/test_vcf_rdfizer_unit.py b/test/test_vcf_rdfizer_unit.py index dca41b5..6e5732e 100644 --- a/test/test_vcf_rdfizer_unit.py +++ b/test/test_vcf_rdfizer_unit.py @@ -1,8 +1,12 @@ +import argparse import csv +import gzip +import importlib.util import json import os import re import shutil +import subprocess import sys import tempfile import unittest @@ -15,7 +19,94 @@ from test.helpers import VerboseTestCase -def invoke_main(argv, *, auto_layout=True): +def emulate_partitioned_runner(cmd): + """Create deterministic final artifacts for the one-shot Docker runner.""" + if vcf_rdfizer.PARTITIONED_COMPRESSION_RUNNER_CONTAINER not in cmd: + return False + out_mount = next( + ( + Path(part.split(":", 1)[0]) + for part in cmd + if isinstance(part, str) and part.endswith(":/data/out") + ), + None, + ) + if out_mount is None: + return False + output_name = cmd[cmd.index("--output-name") + 1] + methods = cmd[cmd.index("--methods") + 1].split(",") + method_results = {} + for method in methods: + artifact_name = vcf_rdfizer.compression_artifact_name_for_method( + Path(f"{output_name}.nt"), method + ) + artifact_path = out_mount / artifact_name + artifact_path.write_text(f"mock-{method}\n") + method_results[method] = { + "exit_code": 0, + "wall_seconds": 0.2, + "user_seconds": 0.1, + "sys_seconds": 0.05, + "max_rss_kb": 4096, + "output_path": f"/data/out/{artifact_name}", + "output_size_bytes": artifact_path.stat().st_size, + "source": "partitioned_generated", + "details": { + "chunk_count": 2, + "merge_rounds": 1, + "validation": { + "valid": True, + "source_triples": 12, + "decoded_triples": 12, + "expected_triples": 12, + "count_match": True, + }, + }, + } + if "hdt" in methods: + (out_mount / f"{output_name}.hdt.index.v1-1").write_text("mock-index\n") + result_container_path = cmd[cmd.index("--result-path") + 1] + result_path = out_mount / result_container_path.replace("/data/out/", "", 1) + result_path.write_text( + json.dumps({"exit_code": 0, "methods": method_results, "stages": []}) + ) + return True + + +def emulate_validation_command(cmd): + """Make mocked Docker validation return a successful one-triple report.""" + rendered = str(cmd[-1]) if cmd else "" + if "validate_compression.py" not in rendered: + return False + out_mount = next( + ( + Path(part.split(":", 1)[0]) + for part in cmd + if isinstance(part, str) and part.endswith(":/data/out") + ), + None, + ) + result_match = re.search(r"--result-path\s+['\"]?(/data/out/[^\s'\";]+)", rendered) + if out_mount is None or result_match is None: + return False + result_path = out_mount / result_match.group(1).replace("/data/out/", "", 1) + result_path.parent.mkdir(parents=True, exist_ok=True) + result_path.write_text( + json.dumps( + { + "valid": True, + "source_triples": 1, + "decoded_triples": 1, + "expected_triples": 1, + "count_match": True, + "validator": "mock", + } + ) + ) + return True + + +def invoke_main(argv, *, auto_storage=True): args = list(argv) normalized = [] skip_next = False @@ -33,18 +124,88 @@ def invoke_main(argv, *, auto_layout=True): normalized.append(token) args = normalized - if auto_layout: + if auto_storage: mode = "full" for index, token in enumerate(args): if token in {"--mode", "-m"} and index + 1 < len(args): mode = args[index + 1] break - if mode == "full" and "--rdf-layout" not in args and "-l" not in args: - args.extend(["--rdf-layout", "aggregate"]) + if ( + mode == "full" + and "--rdf-storage-mode" not in args + ): + args.extend(["--rdf-storage-mode", "plain"]) if "--out" not in args and "-o" not in args: args.extend(["--out", "./out"]) - with mock.patch.object(sys, "argv", ["vcf_rdfizer.py", *args]): + original_run = vcf_rdfizer.run + + def run_with_default_aggregate(cmd, cwd=None, env=None): + """Make successful mocked RMLStreamer calls produce the new aggregate artifact.""" + if isinstance(cmd, list) and "volume" in cmd and "docker" in cmd: + return 0 + if emulate_validation_command(cmd): + return 0 + result = original_run(cmd, cwd=cwd, env=env) + if result == 0 and emulate_partitioned_runner(cmd): + return result + if result == 0 and "/opt/vcf-rdfizer/run_conversion.sh" in cmd: + out_mount = next( + ( + Path(part.split(":", 1)[0]) + for part in cmd + if isinstance(part, str) and part.endswith(":/data/out") + ), + None, + ) + output_name = next( + ( + part.split("=", 1)[1] + for part in cmd + if isinstance(part, str) and part.startswith("OUT_NAME=") + ), + "rdf", + ) + if out_mount is not None: + output_dir = out_mount / output_name + output_dir.mkdir(parents=True, exist_ok=True) + if not any(output_dir.glob("*.nt")) and not any(output_dir.glob("*.nt.gz")): + (output_dir / f"{output_name}.nt").write_text("

.\n") + if result == 0: + rendered = str(cmd[-1]) if cmd else "" + work_mount = next( + ( + Path(part.split(":", 1)[0]) + for part in cmd + if isinstance(part, str) and part.endswith(":/data/work") + ), + None, + ) + if work_mount and ('"$HDT_BIN"' in rendered or '"$HDTCAT_BIN"' in rendered): + hdt_paths = re.findall(r"(/data/work/[^\s;]+\.hdt)", rendered) + if hdt_paths: + hdt_output = work_mount / hdt_paths[-1].replace("/data/work/", "", 1) + hdt_output.parent.mkdir(parents=True, exist_ok=True) + hdt_output.write_text("mock-hdt\n") + out_mount = next( + ( + Path(part.split(":", 1)[0]) + for part in cmd + if isinstance(part, str) and part.endswith(":/data/out") + ), + None, + ) + if out_mount and "ensure_hdt_index.sh" in rendered: + index_paths = re.findall(r"(/data/out/[^\s;]+\.hdt)", rendered) + if index_paths: + hdt_output = out_mount / index_paths[-1].replace("/data/out/", "", 1) + hdt_output.parent.mkdir(parents=True, exist_ok=True) + Path(str(hdt_output) + ".index").write_text("mock-index\n") + return result + + with mock.patch.object(sys, "argv", ["vcf_rdfizer.py", *args]), mock.patch.object( + vcf_rdfizer, "run", side_effect=run_with_default_aggregate + ): return vcf_rdfizer.main() @@ -96,6 +257,156 @@ def latest_metrics_run_dir(metrics_root: Path) -> Path: class WrapperUnitTests(VerboseTestCase): + def test_validator_counts_plain_and_gzip_ntriples(self): + """The Docker validator's fallback source count handles .nt and .nt.gz.""" + validator_path = Path(__file__).parents[1] / "src" / "validate_compression.py" + spec = importlib.util.spec_from_file_location("validate_compression", validator_path) + validator = importlib.util.module_from_spec(spec) + assert spec.loader is not None + spec.loader.exec_module(validator) + with tempfile.TemporaryDirectory() as td: + tmp_path = Path(td) + payload = b"# comment\n

.\n\n

.\n" + plain = tmp_path / "sample.nt" + plain.write_bytes(payload) + compressed = tmp_path / "sample.nt.gz" + with gzip.open(compressed, "wb") as handle: + handle.write(payload) + self.assertEqual(validator.count_nt(plain), 2) + self.assertEqual(validator.count_nt(compressed), 2) + + def test_validator_streams_hdt_decode_to_stdout(self): + """HDT validation uses hdt2rdf's '-' stdout sentinel, not a file path.""" + validator_path = Path(__file__).parents[1] / "src" / "validate_compression.py" + spec = importlib.util.spec_from_file_location("validate_compression", validator_path) + validator = importlib.util.module_from_spec(spec) + assert spec.loader is not None + spec.loader.exec_module(validator) + with tempfile.TemporaryDirectory() as td: + tmp_path = Path(td) + source = tmp_path / "sample.nt" + artifact = tmp_path / "sample.hdt" + source.write_text("

.\n", encoding="utf-8") + artifact.write_bytes(b"mock-hdt") + commands: list[list[str]] = [] + + def count_decoded(command): + commands.append(command) + return 1 + + args = argparse.Namespace( + source=str(source), + artifact=str(artifact), + format="hdt", + source_triples=None, + expected_triples=None, + skip_index_check=True, + ) + with ( + mock.patch.object(validator, "resolve_hdt2rdf", return_value="hdt2rdf"), + mock.patch.object(validator, "count_decoded", side_effect=count_decoded), + ): + report = validator.validate(args) + + self.assertTrue(report["valid"]) + self.assertTrue(report["count_match"]) + self.assertEqual(commands, [["hdt2rdf", str(artifact), "-"]]) + + def test_nt_and_nt_gz_share_a_base_output_directory_and_artifact_names(self): + """Both supported RDF inputs produce artifacts under the same basename.""" + output_dir = Path("results") + for rdf_name in ("test-larger.nt", "test-larger.nt.gz"): + rdf_path = Path(rdf_name) + self.assertEqual(vcf_rdfizer.rdf_output_basename(rdf_path), "test-larger") + planned = vcf_rdfizer.planned_output_paths( + out_dir=output_dir, + output_name=vcf_rdfizer.rdf_output_basename(rdf_path), + rdf_name=None, + methods=["hdt", "hdt_gzip", "cottas"], + partitioned=True, + ) + self.assertIn(output_dir / "test-larger" / "test-larger.hdt", planned) + self.assertIn( + output_dir / "test-larger" / "test-larger.hdt.index.v1-1", + planned, + ) + self.assertIn(output_dir / "test-larger" / "test-larger.hdt.gz", planned) + self.assertIn(output_dir / "test-larger" / "test-larger.cottas", planned) + self.assertFalse(any("test-larger.nt" in str(path) for path in planned)) + + def test_compress_mode_partitioned_nt_gz_uses_base_output_directory(self): + """Partitioned compression does not create a `.nt` output directory.""" + with tempfile.TemporaryDirectory() as td: + tmp_path = Path(td) + rdf_path = tmp_path / "test-larger.nt.gz" + rdf_path.write_bytes(b"not-read-by-this-mock") + out_dir = tmp_path / "out" + with ( + mock.patch.object( + vcf_rdfizer, + "run_partitioned_representation_methods_for_rdf_files", + return_value=(True, {"hdt": {"exit_code": 0}}), + ) as run_partitioned, + mock.patch.object(vcf_rdfizer, "print_nt_hdt_summary"), + ): + rc = vcf_rdfizer.run_compress_mode( + rdf_path=rdf_path, + out_dir=out_dir, + metrics_dir=tmp_path / "metrics", + run_id="run-output-name", + timestamp="2026-08-20T17:00:00", + image_ref="example/vcf-rdfizer:latest", + methods=["hdt"], + hdt_strategy="partitioned", + chunk_target_bytes=40, + chunk_min_bytes=10, + chunk_max_bytes=60, + wrapper_log_path=tmp_path / "wrapper.log", + ) + + self.assertEqual(rc, 0) + self.assertEqual( + run_partitioned.call_args.kwargs["out_dir"], out_dir / "test-larger" + ) + self.assertEqual(run_partitioned.call_args.kwargs["output_name"], "test-larger") + + def test_compress_mode_rejects_existing_artifact_before_docker_preflight(self): + """Existing final outputs are rejected without starting Docker or the pipeline.""" + with tempfile.TemporaryDirectory() as td: + tmp_path = Path(td) + rdf_path = tmp_path / "test-larger.nt.gz" + rdf_path.write_bytes(b"fake-gzip") + out_dir = tmp_path / "out" + artifact = out_dir / "test-larger" / "test-larger.hdt" + artifact.parent.mkdir(parents=True) + artifact.write_bytes(b"existing") + + with mock.patch.object( + vcf_rdfizer, + "check_docker", + side_effect=AssertionError("Docker preflight must not run"), + ), redirect_stderr(StringIO()) as stderr: + rc = invoke_main( + [ + "--mode", + "compress", + "--rdf", + str(rdf_path), + "--representations", + "hdt", + "--rdf-compression", + "none", + "--artifact-compression", + "none", + "--out", + str(out_dir), + ] + ) + + self.assertEqual(rc, 2) + self.assertIn("Refusing to overwrite existing output file", stderr.getvalue()) + self.assertIn(str(artifact), stderr.getvalue()) + def test_print_summary_lists_all_selected_compression_sizes(self): """Summary printer includes one size line per requested compression method.""" with tempfile.TemporaryDirectory() as td: @@ -226,6 +537,8 @@ def fake_run(cmd, cwd=None, env=None): ), None, ) + if emulate_validation_command(cmd): + return 0 script = str(cmd[-1]) if cmd else "" time_match = re.search(r"-o\s+(/data/metrics/raw_metrics/tsv_time/[^\s;]+)", script) if metrics_mount and time_match: @@ -278,6 +591,8 @@ def test_run_compression_methods_persists_raw_metrics_and_time_logs(self): rdf_path.write_text("

.\n") def fake_run(cmd, cwd=None, env=None): + if emulate_validation_command(cmd): + return 0 script = str(cmd[-1]) if cmd else "" time_match = re.search(r"-o\s+(/data/out/[^\s;]+)", script) if time_match: @@ -363,6 +678,8 @@ def test_run_compression_methods_use_wrapper_elapsed_for_wall_seconds(self): rdf_path.write_text("

.\n") def fake_run(cmd, cwd=None, env=None): + if emulate_validation_command(cmd): + return 0 script = str(cmd[-1]) if cmd else "" time_match = re.search(r"-o\s+(/data/out/[^\s;]+)", script) if time_match: @@ -379,7 +696,7 @@ def fake_run(cmd, cwd=None, env=None): return 0 with mock.patch.object(vcf_rdfizer, "run", side_effect=fake_run), mock.patch( - "vcf_rdfizer.time.perf_counter", side_effect=[1000.0, 3700.0] + "vcf_rdfizer.time.perf_counter", side_effect=[1000.0, 3700.0, 3700.0, 3700.0] ): ok, method_results = vcf_rdfizer.run_compression_methods_for_rdf( rdf_path=rdf_path, @@ -407,6 +724,8 @@ def test_run_compression_methods_records_implicit_hdt_in_raw_metrics(self): rdf_path.write_text("

.\n") def fake_run(cmd, cwd=None, env=None): + if emulate_validation_command(cmd): + return 0 script = str(cmd[-1]) if cmd else "" time_match = re.search(r"-o\s+(/data/out/[^\s;]+)", script) if time_match: @@ -460,6 +779,335 @@ def fake_run(cmd, cwd=None, env=None): self.assertIn("hdt", payload["methods"]) self.assertIn("hdt_gzip", payload["methods"]) + def test_run_compression_methods_packages_cottas(self): + """COTTAS packaging reuses one generated COTTAS artifact for gzip and Brotli.""" + with tempfile.TemporaryDirectory() as td: + tmp_path = Path(td) + input_dir = tmp_path / "input" + out_dir = tmp_path / "out" + target_dir = out_dir / "sample" + metrics_dir = tmp_path / "metrics" + input_dir.mkdir() + target_dir.mkdir(parents=True) + rdf_path = input_dir / "sample.nt" + rdf_path.write_text("

.\n") + + def fake_run(cmd, cwd=None, env=None): + if emulate_validation_command(cmd): + return 0 + script = str(cmd[-1]) if cmd else "" + time_match = re.search(r"-o\s+(/data/out/[^\s;]+)", script) + if time_match: + time_log = out_dir / time_match.group(1).replace("/data/out/", "", 1) + time_log.parent.mkdir(parents=True, exist_ok=True) + time_log.write_text( + "User time (seconds): 0.08\n" + "System time (seconds): 0.02\n" + "Elapsed (wall clock) time (h:mm:ss or m:ss): 0:00.14\n" + "Maximum resident set size (kbytes): 1111\n" + ) + + if "cottas_tool.py convert" in script: + (target_dir / "sample.cottas").write_text("cottas\n") + if "gzip -c /data/out/sample/sample.cottas" in script: + (target_dir / "sample.cottas.gz").write_text("gz\n") + if "brotli -q 7 -c /data/out/sample/sample.cottas" in script: + (target_dir / "sample.cottas.br").write_text("br\n") + return 0 + + with mock.patch.object(vcf_rdfizer, "run", side_effect=fake_run): + ok, method_results = vcf_rdfizer.run_compression_methods_for_rdf( + rdf_path=rdf_path, + out_dir=out_dir, + target_out_dir=target_dir, + image_ref="example/vcf-rdfizer:latest", + methods=["cottas", "cottas_gzip", "cottas_brotli"], + wrapper_log_path=tmp_path / "wrapper.log", + status_indent=None, + metrics_dir=metrics_dir, + run_id="run-cottas-package", + timestamp="2026-03-11T10:00:00", + output_name="sample", + ) + + self.assertTrue(ok) + self.assertEqual(method_results["cottas"]["exit_code"], 0) + self.assertEqual(method_results["cottas_gzip"]["exit_code"], 0) + self.assertEqual(method_results["cottas_brotli"]["exit_code"], 0) + self.assertTrue((target_dir / "sample.cottas").exists()) + self.assertTrue((target_dir / "sample.cottas.gz").exists()) + self.assertTrue((target_dir / "sample.cottas.br").exists()) + + raw_json = ( + metrics_dir + / "raw_metrics" + / "compression_metrics" + / "sample" + / "sample.nt" + / "run-cottas-package.json" + ) + self.assertTrue(raw_json.exists()) + payload = json.loads(raw_json.read_text()) + self.assertIn("cottas_gzip", payload["methods"]) + self.assertIn("cottas_brotli", payload["methods"]) + + def test_compression_validation_mismatch_fails_before_success(self): + """A decoded triple-count mismatch fails compression and preserves RDF input.""" + with tempfile.TemporaryDirectory() as td: + tmp_path = Path(td) + out_dir = tmp_path / "out" + target_dir = out_dir / "sample" + target_dir.mkdir(parents=True) + rdf_path = target_dir / "sample.nt" + rdf_path.write_text("

.\n") + + def fake_run(cmd, cwd=None, env=None): + if emulate_validation_command(cmd): + rendered = str(cmd[-1]) + result_match = re.search( + r"--result-path\s+['\"]?(/data/out/[^\s'\";]+)", + rendered, + ) + result_path = out_dir / result_match.group(1).replace( + "/data/out/", "", 1 + ) + result_path.write_text( + json.dumps( + { + "valid": False, + "source_triples": 1, + "decoded_triples": 0, + "count_match": False, + "error": "decoded triple count does not match the source", + } + ) + ) + return 1 + rendered = str(cmd[-1]) + if 'HDT_BIN="${RDF2HDT_BIN' in rendered: + (target_dir / "sample.hdt").write_text("mock-hdt\n") + return 0 + + with mock.patch.object(vcf_rdfizer, "run", side_effect=fake_run): + ok, method_results = vcf_rdfizer.run_compression_methods_for_rdf( + rdf_path=rdf_path, + out_dir=out_dir, + target_out_dir=target_dir, + image_ref="example/vcf-rdfizer:latest", + methods=["hdt"], + wrapper_log_path=tmp_path / "wrapper.log", + status_indent=None, + ) + + self.assertFalse(ok) + self.assertEqual(method_results["hdt"]["exit_code"], 0) + self.assertTrue(rdf_path.exists()) + + def test_plan_partitioned_hdt_chunks_groups_small_inputs_and_splits_large_ones(self): + """Partition planning coalesces small RDF parts and line-splits oversized ones.""" + with tempfile.TemporaryDirectory() as td: + tmp_path = Path(td) + rdf_dir = tmp_path / "rdf" + chunk_dir = tmp_path / "chunks" + rdf_dir.mkdir() + + small_a = rdf_dir / "part-a.nt" + small_b = rdf_dir / "part-b.nt" + large = rdf_dir / "part-c.nt" + small_a.write_text("

.\n") + small_b.write_text("

.\n") + large.write_text(("

.\n".lstrip()) * 8) + + chunk_inputs, plan = vcf_rdfizer.plan_partitioned_hdt_chunks( + [small_a, small_b, large], + chunk_dir, + target_bytes=40, + min_bytes=10, + max_bytes=60, + ) + + self.assertGreaterEqual(len(chunk_inputs), 2) + self.assertEqual(plan["source_file_count"], 3) + self.assertEqual(plan["chunk_count"], len(chunk_inputs)) + self.assertTrue(all(path.exists() for path in chunk_inputs)) + first_chunk_text = chunk_inputs[0].read_text() + self.assertIn("

.", first_chunk_text) + self.assertIn("

.", first_chunk_text) + + def test_record_safe_chunk_planner_reads_gzip_and_writes_boundary_guide(self): + """Gzip-backed chunking preserves complete records and records exact ranges.""" + with tempfile.TemporaryDirectory() as td: + tmp_path = Path(td) + source = tmp_path / "aggregate.nt.gz" + source_payload = ( + b"

.\n" + b"

.\n" + b"

.\n" + ) + import gzip + + with gzip.open(source, "wb") as handle: + handle.write(source_payload) + + chunk_dir = tmp_path / "chunks" + guide = tmp_path / "chunks.json" + chunk_paths, plan = vcf_rdfizer.plan_record_safe_rdf_chunks( + [source], + chunk_dir, + target_bytes=20, + min_bytes=10, + max_bytes=30, + guide_path=guide, + ) + + self.assertTrue(guide.exists()) + self.assertEqual(plan["record_count"], 3) + self.assertEqual(plan["chunk_count"], len(chunk_paths)) + self.assertEqual(b"".join(path.read_bytes() for path in chunk_paths), source_payload) + self.assertTrue(all(path.read_bytes().endswith(b"\n") for path in chunk_paths)) + self.assertEqual(json.loads(guide.read_text())["chunks"], plan["chunks"]) + + def test_run_partitioned_hdt_methods_merges_chunks_and_generates_index(self): + """Partitioned HDT/COTTAS pipelines share chunks in one container.""" + with tempfile.TemporaryDirectory() as td: + tmp_path = Path(td) + out_dir = tmp_path / "out" / "sample" + source = out_dir / "sample.nt" + out_dir.mkdir(parents=True, exist_ok=True) + source.write_text("

.\n" * 12) + commands = [] + + def fake_run(cmd, cwd=None, env=None): + commands.append(cmd) + if vcf_rdfizer.PARTITIONED_COMPRESSION_RUNNER_CONTAINER in cmd: + self.assertTrue(emulate_partitioned_runner(cmd)) + return 0 + + with mock.patch.object(vcf_rdfizer, "run", side_effect=fake_run): + ok, method_results = vcf_rdfizer.run_partitioned_representation_methods_for_rdf_files( + rdf_paths=[], + source_rdf_path=source, + out_dir=out_dir, + image_ref="example/vcf-rdfizer:latest", + methods=["hdt", "hdt_gzip", "cottas", "cottas_gzip"], + wrapper_log_path=tmp_path / "wrapper.log", + metrics_dir=tmp_path / "metrics", + run_id="run-partitioned", + timestamp="2026-08-04T10:00:00", + output_name="sample", + target_chunk_bytes=40, + min_chunk_bytes=10, + max_chunk_bytes=60, + expected_triples=12, + ) + + self.assertTrue(ok) + self.assertEqual(method_results["hdt"]["source"], "partitioned_generated") + self.assertTrue((out_dir / "sample.hdt").exists()) + self.assertTrue((out_dir / "sample.hdt.index.v1-1").exists()) + self.assertEqual( + method_results["hdt"]["details"]["index_path"], + str(out_dir / "sample.hdt.index.v1-1"), + ) + self.assertTrue((out_dir / "sample.hdt.gz").exists()) + self.assertTrue((out_dir / "sample.cottas").exists()) + self.assertTrue((out_dir / "sample.cottas.gz").exists()) + self.assertTrue(any("target=/work" in " ".join(cmd) for cmd in commands)) + runner_command = next( + cmd + for cmd in commands + if vcf_rdfizer.PARTITIONED_COMPRESSION_RUNNER_CONTAINER in cmd + ) + self.assertIn("--expected-triples", runner_command) + self.assertIn("12", runner_command) + self.assertGreaterEqual(method_results["hdt"]["details"]["chunk_count"], 2) + self.assertGreaterEqual(method_results["hdt"]["details"]["merge_rounds"], 1) + + def test_containerized_partitioned_pipeline_cleans_ephemeral_volume(self): + """Source-based partitioning leaves only final artifacts on the host.""" + with tempfile.TemporaryDirectory() as td: + tmp_path = Path(td) + source = tmp_path / "input.nt" + source.write_text("

.\n") + out_dir = tmp_path / "out" / "sample" + commands = [] + + def fake_run(cmd, cwd=None, env=None): + commands.append(cmd) + if vcf_rdfizer.PARTITIONED_COMPRESSION_RUNNER_CONTAINER in cmd: + self.assertTrue(emulate_partitioned_runner(cmd)) + return 0 + + with mock.patch.object(vcf_rdfizer, "run", side_effect=fake_run): + ok, method_results = vcf_rdfizer.run_partitioned_representation_methods_for_rdf_files( + rdf_paths=[], + source_rdf_path=source, + out_dir=out_dir, + image_ref="example/vcf-rdfizer:latest", + methods=["hdt", "cottas"], + wrapper_log_path=tmp_path / "wrapper.log", + output_name="sample", + target_chunk_bytes=40, + min_chunk_bytes=10, + max_chunk_bytes=60, + ) + + self.assertTrue(ok) + self.assertEqual(method_results["hdt"]["details"]["workspace"], "docker-volume") + self.assertEqual(method_results["cottas"]["details"]["workspace_cleanup"], "removed") + self.assertTrue((out_dir / "sample.hdt").exists()) + self.assertTrue((out_dir / "sample.cottas").exists()) + self.assertFalse((out_dir / ".compression_partitioned").exists()) + self.assertTrue(any("volume" in cmd and "create" in cmd for cmd in commands)) + self.assertTrue(any("volume" in cmd and "rm" in cmd for cmd in commands)) + runner_command = next( + cmd for cmd in commands if vcf_rdfizer.PARTITIONED_COMPRESSION_RUNNER_CONTAINER in cmd + ) + self.assertIn(f"{source.parent.resolve()}:/data/in:ro", runner_command) + + def test_containerized_partitioned_pipeline_removes_volume_after_failure(self): + """A failed container run still removes its named workspace volume.""" + with tempfile.TemporaryDirectory() as td: + tmp_path = Path(td) + source = tmp_path / "input.nt" + source.write_text("

.\n") + out_dir = tmp_path / "out" + commands = [] + + def fake_run(cmd, cwd=None, env=None): + commands.append(cmd) + if vcf_rdfizer.PARTITIONED_COMPRESSION_RUNNER_CONTAINER in cmd: + out_mount = next( + Path(part.split(":", 1)[0]) + for part in cmd + if isinstance(part, str) and part.endswith(":/data/out") + ) + result_path = out_mount / cmd[cmd.index("--result-path") + 1].replace( + "/data/out/", "", 1 + ) + result_path.write_text(json.dumps({"exit_code": 1, "methods": {}})) + return 1 + return 0 + + with mock.patch.object(vcf_rdfizer, "run", side_effect=fake_run): + ok, method_results = vcf_rdfizer.run_partitioned_representation_methods_for_rdf_files( + rdf_paths=[], + source_rdf_path=source, + out_dir=out_dir, + image_ref="example/vcf-rdfizer:latest", + methods=["cottas"], + wrapper_log_path=tmp_path / "wrapper.log", + output_name="sample", + target_chunk_bytes=40, + min_chunk_bytes=10, + max_chunk_bytes=60, + ) + + self.assertFalse(ok) + self.assertEqual(method_results, {}) + self.assertTrue(any("volume" in cmd and "rm" in cmd for cmd in commands)) + self.assertFalse(any(path.name.startswith(".") for path in out_dir.glob("*"))) + def test_main_full_mode_records_tsv_metrics_and_raw_artifacts(self): """Full mode stores TSV timing/metrics artifacts and writes TSV fields into metrics.csv.""" with tempfile.TemporaryDirectory() as td: @@ -522,8 +1170,8 @@ def fake_run(cmd, cwd=None, env=None): str(input_dir), "--rules", str(rules_path), - "--rdf-layout", - "aggregate", + "--rdf-storage-mode", + "plain", "--compression", "none", "--out", @@ -664,7 +1312,7 @@ def test_cleanup_interrupted_full_run_removes_intermediates(self): run_tracker=tracker, out_root=out_root, image_ref=None, - keep_rdf=False, + keep_rmlstreamer_rdf_output=False, wrapper_log_path=metrics_dir / "wrapper.log", ) tracker.close() @@ -674,40 +1322,6 @@ def test_cleanup_interrupted_full_run_removes_intermediates(self): self.assertFalse(tsv_file.parent.exists()) self.assertFalse(raw_rdf.exists()) - def test_aggregate_method_results_includes_all_timing_types(self): - """Batch aggregation keeps wall/user/sys and max_rss metrics for each method.""" - aggregated = vcf_rdfizer.aggregate_method_results_across_files( - { - "part1.nt": { - "gzip": { - "exit_code": 0, - "wall_seconds": 1.0, - "user_seconds": 0.7, - "sys_seconds": 0.1, - "max_rss_kb": 100, - "output_size_bytes": 10, - } - }, - "part2.nt": { - "gzip": { - "exit_code": 0, - "wall_seconds": 2.0, - "user_seconds": 1.2, - "sys_seconds": 0.2, - "max_rss_kb": 150, - "output_size_bytes": 20, - } - }, - } - ) - - gzip = aggregated["gzip"] - self.assertEqual(gzip["wall_seconds"], 3.0) - self.assertAlmostEqual(gzip["user_seconds"], 1.9, places=6) - self.assertAlmostEqual(gzip["sys_seconds"], 0.3, places=6) - self.assertEqual(gzip["max_rss_kb"], 150) - self.assertEqual(gzip["output_size_bytes"], 30) - def test_help_flag_prints_usage_guide(self): """Help flag exits cleanly and prints mode usage examples.""" out_buf = StringIO() @@ -718,8 +1332,15 @@ def test_help_flag_prints_usage_guide(self): self.assertEqual(exc.exception.code, 0) text = out_buf.getvalue() self.assertIn("Examples:", text) - self.assertIn("-m {full,compress,decompress,tsv}", text) + self.assertIn("-m {full,compress,decompress,tsv,index}", text) self.assertIn("-i INPUT", text) + self.assertIn("--keep-rmlstreamer-rdf-output", text) + self.assertIn("--remove-rdf-storage-output", text) + self.assertIn("--rdf-compression", text) + self.assertIn("--representations", text) + self.assertIn("--artifact-compression", text) + self.assertNotIn("--keep-rdf", text) + self.assertNotIn("--compression", text) def test_estimate_pipeline_sizes_handles_plain_and_gz_inputs(self): """Size estimation scales gzipped inputs and reports free disk bytes.""" @@ -745,6 +1366,7 @@ def test_main_estimate_size_prints_summary_and_warning(self): with tempfile.TemporaryDirectory() as td: tmp_path = Path(td) input_dir, rules_path = prepare_inputs(tmp_path) + out_dir = tmp_path / "out" commands = [] def fake_run(cmd, cwd=None, env=None): @@ -774,6 +1396,8 @@ def fake_run(cmd, cwd=None, env=None): "--rules", str(rules_path), "--estimate-size", + "--hdt-strategy", + "single", "--keep-tsv", ] ) @@ -956,8 +1580,8 @@ def fake_run(cmd, cwd=None, env=None): self.assertIn("brotli -q 7 -c", commands[1][-1]) self.assertIn("/data/out/sample/sample.nt.br", commands[1][-1]) - def test_main_compress_mode_hdt_gzip_reuses_existing_hdt(self): - """Compound method hdt_gzip reuses a preexisting HDT artifact instead of regenerating it.""" + def test_main_compress_mode_hdt_gzip_rejects_existing_hdt(self): + """Compound HDT packaging refuses to overwrite a preexisting HDT artifact.""" with tempfile.TemporaryDirectory() as td: tmp_path = Path(td) nt_path = tmp_path / "sample.nt" @@ -966,20 +1590,14 @@ def test_main_compress_mode_hdt_gzip_reuses_existing_hdt(self): sample_out = out_dir / "sample" sample_out.mkdir(parents=True, exist_ok=True) (sample_out / "sample.hdt").write_text("prebuilt-hdt\n") - commands = [] - - def fake_run(cmd, cwd=None, env=None): - commands.append(cmd) - return 0 - old_cwd = os.getcwd() os.chdir(tmp_path) try: - with mock.patch.object(vcf_rdfizer, "run", side_effect=fake_run), mock.patch.object( - vcf_rdfizer, "check_docker", return_value=True - ), mock.patch.object( - vcf_rdfizer, "docker_image_exists", return_value=True - ): + with mock.patch.object( + vcf_rdfizer, + "check_docker", + side_effect=AssertionError("Docker preflight must not run"), + ), redirect_stderr(StringIO()) as stderr: rc = invoke_main( [ "--mode", @@ -995,13 +1613,8 @@ def fake_run(cmd, cwd=None, env=None): finally: os.chdir(old_cwd) - self.assertEqual(rc, 0) - self.assertEqual(len(commands), 1) - self.assertIn( - "gzip -c /data/out/sample/sample.hdt > /data/out/sample/sample.hdt.gz", - commands[0][-1], - ) - self.assertNotIn("rdf2hdt", commands[0][-1]) + self.assertEqual(rc, 2) + self.assertIn("Refusing to overwrite existing output file", stderr.getvalue()) def test_main_compress_mode_hdt_brotli_generates_hdt_then_compresses_hdt(self): """Compound method hdt_brotli runs rdf2hdt first, then brotli on the generated HDT file.""" @@ -1147,8 +1760,8 @@ def fake_run(cmd, cwd=None, env=None): str(input_dir), "--rules", str(rules_path), - "--rdf-layout", - "aggregate", + "--rdf-storage-mode", + "plain", "--compression", "none", "--out", @@ -1156,7 +1769,7 @@ def fake_run(cmd, cwd=None, env=None): "--metrics", str(metrics_dir), "--keep-tsv", - "--keep-rdf", + "--keep-rmlstreamer-rdf-output", ] ) finally: @@ -1219,14 +1832,14 @@ def fake_run(cmd, cwd=None, env=None): str(input_dir), "--rules", str(rules_path), - "--rdf-layout", - "aggregate", + "--rdf-storage-mode", + "plain", "--compression", "none", "--out", str(out_dir), "--keep-tsv", - "--keep-rdf", + "--keep-rmlstreamer-rdf-output", ] ) finally: @@ -1286,7 +1899,7 @@ def test_main_rejects_spark_partitions_outside_full_mode(self): """--spark-partitions is rejected for non-full modes.""" rc = invoke_main( ["--mode", "compress", "--spark-partitions", "4", "--rdf", "missing.nt"], - auto_layout=False, + auto_storage=False, ) self.assertEqual(rc, 2) @@ -1304,17 +1917,57 @@ def test_main_full_mode_requires_input_argument(self): rc = invoke_main(["--mode", "full"]) self.assertEqual(rc, 2) - def test_main_full_mode_requires_rdf_layout_argument(self): - """Full mode fails validation when --rdf-layout is omitted.""" + def test_main_full_mode_requires_storage_mode_argument(self): + """Full mode fails validation when --rdf-storage-mode is omitted.""" with tempfile.TemporaryDirectory() as td: tmp_path = Path(td) input_dir, rules_path = prepare_inputs(tmp_path) rc = invoke_main( ["--mode", "full", "--input", str(input_dir), "--rules", str(rules_path)], - auto_layout=False, + auto_storage=False, ) self.assertEqual(rc, 2) + def test_removed_legacy_cli_options_are_rejected(self): + """Removed layout and compatibility aliases are not accepted by the CLI.""" + for legacy_args in ( + ["--rdf-layout", "batch"], + ["--nt", "sample.nt"], + ["--keep_rdf"], + ["--keep-rdf"], + ): + with self.subTest(args=legacy_args), redirect_stderr(StringIO()): + with mock.patch.object( + sys, + "argv", + ["vcf_rdfizer.py", "--mode", "compress", *legacy_args, "--out", "./out"], + ), self.assertRaises(SystemExit) as exc: + vcf_rdfizer.main() + self.assertEqual(exc.exception.code, 2) + + def test_rdf_output_retention_and_removal_flags_are_mutually_exclusive(self): + """RDF output retention and removal cannot be requested together.""" + with redirect_stderr(StringIO()): + with mock.patch.object( + sys, + "argv", + [ + "vcf_rdfizer.py", + "--mode", + "full", + "--input", + "sample.vcf", + "--rdf-storage-mode", + "plain", + "--keep-rmlstreamer-rdf-output", + "--remove-rdf-storage-output", + "--out", + "./out", + ], + ), self.assertRaises(SystemExit) as exc: + vcf_rdfizer.main() + self.assertEqual(exc.exception.code, 2) + def test_main_full_mode_keyboard_interrupt_returns_130_and_writes_progress_log(self): """Keyboard interrupt exits with 130 and records interruption in progress log.""" with tempfile.TemporaryDirectory() as td: @@ -1338,8 +1991,8 @@ def test_main_full_mode_keyboard_interrupt_returns_130_and_writes_progress_log(s str(input_dir), "--rules", str(rules_path), - "--rdf-layout", - "aggregate", + "--rdf-storage-mode", + "plain", "--out", str(out_dir), ] @@ -1466,6 +2119,87 @@ def fake_run(cmd, cwd=None, env=None): self.assertTrue(any(arg.endswith("/out/sample:/data/out") for arg in commands[0])) self.assertIn("/data/out/sample.nt", commands[0][-1]) + def test_main_index_mode_initializes_existing_hdt(self): + """Index mode runs the container helper and records the sidecar artifact.""" + with tempfile.TemporaryDirectory() as td: + tmp_path = Path(td) + hdt_path = tmp_path / "sample.hdt" + hdt_path.write_bytes(b"fake-hdt") + out_dir = tmp_path / "out" + commands = [] + + def fake_run(cmd, cwd=None, env=None): + commands.append(cmd) + rendered = str(cmd[-1]) + if "ensure_hdt_index.sh" in rendered: + Path(str(hdt_path) + ".index.v1-1").write_text("index\n") + return 0 + + old_cwd = os.getcwd() + os.chdir(tmp_path) + try: + with mock.patch.object(vcf_rdfizer, "run", side_effect=fake_run), mock.patch.object( + vcf_rdfizer, "check_docker", return_value=True + ), mock.patch.object( + vcf_rdfizer, "docker_image_exists", return_value=True + ): + rc = invoke_main( + [ + "--mode", + "index", + "--hdt", + str(hdt_path), + "--out", + str(out_dir), + ] + ) + finally: + os.chdir(old_cwd) + + self.assertEqual(rc, 0) + self.assertTrue((tmp_path / "sample.hdt.index.v1-1").exists()) + self.assertEqual(len(commands), 1) + self.assertIn("ensure_hdt_index.sh", commands[0][-1]) + self.assertTrue(any(arg.endswith(":/data/hdt") for arg in commands[0])) + metrics = latest_metrics_run_dir(out_dir / "run_metrics") / "hdt_index_metrics.json" + payload = json.loads(metrics.read_text()) + self.assertEqual(payload["index_status"], "generated") + self.assertEqual( + Path(payload["index_path"]).resolve(), + (tmp_path / "sample.hdt.index.v1-1").resolve(), + ) + + def test_hdt_index_helper_uses_exit_only_and_verifies_sidecar(self): + """The Docker-side helper initializes the index without issuing a query.""" + with tempfile.TemporaryDirectory() as td: + tmp_path = Path(td) + java_home = tmp_path / "java-home" + bin_dir = java_home / "bin" + bin_dir.mkdir(parents=True) + search = bin_dir / "hdtSearch.sh" + search.write_text( + "#!/usr/bin/env bash\n" + "read -r command\n" + "[[ \"$command\" == \"exit\" ]] || exit 3\n" + "printf 'index\\n' > \"${1}.index.v1-1\"\n", + encoding="utf-8", + ) + search.chmod(0o755) + hdt_path = tmp_path / "sample.hdt" + hdt_path.write_bytes(b"fake-hdt") + helper = Path(__file__).parents[1] / "src" / "ensure_hdt_index.sh" + + result = subprocess.run( + ["bash", str(helper), str(hdt_path)], + env={**os.environ, "HDT_JAVA_HOME": str(java_home)}, + capture_output=True, + text=True, + ) + + self.assertEqual(result.returncode, 0, msg=result.stderr) + self.assertTrue(Path(str(hdt_path) + ".index.v1-1").exists()) + self.assertIn(f"HDT index ready: {hdt_path}.index.v1-1", result.stdout) + def test_main_decompress_mode_rejects_unknown_extension(self): """Decompression mode rejects unsupported compressed RDF extensions.""" with tempfile.TemporaryDirectory() as td: @@ -1545,79 +2279,65 @@ def fake_run(cmd, cwd=None, env=None): self.assertTrue(any(str(arg).endswith(":/data/in:ro") for arg in commands[1])) self.assertTrue(any(str(arg).startswith("IN_VCF=/data/in/") for arg in commands[1])) - def test_main_full_mode_batch_layout_compresses_each_rml_part(self): - """Batch layout compresses each part and prints one consolidated size summary.""" + def test_main_full_mode_plain_hdt_uses_partitioned_merge_pipeline(self): + """Plain aggregate HDT uses the ephemeral container-side merge pipeline.""" with tempfile.TemporaryDirectory() as td: tmp_path = Path(td) input_dir, rules_path = prepare_inputs(tmp_path) out_dir = tmp_path / "out" commands = [] - out_buf = StringIO() def fake_run(cmd, cwd=None, env=None): commands.append(cmd) + rendered = str(cmd[-1]) if cmd else "" if "/opt/vcf-rdfizer/run_conversion.sh" in cmd: sample_dir = out_dir / "sample" sample_dir.mkdir(parents=True, exist_ok=True) - (sample_dir / "part-00000.nt").write_text("

.\n") - (sample_dir / "part-00001.nt").write_text("

.\n") - return 0 - - old_cwd = os.getcwd() - os.chdir(tmp_path) - try: - with mock.patch.object(vcf_rdfizer, "run", side_effect=fake_run), mock.patch.object( - vcf_rdfizer, "check_docker", return_value=True - ), mock.patch.object( - vcf_rdfizer, "docker_image_exists", return_value=True - ), mock.patch.object( - vcf_rdfizer, "discover_tsv_triplets", return_value=mocked_triplets() - ), redirect_stdout(out_buf): - rc = invoke_main( - [ - "--input", - str(input_dir), - "--rules", - str(rules_path), - "--rdf-layout", - "batch", - "--compression", - "gzip", - "--out", - str(out_dir), - "--keep-tsv", - ] + (sample_dir / "sample.nt").write_text( + "

.\n" * 4 + "

.\n" * 4 ) - finally: - os.chdir(old_cwd) - - self.assertEqual(rc, 0) - self.assertIn("AGGREGATE_RDF=0", commands[1]) - gzip_cmds = [cmd for cmd in commands if isinstance(cmd, list) and cmd and "gzip -c" in cmd[-1]] - self.assertEqual(len(gzip_cmds), 2) - self.assertIn("/data/in/part-00000.nt", gzip_cmds[0][-1]) - self.assertIn("/data/in/part-00001.nt", gzip_cmds[1][-1]) - self.assertEqual(out_buf.getvalue().count("* Output directory:"), 1) + return 0 - def test_main_full_mode_batch_metrics_upsert_is_sample_scoped(self): - """Batch layout writes compression CSV metrics once per sample, not once per RDF part.""" - with tempfile.TemporaryDirectory() as td: - tmp_path = Path(td) - input_dir, rules_path = prepare_inputs(tmp_path) - out_dir = tmp_path / "out" - seen_output_names = [] + work_mount = next( + (part.split(":", 1)[0] for part in cmd if isinstance(part, str) and part.endswith(":/data/work")), + None, + ) + out_mount = next( + (part.split(":", 1)[0] for part in cmd if isinstance(part, str) and part.endswith(":/data/out")), + None, + ) + time_match = re.search(r"-o\s+(/data/work/[^\s;]+)", rendered) + if work_mount and time_match: + time_log = Path(work_mount) / time_match.group(1).replace("/data/work/", "", 1) + time_log.parent.mkdir(parents=True, exist_ok=True) + time_log.write_text( + "User time (seconds): 0.10\n" + "System time (seconds): 0.02\n" + "Elapsed (wall clock) time (h:mm:ss or m:ss): 0:00.05\n" + "Maximum resident set size (kbytes): 1024\n" + ) - def fake_run(cmd, cwd=None, env=None): - if "/opt/vcf-rdfizer/run_conversion.sh" in cmd: - sample_dir = out_dir / "sample" - sample_dir.mkdir(parents=True, exist_ok=True) - (sample_dir / "part-00000.nt").write_text("

.\n") - (sample_dir / "part-00001.nt").write_text("

.\n") + if work_mount and '"$HDT_BIN"' in rendered: + outputs = re.findall(r"(/data/work/[^\s;]+\.hdt)", rendered) + if outputs: + output_path = Path(work_mount) / outputs[-1].replace("/data/work/", "", 1) + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text("chunk-hdt\n") + + if work_mount and '"$HDTCAT_BIN"' in rendered: + outputs = re.findall(r"(/data/work/[^\s;]+\.hdt)", rendered) + if outputs: + output_path = Path(work_mount) / outputs[-1].replace("/data/work/", "", 1) + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text("merged-hdt\n") + + if out_mount and "ensure_hdt_index.sh" in rendered: + index_match = re.search(r"ensure_hdt_index\.sh\s+(/data/out/[^\s;]+\.hdt)", rendered) + if index_match: + hdt_path = Path(out_mount) / index_match.group(1).replace("/data/out/", "", 1) + Path(str(hdt_path) + ".index.v1-1").write_text("index\n") return 0 - def fake_update_metrics_csv_with_compression(**kwargs): - seen_output_names.append(kwargs["output_name"]) - old_cwd = os.getcwd() os.chdir(tmp_path) try: @@ -1627,8 +2347,6 @@ def fake_update_metrics_csv_with_compression(**kwargs): vcf_rdfizer, "docker_image_exists", return_value=True ), mock.patch.object( vcf_rdfizer, "discover_tsv_triplets", return_value=mocked_triplets() - ), mock.patch.object( - vcf_rdfizer, "update_metrics_csv_with_compression", side_effect=fake_update_metrics_csv_with_compression ): rc = invoke_main( [ @@ -1636,10 +2354,16 @@ def fake_update_metrics_csv_with_compression(**kwargs): str(input_dir), "--rules", str(rules_path), - "--rdf-layout", - "batch", + "--rdf-storage-mode", + "plain", "--compression", - "gzip", + "hdt", + "--chunk-target-bytes", + "40", + "--chunk-min-bytes", + "10", + "--chunk-max-bytes", + "60", "--out", str(out_dir), "--keep-tsv", @@ -1649,17 +2373,31 @@ def fake_update_metrics_csv_with_compression(**kwargs): os.chdir(old_cwd) self.assertEqual(rc, 0) - self.assertEqual(seen_output_names, ["sample"]) + runner_commands = [ + cmd + for cmd in commands + if vcf_rdfizer.PARTITIONED_COMPRESSION_RUNNER_CONTAINER in cmd + ] + self.assertEqual(len(runner_commands), 1) + self.assertIn("target=/work", " ".join(runner_commands[0])) + self.assertIn("--methods hdt", " ".join(runner_commands[0])) + self.assertFalse((out_dir / "sample" / ".compression_partitioned").exists()) + self.assertTrue((out_dir / "sample" / "sample.hdt").exists()) - def test_main_full_mode_aggregate_layout_sets_merge_flag(self): - """Aggregate layout passes AGGREGATE_RDF=1 to conversion step.""" + def test_main_full_mode_plain_storage_sets_storage_mode(self): + """Plain storage passes the canonical storage mode to conversion.""" with tempfile.TemporaryDirectory() as td: tmp_path = Path(td) input_dir, rules_path = prepare_inputs(tmp_path) + out_dir = tmp_path / "out" commands = [] def fake_run(cmd, cwd=None, env=None): commands.append(cmd) + if "/opt/vcf-rdfizer/run_conversion.sh" in cmd: + sample_dir = out_dir / "sample" + sample_dir.mkdir(parents=True, exist_ok=True) + (sample_dir / "sample.nt").write_text("

.\n") return 0 old_cwd = os.getcwd() @@ -1678,8 +2416,8 @@ def fake_run(cmd, cwd=None, env=None): str(input_dir), "--rules", str(rules_path), - "--rdf-layout", - "aggregate", + "--rdf-storage-mode", + "plain", "--compression", "none", "--keep-tsv", @@ -1689,7 +2427,7 @@ def fake_run(cmd, cwd=None, env=None): os.chdir(old_cwd) self.assertEqual(rc, 0) - self.assertIn("AGGREGATE_RDF=1", commands[1]) + self.assertIn("RDF_STORAGE_MODE=plain", commands[1]) def test_main_full_mode_passes_spark_partition_hint_to_conversion(self): """Full mode forwards --spark-partitions to run_conversion as SPARK_PARTITIONS.""" @@ -1718,8 +2456,8 @@ def fake_run(cmd, cwd=None, env=None): str(input_dir), "--rules", str(rules_path), - "--rdf-layout", - "aggregate", + "--rdf-storage-mode", + "plain", "--compression", "none", "--spark-partitions", @@ -1774,7 +2512,17 @@ def fake_run(cmd, cwd=None, env=None): ), mock.patch.object( vcf_rdfizer, "discover_tsv_triplets", return_value=multi_triplets ): - rc = invoke_main(["--input", str(input_dir), "--rules", str(rules_path), "--keep-tsv"]) + rc = invoke_main( + [ + "--input", + str(input_dir), + "--rules", + str(rules_path), + "--hdt-strategy", + "single", + "--keep-tsv", + ] + ) finally: os.chdir(old_cwd) @@ -1871,8 +2619,8 @@ def fake_run(cmd, cwd=None, env=None): self.assertIn("sample_a", report_text) self.assertIn("rdf-conversion", report_text) - def test_main_full_mode_deletes_nt_after_compression_by_default(self): - """Full mode removes merged .nt outputs after successful compression unless --keep-rdf is set.""" + def test_main_full_mode_remove_rdf_storage_output_deletes_nt(self): + """The explicit RDF storage cleanup flag removes the aggregate after compression.""" with tempfile.TemporaryDirectory() as td: tmp_path = Path(td) input_dir, rules_path = prepare_inputs(tmp_path) @@ -1910,6 +2658,7 @@ def fake_run(cmd, cwd=None, env=None): "--out", str(out_dir), "--keep-tsv", + "--remove-rdf-storage-output", ] ) finally: @@ -1919,8 +2668,8 @@ def fake_run(cmd, cwd=None, env=None): self.assertFalse((out_dir / "sample" / "sample.nt").exists()) self.assertTrue((out_dir / "sample" / "sample.hdt").exists()) - def test_main_full_mode_keep_rdf_preserves_nt_after_compression(self): - """Full mode keeps merged .nt outputs when --keep-rdf is provided.""" + def test_main_full_mode_keep_rmlstreamer_rdf_output_preserves_nt_after_compression(self): + """Full mode keeps merged .nt outputs with the RMLStreamer retention flag.""" with tempfile.TemporaryDirectory() as td: tmp_path = Path(td) input_dir, rules_path = prepare_inputs(tmp_path) @@ -1958,7 +2707,7 @@ def fake_run(cmd, cwd=None, env=None): "--out", str(out_dir), "--keep-tsv", - "--keep-rdf", + "--keep-rmlstreamer-rdf-output", ] ) finally: @@ -2079,7 +2828,7 @@ def fake_run(cmd, cwd=None, env=None): "--compression", "hdt", "--keep-tsv", - "--keep-rdf", + "--keep-rmlstreamer-rdf-output", ] ) finally: @@ -2246,7 +2995,17 @@ def fake_run(cmd, cwd=None, env=None): ), mock.patch.object( vcf_rdfizer, "discover_tsv_triplets", return_value=triplets ): - rc = invoke_main(["--input", str(input_dir), "--rules", str(rules_path), "--keep-tsv"]) + rc = invoke_main( + [ + "--input", + str(input_dir), + "--rules", + str(rules_path), + "--hdt-strategy", + "single", + "--keep-tsv", + ] + ) finally: os.chdir(old_cwd) diff --git a/vcf_rdfizer.py b/vcf_rdfizer.py index 4efacc1..4c509fa 100644 --- a/vcf_rdfizer.py +++ b/vcf_rdfizer.py @@ -14,6 +14,7 @@ import argparse import csv +import gzip import importlib.resources as importlib_resources import json import os @@ -107,6 +108,9 @@ "user_seconds_hdt", "sys_seconds_hdt", "max_rss_kb_hdt", + "source_triples_hdt", + "decoded_triples_hdt", + "validation_hdt", ], "hdt_gzip": [ "gzip_on_hdt_size_bytes", @@ -124,11 +128,63 @@ "sys_seconds_brotli_on_hdt", "max_rss_kb_brotli_on_hdt", ], + "cottas_gzip": [ + "gzip_on_cottas_size_bytes", + "exit_code_gzip_on_cottas", + "wall_seconds_gzip_on_cottas", + "user_seconds_gzip_on_cottas", + "sys_seconds_gzip_on_cottas", + "max_rss_kb_gzip_on_cottas", + ], + "cottas_brotli": [ + "brotli_on_cottas_size_bytes", + "exit_code_brotli_on_cottas", + "wall_seconds_brotli_on_cottas", + "user_seconds_brotli_on_cottas", + "sys_seconds_brotli_on_cottas", + "max_rss_kb_brotli_on_cottas", + ], + "cottas": [ + "cottas_size_bytes", + "exit_code_cottas", + "wall_seconds_cottas", + "user_seconds_cottas", + "sys_seconds_cottas", + "max_rss_kb_cottas", + "source_triples_cottas", + "decoded_triples_cottas", + "validation_cottas", + ], } HDT_SOURCE_COLUMN = "hdt_source" -VALID_COMPRESSION_METHODS = {"gzip", "brotli", "hdt", "hdt_gzip", "hdt_brotli"} +VALID_COMPRESSION_METHODS = { + "gzip", + "brotli", + "hdt", + "hdt_gzip", + "hdt_brotli", + "cottas", + "cottas_gzip", + "cottas_brotli", +} HDT_COMPRESSION_METHODS = {"hdt", "hdt_gzip", "hdt_brotli"} +COTTAS_COMPRESSION_METHODS = {"cottas", "cottas_gzip", "cottas_brotli"} +PARTITIONED_COMPRESSION_METHODS = HDT_COMPRESSION_METHODS | COTTAS_COMPRESSION_METHODS +RDF_COMPRESSION_CHOICES = {"gzip", "brotli"} +REPRESENTATION_CHOICES = {"hdt", "cottas"} +ARTIFACT_COMPRESSION_CHOICES = {"gzip", "brotli"} +DEFAULT_RDF_COMPRESSION = "gzip,brotli" +DEFAULT_REPRESENTATIONS = "hdt" +DEFAULT_ARTIFACT_COMPRESSION = "none" +HDT_STRATEGY_CHOICES = {"auto", "single", "partitioned"} +DEFAULT_HDT_STRATEGY = "auto" +RDF_STORAGE_MODES = {"space-optimized", "plain"} +DEFAULT_CHUNK_TARGET_BYTES = 512 * 1024 * 1024 +DEFAULT_CHUNK_MIN_BYTES = 128 * 1024 * 1024 +DEFAULT_CHUNK_MAX_BYTES = 1024 * 1024 * 1024 +HDT_INDEX_HELPER_CONTAINER = "/opt/vcf-rdfizer/ensure_hdt_index.sh" +PARTITIONED_COMPRESSION_RUNNER_CONTAINER = "/opt/vcf-rdfizer/partitioned_compression.py" # --------------------------------------------------------------------------- @@ -578,6 +634,257 @@ def file_size_bytes(path: Path): return path.stat().st_size +def find_hdt_index_sidecar(hdt_path: Path) -> Path | None: + """Return HDT Java's non-empty versioned index sidecar.""" + for candidate in sorted(hdt_path.parent.glob(f"{hdt_path.name}.index.*")): + size = file_size_bytes(candidate) + if size is not None and size > 0: + return candidate + return None + + +def write_nt_chunk(chunk_path: Path, source_paths: list[Path]) -> int: + """Concatenate one or more RDF files into a chunk-local `.nt` input.""" + ensure_dir(chunk_path.parent) + total_bytes = 0 + with chunk_path.open("w", encoding="utf-8") as out_handle: + for source_path in source_paths: + with source_path.open("r", encoding="utf-8", errors="replace") as in_handle: + for line in in_handle: + out_handle.write(line) + total_bytes += len(line.encode("utf-8")) + return total_bytes + + +def split_nt_file_for_hdt( + source_path: Path, + chunk_dir: Path, + *, + target_bytes: int, + max_bytes: int, +) -> list[Path]: + """Split an oversized RDF file into line-preserving chunk files for HDT conversion.""" + ensure_dir(chunk_dir) + chunk_paths: list[Path] = [] + chunk_handle = None + chunk_path = None + chunk_size = 0 + + def open_chunk(index: int): + path = chunk_dir / f"{source_path.stem}.split-{index:05d}.nt" + return path, path.open("w", encoding="utf-8") + + try: + with source_path.open("r", encoding="utf-8", errors="replace") as in_handle: + chunk_index = 0 + for line in in_handle: + line_size = len(line.encode("utf-8")) + if chunk_handle is None: + chunk_path, chunk_handle = open_chunk(chunk_index) + chunk_paths.append(chunk_path) + chunk_size = 0 + chunk_index += 1 + elif chunk_size > 0 and ( + chunk_size >= target_bytes or chunk_size + line_size > max_bytes + ): + chunk_handle.close() + chunk_path, chunk_handle = open_chunk(chunk_index) + chunk_paths.append(chunk_path) + chunk_size = 0 + chunk_index += 1 + + chunk_handle.write(line) + chunk_size += line_size + finally: + if chunk_handle is not None and not chunk_handle.closed: + chunk_handle.close() + + return chunk_paths or [source_path] + + +def iter_rdf_binary_lines(path: Path): + """Yield RDF records from plain or gzip-compressed line-oriented RDF.""" + opener = gzip.open if path.name.endswith(".gz") else Path.open + with opener(path, "rb") as handle: + for line in handle: + yield line + + +def plan_record_safe_rdf_chunks( + source_paths: list[Path], + chunk_dir: Path, + *, + target_bytes: int, + min_bytes: int, + max_bytes: int, + guide_path: Path | None = None, +) -> tuple[list[Path], dict]: + """Create bounded RDF chunks without splitting a line-level statement. + + The guide is written as boundaries are discovered during this single + sequential pass. A separate pre-scan would read/decompress the complete + aggregate twice, so the guide and chunk files are produced together. + Logical offsets are uncompressed offsets and therefore work for both plain + and gzip-backed aggregate sources. + """ + if not source_paths: + return [], {"source_file_count": 0, "chunk_count": 0, "chunk_input_bytes": 0} + if target_bytes <= 0 or min_bytes <= 0 or max_bytes <= 0: + raise ValueError("RDF chunk sizes must be positive.") + if min_bytes > target_bytes or target_bytes > max_bytes: + raise ValueError("RDF chunk sizes must satisfy min <= target <= max.") + + ensure_dir(chunk_dir) + chunk_paths: list[Path] = [] + guide_chunks: list[dict] = [] + chunk_handle = None + chunk_path = None + chunk_size = 0 + chunk_start_offset = 0 + chunk_start_record = 0 + logical_offset = 0 + record_count = 0 + total_bytes = 0 + chunk_index = 0 + + def close_chunk(): + nonlocal chunk_handle, chunk_path, chunk_size + if chunk_handle is None or chunk_path is None: + return + chunk_handle.close() + chunk_paths.append(chunk_path) + guide_chunks.append( + { + "chunk_id": len(guide_chunks), + "path": str(chunk_path), + "start_record": chunk_start_record, + "end_record": record_count, + "start_uncompressed_byte": chunk_start_offset, + "end_uncompressed_byte": logical_offset, + "record_count": record_count - chunk_start_record, + "payload_bytes": chunk_size, + } + ) + chunk_handle = None + chunk_path = None + chunk_size = 0 + + try: + for source_path in source_paths: + for line in iter_rdf_binary_lines(source_path): + if not line.endswith(b"\n"): + raise ValueError( + f"RDF source contains a non-line-terminated record: {source_path}" + ) + line_size = len(line) + if chunk_handle is None: + chunk_path = chunk_dir / f"chunk-{chunk_index:05d}.nt" + chunk_index += 1 + chunk_handle = chunk_path.open("wb") + chunk_start_offset = logical_offset + chunk_start_record = record_count + elif chunk_size > 0 and ( + (chunk_size >= target_bytes and chunk_size >= min_bytes) + or chunk_size + line_size > max_bytes + ): + close_chunk() + chunk_path = chunk_dir / f"chunk-{chunk_index:05d}.nt" + chunk_index += 1 + chunk_handle = chunk_path.open("wb") + chunk_start_offset = logical_offset + chunk_start_record = record_count + + chunk_handle.write(line) + chunk_size += line_size + logical_offset += line_size + total_bytes += line_size + record_count += 1 + finally: + close_chunk() + + plan = { + "source_file_count": len(source_paths), + "source_paths": [str(path) for path in source_paths], + "chunk_count": len(chunk_paths), + "chunk_input_bytes": total_bytes, + "record_count": record_count, + "target_chunk_bytes": target_bytes, + "min_chunk_bytes": min_bytes, + "max_chunk_bytes": max_bytes, + "chunks": guide_chunks, + } + if guide_path is not None: + ensure_dir(guide_path.parent) + plan["guide_path"] = str(guide_path) + guide_path.write_text(json.dumps(plan, indent=2) + "\n", encoding="utf-8") + return chunk_paths, plan + + +def plan_partitioned_hdt_chunks( + rdf_paths: list[Path], + chunk_dir: Path, + *, + target_bytes: int, + min_bytes: int, + max_bytes: int, +) -> tuple[list[Path], dict]: + """Plan chunk-local `.nt` inputs for partitioned HDT generation. + + The goal is to keep HDT conversion work units small enough to be fast, + while also avoiding the pathological "many tiny HDTs" case. Existing RDF + part files are treated as the first split boundary, and only oversized + parts are re-split on line boundaries. + """ + ensure_dir(chunk_dir) + + prepared_inputs: list[tuple[Path, int]] = [] + for rdf_path in rdf_paths: + size = int(file_size_bytes(rdf_path) or 0) + if size <= max_bytes: + prepared_inputs.append((rdf_path, size)) + continue + for split_path in split_nt_file_for_hdt( + rdf_path, + chunk_dir / "_split_inputs", + target_bytes=target_bytes, + max_bytes=max_bytes, + ): + prepared_inputs.append((split_path, int(file_size_bytes(split_path) or 0))) + + chunk_groups: list[list[tuple[Path, int]]] = [] + current_group: list[tuple[Path, int]] = [] + current_size = 0 + for path, size in prepared_inputs: + if not current_group: + current_group = [(path, size)] + current_size = size + continue + if current_size < min_bytes or current_size + size <= target_bytes: + current_group.append((path, size)) + current_size += size + continue + chunk_groups.append(current_group) + current_group = [(path, size)] + current_size = size + if current_group: + chunk_groups.append(current_group) + + chunk_inputs: list[Path] = [] + chunk_input_bytes = 0 + for chunk_index, group in enumerate(chunk_groups): + chunk_path = chunk_dir / f"chunk-{chunk_index:05d}.nt" + chunk_input_bytes += write_nt_chunk(chunk_path, [path for path, _size in group]) + chunk_inputs.append(chunk_path) + + plan = { + "source_file_count": len(rdf_paths), + "prepared_input_count": len(prepared_inputs), + "chunk_count": len(chunk_inputs), + "chunk_input_bytes": chunk_input_bytes, + } + return chunk_inputs, plan + + def count_triples_in_nt_files(paths: list[Path]) -> int | None: """Count triples in RDF line-oriented files as a fallback when metrics are missing.""" total = 0 @@ -587,7 +894,8 @@ def count_triples_in_nt_files(paths: list[Path]) -> int | None: if not path.exists() or not path.is_file(): continue try: - with path.open("r", encoding="utf-8", errors="replace") as handle: + opener = gzip.open if path.name.endswith(".gz") else Path.open + with opener(path, "rt", encoding="utf-8", errors="replace") as handle: for line in handle: if pattern.match(line): total += 1 @@ -805,9 +1113,9 @@ def print_nt_hdt_summary( else: artifact_text = f"{format_bytes(int(size))} ({artifact_path})" - if method in {"hdt", "hdt_gzip", "hdt_brotli"}: + if method in HDT_COMPRESSION_METHODS | COTTAS_COMPRESSION_METHODS: source = str(result.get("source", "")).strip() - if not source and "hdt" in results: + if not source and method in HDT_COMPRESSION_METHODS and "hdt" in results: source = str(results.get("hdt", {}).get("source", "")).strip() if source == "existing": artifact_text = f"{artifact_text} (reused existing HDT)" @@ -824,10 +1132,23 @@ def rdf_label_for_path(path: Path) -> str: return "RDF" +def rdf_output_basename(path: Path) -> str: + """Return the common output basename for ``.nt`` and ``.nt.gz`` RDF.""" + if path.name.endswith(".nt.gz"): + return path.name[: -len(".nt.gz")] + if path.name.endswith(".nt"): + return path.name[: -len(".nt")] + return path.stem + + def compression_artifact_name_for_method(path: Path, method: str) -> str: """Compute expected compressed artifact filename for a method.""" - stem = path.stem - ext = path.suffix.lstrip(".") or "nt" + if path.name.endswith(".nt.gz"): + stem = rdf_output_basename(path) + ext = "nt" + else: + stem = rdf_output_basename(path) + ext = path.suffix.lstrip(".") or "nt" if method == "gzip": return f"{stem}.{ext}.gz" if method == "brotli": @@ -838,18 +1159,87 @@ def compression_artifact_name_for_method(path: Path, method: str) -> str: return f"{stem}.hdt.gz" if method == "hdt_brotli": return f"{stem}.hdt.br" + if method == "cottas": + return f"{stem}.cottas" + if method == "cottas_gzip": + return f"{stem}.cottas.gz" + if method == "cottas_brotli": + return f"{stem}.cottas.br" return f"{stem}.{method}" +def planned_output_paths( + *, + out_dir: Path, + output_name: str, + rdf_name: str | None, + methods: list[str], + partitioned: bool, +) -> set[Path]: + """List final output paths that a compression plan would create.""" + target_dir = out_dir / output_name + planned: set[Path] = set() + if rdf_name is not None: + planned.add(target_dir / rdf_name) + + rdf_path = Path(rdf_name or f"{output_name}.nt") + planned.update( + target_dir / compression_artifact_name_for_method(rdf_path, method) + for method in methods + ) + if any(method in HDT_COMPRESSION_METHODS for method in methods): + planned.add(target_dir / f"{output_name}.hdt") + # The pinned HDT Java package generates this versioned sidecar. + planned.add(target_dir / f"{output_name}.hdt.index.v1-1") + if any(method in COTTAS_COMPRESSION_METHODS for method in methods): + planned.add(target_dir / f"{output_name}.cottas") + if partitioned: + planned.add(target_dir / f".{safe_metrics_name(output_name)}.partitioned-results.json") + return planned + + +def validate_no_output_collisions(plans: dict[str, set[Path]]): + """Fail before execution rather than overwriting planned output artifacts.""" + claimed_by: dict[Path, list[str]] = {} + existing: set[Path] = set() + for owner, paths in plans.items(): + for path in paths: + claimed_by.setdefault(path, []).append(owner) + if path.exists(): + existing.add(path) + if path.parent.exists() and not path.parent.is_dir(): + existing.add(path.parent) + if ".hdt.index." in path.name: + hdt_name = path.name.split(".index.", 1)[0] + existing.update(path.parent.glob(f"{hdt_name}.index.*")) + + duplicate_plans = [path for path, owners in claimed_by.items() if len(owners) > 1] + if not existing and not duplicate_plans: + return + + conflicts = sorted({*existing, *duplicate_plans}, key=lambda path: str(path)) + listed = ", ".join(str(path) for path in conflicts) + raise ValueError( + "Refusing to overwrite existing output file(s): " + f"{listed}. VCF-RDFizer does not overwrite outputs; choose a different " + "--out directory or rename/remove the conflicting file(s) and try again." + ) + + def compression_method_label_for_path(path: Path, method: str) -> str: """Return human-readable compression method label for a path.""" ext = path.suffix.lstrip(".") or "nt" + if path.name.endswith(".nt.gz"): + ext = "nt" labels = { "gzip": f"gzip (.{ext}.gz)", "brotli": f"brotli (.{ext}.br)", "hdt": "HDT (.hdt)", "hdt_gzip": "gzip-on-HDT (.hdt.gz)", "hdt_brotli": "brotli-on-HDT (.hdt.br)", + "cottas": "COTTAS (.cottas)", + "cottas_gzip": "gzip-on-COTTAS (.cottas.gz)", + "cottas_brotli": "brotli-on-COTTAS (.cottas.br)", } return labels.get(method, method) @@ -956,16 +1346,17 @@ def cleanup_interrupted_full_run( run_tracker: RunTracker, out_root: Path, image_ref: str | None, - keep_rdf: bool, + keep_rmlstreamer_rdf_output: bool, wrapper_log_path: Path, ): """Best-effort cleanup for full-mode interruption. - Removes tracked intermediates (and raw RDF artifacts when `keep_rdf` is not set), + Removes tracked intermediates (and raw RDF artifacts when + `keep_rmlstreamer_rdf_output` is not set), then records a compact cleanup summary in the run progress log. """ targets: set[Path] = set(run_tracker.intermediate_paths) - if not keep_rdf: + if not keep_rmlstreamer_rdf_output: targets.update(run_tracker.raw_rdf_paths) removed = 0 @@ -985,7 +1376,10 @@ def cleanup_interrupted_full_run( failed += 1 run_tracker.mark( - f"Interrupt cleanup finished: removed={removed}, failed={failed}, keep_rdf={str(keep_rdf).lower()}" + "Interrupt cleanup finished: " + f"removed={removed}, failed={failed}, " + "keep_rmlstreamer_rdf_output=" + f"{str(keep_rmlstreamer_rdf_output).lower()}" ) return removed, failed @@ -1301,7 +1695,7 @@ def resolve_image_ref(image: str, image_version: str | None): def parse_compression_methods(raw: str): - """Parse and validate compression method selection from CLI.""" + """Parse internal compression stage names used by the execution helpers.""" value = (raw or "").strip() if value == "" or value == "none": return [] @@ -1313,14 +1707,79 @@ def parse_compression_methods(raw: str): continue if method not in VALID_COMPRESSION_METHODS: raise ValueError( - "Unsupported compression method " - f"'{method}'. Use gzip,brotli,hdt,hdt_gzip,hdt_brotli, or none." + "Unsupported internal compression stage " + f"'{method}'." ) if method not in methods: methods.append(method) return methods +def parse_compression_plan_option(raw: str, *, option_name: str, choices: set[str]): + """Parse one comma-separated public compression-plan option.""" + value = (raw or "").strip() + if value == "" or value == "none": + return [] + + values = [] + for token in value.split(","): + choice = token.strip() + if not choice: + continue + if choice == "none" or choice not in choices: + allowed = ",".join(sorted(choices)) + raise ValueError( + f"Unsupported value '{choice}' for {option_name}. " + f"Use {allowed}, or none." + ) + if choice not in values: + values.append(choice) + return values + + +def build_compression_methods( + *, + rdf_compression: str, + representations: str, + artifact_compression: str, +): + """Translate the public compression plan into internal execution stages. + + Raw RDF codecs are independent from indexed representations. Packaging + codecs are applied to every selected representation, so users do not need + compound names such as ``hdt_gzip`` or ``cottas_brotli``. + """ + raw_methods = parse_compression_plan_option( + rdf_compression, + option_name="--rdf-compression", + choices=RDF_COMPRESSION_CHOICES, + ) + selected_representations = parse_compression_plan_option( + representations, + option_name="--representations", + choices=REPRESENTATION_CHOICES, + ) + packaging_methods = parse_compression_plan_option( + artifact_compression, + option_name="--artifact-compression", + choices=ARTIFACT_COMPRESSION_CHOICES, + ) + if packaging_methods and not selected_representations: + raise ValueError( + "--artifact-compression requires at least one value in --representations." + ) + + methods = list(raw_methods) + for representation in selected_representations: + if representation not in methods: + methods.append(representation) + for packaging_method in packaging_methods: + compound = f"{representation}_{packaging_method}" + if compound not in methods: + methods.append(compound) + return methods + + def parse_positive_int(value: str, *, name: str) -> int: """Parse a strictly positive integer CLI value.""" try: @@ -1332,6 +1791,35 @@ def parse_positive_int(value: str, *, name: str) -> int: return parsed +def compression_uses_partitioning(methods: list[str]) -> bool: + """Return whether selected methods need bounded RDF chunks.""" + return any(method in PARTITIONED_COMPRESSION_METHODS for method in methods) + + +def compression_uses_hdt(methods: list[str]) -> bool: + """Return whether any selected compression step depends on HDT generation.""" + return any(method in HDT_COMPRESSION_METHODS for method in methods) + + +def should_use_partitioned_hdt( + *, + mode: str, + methods: list[str], + hdt_strategy: str, + rdf_storage_mode: str | None = None, +) -> bool: + """Resolve whether the HDT pipeline should use chunked generation + HDTCat.""" + if not compression_uses_hdt(methods): + return False + if hdt_strategy == "single": + return False + if hdt_strategy == "partitioned": + return True + # `auto` uses partitioned conversion for the aggregate storage modes, while + # ordinary compression-only inputs remain single-pass unless requested. + return mode == "full" and rdf_storage_mode in RDF_STORAGE_MODES + + def safe_metrics_name(value: str) -> str: """Sanitize names used in metrics artifact filenames.""" safe = re.sub(r"[^A-Za-z0-9._-]+", "_", value).strip("_") @@ -1351,6 +1839,10 @@ def metrics_header_for_methods(selected_methods: list[str]) -> list[str]: if "brotli" in methods: header.extend(COMPRESSION_METHOD_COLUMNS["brotli"]) + uses_cottas = any(method in COTTAS_COMPRESSION_METHODS for method in methods) + if uses_cottas: + header.extend(COMPRESSION_METHOD_COLUMNS["cottas"]) + uses_hdt = any(method in HDT_COMPRESSION_METHODS for method in methods) if uses_hdt: header.extend(COMPRESSION_METHOD_COLUMNS["hdt"]) @@ -1359,6 +1851,10 @@ def metrics_header_for_methods(selected_methods: list[str]) -> list[str]: header.extend(COMPRESSION_METHOD_COLUMNS["hdt_gzip"]) if "hdt_brotli" in methods: header.extend(COMPRESSION_METHOD_COLUMNS["hdt_brotli"]) + if "cottas_gzip" in methods: + header.extend(COMPRESSION_METHOD_COLUMNS["cottas_gzip"]) + if "cottas_brotli" in methods: + header.extend(COMPRESSION_METHOD_COLUMNS["cottas_brotli"]) return unique_in_order(header) @@ -1450,6 +1946,18 @@ def update_metrics_csv_with_compression( "max_rss_kb_brotli": "null", "max_rss_kb_hdt": "null", "hdt_source": "not_used", + "source_triples_hdt": "null", + "decoded_triples_hdt": "null", + "validation_hdt": "null", + "cottas_size_bytes": "0", + "exit_code_cottas": "0", + "wall_seconds_cottas": "null", + "user_seconds_cottas": "null", + "sys_seconds_cottas": "null", + "max_rss_kb_cottas": "null", + "source_triples_cottas": "null", + "decoded_triples_cottas": "null", + "validation_cottas": "null", "gzip_on_hdt_size_bytes": "0", "brotli_on_hdt_size_bytes": "0", "exit_code_gzip_on_hdt": "0", @@ -1507,7 +2015,23 @@ def assign_timing(prefix: str, result: dict): if rss_col in row: row[rss_col] = "null" if rss_val is None else str(int(rss_val)) - for method in ("gzip", "brotli", "hdt"): + def assign_validation(method: str, result: dict): + validation = result.get("validation") or result.get("details", {}).get("validation") or {} + if not validation: + return + source_key = f"source_triples_{method}" + decoded_key = f"decoded_triples_{method}" + valid_key = f"validation_{method}" + if source_key in row: + source = validation.get("source_triples") + row[source_key] = "null" if source is None else str(int(source)) + if decoded_key in row: + decoded = validation.get("decoded_triples") + row[decoded_key] = "null" if decoded is None else str(int(decoded)) + if valid_key in row: + row[valid_key] = "true" if validation.get("valid") and validation.get("count_match") else "false" + + for method in ("gzip", "brotli", "hdt", "cottas"): result = method_results.get(method) if result is None: continue @@ -1518,6 +2042,8 @@ def assign_timing(prefix: str, result: dict): if exit_key in row: row[exit_key] = str(int(result.get("exit_code") or 0)) assign_timing(method, result) + if method in {"hdt", "cottas"}: + assign_validation(method, result) hdt_result = method_results.get("hdt") if hdt_result is not None and "hdt_source" in row: @@ -1539,6 +2065,22 @@ def assign_timing(prefix: str, result: dict): row["exit_code_brotli_on_hdt"] = str(int(hdt_brotli_result.get("exit_code") or 0)) assign_timing("brotli_on_hdt", hdt_brotli_result) + cottas_gzip_result = method_results.get("cottas_gzip") + if cottas_gzip_result is not None: + if "gzip_on_cottas_size_bytes" in row: + row["gzip_on_cottas_size_bytes"] = str(int(cottas_gzip_result.get("output_size_bytes") or 0)) + if "exit_code_gzip_on_cottas" in row: + row["exit_code_gzip_on_cottas"] = str(int(cottas_gzip_result.get("exit_code") or 0)) + assign_timing("gzip_on_cottas", cottas_gzip_result) + + cottas_brotli_result = method_results.get("cottas_brotli") + if cottas_brotli_result is not None: + if "brotli_on_cottas_size_bytes" in row: + row["brotli_on_cottas_size_bytes"] = str(int(cottas_brotli_result.get("output_size_bytes") or 0)) + if "exit_code_brotli_on_cottas" in row: + row["exit_code_brotli_on_cottas"] = str(int(cottas_brotli_result.get("exit_code") or 0)) + assign_timing("brotli_on_cottas", cottas_brotli_result) + with metrics_csv.open("w", newline="", encoding="utf-8") as handle: writer = csv.DictWriter(handle, fieldnames=target_header) writer.writeheader() @@ -1574,6 +2116,16 @@ def write_compression_metrics_artifacts( f"output_path={result.get('output_path', '')}", f"output_size_bytes={result.get('output_size_bytes', 0)}", ] + validation = result.get("validation") or result.get("details", {}).get("validation") + if validation: + lines.extend( + [ + f"validation_valid={str(bool(validation.get('valid'))).lower()}", + f"validation_count_match={str(bool(validation.get('count_match'))).lower()}", + f"source_triples={validation.get('source_triples', 'null')}", + f"decoded_triples={validation.get('decoded_triples', 'null')}", + ] + ) time_log.write_text("\n".join(lines) + "\n", encoding="utf-8") gzip_result = method_results.get("gzip", {}) @@ -1581,6 +2133,9 @@ def write_compression_metrics_artifacts( hdt_result = method_results.get("hdt", {}) hdt_gzip_result = method_results.get("hdt_gzip", {}) hdt_brotli_result = method_results.get("hdt_brotli", {}) + cottas_result = method_results.get("cottas", {}) + cottas_gzip_result = method_results.get("cottas_gzip", {}) + cottas_brotli_result = method_results.get("cottas_brotli", {}) def timing_payload(result: dict): return { @@ -1611,11 +2166,21 @@ def timing_payload(result: dict): "exit_code": int(brotli_result.get("exit_code") or 0), "timing": timing_payload(brotli_result), }, + "cottas_conversion": { + "output_cottas_path": cottas_result.get("output_path", ""), + "output_cottas_size_bytes": int(cottas_result.get("output_size_bytes") or 0), + "exit_code": int(cottas_result.get("exit_code") or 0), + "timing": timing_payload(cottas_result), + "validation": cottas_result.get("validation") + or cottas_result.get("details", {}).get("validation"), + }, "hdt_conversion": { "output_hdt_path": hdt_result.get("output_path", ""), "output_hdt_size_bytes": int(hdt_result.get("output_size_bytes") or 0), "exit_code": int(hdt_result.get("exit_code") or 0), "timing": timing_payload(hdt_result), + "validation": hdt_result.get("validation") + or hdt_result.get("details", {}).get("validation"), }, "gzip_on_hdt": { "output_hdt_gz_path": hdt_gzip_result.get("output_path", ""), @@ -1629,6 +2194,18 @@ def timing_payload(result: dict): "exit_code": int(hdt_brotli_result.get("exit_code") or 0), "timing": timing_payload(hdt_brotli_result), }, + "gzip_on_cottas": { + "output_cottas_gz_path": cottas_gzip_result.get("output_path", ""), + "output_cottas_gz_size_bytes": int(cottas_gzip_result.get("output_size_bytes") or 0), + "exit_code": int(cottas_gzip_result.get("exit_code") or 0), + "timing": timing_payload(cottas_gzip_result), + }, + "brotli_on_cottas": { + "output_cottas_br_path": cottas_brotli_result.get("output_path", ""), + "output_cottas_br_size_bytes": int(cottas_brotli_result.get("output_size_bytes") or 0), + "exit_code": int(cottas_brotli_result.get("exit_code") or 0), + "timing": timing_payload(cottas_brotli_result), + }, } metrics_json_dir = metrics_dir / "compression_metrics" / safe_name @@ -1665,6 +2242,7 @@ def write_raw_compression_metrics_artifact( } for method, result in method_results.items(): + details = result.get("details", {}) payload["methods"][method] = { "exit_code": int(result.get("exit_code") or 0), "wall_seconds": result.get("wall_seconds"), @@ -1674,83 +2252,14 @@ def write_raw_compression_metrics_artifact( "output_path": result.get("output_path", ""), "output_size_bytes": int(result.get("output_size_bytes") or 0), "source": result.get("source"), + "details": details, + "validation": result.get("validation") or details.get("validation"), } raw_json = raw_json_dir / f"{run_id}.json" raw_json.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") -def aggregate_method_results_across_files(method_results_by_file: dict[str, dict[str, dict]]): - """Aggregate per-file compression results into one per-method summary. - - This is used for batch RDF layout where one VCF can produce many raw RDF - part files. Aggregation keeps metrics comparable with aggregate layout by - storing one row per sample (not one row per part file). - """ - aggregated: dict[str, dict] = {} - for file_results in method_results_by_file.values(): - for method, result in file_results.items(): - current = aggregated.setdefault( - method, - { - "exit_code": 0, - "wall_seconds": 0.0, - "user_seconds": 0.0, - "sys_seconds": 0.0, - "max_rss_kb": 0, - "output_size_bytes": 0, - "_seen_wall": False, - "_seen_user": False, - "_seen_sys": False, - "_seen_rss": False, - }, - ) - current["exit_code"] = max( - int(current.get("exit_code", 0)), - int(result.get("exit_code") or 0), - ) - wall = result.get("wall_seconds") - if wall is not None: - current["wall_seconds"] = float(current.get("wall_seconds", 0.0)) + float(wall) - current["_seen_wall"] = True - user = result.get("user_seconds") - if user is not None: - current["user_seconds"] = float(current.get("user_seconds", 0.0)) + float(user) - current["_seen_user"] = True - sys_seconds = result.get("sys_seconds") - if sys_seconds is not None: - current["sys_seconds"] = float(current.get("sys_seconds", 0.0)) + float(sys_seconds) - current["_seen_sys"] = True - max_rss = result.get("max_rss_kb") - if max_rss is not None: - current["max_rss_kb"] = max(int(current.get("max_rss_kb", 0)), int(max_rss)) - current["_seen_rss"] = True - current["output_size_bytes"] = int(current.get("output_size_bytes", 0)) + int( - result.get("output_size_bytes") or 0 - ) - - source = result.get("source") - if source is not None: - source = str(source) - prior = current.get("source") - if prior is None: - current["source"] = source - elif prior != source: - current["source"] = "mixed" - - for current in aggregated.values(): - if not current.pop("_seen_wall", False): - current["wall_seconds"] = None - if not current.pop("_seen_user", False): - current["user_seconds"] = None - if not current.pop("_seen_sys", False): - current["sys_seconds"] = None - if not current.pop("_seen_rss", False): - current["max_rss_kb"] = None - - return aggregated - - def validate_mode_dirs(paths): """Validate that expected directory arguments are not file paths.""" for p in paths: @@ -2125,17 +2634,18 @@ def run_compression_methods_for_rdf( run_id: str | None = None, timestamp: str | None = None, output_name: str | None = None, + expected_triples: int | None = None, ): - """Run selected compression methods for a single RDF file. + """Run selected compression stages for a single RDF file. - Supports compound HDT-first methods by reusing an existing `.hdt` when - present, or generating it once and reusing it for subsequent steps. + Representation packaging stages reuse an existing `.hdt` or `.cottas` when + present, or generate it once and reuse it for subsequent packaging steps. Returns `(ok, method_results)`. """ in_dir = rdf_path.parent input_container = f"/data/in/{rdf_path.name}" - input_stem = rdf_path.stem - input_ext = rdf_path.suffix.lstrip(".") or "nt" + input_stem = rdf_output_basename(rdf_path) + input_ext = "nt" if rdf_path.name.endswith(".nt.gz") else rdf_path.suffix.lstrip(".") or "nt" if target_out_dir is None: target_out_dir = out_dir / input_stem ensure_dir(target_out_dir) @@ -2164,11 +2674,21 @@ def run_compression_methods_for_rdf( hdt_container = f"{target_out_container}/{hdt_name}" hdt_is_ready = False hdt_source = "generated" + cottas_name = f"{input_stem}.cottas" + cottas_path = target_out_dir / cottas_name + cottas_container = f"{target_out_container}/{cottas_name}" + cottas_is_ready = False metrics_output_name = output_name or target_out_dir.name safe_output_name = safe_metrics_name(metrics_output_name) safe_rdf_name = safe_metrics_name(rdf_path.name) - def run_container_command(*, method: str, artifact_name: str, command: str): + def run_container_command( + *, + method: str, + artifact_name: str, + command: str, + record_method: bool = True, + ): """Execute one compression command in Docker and capture timing/size.""" timing_name = f".{input_stem}.{method}.time" timing_container = f"{target_out_container}/{timing_name}" @@ -2198,7 +2718,7 @@ def run_container_command(*, method: str, artifact_name: str, command: str): elapsed = time.perf_counter() - started timing = parse_time_log_metrics(timing_host) output_path = target_out_dir / artifact_name - method_results[method] = { + result = { "exit_code": exit_code, # Prefer the wrapper-observed docker runtime over the inner # `/usr/bin/time` elapsed field for long-running jobs. @@ -2209,7 +2729,9 @@ def run_container_command(*, method: str, artifact_name: str, command: str): "output_path": str(output_path), "output_size_bytes": int(file_size_bytes(output_path) or 0), } - if metrics_dir is not None and run_id is not None and timing_host.exists(): + if record_method: + method_results[method] = result + if record_method and metrics_dir is not None and run_id is not None and timing_host.exists(): raw_time_dir = ( metrics_dir / "raw_metrics" @@ -2229,20 +2751,78 @@ def run_container_command(*, method: str, artifact_name: str, command: str): timing_host.unlink() except OSError: pass - if method == "hdt": + if record_method and method == "hdt": method_results[method]["source"] = "generated" - if exit_code != 0: + if exit_code != 0 and record_method: eprint(f"Error: {method} compression failed. See log: {wrapper_log_path}") return False return True + def validate_container_artifact( + *, + method: str, + artifact_name: str, + artifact_format: str, + artifact_container: str, + ) -> tuple[bool, dict]: + """Validate a base representation before any packaging or cleanup.""" + report_name = f".{input_stem}.{artifact_format}.validation.json" + report_path = target_out_dir / report_name + report_container = f"{target_out_container}/{report_name}" + command_parts = [ + "set -euo pipefail;", + f"rm -f {shlex.quote(report_container)};", + 'PYTHON_BIN="${COTTAS_PYTHON_BIN:-$(command -v python3 || true)}";', + 'if [[ -z "$PYTHON_BIN" || ! -x "$PYTHON_BIN" ]]; then ', + 'echo "Missing Python executable in container" >&2; exit 127; fi;', + 'VALIDATOR="/opt/vcf-rdfizer/validate_compression.py";', + 'if [[ ! -f "$VALIDATOR" ]]; then ', + 'echo "Missing compression validator in container" >&2; exit 127; fi;', + '"$PYTHON_BIN" "$VALIDATOR"', + f"--source {shlex.quote(input_container)}", + f"--artifact {shlex.quote(artifact_container)}", + f"--format {shlex.quote(artifact_format)}", + f"--result-path {shlex.quote(report_container)}", + ] + if expected_triples is not None: + command_parts.append(f"--expected-triples {int(expected_triples)}") + command = " ".join(command_parts) + if not run_container_command( + method=f"{method}-validation", + artifact_name=report_name, + command=command, + record_method=False, + ): + report = { + "valid": False, + "count_match": False, + "error": f"{artifact_format.upper()} validation command failed", + } + else: + try: + report = json.loads(report_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + report = { + "valid": False, + "count_match": False, + "error": f"validator did not produce a valid report: {exc}", + } + report_path.unlink(missing_ok=True) + valid = bool(report.get("valid")) and bool(report.get("count_match")) + if not valid: + eprint( + f"Error: {artifact_format.upper()} validation failed for {artifact_name}: " + f"{report.get('error', 'decoded triple count mismatch')}. " + f"See log: {wrapper_log_path}" + ) + return valid, report + def ensure_hdt_available(): """Ensure `.hdt` exists for HDT-based compound methods.""" nonlocal hdt_is_ready, hdt_source if hdt_is_ready: return True if hdt_path.exists(): - hdt_is_ready = True hdt_source = "existing" method_results.setdefault( "hdt", @@ -2256,6 +2836,16 @@ def ensure_hdt_available(): "output_size_bytes": int(file_size_bytes(hdt_path) or 0), }, ) + valid, report = validate_container_artifact( + method="hdt", + artifact_name=hdt_name, + artifact_format="hdt", + artifact_container=hdt_container, + ) + if not valid: + return False + method_results["hdt"]["validation"] = report + hdt_is_ready = True return True hdt_command = ( "set -euo pipefail; " @@ -2274,14 +2864,99 @@ def ensure_hdt_available(): ) if not run_container_command(method="hdt", artifact_name=hdt_name, command=hdt_command): return False + valid, report = validate_container_artifact( + method="hdt", + artifact_name=hdt_name, + artifact_format="hdt", + artifact_container=hdt_container, + ) + if not valid: + return False + method_results["hdt"]["validation"] = report hdt_is_ready = True hdt_source = "generated" return True + def ensure_cottas_available(): + """Ensure `.cottas` exists for COTTAS packaging stages.""" + nonlocal cottas_is_ready + if cottas_is_ready: + return True + if cottas_path.exists(): + method_results.setdefault( + "cottas", + { + "exit_code": 0, + "wall_seconds": 0.0, + "user_seconds": 0.0, + "sys_seconds": 0.0, + "max_rss_kb": 0, + "output_path": str(cottas_path), + "output_size_bytes": int(file_size_bytes(cottas_path) or 0), + "source": "existing", + }, + ) + valid, report = validate_container_artifact( + method="cottas", + artifact_name=cottas_name, + artifact_format="cottas", + artifact_container=cottas_container, + ) + if not valid: + return False + method_results["cottas"]["validation"] = report + cottas_is_ready = True + return True + cottas_command = ( + "set -euo pipefail; " + f"rm -f {shlex.quote(cottas_container)}; " + 'PYTHON_BIN="${COTTAS_PYTHON_BIN:-$(command -v python3 || true)}"; ' + 'if [[ -z "$PYTHON_BIN" || ! -x "$PYTHON_BIN" ]]; then ' + 'echo "Missing pycottas Python executable in container" >&2; exit 127; fi; ' + f'"$PYTHON_BIN" /opt/vcf-rdfizer/cottas_tool.py convert ' + f"{shlex.quote(input_container)} {shlex.quote(cottas_container)} spo" + ) + if not run_container_command( + method="cottas", + artifact_name=cottas_name, + command=cottas_command, + ): + return False + valid, report = validate_container_artifact( + method="cottas", + artifact_name=cottas_name, + artifact_format="cottas", + artifact_container=cottas_container, + ) + if not valid: + return False + method_results["cottas"]["validation"] = report + cottas_is_ready = True + return True + for method in methods: if method == "gzip": artifact_name = f"{input_stem}.{input_ext}.gz" out_container = f"{target_out_container}/{artifact_name}" + if rdf_path.name.endswith(".gz"): + # The space-optimized aggregate is already gzip-compressed. + # Keep it as the selected gzip artifact rather than producing + # a redundant second gzip layer. + if rdf_path.resolve() != (target_out_dir / artifact_name).resolve(): + shutil.copyfile(rdf_path, target_out_dir / artifact_name) + method_results[method] = { + "exit_code": 0, + "wall_seconds": 0.0, + "user_seconds": 0.0, + "sys_seconds": 0.0, + "max_rss_kb": 0, + "output_path": str(target_out_dir / artifact_name), + "output_size_bytes": int(file_size_bytes(target_out_dir / artifact_name) or 0), + "source": "already_compressed", + } + if status_indent is not None: + print(f"{status_indent}- {method}: {artifact_name} {success_symbol()} (already compressed)") + continue command = ( "set -euo pipefail; " f"rm -f {shlex.quote(out_container)}; " @@ -2296,10 +2971,15 @@ def ensure_hdt_available(): if method == "brotli": artifact_name = f"{input_stem}.{input_ext}.br" out_container = f"{target_out_container}/{artifact_name}" + input_command = ( + f"gzip -dc {shlex.quote(input_container)}" + if rdf_path.name.endswith(".gz") + else f"cat {shlex.quote(input_container)}" + ) command = ( "set -euo pipefail; " f"rm -f {shlex.quote(out_container)}; " - f"brotli -q 7 -c {shlex.quote(input_container)} > {shlex.quote(out_container)}" + f"{input_command} | brotli -q 7 -c > {shlex.quote(out_container)}" ) if not run_container_command(method=method, artifact_name=artifact_name, command=command): return False, method_results @@ -2307,6 +2987,14 @@ def ensure_hdt_available(): print(f"{status_indent}- {method}: {artifact_name} {success_symbol()}") continue + if method == "cottas": + if not ensure_cottas_available(): + return False, method_results + if status_indent is not None: + suffix = " (reused existing COTTAS)" if method_results[method].get("source") == "existing" else "" + print(f"{status_indent}- {method}: {cottas_name} {success_symbol()}{suffix}") + continue + if method == "hdt": if not ensure_hdt_available(): return False, method_results @@ -2349,6 +3037,38 @@ def ensure_hdt_available(): print(f"{status_indent}- {method}: {artifact_name} {success_symbol()}{suffix}") continue + if method == "cottas_gzip": + if not ensure_cottas_available(): + return False, method_results + artifact_name = f"{input_stem}.cottas.gz" + out_container = f"{target_out_container}/{artifact_name}" + command = ( + "set -euo pipefail; " + f"rm -f {shlex.quote(out_container)}; " + f"gzip -c {shlex.quote(cottas_container)} > {shlex.quote(out_container)}" + ) + if not run_container_command(method=method, artifact_name=artifact_name, command=command): + return False, method_results + if status_indent is not None: + print(f"{status_indent}- {method}: {artifact_name} {success_symbol()}") + continue + + if method == "cottas_brotli": + if not ensure_cottas_available(): + return False, method_results + artifact_name = f"{input_stem}.cottas.br" + out_container = f"{target_out_container}/{artifact_name}" + command = ( + "set -euo pipefail; " + f"rm -f {shlex.quote(out_container)}; " + f"brotli -q 7 -c {shlex.quote(cottas_container)} > {shlex.quote(out_container)}" + ) + if not run_container_command(method=method, artifact_name=artifact_name, command=command): + return False, method_results + if status_indent is not None: + print(f"{status_indent}- {method}: {artifact_name} {success_symbol()}") + continue + if metrics_dir is not None and run_id is not None and timestamp is not None: write_raw_compression_metrics_artifact( metrics_dir=metrics_dir, @@ -2364,6 +3084,225 @@ def ensure_hdt_available(): return True, method_results +def run_containerized_partitioned_representation_methods( + *, + source_rdf_path: Path, + out_dir: Path, + image_ref: str, + methods: list[str], + wrapper_log_path: Path, + metrics_dir: Path | None = None, + run_id: str | None = None, + timestamp: str | None = None, + output_name: str, + target_chunk_bytes: int, + min_chunk_bytes: int, + max_chunk_bytes: int, + expected_triples: int | None = None, +): + """Run partitioned compression in an ephemeral Docker-managed volume. + + The source and final outputs are the only bind mounts. Chunks, DuckDB + scratch data, HDT/COTTAS merge intermediates, and stage timing files stay + in a named volume that is removed in ``finally`` on both success and + failure. A short JSON handoff carries the container-side metrics back to + the host without exposing the temporary workspace. + """ + ensure_dir(out_dir) + if not source_rdf_path.is_file(): + eprint(f"Error: RDF source file not found: {source_rdf_path}. See log: {wrapper_log_path}") + return False, {} + + safe_output_name = safe_metrics_name(output_name) + volume_name = ( + f"vcf-rdfizer-{safe_output_name[:40]}-{os.getpid()}-{time.time_ns()}" + ) + result_path = out_dir / f".{safe_output_name}.partitioned-results.json" + method_results: dict[str, dict] = {} + volume_created = False + + source_resolved = source_rdf_path.resolve() + out_resolved = out_dir.resolve() + try: + # Mount the source separately, even when it lives below the output + # directory. This keeps the aggregate read-only inside the runner. + source_mount = f"{source_resolved.parent}:/data/in:ro" + source_container = f"/data/in/{source_resolved.name}" + + if run([*docker_cmd_prefix(), "volume", "create", volume_name]) != 0: + eprint(f"Error: unable to create temporary Docker volume '{volume_name}'. See log: {wrapper_log_path}") + return False, {} + volume_created = True + + # A named volume is normally initialized by Docker as root. Make its + # workspace writable for the mapped host user before running COTTAS. + init_cmd = [ + *docker_run_base(as_user=False), + "--mount", + f"type=volume,source={volume_name},target=/work", + image_ref, + "bash", + "-lc", + "chmod 1777 /work", + ] + if run(init_cmd) != 0: + eprint(f"Error: unable to initialize temporary Docker volume '{volume_name}'. See log: {wrapper_log_path}") + return False, {} + + command = [ + *docker_run_base(), + "--mount", + f"type=volume,source={volume_name},target=/work", + ] + if source_mount is not None: + command.extend(["-v", source_mount]) + command.extend( + [ + "-v", + f"{out_resolved}:/data/out", + image_ref, + "python3", + PARTITIONED_COMPRESSION_RUNNER_CONTAINER, + "--source", + source_container, + "--output-dir", + "/data/out", + "--output-name", + output_name, + "--methods", + ",".join(methods), + "--target-chunk-bytes", + str(target_chunk_bytes), + "--min-chunk-bytes", + str(min_chunk_bytes), + "--max-chunk-bytes", + str(max_chunk_bytes), + *(["--expected-triples", str(expected_triples)] if expected_triples is not None else []), + "--result-path", + f"/data/out/{result_path.name}", + ] + ) + run_exit_code = run(command) + payload = None + if result_path.is_file(): + try: + payload = json.loads(result_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + eprint(f"Error: invalid partitioned compression result: {exc}. See log: {wrapper_log_path}") + + if payload is not None: + method_results = payload.get("methods", {}) + for method, result in method_results.items(): + artifact_path = out_dir / compression_artifact_name_for_method( + Path(f"{output_name}.nt"), method + ) + result["output_path"] = str(artifact_path) + result["output_size_bytes"] = int(file_size_bytes(artifact_path) or 0) + details = result.setdefault("details", {}) + if method == "hdt": + index_path = find_hdt_index_sidecar(artifact_path) + details["index_path"] = str(index_path) if index_path else "" + details["index_size_bytes"] = int(file_size_bytes(index_path) or 0) if index_path else 0 + if "source_paths" in details: + details["source_paths"] = [str(source_rdf_path)] + for chunk in details.get("chunks", []): + if isinstance(chunk, dict) and chunk.get("path"): + chunk["path"] = Path(str(chunk["path"])).name + details["workspace"] = "docker-volume" + details["workspace_cleanup"] = "removed" + + missing_methods = set(methods) - set(method_results) + if missing_methods and payload is not None and int(payload.get("exit_code", 1)) == 0: + eprint( + "Error: partitioned compression did not return results for: " + + ", ".join(sorted(missing_methods)) + + f". See log: {wrapper_log_path}" + ) + return False, method_results + + if run_exit_code != 0 or payload is None or int(payload.get("exit_code", 1)) != 0: + error = payload.get("error") if payload else "container did not return a result" + eprint(f"Error: partitioned compression failed: {error}. See log: {wrapper_log_path}") + return False, method_results + + if metrics_dir is not None and run_id is not None and timestamp is not None: + write_raw_compression_metrics_artifact( + metrics_dir=metrics_dir, + run_id=run_id, + timestamp=timestamp, + output_name=output_name, + rdf_name="__partitioned_compression__", + source_rdf_path=out_dir, + selected_methods=methods, + method_results=method_results, + ) + return True, method_results + finally: + # The result handoff is not a user artifact; remove it before volume + # cleanup so only final compression outputs and normal metrics remain. + try: + result_path.unlink() + except OSError: + pass + if volume_created: + cleanup_exit_code = run([*docker_cmd_prefix(), "volume", "rm", "-f", volume_name]) + if cleanup_exit_code != 0: + eprint( + f"Warning: failed to remove temporary Docker volume '{volume_name}'. " + f"See log: {wrapper_log_path}" + ) + + +def run_partitioned_representation_methods_for_rdf_files( + *, + rdf_paths: list[Path], + source_rdf_path: Path | None = None, + out_dir: Path, + image_ref: str, + methods: list[str], + wrapper_log_path: Path, + metrics_dir: Path | None = None, + run_id: str | None = None, + timestamp: str | None = None, + output_name: str, + target_chunk_bytes: int, + min_chunk_bytes: int, + max_chunk_bytes: int, + expected_triples: int | None = None, +): + """Dispatch aggregate RDF to the ephemeral container pipeline. + + Partitioned compression intentionally accepts one logical aggregate. The + full and compression CLI modes create that aggregate before this call; + callers using this helper directly must provide exactly one source path. + This prevents a second host-side implementation from reintroducing large + intermediate files. + """ + if source_rdf_path is None: + if len(rdf_paths) != 1: + eprint( + "Error: partitioned compression requires one aggregate RDF source; " + "combine RDF parts before calling this helper." + ) + return False, {} + source_rdf_path = rdf_paths[0] + return run_containerized_partitioned_representation_methods( + source_rdf_path=source_rdf_path, + out_dir=out_dir, + image_ref=image_ref, + methods=methods, + wrapper_log_path=wrapper_log_path, + metrics_dir=metrics_dir, + run_id=run_id, + timestamp=timestamp, + output_name=output_name, + target_chunk_bytes=target_chunk_bytes, + min_chunk_bytes=min_chunk_bytes, + max_chunk_bytes=max_chunk_bytes, + expected_triples=expected_triples, + ) + + def run_full_mode( *, input_mount_dir: Path, @@ -2376,11 +3315,16 @@ def run_full_mode( metrics_dir: Path, image_ref: str, out_name: str, - rdf_layout: str, - compression: str, + rdf_storage_mode: str, + methods: list[str], + hdt_strategy: str, + chunk_target_bytes: int, + chunk_min_bytes: int, + chunk_max_bytes: int, spark_partitions: int | None, keep_tsv: bool, - keep_rdf: bool, + keep_rmlstreamer_rdf_output: bool, + remove_rdf_storage_output: bool, run_id: str, timestamp: str, wrapper_log_path: Path, @@ -2395,7 +3339,23 @@ def run_full_mode( ensure_dir(out_dir) ensure_dir(metrics_dir) - selected_methods = parse_compression_methods(compression) + selected_methods = list(methods) + use_partitioned_hdt = should_use_partitioned_hdt( + mode="full", + methods=selected_methods, + hdt_strategy=hdt_strategy, + rdf_storage_mode=rdf_storage_mode, + ) + partitioned_methods = [ + method for method in selected_methods if method in PARTITIONED_COMPRESSION_METHODS + ] + non_partitioned_methods = [ + method for method in selected_methods if method not in PARTITIONED_COMPRESSION_METHODS + ] + use_partitioned_compression = ( + use_partitioned_hdt + or any(method in COTTAS_COMPRESSION_METHODS for method in selected_methods) + ) generated_rules_dir = metrics_dir / "_generated_rules" if generated_rules_dir.exists(): @@ -2576,7 +3536,7 @@ def fail_current(stage: str, message: str): "-e", f"OUT_NAME={output_name}", "-e", - f"AGGREGATE_RDF={'1' if rdf_layout == 'aggregate' else '0'}", + f"RDF_STORAGE_MODE={rdf_storage_mode}", "-e", f"SPARK_PARTITIONS={spark_partitions or ''}", "-e", @@ -2654,23 +3614,23 @@ def fail_current(stage: str, message: str): triples_produced = read_conversion_total_triples(metrics_dir, output_name, run_id) - if rdf_layout == "aggregate": - # Aggregate mode yields one merged RDF artifact per sample. - nt_path = out_dir / output_name / f"{output_name}.nt" - if nt_path.exists(): - raw_rdf_files = [nt_path] - else: - raw_rdf_files = [nt_path] + if rdf_storage_mode == "space-optimized": + # The optimized mode leaves one gzip stream assembled from the + # RMLStreamer parts; chunk planning will decompress it incrementally. + raw_rdf_files = [out_dir / output_name / f"{output_name}.nt.gz"] else: - # Batch mode keeps each RMLStreamer part as its own RDF artifact. - raw_rdf_files = sorted((out_dir / output_name).glob("*.nt")) - if not raw_rdf_files: - fail_current( - "rdf-discovery", - f"no RDF part files produced in batch mode for '{output_name}'. " - f"Expected .nt files in {out_dir / output_name}. See log: {wrapper_log_path}", - ) - continue + # Plain storage yields one merged RDF artifact per sample. + raw_rdf_files = [out_dir / output_name / f"{output_name}.nt"] + + missing_raw_rdf = [path for path in raw_rdf_files if not path.is_file()] + if missing_raw_rdf: + fail_current( + "rdf-discovery", + "expected RDF output was not produced: " + + ", ".join(str(path) for path in missing_raw_rdf) + + f". See log: {wrapper_log_path}", + ) + continue if triples_produced is None: triples_produced = count_triples_in_nt_files(raw_rdf_files) @@ -2684,21 +3644,31 @@ def fail_current(stage: str, message: str): run_tracker.track_raw_rdf(raw_rdf_path) method_results_by_file: dict[str, dict[str, dict]] = {} + partitioned_representation_results: dict[str, dict] = {} if selected_methods: - # Compress each produced RDF artifact independently. + # Ordinary methods process the single aggregate. HDT and COTTAS + # consume one shared, record-safe chunk stream when partitioning is active. + per_file_methods = selected_methods + if use_partitioned_compression: + per_file_methods = non_partitioned_methods + for raw_rdf_path in raw_rdf_files: + if not per_file_methods: + method_results_by_file[raw_rdf_path.name] = {} + continue ok, method_results = run_compression_methods_for_rdf( rdf_path=raw_rdf_path, out_dir=out_dir / output_name, target_out_dir=out_dir / output_name, image_ref=image_ref, - methods=selected_methods, + methods=per_file_methods, wrapper_log_path=wrapper_log_path, status_indent=None, metrics_dir=metrics_dir, run_id=run_id, timestamp=timestamp, output_name=output_name, + expected_triples=triples_produced, ) if not ok: fail_current( @@ -2707,6 +3677,29 @@ def fail_current(stage: str, message: str): ) break method_results_by_file[raw_rdf_path.name] = method_results + + if not input_failed and use_partitioned_compression and partitioned_methods: + ok, partitioned_representation_results = run_partitioned_representation_methods_for_rdf_files( + rdf_paths=[], + source_rdf_path=raw_rdf_files[0], + out_dir=out_dir / output_name, + image_ref=image_ref, + methods=partitioned_methods, + wrapper_log_path=wrapper_log_path, + metrics_dir=metrics_dir, + run_id=run_id, + timestamp=timestamp, + output_name=output_name, + target_chunk_bytes=chunk_target_bytes, + min_chunk_bytes=chunk_min_bytes, + max_chunk_bytes=chunk_max_bytes, + expected_triples=triples_produced, + ) + if not ok: + fail_current( + "compression", + f"partitioned compression failed for '{output_name}'. See log: {wrapper_log_path}", + ) if input_failed: continue print(f" * Compression {success_symbol()}") @@ -2718,58 +3711,32 @@ def fail_current(stage: str, message: str): } try: # Persist machine-readable metrics after compression succeeds. - if rdf_layout == "batch": - # Batch mode produces many RDF parts for a single sample. Keep - # one metrics row per sample by aggregating per-part compression - # outputs and timings. - aggregated_results = aggregate_method_results_across_files(method_results_by_file) - combined_size_before_cleanup = sum(raw_size_before_cleanup_by_file.values()) - write_compression_metrics_artifacts( - metrics_dir=metrics_dir, - run_id=run_id, - timestamp=timestamp, - output_name=output_name, - source_rdf_path=out_dir / output_name, - combined_size_bytes=combined_size_before_cleanup, - selected_methods=selected_methods, - method_results=aggregated_results, - ) - update_metrics_csv_with_compression( - metrics_csv=metrics_dir / "metrics.csv", - run_id=run_id, - timestamp=timestamp, - output_name=output_name, - output_dir=out_dir / output_name, - combined_size_bytes=combined_size_before_cleanup, - selected_methods=selected_methods, - method_results=aggregated_results, - tsv_metrics=tsv_metrics, - ) - else: - for raw_rdf_path in raw_rdf_files: - method_results = method_results_by_file.get(raw_rdf_path.name, {}) - source_size_before_cleanup = raw_size_before_cleanup_by_file[raw_rdf_path.name] - write_compression_metrics_artifacts( - metrics_dir=metrics_dir, - run_id=run_id, - timestamp=timestamp, - output_name=raw_rdf_path.stem, - source_rdf_path=raw_rdf_path, - combined_size_bytes=source_size_before_cleanup, - selected_methods=selected_methods, - method_results=method_results, - ) - update_metrics_csv_with_compression( - metrics_csv=metrics_dir / "metrics.csv", - run_id=run_id, - timestamp=timestamp, - output_name=raw_rdf_path.stem, - output_dir=out_dir / output_name, - combined_size_bytes=source_size_before_cleanup, - selected_methods=selected_methods, - method_results=method_results, - tsv_metrics=tsv_metrics, - ) + # Full mode always has one aggregate RDF source, so metrics are + # written once per VCF input regardless of the storage mode. + aggregated_results = dict(method_results_by_file.get(raw_rdf_files[0].name, {})) + aggregated_results.update(partitioned_representation_results) + combined_size_before_cleanup = sum(raw_size_before_cleanup_by_file.values()) + write_compression_metrics_artifacts( + metrics_dir=metrics_dir, + run_id=run_id, + timestamp=timestamp, + output_name=output_name, + source_rdf_path=raw_rdf_files[0], + combined_size_bytes=combined_size_before_cleanup, + selected_methods=selected_methods, + method_results=aggregated_results, + ) + update_metrics_csv_with_compression( + metrics_csv=metrics_dir / "metrics.csv", + run_id=run_id, + timestamp=timestamp, + output_name=output_name, + output_dir=out_dir / output_name, + combined_size_bytes=combined_size_before_cleanup, + selected_methods=selected_methods, + method_results=aggregated_results, + tsv_metrics=tsv_metrics, + ) except PermissionError as exc: blocked_path = exc.filename or str(metrics_dir) eprint("Error: unable to write compression metrics due to file permissions.") @@ -2780,14 +3747,36 @@ def fail_current(stage: str, message: str): ) return 1 - if not keep_rdf and selected_methods: + rdf_storage_removed = False + if selected_methods and ( + remove_rdf_storage_output or not keep_rmlstreamer_rdf_output + ): # Cleanup raw RDF only after every selected compression method has # completed successfully for that specific RDF artifact. cleanup_failed = False + if use_partitioned_compression: + missing_or_failed_hdt = [] + for method in partitioned_methods: + result = partitioned_representation_results.get(method) + if result is None or int(result.get("exit_code", 1)) != 0: + missing_or_failed_hdt.append(method) + if missing_or_failed_hdt: + fail_current( + "rdf-cleanup-validation", + "refusing to remove raw RDF before the final merged compression pipeline " + f"completed successfully for '{output_name}'. Pending/failed: " + f"{', '.join(missing_or_failed_hdt)}. See log: {wrapper_log_path}", + ) + cleanup_failed = True for raw_rdf_path in raw_rdf_files: + if cleanup_failed: + break method_results = method_results_by_file.get(raw_rdf_path.name, {}) + methods_to_validate = selected_methods + if use_partitioned_compression: + methods_to_validate = non_partitioned_methods missing_or_failed = [] - for method in selected_methods: + for method in methods_to_validate: result = method_results.get(method) if result is None or int(result.get("exit_code", 1)) != 0: missing_or_failed.append(method) @@ -2802,6 +3791,13 @@ def fail_current(stage: str, message: str): cleanup_failed = True break + # In space-optimized mode the aggregate `.nt.gz` is itself + # the gzip artifact. Do not remove it when gzip was selected. + if ( + rdf_storage_mode == "space-optimized" + and method_results.get("gzip", {}).get("output_path") == str(raw_rdf_path) + ): + continue if raw_rdf_path.exists(): if not remove_file_with_docker_fallback( path=raw_rdf_path, @@ -2818,30 +3814,47 @@ def fail_current(stage: str, message: str): break if cleanup_failed: continue + rdf_storage_removed = not any(path.exists() for path in raw_rdf_files) - if rdf_layout == "batch" and raw_rdf_files: + if raw_rdf_files: output_root = out_dir / output_name - part_count = len(raw_rdf_files) raw_total_size = sum(raw_size_before_cleanup_by_file.values()) - if keep_rdf: - raw_note = "retained via --keep-rdf" + if keep_rmlstreamer_rdf_output: + raw_note = "retained via --keep-rmlstreamer-rdf-output" + elif rdf_storage_removed and remove_rdf_storage_output: + raw_note = "removed via --remove-rdf-storage-output" + elif rdf_storage_removed and selected_methods: + raw_note = "removed after successful compression" + elif selected_methods and rdf_storage_mode == "space-optimized" and "gzip" in selected_methods: + raw_note = "retained because it is also the selected gzip artifact" elif selected_methods: - raw_note = "removed, set --keep-rdf to retain" + raw_note = "retained" else: raw_note = "kept (compression methods set to none)" first_path = raw_rdf_files[0] print(f" * Output directory: {output_root}") - print(f" - RDF part files: {part_count}") - raw_text = f"{format_bytes(raw_total_size)} across {part_count} files" + print(f" - RDF aggregate: {first_path.name}") + raw_text = format_bytes(raw_total_size) print( - f" - {rdf_label_for_path(first_path)} total: {raw_text} " + f" - {rdf_label_for_path(first_path)}: {raw_text} " f"({raw_note})" ) if selected_methods: for method in selected_methods: + if use_partitioned_compression and method in PARTITIONED_COMPRESSION_METHODS: + result = partitioned_representation_results.get(method) + label = compression_method_label_for_path(first_path, method) + if not result or int(result.get("exit_code", 1)) != 0: + print(f" - {label}: not generated") + else: + print( + f" - {label}: {format_bytes(int(result.get('output_size_bytes') or 0))} " + f"({result.get('output_path', '')})" + ) + continue method_total = 0 method_count = 0 for raw_rdf_path in raw_rdf_files: @@ -2855,10 +3868,7 @@ def fail_current(stage: str, message: str): if method_count == 0: print(f" - {label}: not generated") else: - print( - f" - {label}: {format_bytes(method_total)} " - f"across {method_count} files" - ) + print(f" - {label}: {format_bytes(method_total)}") else: print(" - Compression: none selected") print(f" - Final RDF size (no compression): {format_bytes(raw_total_size)}") @@ -2868,12 +3878,18 @@ def fail_current(stage: str, message: str): rdf_size = file_size_bytes(raw_rdf_path) nt_note = None method_results = method_results_by_file.get(raw_rdf_path.name, {}) - if raw_rdf_path.exists(): - nt_note = "retained via --keep-rdf" if keep_rdf else "retained" - elif not keep_rdf and selected_methods: - nt_note = "removed, set --keep-rdf to retain" - elif not keep_rdf and not selected_methods: + if raw_rdf_path.exists() and keep_rmlstreamer_rdf_output: + nt_note = "retained via --keep-rmlstreamer-rdf-output" + elif raw_rdf_path.exists() and selected_methods and rdf_storage_mode == "space-optimized" and "gzip" in selected_methods: + nt_note = "retained because it is also the selected gzip artifact" + elif not raw_rdf_path.exists() and remove_rdf_storage_output: + nt_note = "removed via --remove-rdf-storage-output" + elif not raw_rdf_path.exists() and selected_methods: + nt_note = "removed after successful compression" + elif not raw_rdf_path.exists() and not selected_methods: nt_note = "kept (compression methods set to none)" + else: + nt_note = "retained" print_nt_hdt_summary( output_root=out_dir / output_name, nt_path=raw_rdf_path, @@ -2969,6 +3985,10 @@ def run_compress_mode( timestamp: str, image_ref: str, methods: list[str], + hdt_strategy: str, + chunk_target_bytes: int, + chunk_min_bytes: int, + chunk_max_bytes: int, wrapper_log_path: Path, ): """Execute compression-only mode for a designated RDF file.""" @@ -2977,6 +3997,13 @@ def run_compress_mode( print("No compression methods selected (`none`). Nothing to do.") return 0 + if rdf_path.name.endswith(".gz") and compression_uses_hdt(methods) and hdt_strategy == "single": + eprint( + "Error: --hdt-strategy single cannot read a gzip aggregate without materializing " + "an uncompressed RDF file; use --hdt-strategy partitioned." + ) + return 2 + if any(method in HDT_COMPRESSION_METHODS for method in methods): file_size = file_size_bytes(rdf_path) or 0 if file_size > 5 * 1024 * 1024 * 1024: @@ -2986,21 +4013,68 @@ def run_compress_mode( ) ensure_dir(out_dir) - input_stem = rdf_path.stem - ok, method_results = run_compression_methods_for_rdf( - rdf_path=rdf_path, - out_dir=out_dir, - image_ref=image_ref, - methods=methods, - wrapper_log_path=wrapper_log_path, - status_indent=" ", - metrics_dir=metrics_dir, - run_id=run_id, - timestamp=timestamp, - output_name=input_stem, + input_stem = rdf_output_basename(rdf_path) + use_partitioned_compression = compression_uses_partitioning(methods) and ( + any(method in COTTAS_COMPRESSION_METHODS for method in methods) + or should_use_partitioned_hdt( + mode="compress", + methods=methods, + hdt_strategy=hdt_strategy, + ) ) - if not ok: - return 1 + if use_partitioned_compression: + raw_methods = [method for method in methods if method not in PARTITIONED_COMPRESSION_METHODS] + partitioned_methods = [method for method in methods if method in PARTITIONED_COMPRESSION_METHODS] + method_results = {} + if raw_methods: + ok, raw_method_results = run_compression_methods_for_rdf( + rdf_path=rdf_path, + out_dir=out_dir, + image_ref=image_ref, + methods=raw_methods, + wrapper_log_path=wrapper_log_path, + status_indent=" ", + metrics_dir=metrics_dir, + run_id=run_id, + timestamp=timestamp, + output_name=input_stem, + ) + if not ok: + return 1 + method_results.update(raw_method_results) + ok, partitioned_results = run_partitioned_representation_methods_for_rdf_files( + rdf_paths=[], + source_rdf_path=rdf_path, + out_dir=out_dir / input_stem, + image_ref=image_ref, + methods=partitioned_methods, + wrapper_log_path=wrapper_log_path, + metrics_dir=metrics_dir, + run_id=run_id, + timestamp=timestamp, + output_name=input_stem, + target_chunk_bytes=chunk_target_bytes, + min_chunk_bytes=chunk_min_bytes, + max_chunk_bytes=chunk_max_bytes, + ) + if not ok: + return 1 + method_results.update(partitioned_results) + else: + ok, method_results = run_compression_methods_for_rdf( + rdf_path=rdf_path, + out_dir=out_dir, + image_ref=image_ref, + methods=methods, + wrapper_log_path=wrapper_log_path, + status_indent=" ", + metrics_dir=metrics_dir, + run_id=run_id, + timestamp=timestamp, + output_name=input_stem, + ) + if not ok: + return 1 target_out_dir = out_dir / input_stem hdt_path = target_out_dir / f"{input_stem}.hdt" @@ -3040,6 +4114,63 @@ def default_decompressed_name(path: Path, fmt: str): return f"{path.stem}.nt" +def run_hdt_index_mode( + *, + hdt_path: Path, + metrics_dir: Path, + image_ref: str, + wrapper_log_path: Path, +): + """Eagerly create the HDT Java index beside an existing HDT file.""" + print("Step 3/3: Initializing HDT index") + ensure_dir(metrics_dir) + existing_index_path = find_hdt_index_sidecar(hdt_path) + index_existed = existing_index_path is not None + source_container = f"/data/hdt/{hdt_path.name}" + command = ( + "set -euo pipefail; " + f"{shlex.quote(HDT_INDEX_HELPER_CONTAINER)} {shlex.quote(source_container)}" + ) + cmd = [ + *docker_run_base(), + "-v", + f"{str(hdt_path.parent)}:/data/hdt", + image_ref, + "bash", + "-lc", + command, + ] + + started = time.perf_counter() + exit_code = run(cmd) + elapsed = time.perf_counter() - started + index_path = find_hdt_index_sidecar(hdt_path) + index_ready = index_path is not None + final_code = int(exit_code) if int(exit_code) != 0 else (0 if index_ready else 1) + payload = { + "hdt_path": str(hdt_path), + "index_path": str(index_path) if index_path else "", + "exit_code": final_code, + "wall_seconds": elapsed, + "index_status": ( + "existing" if index_existed else "generated" + ) if index_ready else "failed", + "index_size_bytes": file_size_bytes(index_path) if index_path else 0, + } + (metrics_dir / "hdt_index_metrics.json").write_text( + json.dumps(payload, indent=2) + "\n", + encoding="utf-8", + ) + + if final_code != 0: + eprint(f"Error: HDT index initialization failed. See log: {wrapper_log_path}") + return 1 + + print(f"HDT index ready: {index_path}") + print(f"HDT index metrics: {metrics_dir / 'hdt_index_metrics.json'}") + return 0 + + def run_decompress_mode( *, compressed_path: Path, @@ -3114,26 +4245,40 @@ def main(): formatter_class=argparse.RawTextHelpFormatter, epilog=( "Examples:\n" - " Full pipeline:\n" - " vcf_rdfizer.py -m full -i ./vcf_files --rdf-layout aggregate -o ./results\n" - " Full pipeline (batch RDF outputs, compress each part):\n" - " vcf_rdfizer.py -m full -i ./vcf_files --rdf-layout batch -c hdt -o ./results\n" - " Full pipeline with partition hint:\n" - " vcf_rdfizer.py -m full -i ./vcf_files --rdf-layout batch -P 8 -c hdt -o ./results\n" + " Full pipeline (plain aggregate):\n" + " vcf_rdfizer.py -m full -i ./vcf_files --rdf-storage-mode plain " + "--representations hdt -o ./results\n" + " Full pipeline (space-optimized aggregate):\n" + " vcf_rdfizer.py -m full -i ./vcf_files --rdf-storage-mode space-optimized " + "--representations hdt,cottas --rdf-compression none -o ./results\n" + " Space-optimized full pipeline with shared HDT/COTTAS chunks:\n" + " vcf_rdfizer.py -m full -i ./vcf_files --rdf-storage-mode space-optimized " + "--representations hdt,cottas --rdf-compression none " + "--chunk-target-bytes 536870912 -o ./results\n" + " Ultra-small full pipeline (remove aggregate RDF after compression):\n" + " vcf_rdfizer.py -m full -i ./vcf_files --rdf-storage-mode space-optimized " + "--representations hdt --rdf-compression none --hdt-strategy partitioned " + "--remove-rdf-storage-output -o ./results\n" + " Queryable HDT plus gzip-packaged HDT:\n" + " vcf_rdfizer.py -m full -i ./vcf_files --rdf-storage-mode space-optimized " + "--rdf-compression none --representations hdt --artifact-compression gzip -o ./results\n" " TSV-only benchmark:\n" " vcf_rdfizer.py -m tsv -i ./vcf_files -o ./results\n" " Compression-only:\n" - " vcf_rdfizer.py -m compress -q ./results/out/sample/sample.nt -c gzip,hdt_gzip -o ./results\n" + " vcf_rdfizer.py -m compress --rdf ./results/out/sample/sample.nt " + "--rdf-compression gzip --representations hdt --artifact-compression gzip -o ./results\n" " Decompression-only:\n" " vcf_rdfizer.py -m decompress -C ./results/out/sample/sample.nt.gz -o ./results\n" + " Initialize an existing HDT index:\n" + " vcf_rdfizer.py -m index -H ./results/sample/sample.hdt -o ./results\n" ), ) parser.add_argument( "-m", "--mode", - choices=["full", "compress", "decompress", "tsv"], + choices=["full", "compress", "decompress", "tsv", "index"], default="full", - help="Run mode: full VCF->RDF pipeline, TSV-only benchmark, compression-only, or decompression-only", + help="Run mode: full pipeline, TSV benchmark, compression, decompression, or HDT index initialization", ) parser.add_argument( "-i", @@ -3142,12 +4287,9 @@ def main(): help="VCF file or directory (required for --mode full and --mode tsv)", ) parser.add_argument( - "-q", "--rdf", - "--nt", - dest="rdf", default=None, - help="Input RDF file (.nt) for --mode compress", + help="Input RDF file (.nt or .nt.gz) for --mode compress", ) parser.add_argument( "-C", @@ -3155,6 +4297,12 @@ def main(): default=None, help="Compressed RDF input (.gz/.br/.hdt) for --mode decompress", ) + parser.add_argument( + "-H", + "--hdt", + default=None, + help="Existing HDT file for --mode index; HDT Java creates a versioned .hdt.index.* sidecar beside it", + ) parser.add_argument( "-d", "--decompress-out", @@ -3168,11 +4316,13 @@ def main(): help="RML mapping rules .ttl (default: /rules/default_rules.ttl)", ) parser.add_argument( - "-l", - "--rdf-layout", - choices=["aggregate", "batch"], + "--rdf-storage-mode", + choices=sorted(RDF_STORAGE_MODES), default=None, - help="Full mode required: aggregate merges RML output parts into one .nt; batch keeps part files separate", + help=( + "Full-mode aggregate storage: plain keeps one .nt; space-optimized builds one .nt.gz " + "from RMLStreamer parts before record-safe HDT/COTTAS chunking" + ), ) parser.add_argument( "-P", @@ -3210,11 +4360,64 @@ def main(): default="rdf", help="Fallback output directory/file basename when a TSV basename cannot be inferred", ) + parser.add_argument( + "--rdf-compression", + default=DEFAULT_RDF_COMPRESSION, + help=( + "Final raw RDF codecs (comma-separated): gzip,brotli, or none " + f"(default: {DEFAULT_RDF_COMPRESSION})" + ), + ) + parser.add_argument( + "--representations", + default=DEFAULT_REPRESENTATIONS, + help=( + "Queryable RDF representations (comma-separated): hdt,cottas, or none " + f"(default: {DEFAULT_REPRESENTATIONS})" + ), + ) + parser.add_argument( + "--artifact-compression", + default=DEFAULT_ARTIFACT_COMPRESSION, + help=( + "Packaging codecs applied to each selected representation (comma-separated): " + "gzip,brotli, or none " + f"(default: {DEFAULT_ARTIFACT_COMPRESSION})" + ), + ) + # Keep the old flat selector only for existing automation. It is hidden + # from help and documentation; new callers should use the three explicit + # compression-plan options above. parser.add_argument( "-c", "--compression", - default="gzip,brotli,hdt", - help="Compression methods (gzip,brotli,hdt,hdt_gzip,hdt_brotli,none)", + dest="legacy_compression", + default=None, + help=argparse.SUPPRESS, + ) + parser.add_argument( + "--hdt-strategy", + choices=sorted(HDT_STRATEGY_CHOICES), + default=DEFAULT_HDT_STRATEGY, + help=( + "HDT generation strategy: auto uses partitioned HDT+HDTCat for full-mode aggregate storage, " + "single uses one rdf2hdt run, partitioned forces chunked HDT generation" + ), + ) + parser.add_argument( + "--chunk-target-bytes", + default=str(DEFAULT_CHUNK_TARGET_BYTES), + help="Target uncompressed RDF bytes per HDT/COTTAS chunk (default: 536870912)", + ) + parser.add_argument( + "--chunk-min-bytes", + default=str(DEFAULT_CHUNK_MIN_BYTES), + help="Minimum uncompressed RDF bytes before flushing a chunk group (default: 134217728)", + ) + parser.add_argument( + "--chunk-max-bytes", + default=str(DEFAULT_CHUNK_MAX_BYTES), + help="Maximum uncompressed RDF bytes allowed in one chunk (default: 1073741824)", ) parser.add_argument("-k", "--keep-tsv", action="store_true", help="Keep TSV intermediates") parser.add_argument( @@ -3223,12 +4426,18 @@ def main(): action="store_true", help="Print a rough storage estimate before running conversion", ) - parser.add_argument( + rdf_output_group = parser.add_mutually_exclusive_group() + rdf_output_group.add_argument( "-R", - "--keep-rdf", - "--keep_rdf", + "--keep-rmlstreamer-rdf-output", + dest="keep_rmlstreamer_rdf_output", + action="store_true", + help="Keep the aggregate RDF output produced by RMLStreamer in full mode", + ) + rdf_output_group.add_argument( + "--remove-rdf-storage-output", action="store_true", - help="Keep raw N-Triples outputs after compression in full mode", + help="Explicitly remove the aggregate .nt/.nt.gz output after successful compression", ) args = parser.parse_args() @@ -3246,16 +4455,33 @@ def main(): metrics_dir = metrics_root / run_id mode = args.mode spark_partitions = None + chunk_target_bytes = DEFAULT_CHUNK_TARGET_BYTES + chunk_min_bytes = DEFAULT_CHUNK_MIN_BYTES + chunk_max_bytes = DEFAULT_CHUNK_MAX_BYTES step1_label = "Step 1/5" if mode == "full" else "Step 1/3" try: + chunk_target_bytes = parse_positive_int( + args.chunk_target_bytes, name="--chunk-target-bytes" + ) + chunk_min_bytes = parse_positive_int( + args.chunk_min_bytes, name="--chunk-min-bytes" + ) + chunk_max_bytes = parse_positive_int( + args.chunk_max_bytes, name="--chunk-max-bytes" + ) + if chunk_min_bytes > chunk_target_bytes: + raise ValueError("--chunk-min-bytes must be <= --chunk-target-bytes") + if chunk_target_bytes > chunk_max_bytes: + raise ValueError("--chunk-target-bytes must be <= --chunk-max-bytes") + # Mode-specific argument validation and canonical path resolution. if mode == "full": if args.input is None: raise ValueError("--input is required in --mode full") - if args.rdf_layout is None: - raise ValueError("--rdf-layout is required in --mode full (aggregate|batch)") + if args.rdf_storage_mode is None: + raise ValueError("--rdf-storage-mode is required in --mode full") input_path = Path(args.input).expanduser().resolve() ( input_mount_dir, @@ -3270,11 +4496,59 @@ def main(): if not rules_path.exists() or not rules_path.is_file(): raise ValueError(f"rules file not found: {rules_path}") validate_mode_dirs([out_root, out_dir, tsv_dir, metrics_root]) - parse_compression_methods(args.compression) + if args.legacy_compression is not None: + if ( + args.rdf_compression != DEFAULT_RDF_COMPRESSION + or args.representations != DEFAULT_REPRESENTATIONS + or args.artifact_compression != DEFAULT_ARTIFACT_COMPRESSION + ): + raise ValueError( + "--compression cannot be combined with --rdf-compression, " + "--representations, or --artifact-compression." + ) + full_methods = parse_compression_methods(args.legacy_compression) + else: + full_methods = build_compression_methods( + rdf_compression=args.rdf_compression, + representations=args.representations, + artifact_compression=args.artifact_compression, + ) + if ( + args.rdf_storage_mode == "space-optimized" + and args.hdt_strategy == "single" + and compression_uses_hdt(full_methods) + ): + raise ValueError( + "--hdt-strategy single cannot be used with space-optimized HDT input; " + "use --hdt-strategy partitioned" + ) if args.spark_partitions is not None: spark_partitions = parse_positive_int( args.spark_partitions, name="--spark-partitions" ) + full_uses_partitioning = compression_uses_partitioning(full_methods) and ( + any(method in COTTAS_COMPRESSION_METHODS for method in full_methods) + or should_use_partitioned_hdt( + mode="full", + methods=full_methods, + hdt_strategy=args.hdt_strategy, + rdf_storage_mode=args.rdf_storage_mode, + ) + ) + output_plans = {} + for index, prefix in enumerate(expected_prefixes, start=1): + output_name = slugify(prefix) or slugify(args.out_name) + rdf_name = f"{output_name}.nt" + if args.rdf_storage_mode == "space-optimized": + rdf_name += ".gz" + output_plans[f"input {index} ({prefix})"] = planned_output_paths( + out_dir=out_dir, + output_name=output_name, + rdf_name=rdf_name, + methods=full_methods, + partitioned=full_uses_partitioning, + ) + validate_no_output_collisions(output_plans) elif mode == "tsv": if args.spark_partitions is not None: raise ValueError("--spark-partitions is only valid in --mode full") @@ -3296,10 +4570,74 @@ def main(): rdf_path = Path(args.rdf).expanduser().resolve() if not rdf_path.exists() or not rdf_path.is_file(): raise ValueError(f"RDF input file not found: {rdf_path}") - if rdf_path.suffix != ".nt": - raise ValueError("Compression input must be a .nt file") - methods = parse_compression_methods(args.compression) + if rdf_path.suffix != ".nt" and not rdf_path.name.endswith(".nt.gz"): + raise ValueError("Compression input must be a .nt or .nt.gz file") + if args.legacy_compression is not None: + if ( + args.rdf_compression != DEFAULT_RDF_COMPRESSION + or args.representations != DEFAULT_REPRESENTATIONS + or args.artifact_compression != DEFAULT_ARTIFACT_COMPRESSION + ): + raise ValueError( + "--compression cannot be combined with --rdf-compression, " + "--representations, or --artifact-compression." + ) + methods = parse_compression_methods(args.legacy_compression) + else: + methods = build_compression_methods( + rdf_compression=args.rdf_compression, + representations=args.representations, + artifact_compression=args.artifact_compression, + ) + if ( + rdf_path.name.endswith(".gz") + and args.hdt_strategy == "single" + and compression_uses_hdt(methods) + ): + raise ValueError( + "--hdt-strategy single cannot be used with a .nt.gz HDT input; " + "use --hdt-strategy partitioned" + ) validate_mode_dirs([out_root, out_dir, metrics_root]) + compression_uses_partitioning_for_input = compression_uses_partitioning(methods) and ( + any(method in COTTAS_COMPRESSION_METHODS for method in methods) + or should_use_partitioned_hdt( + mode="compress", + methods=methods, + hdt_strategy=args.hdt_strategy, + ) + ) + output_name = rdf_output_basename(rdf_path) + validate_no_output_collisions( + { + f"RDF input {rdf_path.name}": planned_output_paths( + out_dir=out_dir, + output_name=output_name, + rdf_name=None, + methods=methods, + partitioned=compression_uses_partitioning_for_input, + ) + } + ) + elif mode == "index": + if args.spark_partitions is not None: + raise ValueError("--spark-partitions is only valid in --mode full") + if not args.hdt: + raise ValueError("--hdt is required in --mode index") + hdt_path = Path(args.hdt).expanduser().resolve() + if not hdt_path.exists() or not hdt_path.is_file(): + raise ValueError(f"HDT input file not found: {hdt_path}") + if hdt_path.suffix != ".hdt": + raise ValueError("HDT index input must end with .hdt") + validate_mode_dirs([out_root, out_dir, metrics_root]) + existing_indexes = sorted(hdt_path.parent.glob(f"{hdt_path.name}.index.*")) + if existing_indexes: + raise ValueError( + "Refusing to overwrite existing output file(s): " + + ", ".join(str(path) for path in existing_indexes) + + ". VCF-RDFizer does not overwrite outputs; rename/remove the " + "existing index and try again." + ) else: if args.spark_partitions is not None: raise ValueError("--spark-partitions is only valid in --mode full") @@ -3321,6 +4659,11 @@ def main(): ) if decompressed_out.exists() and decompressed_out.is_dir(): raise ValueError(f"decompression output path is a directory: {decompressed_out}") + if decompressed_out.exists(): + raise ValueError( + f"Refusing to overwrite existing output file: {decompressed_out}. " + "Choose a different --decompress-out path and try again." + ) if decompressed_out.parent.exists() and not decompressed_out.parent.is_dir(): raise ValueError( f"decompression output parent is not a directory: {decompressed_out.parent}" @@ -3423,6 +4766,12 @@ def execute_mode(): (out_write_target, True), (metrics_write_target, True), ] + elif mode == "index": + metrics_write_target = metrics_dir if metrics_dir.exists() else metrics_dir.parent + writable_targets = [ + (hdt_path.parent, True), + (metrics_write_target, True), + ] else: decompress_parent = ( decompressed_out.parent @@ -3465,11 +4814,16 @@ def execute_mode(): metrics_dir=metrics_dir, image_ref=image_ref, out_name=args.out_name, - rdf_layout=args.rdf_layout, - compression=args.compression, + rdf_storage_mode=args.rdf_storage_mode, + methods=full_methods, + hdt_strategy=args.hdt_strategy, + chunk_target_bytes=chunk_target_bytes, + chunk_min_bytes=chunk_min_bytes, + chunk_max_bytes=chunk_max_bytes, spark_partitions=spark_partitions, keep_tsv=args.keep_tsv, - keep_rdf=args.keep_rdf, + keep_rmlstreamer_rdf_output=args.keep_rmlstreamer_rdf_output, + remove_rdf_storage_output=args.remove_rdf_storage_output, run_id=run_id, timestamp=timestamp, wrapper_log_path=wrapper_log_path, @@ -3499,6 +4853,17 @@ def execute_mode(): timestamp=timestamp, image_ref=image_ref, methods=methods, + hdt_strategy=args.hdt_strategy, + chunk_target_bytes=chunk_target_bytes, + chunk_min_bytes=chunk_min_bytes, + chunk_max_bytes=chunk_max_bytes, + wrapper_log_path=wrapper_log_path, + ) + if mode == "index": + return run_hdt_index_mode( + hdt_path=hdt_path, + metrics_dir=metrics_dir, + image_ref=image_ref, wrapper_log_path=wrapper_log_path, ) # Decompression-only mode. @@ -3528,12 +4893,15 @@ def _interrupt_handler(_signum, _frame): run_tracker=run_tracker, out_root=out_root, image_ref=resolved_image_ref, - keep_rdf=args.keep_rdf, + keep_rmlstreamer_rdf_output=args.keep_rmlstreamer_rdf_output, wrapper_log_path=wrapper_log_path, ) eprint( "Interrupt cleanup summary: " - f"removed={removed}, failed={failed}, keep_rdf={str(args.keep_rdf).lower()}" + "removed=" + f"{removed}, failed={failed}, " + "keep_rmlstreamer_rdf_output=" + f"{str(args.keep_rmlstreamer_rdf_output).lower()}" ) eprint(f"Progress log: {progress_log_path}") finally: diff --git a/vcf_rdfizer_data/rules/__init__.py b/vcf_rdfizer_data/rules/__init__.py new file mode 100644 index 0000000..46df930 --- /dev/null +++ b/vcf_rdfizer_data/rules/__init__.py @@ -0,0 +1 @@ +"""Packaged default RML mapping rules."""