diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index a85a556..14356c7 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -1,6 +1,6 @@ -# Contributing to REPO-NAME +# Contributing to biosim-extractor -Thank you for your interest in contributing to **REPO-NAME**! +Thank you for your interest in contributing to **biosim-extractor**! We’re excited to collaborate with developers, researchers, and community members to make REPO-NAME better for everyone. @@ -62,4 +62,4 @@ Well-documented issues help us address problems faster and keep REPO-NAME stable --- -Thank you for helping improve **REPO-NAME**, your contributions make open source stronger! +Thank you for helping improve **biosim-extractor**, your contributions make open source stronger! diff --git a/.gitignore b/.gitignore index f0fdc86..d81aff7 100644 --- a/.gitignore +++ b/.gitignore @@ -70,6 +70,7 @@ instance/ # Sphinx documentation docs/_build/ +.DS_Store # PyBuilder .pybuilder/ diff --git a/README.md b/README.md index 76d1f75..efc40c4 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,14 @@ biosim-extractor ================ -A repository for extracting simulation data from output files produced from molecular dynamics (MD) simulations of biomolecules and validating against biosim-schem. + +

+ biosim-extractor logo + biosim-extractor logo +

+ +biosim-extractor is a repository for extracting simulation data from output files produced from molecular dynamics (MD) simulations of biomolecules and validating against [biosim-schema](https://github.com/CCPBioSim/biosim-schema). + +See the [biosim-extractor documentation](https://biosim-extractor.readthedocs.io/en/latest/) for more information. ## Project Status diff --git a/biosim_extractor/metadata/filemetadata.py b/biosim_extractor/metadata/filemetadata.py new file mode 100644 index 0000000..ba7b8c7 --- /dev/null +++ b/biosim_extractor/metadata/filemetadata.py @@ -0,0 +1,68 @@ +"""Utility to extract binary file metadata (size, hash).""" + +import hashlib +from pathlib import Path + + +def file_metadata(path, role="other", hash_algorithm="md5"): + """Extract name, size (MB), and hash of a single file. + + Args: + path: Input file path (str or :class:`pathlib.Path`). + role: Optional category label for the file. Defaults to "other". + hash_algorithm: Hash function identifier ('md5', 'sha256'). Default is 'md5'. + + Returns: + Dictionary containing 'file_name', 'file_size' (dict), 'file_hash', + 'file_hash_algorithm', and 'file_role'. + """ + path = Path(path) + digest = hashlib.new(hash_algorithm) + size_mb = path.stat().st_size / 1_000_000 + + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + + return { + "file_name": path.name, + "file_size": {"value": round(size_mb, 6), "value_unit": "MB"}, + "file_hash": digest.hexdigest(), + "file_hash_algorithm": hash_algorithm, + "file_role": role, + } + + +def files_metadata(saved_files): + """Compile metadata for multiple files grouped by role. + + Args: + saved_files: Dictionary mapping roles to lists of file paths + (:data:`str` or :class:`pathlib.Path`). + + Returns: + Flat list of ``file_metadata`` dictionaries for all provided paths. + """ + return [ + file_metadata(path, role) + for role, paths in saved_files.items() + for path in paths + ] + + +def group_files(filepaths, saved_files=None, role="other"): + """Add file paths to a role-based dictionary. + + Args: + filepaths: Iterable of file paths to add. + saved_files: Dictionary mapping roles to lists of file paths. + role: Role name under which to store the file paths. + + Returns: + Updated dictionary of file paths grouped by role. + """ + if saved_files is None: + saved_files = {} + for filepath in filepaths: + saved_files.setdefault(role, []).append(filepath) + return saved_files diff --git a/biosim_extractor/metadata/populatemetadata.py b/biosim_extractor/metadata/populatemetadata.py index 92b044f..a0f64fe 100644 --- a/biosim_extractor/metadata/populatemetadata.py +++ b/biosim_extractor/metadata/populatemetadata.py @@ -13,6 +13,7 @@ from biosim_extractor.helpers.metadata_utils import round_floats from biosim_extractor.mdanalysis.toptraj import TopTrajParser from biosim_extractor.metadata.fetchschema import get_schema, update_schema +from biosim_extractor.metadata.filemetadata import files_metadata, group_files from biosim_extractor.metadata.validatemetadata import validate_metadata from biosim_extractor.units.unitconversion import UnitConverter @@ -200,19 +201,34 @@ def __init__( engine=None, top_file=None, traj_file=None, + store_file_metadata=True, ): - """ - Args: - schema_path: Path to the extraction schema JSON file. - log_file: Path to the MD engine log file. - engine: MD engine name (``"amber"``, ``"gromacs"``). - top_file: Path to the topology file. - traj_file: Path to the trajectory file (or list of paths). + """Orchestrates extraction of MD engine metadata from log files (Amber/GROMACS) + or topology/trajectory inputs, mapping results to the `biosim-schema` format. + + Features supported by this class: + - Reads logs via specialized parsers (`AmberLogParser`, etc.). + - Flattens nested structures and applies reverse-forward schema mappings. + - Handles unit conversions where required (e.g., kcal/mol → eV). + - Stores metadata about input files if `store_file_metadata=True`. + + Use cases: + 1. Batch processing logs via CLI (`--logfile`, `--engine`). + 2. Populating a single simulation from top/traj without logs via MDAnalysis integration. + + Args (constructor): + schema_path: Path to the engine mapping JSON file. + log_file: Optional MD engine log file path. + engine: MD engine name, such as "amber" or "gromacs". + top_file: Optional topology file path. + traj_file: Optional trajectory file path or list of trajectory paths. + store_file_metadata: If True, include input file metadata in the output. """ self.schema_path = schema_path self.log_file = log_file self.top_file = top_file self.traj_file = traj_file + self.store_file_metadata = store_file_metadata self.engine = engine if engine: self.engine = engine.lower() @@ -229,7 +245,6 @@ def populate(self): entries removed. """ self.load_schema() - if self.engine: self.engine_data = self.parse_log() @@ -247,6 +262,18 @@ def populate(self): # self.data["SimulationMetadata"]["@type"] = "SimulationMetadata" result = self.data["SimulationMetadata"] + # save file metadata in dict + if self.store_file_metadata: + saved_files = {} + if self.log_file: + saved_files = group_files([self.log_file], saved_files, role="log") + if self.top_file and self.traj_file: + saved_files = group_files([self.top_file], saved_files, role="topology") + saved_files = group_files( + self.traj_file, saved_files, role="trajectory" + ) + result["files"] = files_metadata(saved_files) + # Remove any dict that contains a None field result = remove_null_parents(result) or {} # Round all floats @@ -490,7 +517,12 @@ def convert_values(self, value, term, is_vector=False): def resolve_schema_inputs(args): - """Resolve mapping and biosim schema paths from args or remote schema bundle.""" + """Resolve mapping and biosim schema paths from args or remote schema bundle. + + If either path argument (`mappingschema`, `biosimschema`) is missing, the function + fetches a bundled schema release (optionally updating if requested). This ensures + downstream processing has valid JSON/YAML sources without requiring manual caching setup + """ mapping_path = args.mappingschema biosim_path = args.biosimschema @@ -559,6 +591,18 @@ def parse_args(): parser.add_argument("--top", help="Topology file path") parser.add_argument("--traj", nargs="+", help="Trajectory file path") parser.add_argument("--config", help="Configuration file path") + parser.add_argument( + "--exclude-file-metadata", + action="store_false", + help="Include flag to include file metadata in metadata output", + ) + parser.add_argument( + "--file-metadata", + dest="store_file_metadata", + action=argparse.BooleanOptionalAction, + default=True, + help="Include file metadata in output (use --no-file-metadata to disable).", + ) parser.add_argument("--output", "-o", help="Output file path") return parser.parse_args() @@ -576,6 +620,7 @@ def main(): engine=args.engine, top_file=args.top, traj_file=args.traj, + store_file_metadata=args.store_file_metadata, ) result = populator.populate() diff --git a/biosim_extractor/metadata/validatemetadata.py b/biosim_extractor/metadata/validatemetadata.py index e81d0f5..ea6ca0b 100644 --- a/biosim_extractor/metadata/validatemetadata.py +++ b/biosim_extractor/metadata/validatemetadata.py @@ -4,11 +4,38 @@ """ import os +import re import warnings +from pathlib import Path from linkml.validator import validate +def extract_schema_version(schema_path): + """Extract the biosim schema version from a local schema YAML file. + + Args: + schema_path (str | Path): Path to the biosim schema YAML file. + + Returns: + str | None: Parsed schema version string, or None if the path is missing, + points to a URL, the file does not exist, or no version field is found. + """ + if not schema_path or str(schema_path).startswith("http"): + return None + + p = Path(schema_path) + if not p.exists(): + return None + + text = p.read_text(encoding="utf-8") + m = re.search( + r"(?m)^version:\s*['\"]?([^\r\n'\"#]+)['\"]?\s*(?:#.*)?$", + text, + ) + return m.group(1).strip() if m else None + + def validate_metadata(result, biosimschema_path=None, strict=False): """Validate a populated metadata dict, optionally against a biosim schema. @@ -48,6 +75,11 @@ def validate_extracted(instance, schema_path): Returns: list: Validation error messages. An empty list means the instance is valid. """ + + schema_version = extract_schema_version(schema_path) + if schema_version and isinstance(instance, dict): + instance.setdefault("biosim_schema_version", schema_version) + errors = [] errors.extend(_validate_all_vector_values(instance)) diff --git a/docs/source/_static/logos/CCPBioSim-logo.png b/docs/source/_static/logos/CCPBioSim-logo.png new file mode 100644 index 0000000..7707537 Binary files /dev/null and b/docs/source/_static/logos/CCPBioSim-logo.png differ diff --git a/docs/source/_static/logos/biosim-extractor-logo-dark-text.png b/docs/source/_static/logos/biosim-extractor-logo-dark-text.png new file mode 100644 index 0000000..630a5c2 Binary files /dev/null and b/docs/source/_static/logos/biosim-extractor-logo-dark-text.png differ diff --git a/docs/source/_static/logos/biosim-extractor-logo-dark.png b/docs/source/_static/logos/biosim-extractor-logo-dark.png new file mode 100644 index 0000000..aa84c9b Binary files /dev/null and b/docs/source/_static/logos/biosim-extractor-logo-dark.png differ diff --git a/docs/source/_static/logos/biosim-extractor-logo-light-text.png b/docs/source/_static/logos/biosim-extractor-logo-light-text.png new file mode 100644 index 0000000..5e4b647 Binary files /dev/null and b/docs/source/_static/logos/biosim-extractor-logo-light-text.png differ diff --git a/docs/source/_static/logos/biosim-extractor-logo-light.png b/docs/source/_static/logos/biosim-extractor-logo-light.png new file mode 100644 index 0000000..7257e74 Binary files /dev/null and b/docs/source/_static/logos/biosim-extractor-logo-light.png differ diff --git a/docs/source/_static/logos/cosec-logo.png b/docs/source/_static/logos/cosec-logo.png new file mode 100644 index 0000000..f46ec6f Binary files /dev/null and b/docs/source/_static/logos/cosec-logo.png differ diff --git a/docs/source/_static/logos/psdi-logo.png b/docs/source/_static/logos/psdi-logo.png new file mode 100644 index 0000000..5513421 Binary files /dev/null and b/docs/source/_static/logos/psdi-logo.png differ diff --git a/docs/source/conf.py b/docs/source/conf.py index aeaccb1..b615c35 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -40,3 +40,7 @@ html_theme = 'furo' html_static_path = ['_static'] +html_theme_options = { + "dark_logo": "logos/biosim-extractor-logo-light-text.png", + "light_logo": "logos/biosim-extractor-logo-dark-text.png", +} diff --git a/docs/source/index.rst b/docs/source/index.rst index 6629fde..8c9311f 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -1,37 +1,32 @@ -.. CCPBioSim-Python-Template documentation master file, created by - sphinx-quickstart on Wed Dec 10 16:51:51 2025. - You can adapt this file completely to your liking, but it should at least - contain the root `toctree` directive. - BioSim Extractor Documentation ============================== -A toolkit for extracting and standardizing metadata from molecular dynamics simulations. +``biosim-extractor`` is available at http://github.com/CCPBioSim/biosim-extractor -Contents --------- +.. image:: /_static/logos/biosim-extractor-logo-dark-text.png + :width: 250px + :align: center + :class: only-light -.. toctree:: - :maxdepth: 2 - :caption: Contents: +.. image:: /_static/logos/biosim-extractor-logo-light-text.png + :width: 250px + :align: center + :class: only-dark - installation - usage - contributing - api +A toolkit for extracting standardised metadata from molecular dynamics (MD) simulations. Overview -------- -BioSim Extractor provides tools to parse, extract, and standardize metadata from MD simulation outputs (AMBER, GROMACS, LAMMPS, etc.), and validate against biosim-schema. +BioSim Extractor provides tools to parse, extract, and standardize metadata from MD simulation outputs (AMBER, GROMACS, etc.), and validate against `biosim-schema `_. Features -------- - Parse log files from multiple MD engines - Extract topology and trajectory metadata -- Standardize units and schema -- Validate metadata against biosim-schema +- Standardise units and schema +- Validate metadata against biosim-schema `biosim-schema `_ Getting Started --------------- @@ -43,9 +38,37 @@ API Reference The full API documentation is available in :doc:`api`. +Contents +-------- + +.. toctree:: + :maxdepth: 2 + :caption: Contents: + + installation + usage + contributing + api + +Funding +======= + +Contributors to biosim-extractor were supported by + +.. image:: _static/logos/psdi-logo.png + :height: 100px + :target: https://www.psdi.ac.uk/ + +.. image:: _static/logos/cosec-logo.png + :height: 100px + :target: https://www.cosec.ac.uk/ + +.. image:: _static/logos/CCPBioSim-logo.png + :height: 100px + :target: https://www.ccpbiosim.ac.uk/ + Indices and tables ------------------ * :ref:`genindex` * :ref:`modindex` -* :ref:`search` diff --git a/docs/source/usage.rst b/docs/source/usage.rst index e07af25..a55afd8 100644 --- a/docs/source/usage.rst +++ b/docs/source/usage.rst @@ -4,31 +4,100 @@ Usage Command Line ------------ -To extract metadata from a log file: +The ``biosim-extractor`` CLI supports both local schema files and auto-fetched schema bundles. + +Basic log-file extraction: + +.. code-block:: bash + + biosim-extractor mappings.json --engine gromacs --logfile md.log --output metadata.json + +The mappings.json file used is created from ``biosim-schema`` `here `_. + +Using an explicit biosim schema file for validation: + +.. code-block:: bash + + biosim-extractor mappings.json \ + --biosimschema biosim_schema.yaml \ + --engine amber \ + --logfile md.log \ + --output metadata.json + +Using fetched schema bundle (no local schema paths required): + +.. code-block:: bash + + biosim-extractor \ + --schema-version latest \ + --schema-cache-dir /tmp/biosim-schema-cache \ + --engine gromacs \ + --logfile md.log + +Force refresh of cached schema bundle: + +.. code-block:: bash + + biosim-extractor --update-schema --engine gromacs --logfile md.log + +Topology/trajectory mode via ``biosim-extractor``: .. code-block:: bash - biosim-extractor --engine gromacs --logfile md.log --output metadata.json + biosim-extractor mappings.json --top topology.top --traj traj1.xtc traj2.xtc --output metadata.json -To extract topology and trajectory metadata: +Control file metadata inclusion: .. code-block:: bash - python -m biosim_extractor.mdanalysis.toptraj topology.top trajectory.xtc + biosim-extractor mappings.json --engine gromacs --logfile md.log --no-file-metadata + biosim-extractor mappings.json --engine gromacs --logfile md.log --file-metadata + +Other available options: + +- ``--config``: path to configuration file. +- ``-o, --output``: output file path (prints to stdout when omitted). + +Toptraj standalone parser: + +.. code-block:: bash + + python -m biosim_extractor.mdanalysis.toptraj topology.top trajectory.xtc --output toptraj.json Python API ---------- -You can also use the API in your own scripts: +You can also use the API directly in your own scripts. + +Log-file mode (Amber/GROMACS): .. code-block:: python - from biosim_extractor.schema.populatemetadata import MetadataPopulator + from biosim_extractor.metadata.populatemetadata import MetadataPopulator populator = MetadataPopulator( - schema_path="schema.json", + schema_path="mappings.json", log_file="md.log", - engine="gromacs" + engine="gromacs", + store_file_metadata=True, # set False to skip file hash/size collection + ) + + metadata = populator.populate() + populator.validate(metadata, biosimschema_path="biosim_schema.yaml") + print(metadata) + +Topology/trajectory mode: + +.. code-block:: python + + from biosim_extractor.metadata.populatemetadata import MetadataPopulator + + populator = MetadataPopulator( + schema_path="mappings.json", + top_file="topology.top", + traj_file=["traj1.xtc", "traj2.xtc"], + store_file_metadata=False, ) + metadata = populator.populate() print(metadata) diff --git a/pyproject.toml b/pyproject.toml index 9641ff4..b5d46ec 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,7 +19,7 @@ license = { file = "LICENSE" } requires-python = ">=3.12" classifiers = [ - "Development Status :: 3 - Alpha", + "Development Status :: 4 - Beta", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", diff --git a/tests/test_gromacs/test_gromacslog.py b/tests/test_gromacs/test_gromacslog.py index eaa0556..2ffef60 100644 --- a/tests/test_gromacs/test_gromacslog.py +++ b/tests/test_gromacs/test_gromacslog.py @@ -100,3 +100,61 @@ def test_parse_with_nonexistent_file(tmp_path): fake_path = tmp_path / "does_not_exist.log" with pytest.raises(FileNotFoundError): GromacsLogParser(str(fake_path)).parse() + + +def test_energy_timeseries_block_parsing(tmp_path): + log = ( + "Step Time\n" + "1 0.002\n" + "note\n" + "Energies (kJ/mol)\n" + "Bond Angle\n" + "1.0 2.0\n" + "LJ-14 Coulomb-14\n" + "3.0 4.0\n" + "LJ-(SR) Coulomb-(SR)\n" + "5.0 6.0\n" + "Coul.-recip. Potential\n" + "7.0 8.0\n" + "9.0 10.0 11.0\n" + ) + p = tmp_path / "energy.log" + p.write_text(log) + + parser = GromacsLogParser(str(p)) + parser.lines = log.splitlines(True) + parser._parse_energy_timeseries() + + assert len(parser.energy_timeseries) == 1 + entry = parser.energy_timeseries[0] + assert entry["Step"] == 1 + assert entry["Time"] == 0.002 + assert entry["Bond"] == 1.0 + assert entry["Pres."] == 9.0 + assert entry["(bar)"] == 11.0 + + +def test_energy_timeseries_missing_energies_block(tmp_path): + log = "Step Time\n1 0.002\n" + p = tmp_path / "no_energy.log" + p.write_text(log) + + parser = GromacsLogParser(str(p)) + parser.lines = log.splitlines(True) + parser._parse_energy_timeseries() + + assert parser.energy_timeseries == [] + + +def test_main_prints_json_without_output_arg(tmp_path, monkeypatch, capsys): + log = "GROMACS version: 2022.1\n" + p = tmp_path / "cli_stdout.log" + p.write_text(log) + + monkeypatch.setattr("sys.argv", ["gromacslog.py", str(p)]) + main() + + out = capsys.readouterr().out + data = json.loads(out) + + assert data["GROMACS version"] == 2022.1 diff --git a/tests/test_mdanalysis/test_toptraj.py b/tests/test_mdanalysis/test_toptraj.py index deb491d..18f7c1f 100644 --- a/tests/test_mdanalysis/test_toptraj.py +++ b/tests/test_mdanalysis/test_toptraj.py @@ -1,10 +1,99 @@ +from types import SimpleNamespace from unittest.mock import MagicMock, patch import pytest import biosim_extractor.mdanalysis.toptraj as toptraj -# --- Helpers --- + +def test_get_protein_sequence_returns_none_on_exception(): + fragment = MagicMock() + fragment.select_atoms.side_effect = RuntimeError("boom") + assert toptraj.get_protein_sequence(fragment) is None + + +def test_get_nucleic_sequence_returns_none_on_exception(): + fragment = MagicMock() + fragment.select_atoms.side_effect = RuntimeError("boom") + assert toptraj.get_nucleic_sequence(fragment) is None + + +def test_classify_box_hexagonal_angle_branch(): + # Covers the 90,90,120 branch + assert toptraj.classify_box([10, 20, 30, 90, 90, 120]) == "orthorhombic" + + +@patch("biosim_extractor.mdanalysis.toptraj.Universe") +def test_parse_handles_non_callable_toptraj_extract(mock_universe, monkeypatch): + u = MagicMock() + u.atoms.fragments = [] + mock_universe.return_value = u + monkeypatch.setattr(toptraj, "TOPTRAJ_AUTO_EXTRACT", {"constant_field": 123}) + + parser = toptraj.TopTrajParser("top", "traj") + out = parser.parse() + assert out["constant_field"] == 123 + + +def test_find_molecule_ids_atom_formula_fallback(monkeypatch): + parser = object.__new__(toptraj.TopTrajParser) + parser.data = {} + atom = SimpleNamespace(name="C12") # no .element -> fallback strips digits + frag = MagicMock() + frag.atoms = MagicMock() + frag.atoms.__getitem__.return_value = atom + + parser.molecule_types = {(("X",), ("C12",)): {"count": 1, "fragment": frag}} + + monkeypatch.setattr(toptraj, "MOLID_AUTO_EXTRACT", {}) + parser._find_molecule_IDs() + assert parser.data["molecule_ids"][0]["molecular_formula"] == "C" + + +def test_find_molecule_ids_rdkit_failure_continue(monkeypatch): + parser = object.__new__(toptraj.TopTrajParser) + parser.data = {} + frag = MagicMock() + frag.convert_to.side_effect = ValueError("rdkit fail") + + parser.molecule_types = {(("ALA",), ("N", "CA")): {"count": 1, "fragment": frag}} + + monkeypatch.setattr(toptraj, "MOLID_AUTO_EXTRACT", {}) + parser._find_molecule_IDs() + assert "InChIKey" not in parser.data["molecule_ids"][0] + + +def test_find_molecule_ids_non_callable_molid_and_sequence(monkeypatch): + parser = object.__new__(toptraj.TopTrajParser) + parser.data = {} + frag = MagicMock() + + parser.molecule_types = { + (("ALA", "GLY"), ("N", "CA")): {"count": 1, "fragment": frag} + } + + monkeypatch.setattr(toptraj, "MOLID_AUTO_EXTRACT", {"kind": "peptide"}) + monkeypatch.setattr(toptraj, "SEQUENCE_AUTO_EXTRACT", {"source": "fallback"}) + parser._find_molecule_IDs() + + m = parser.data["molecule_ids"][0] + assert m["kind"] == "peptide" + assert m["source"] == "fallback" + + +def test_main_writes_output_file(monkeypatch, tmp_path): + out = tmp_path / "result.json" + args = SimpleNamespace(topology="top", trajectory=["traj"], output=str(out)) + + monkeypatch.setattr(toptraj, "parse_args", lambda: args) + + fake_parser = MagicMock() + fake_parser.parse.return_value = {"foo": "bar"} + monkeypatch.setattr(toptraj, "TopTrajParser", lambda *_: fake_parser) + + toptraj.main() + assert out.exists() + assert '"foo": "bar"' in out.read_text() def make_mock_atoms(length, names=None): @@ -26,9 +115,6 @@ def make_mock_residues(length, resnames=None): return residues -# --- Tests --- - - @pytest.mark.parametrize( "dims, expected", [ diff --git a/tests/test_metadata/test_fetchschema.py b/tests/test_metadata/test_fetchschema.py index 2ba38d8..464f7ae 100644 --- a/tests/test_metadata/test_fetchschema.py +++ b/tests/test_metadata/test_fetchschema.py @@ -1,10 +1,272 @@ +import io +import json +import sys +import tarfile from pathlib import Path from types import SimpleNamespace +import pytest + import biosim_extractor.metadata.fetchschema as fs import biosim_extractor.metadata.populatemetadata as pop +class _Ctx: + def __init__(self, obj): + self.obj = obj + + def __enter__(self): + return self.obj + + def __exit__(self, exc_type, exc, tb): + return False + + +def test_latest_release_tag_reads_tag_name(monkeypatch): + stream = io.StringIO('{"tag_name": "v9.9.9"}') + + def _fake_urlopen(url, timeout): + assert url.endswith("/releases/latest") + assert timeout == 30 + stream.seek(0) + return _Ctx(stream) + + monkeypatch.setattr(fs.urllib.request, "urlopen", _fake_urlopen) + + assert fs._latest_release_tag("CCPBioSim", "biosim-schema") == "v9.9.9" + + +def test_download_release_tarball_writes_content(monkeypatch, tmp_path): + out = tmp_path / "release.tar.gz" + + class _Resp: + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, tb): + return False + + def read(self): + return b"tar-bytes" + + def _fake_urlopen(url, timeout): + assert url.endswith("/v1.2.3.tar.gz") + assert timeout == 60 + return _Resp() + + monkeypatch.setattr(fs.urllib.request, "urlopen", _fake_urlopen) + + fs._download_release_tarball("CCPBioSim", "biosim-schema", "v1.2.3", out) + + assert out.read_bytes() == b"tar-bytes" + + +def test_extract_needed_extracts_and_returns_root(tmp_path): + source_root = tmp_path / "biosim-schema-1.0.0" + source_root.mkdir() + (source_root / "dummy.txt").write_text("x", encoding="utf-8") + + tar_path = tmp_path / "bundle.tar.gz" + with tarfile.open(tar_path, "w:gz") as tf: + tf.add(source_root, arcname=source_root.name) + + target = tmp_path / "extract" + + root = fs._extract_needed(tar_path, target) + + assert root.exists() + assert root.name == "biosim-schema-1.0.0" + + +def test_extract_needed_raises_when_archive_has_no_root_dirs(tmp_path): + tar_path = tmp_path / "empty.tar.gz" + with tarfile.open(tar_path, "w:gz"): + pass + + with pytest.raises(RuntimeError, match="Release archive extraction failed"): + fs._extract_needed(tar_path, tmp_path / "extract") + + +def test_get_schema_force_removes_existing_bundle(monkeypatch, tmp_path): + base = tmp_path / "cache" + tag = "v1.2.3" + bundle_dir = base / tag + bundle_dir.mkdir(parents=True, exist_ok=True) + stale = bundle_dir / "stale.txt" + stale.write_text("old", encoding="utf-8") + + release_root = tmp_path / "release" / "biosim-schema-1.2.3" + _write_bundle(release_root) + + def _fake_download(_owner, _repo, _tag, out_path): + # stale marker should be gone after force cleanup + assert not stale.exists() + out_path.write_bytes(b"x") + + monkeypatch.setattr(fs, "_download_release_tarball", _fake_download) + monkeypatch.setattr(fs, "_extract_needed", lambda *_a, **_k: release_root) + + bundle = fs.get_schema(version="1.2.3", cache_dir=str(base), force=True) + + assert bundle.version == tag + assert bundle.root == release_root + + +def test_get_schema_raises_for_incomplete_cached_bundle(tmp_path): + base = tmp_path / "cache" + tag = "v0.1.0" + (base / tag / "src").mkdir(parents=True, exist_ok=True) + + with pytest.raises(RuntimeError, match="Cached schema bundle is incomplete"): + fs.get_schema(version="0.1.0", cache_dir=str(base)) + + +def test_get_schema_raises_when_mapping_missing(tmp_path): + base = tmp_path / "cache" + tag = "v0.0.2" + release_root = base / tag / "src" / "biosim-schema-0.0.2" + _write_bundle(release_root) + (release_root / "project" / "schema_enginemappings.json").unlink() + + with pytest.raises(FileNotFoundError, match="Missing mapping json"): + fs.get_schema(version="0.0.2", cache_dir=str(base)) + + +def test_get_schema_raises_when_schema_yaml_missing(tmp_path): + base = tmp_path / "cache" + tag = "v0.0.2" + release_root = base / tag / "src" / "biosim-schema-0.0.2" + _write_bundle(release_root) + (release_root / "biosim_schema" / "schema" / "biosim_schema.yaml").unlink() + + with pytest.raises(FileNotFoundError, match="Missing schema yaml"): + fs.get_schema(version="0.0.2", cache_dir=str(base)) + + +def test_parse_args_get_defaults(monkeypatch): + monkeypatch.setattr(sys, "argv", ["fetchschema.py", "get"]) + args = fs.parse_args() + + assert args.command == "get" + assert args.version == "latest" + assert args.cache_dir is None + assert args.owner == "CCPBioSim" + assert args.repo == "biosim-schema" + assert args.json is False + + +def test_parse_args_update_with_options(monkeypatch): + monkeypatch.setattr( + sys, + "argv", + [ + "fetchschema.py", + "update", + "--version", + "0.0.2", + "--cache-dir", + "/tmp/cache", + "--owner", + "X", + "--repo", + "Y", + "--json", + ], + ) + args = fs.parse_args() + + assert args.command == "update" + assert args.version == "0.0.2" + assert args.cache_dir == "/tmp/cache" + assert args.owner == "X" + assert args.repo == "Y" + assert args.json is True + + +def test_bundle_payload_serializes_paths(tmp_path): + bundle = fs.SchemaBundle( + version="v1", + root=tmp_path / "root", + mapping_json=tmp_path / "root" / "project" / "schema_enginemappings.json", + schema_yaml=tmp_path + / "root" + / "biosim_schema" + / "schema" + / "biosim_schema.yaml", + ) + + payload = fs._bundle_payload(bundle) + + assert payload["version"] == "v1" + assert isinstance(payload["root"], str) + assert isinstance(payload["mapping_json"], str) + assert isinstance(payload["schema_yaml"], str) + + +def test_main_prints_json_for_get(monkeypatch, tmp_path): + args = SimpleNamespace( + command="get", + version="latest", + cache_dir=None, + owner="CCPBioSim", + repo="biosim-schema", + json=True, + ) + bundle = fs.SchemaBundle( + version="v1", + root=tmp_path / "r", + mapping_json=tmp_path / "r" / "project" / "schema_enginemappings.json", + schema_yaml=tmp_path / "r" / "biosim_schema" / "schema" / "biosim_schema.yaml", + ) + + monkeypatch.setattr(fs, "parse_args", lambda: args) + monkeypatch.setattr(fs, "get_schema", lambda **_k: bundle) + + printed = [] + monkeypatch.setattr( + "builtins.print", lambda *a, **k: printed.append(" ".join(map(str, a))) + ) + + fs.main() + + assert len(printed) == 1 + payload = json.loads(printed[0]) + assert payload["version"] == "v1" + + +def test_main_prints_text_for_update(monkeypatch, tmp_path): + args = SimpleNamespace( + command="update", + version="0.0.2", + cache_dir="/tmp/cache", + owner="CCPBioSim", + repo="biosim-schema", + json=False, + ) + bundle = fs.SchemaBundle( + version="v0.0.2", + root=tmp_path / "r", + mapping_json=tmp_path / "r" / "project" / "schema_enginemappings.json", + schema_yaml=tmp_path / "r" / "biosim_schema" / "schema" / "biosim_schema.yaml", + ) + + monkeypatch.setattr(fs, "parse_args", lambda: args) + monkeypatch.setattr(fs, "update_schema", lambda **_k: bundle) + + printed = [] + monkeypatch.setattr( + "builtins.print", lambda *a, **k: printed.append(" ".join(map(str, a))) + ) + + fs.main() + + assert len(printed) == 4 + assert printed[0].startswith("version:") + assert printed[1].startswith("root:") + assert printed[2].startswith("mapping_json:") + assert printed[3].startswith("schema_yaml:") + + def _args(**overrides): defaults = { "mappingschema": None, diff --git a/tests/test_metadata/test_filemetadata.py b/tests/test_metadata/test_filemetadata.py new file mode 100644 index 0000000..c57d083 --- /dev/null +++ b/tests/test_metadata/test_filemetadata.py @@ -0,0 +1,56 @@ +from pathlib import Path + +from biosim_extractor.metadata.filemetadata import ( + file_metadata, + files_metadata, + group_files, +) + + +def test_group_files_creates_dict_when_none(): + out = group_files(["a.log"], saved_files=None, role="log") + assert out == {"log": ["a.log"]} + + +def test_group_files_appends_to_existing_role(): + out = group_files(["b.log"], saved_files={"log": ["a.log"]}, role="log") + assert out == {"log": ["a.log", "b.log"]} + + +def test_single_file(tmp_path): + path = tmp_path / "test.bin" + path.write_bytes(b"\x00\x01\x02") # Known small content + + meta = file_metadata(str(path), role="other", hash_algorithm="sha256") + + assert meta["file_name"] == "test.bin" + assert meta["file_role"] == "other" + assert len(meta["file_hash"]) == 64 # SHA-256 hex length + assert "value_unit" in meta["file_size"] + + +def test_multiple_files(tmp_path): + p1 = tmp_path / "a.bin" + p2 = tmp_path / "b.bin" + p1.write_bytes(b"A") + p2.write_bytes(b"B") + + inputs: dict[str, list[Path]] = { + "group_1": [p1], + "group_2": [p2], + } + + metas = files_metadata(inputs) + + assert len(metas) == 2 + + +def test_default_algorithm(tmp_path): + p = tmp_path / "d.bin" + p.write_bytes(b"D") + + # Ensure MD5 is default (no arg provided in call if desired, + # but explicit check ensures correct hash length for MD5=32) + meta = file_metadata(str(p)) + + assert len(meta["file_hash"]) == 32 # MD5 hex length diff --git a/tests/test_metadata/test_populatemetadata.py b/tests/test_metadata/test_populatemetadata.py index 911637a..e5b01b7 100644 --- a/tests/test_metadata/test_populatemetadata.py +++ b/tests/test_metadata/test_populatemetadata.py @@ -1,4 +1,171 @@ +from types import SimpleNamespace + +import pytest + from biosim_extractor.metadata import populatemetadata +from biosim_extractor.metadata import populatemetadata as pop + + +class _NoConvert: + def get_target_unit(self, u): + return u + + def needs_conversion(self, u): + return False + + def convert(self, v, u): + return v + + +def test_parse_log_unsupported_engine_raises(): + p = pop.MetadataPopulator(engine="unknown") + with pytest.raises(ValueError, match="Unsupported engine"): + p.parse_log() + + +def test_populate_includes_file_metadata(monkeypatch): + p = pop.MetadataPopulator( + schema_path="ignored.json", + log_file="dummy.log", + engine="amber", + store_file_metadata=True, + ) + + monkeypatch.setattr( + p, + "load_schema", + lambda: setattr( + p, "schema", {"reverse": {"amber": {}}, "forward": {"amber": {}}} + ), + ) + monkeypatch.setattr(p, "parse_log", lambda: {}) + monkeypatch.setattr(p, "apply_mapping", lambda: {"SimulationMetadata": {}}) + monkeypatch.setattr( + pop, "group_files", lambda files, saved, role="other": {"log": files} + ) + monkeypatch.setattr( + pop, "files_metadata", lambda saved: [{"file_name": saved["log"][0]}] + ) + + result = p.populate() + assert result["files"][0]["file_name"] == "dummy.log" + + +def test_populate_uses_toptraj_branch(monkeypatch): + p = pop.MetadataPopulator( + schema_path="ignored.json", + top_file="a.top", + traj_file=["a.xtc"], + store_file_metadata=False, + ) + monkeypatch.setattr(p, "load_schema", lambda: None) + monkeypatch.setattr( + p, "populate_toptraj", lambda: {"SimulationMetadata": {"ok": 1}} + ) + + assert p.populate()["ok"] == 1 + + +def test_apply_mapping_promotes_scalar_to_vector_and_appends(): + p = pop.MetadataPopulator(engine="amber") + p.converter = _NoConvert() + p.schema = { + "reverse": { + "amber": { + "x": {"by_path": {"SimulationMetadata.v": {}}}, + "y": {"by_path": {"SimulationMetadata.v": {}}}, + "z": {"by_path": {"SimulationMetadata.v": {}}}, + } + }, + "forward": { + "amber": { + "SimulationMetadata.v": [ + {"key": "x", "unit": "nm"}, + {"key": "y", "unit": "nm"}, + {"key": "z", "unit": "nm"}, + ] + } + }, + } + p.engine_data = {"x": 1.0, "y": 2.0, "z": 3.0} + p.data = {"SimulationMetadata": {}} + + out = p.apply_mapping() + assert out["SimulationMetadata"]["v"]["vector_value"] == [1.0, 2.0, 3.0] + + +def test_apply_mapping_handles_molecule_ids(): + p = pop.MetadataPopulator(engine="amber") + p.converter = _NoConvert() + p.schema = { + "reverse": { + "amber": {"charge": {"by_path": {"SimulationMetadata.any.charge": {}}}} + }, + "forward": { + "amber": {"SimulationMetadata.any.charge": [{"key": "charge", "unit": "e"}]} + }, + } + p.engine_data = {"molecule_ids": {"1": {"charge": 1.0, "ignore_me": "x"}}} + p.data = {"SimulationMetadata": {}} + + out = p.apply_mapping() + mol = out["SimulationMetadata"]["composition"]["molecule_ID"][0] + assert mol["charge"]["value"] == 1.0 + + +def test_parse_args_no_file_metadata(monkeypatch): + monkeypatch.setattr( + "sys.argv", + [ + "prog", + "m.json", + "--no-file-metadata", + "--engine", + "amber", + "--logfile", + "a.log", + ], + ) + args = pop.parse_args() + assert args.store_file_metadata is False + + +def test_main_prints_json_when_no_output(monkeypatch): + args = SimpleNamespace( + mappingschema="m.json", + biosimschema="b.yaml", + schema_version="latest", + schema_cache_dir=None, + update_schema=False, + engine="amber", + logfile="a.log", + top=None, + traj=None, + config=None, + store_file_metadata=False, + output=None, + ) + + class DummyPopulator: + def __init__(self, **kwargs): + pass + + def populate(self): + return {"k": 1} + + def validate(self, result, biosimschema_path=None): + assert biosimschema_path == "b.yaml" + + printed = [] + monkeypatch.setattr(pop, "parse_args", lambda: args) + monkeypatch.setattr( + pop, "resolve_schema_inputs", lambda _args: ("m.json", "b.yaml") + ) + monkeypatch.setattr(pop, "MetadataPopulator", DummyPopulator) + monkeypatch.setattr("builtins.print", lambda s: printed.append(s)) + + pop.main() + assert printed and '"k": 1' in printed[0] def test_flatten_dict(): @@ -74,7 +241,10 @@ def parse(self): monkeypatch.setattr(populatemetadata, "validate_metadata", lambda *a, **kw: None) pop = populatemetadata.MetadataPopulator( - schema_path=str(schema_path), log_file="dummy.log", engine="amber" + schema_path=str(schema_path), + log_file="dummy.log", + engine="amber", + store_file_metadata=False, ) pop.load_schema() assert pop.schema["reverse"]["amber"] @@ -87,7 +257,10 @@ def parse(self): # Test populate (integration) pop = populatemetadata.MetadataPopulator( - schema_path=str(schema_path), log_file="dummy.log", engine="amber" + schema_path=str(schema_path), + log_file="dummy.log", + engine="amber", + store_file_metadata=False, ) pop.data = {"SimulationMetadata": {}} monkeypatch.setattr(pop, "parse_log", lambda: engine_data)