diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index 4000581fd..f11656480 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -12,8 +12,3 @@ jobs: wheel-wrapper: uses: ./.github/workflows/build-wheel-wrapper.yml secrets: inherit - wheel-python: - uses: ecmwf/reusable-workflows/.github/workflows/cd-pypi.yml@v2 - secrets: inherit - needs: - - wheel-wrapper diff --git a/.github/workflows/pymetkit.yml b/.github/workflows/pymetkit.yml new file mode 100644 index 000000000..0e9f2efdc --- /dev/null +++ b/.github/workflows/pymetkit.yml @@ -0,0 +1,143 @@ +name: Build and Test PyMetkit + +on: + # Trigger the workflow on push to master or develop, except tag creation + push: + branches: + - 'master' + - 'develop' + tags-ignore: + - '**' + + # Trigger the workflow on pull request + pull_request: ~ + + # Trigger the workflow manually + workflow_dispatch: ~ + + # Trigger after public PR approved for CI + pull_request_target: + types: [labeled] + +jobs: + prepare-deps: + runs-on: ubuntu-latest + if: ${{ (success() || failure()) && (!github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci') }} + steps: + - name: Get ecbuild + uses: actions/checkout@v5 + with: + repository: ecmwf/ecbuild + ref: develop + path: ecbuild + - name: Get stack-dependencies + uses: actions/checkout@v5 + with: + repository: ecmwf/stack-dependencies + ref: master + path: stack-dependencies-src + token: ${{ secrets.GH_REPO_READ_TOKEN }} + submodules: recursive + - name: Install dependencies + run: | + mkdir stack-dependencies-build + stack-dependencies-src/build.sh --build-path stack-dependencies-build --install-path dependencies --with-deps libaec,pybind11 + - name: Get eccodes + uses: actions/checkout@v5 + with: + repository: ecmwf/eccodes + ref: develop + path: eccodes-src + - name: Install eccodes + run: | + mkdir eccodes-build + cmake \ + -B eccodes-build \ + -S eccodes-src \ + -GNinja \ + -DCMAKE_INSTALL_PREFIX=dependencies \ + -DCMAKE_PREFIX_PATH=dependencies \ + -DCMAKE_BUILD_TYPE=RelWithDebInfo \ + -DENABLE_MEMFS=ON \ + -DENABLE_AEC=ON + cmake --build eccodes-build -j -t install + - name: Get eckit + uses: actions/checkout@v5 + with: + repository: ecmwf/eckit + ref: develop + path: eckit-src + - name: Install eckit + run: | + mkdir eckit-build + cmake \ + -B eckit-build \ + -S eckit-src \ + -GNinja \ + -DCMAKE_INSTALL_PREFIX=dependencies \ + -DCMAKE_PREFIX_PATH=dependencies \ + -DCMAKE_BUILD_TYPE=RelWithDebInfo + cmake --build eckit-build -j -t install + - name: Archive with permissions preserved + run: tar --zstd -cpf files.tar.zst dependencies/ ecbuild/ + - name: Upload dependencies + uses: actions/upload-artifact@v4 + with: + name: deps + path: files.tar.zst + retention-days: 1 + build-wheels: + needs: prepare-deps + runs-on: ubuntu-latest + if: ${{ (success() || failure()) && (!github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci') }} + strategy: + matrix: + python-version: ['3.11', '3.12', '3.13', '3.14'] + fail-fast: false # Continue running other versions if one fails + + steps: + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + - name: Download dependencies + uses: actions/download-artifact@v4 + with: + name: deps + - name: Extract with zstd + run: tar --zstd -xpf files.tar.zst + - name: Get metkit + uses: actions/checkout@v5 + with: + repository: ecmwf/metkit + path: metkit-src + - name: Display Python version + run: python --version + - name: Install 'build' + run: pip install build pytest findlibs Sybil[pytest] + - name: Build metkit + run: | + export PATH=$(pwd)/dependencies/bin:$PATH + export ECCODES_HOME=$(pwd)/dependencies + export FINDLIBS_DISABLE_PACKAGE=yes + export FINDLIBS_DISABLE_PYTHON=yes + export ECCODES_PYTHON_USE_FINDLIBS=1 + mkdir metkit-build + cmake \ + -B metkit-build \ + -S metkit-src \ + -GNinja \ + -DCMAKE_INSTALL_PREFIX=dependencies \ + -DCMAKE_PREFIX_PATH=dependencies \ + -DCMAKE_BUILD_TYPE=RelWithDebInfo \ + -DENABLE_PYTHON_METKIT_INTERFACE=ON + cmake --build metkit-build -j + cd metkit-build + ctest -j $(nproc) --output-on-failure -L pymetkit + - name: Upload wheel + uses: actions/upload-artifact@v4 + with: + name: pymetkit-wheel-py${{ matrix.python-version }} + path: "metkit-build/pymetkit-*.whl" + retention-days: 10 + if-no-files-found: error # 'warn' or 'ignore' are also available, defaults to `warn` diff --git a/CMakeLists.txt b/CMakeLists.txt index cec3cc2fb..55431a634 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -96,6 +96,15 @@ ecbuild_add_option( FEATURE MARS2GRIB_PYTHON "NAME Python VERSION 3.11 COMPONENTS Interpreter Development" "NAME pybind11 VERSION 3.0.1" ) +# Pythonic pybind11 interface (pymetkit) + +ecbuild_add_option( FEATURE PYTHON_METKIT_INTERFACE + DEFAULT OFF + DESCRIPTION "Build the pybind11 pymetkit interface" + REQUIRED_PACKAGES + "NAME Python VERSION 3.11 COMPONENTS Interpreter Development" + "NAME pybind11 VERSION 3.0.1" ) + # METKIT config files support ecbuild_add_option( FEATURE METKIT_CONFIG diff --git a/cmake/pymetkit_setup.cfg.in b/cmake/pymetkit_setup.cfg.in new file mode 100644 index 000000000..d4aca7429 --- /dev/null +++ b/cmake/pymetkit_setup.cfg.in @@ -0,0 +1,38 @@ +[metadata] +name = pymetkit +description = Python interface to metkit +long_description = file: README.md +long_description_content_type = text/markdown +author_email = European Centre for Medium-Range Weather Forecasts (ECMWF) +project_urls = + Documentation = https://github.com/ecmwf/metkit + Homepage = https://github.com/ecmwf/metkit + Issues = https://github.com/ecmwf/metkit/issues + Repository = https://github.com/ecmwf/metkit +keywords = python, metkit, mars, tools +license_expression = Apache-2.0 +license_file = LICENSE + +[options] +python_requires = >=3.11 +install_requires = + findlibs>=0.1.2 + +[options.extras_require] +test = + pytest + pytest-cov + pytest-flakes + Sybil[pytest] +docs = + Sphinx + breathe + sphinx-book-theme + requests + sphinxcontrib-mermaid + autoapi + sphinx-autoapi +dev = + isort + black + flake8 diff --git a/cmake/pymetkit_setup.py.in b/cmake/pymetkit_setup.py.in new file mode 100644 index 000000000..b181166dc --- /dev/null +++ b/cmake/pymetkit_setup.py.in @@ -0,0 +1,57 @@ +import os +import platform +import sys +from pathlib import Path + +from setuptools import setup +from wheel.bdist_wheel import bdist_wheel + +# NOTE this we need to correctly link with metkitlib version on cd. For local builds feel free to ignore +version_suffix = os.environ.get("VERSION_SUFFIX", "") +if version_suffix: + version_suffix = f".{version_suffix}" + requires_extra = [f"metkitlib==@metkit_VERSION_STR@{version_suffix}"] +else: + requires_extra = [] + + +# NOTE see ci-utils/wheelmaker/buildscripts/setup_utils, we need to get the right abi compat tag +class bdist_wheel_ext(bdist_wheel): + def get_tag(self): + python, abi, plat = bdist_wheel.get_tag(self) + return python, abi, f"manylinux_2_28_{platform.machine()}" + + +ext_kwargs = { + "darwin": {}, + "linux": {"cmdclass": {"bdist_wheel": bdist_wheel_ext}}, +} + +setup( + name="pymetkit", + version=f"@metkit_VERSION_STR@{version_suffix}", + packages=["pymetkit", "pymetkit._internal", "pymetkit_bindings"], + package_data={ + "pymetkit_bindings": ["*.so", "*.pyd"], + }, + license="Apache 2.0", + license_files=["LICENSE"], + long_description=Path("README.md").read_text(), + long_description_content_type="text/markdown", + install_requires=["findlibs>=0.1.2"] + requires_extra, + python_requires=">3.10", + classifiers=[ + "Development Status :: 3 - Alpha", + "Intended Audience :: Developers", + "Intended Audience :: Science/Research", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", + "Operating System :: OS Independent", + "Topic :: Software Development :: Libraries", + ], + has_ext_modules=lambda: True, + **ext_kwargs[sys.platform], +) diff --git a/docs/conf.py b/docs/conf.py index 766f80d7c..f6c77ee8b 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -34,7 +34,7 @@ templates_path = ["_templates"] exclude_patterns = ["_build", "Thumbs.db", ".DS_Store", "_internal"] -autoapi_dirs = ["../python/pymetkit/src/pymetkit"] +autoapi_dirs = ["../src/pymetkit"] autoapi_type = "python" autoapi_generate_api_docs = True autoapi_add_toctree_entry = False diff --git a/docs/pymetkit/api.rst b/docs/pymetkit/api.rst new file mode 100644 index 000000000..7b55c482b --- /dev/null +++ b/docs/pymetkit/api.rst @@ -0,0 +1,32 @@ +API +=== + +The ``PyMetKit`` API provides a Pythonic interface to ``metkit``'s MARS request +model. A :class:`~pymetkit.pymetkit.MarsRequest` is a verb together with a +:data:`~pymetkit.pymetkit_type.MarsSelection` — a type alias for a user-supplied +key-value mapping. Values are normalised automatically to the internal +``dict[str, list[str]]`` representation used by the bindings layer. Operations +that require the MARS language engine (expansion, validation, merging and parsing) +are delegated to the underlying ``metkit`` library through the :doc:`bindings` layer. + +MarsRequest +----------- +.. autoapiclass:: pymetkit.pymetkit.MarsRequest + :members: + +Parsing +------- +.. autoapifunction:: pymetkit.pymetkit.parse_mars_request + +MarsSelection +------------- +.. autoapidata:: pymetkit.pymetkit_type.MarsSelection + +Exceptions +---------- +.. py:exception:: pymetkit.MetKitException + + Raised when the underlying ``metkit`` library reports an error, for example when + :meth:`~pymetkit.pymetkit.MarsRequest.validate` or + :meth:`~pymetkit.pymetkit.MarsRequest.expand` encounters a request that is + incompatible with the MARS language definition. Subclasses :class:`RuntimeError`. diff --git a/docs/pymetkit/bindings.rst b/docs/pymetkit/bindings.rst new file mode 100644 index 000000000..7616dd540 --- /dev/null +++ b/docs/pymetkit/bindings.rst @@ -0,0 +1,142 @@ +pybind11 bindings +================= + +``pymetkit_bindings`` is the compiled `pybind11 `__ +extension module that binds the ``metkit`` C++ classes directly. It is the lowest layer +of ``PyMetKit`` and is consumed by :mod:`pymetkit._internal`; the raw ``MarsRequest`` +class it exposes is re-exported there as ``_MarsRequest``. + +.. warning:: + + This is an internal, low-level layer. Application code should use the Pythonic + :doc:`api` (:class:`pymetkit.pymetkit.MarsRequest`, + :data:`pymetkit.pymetkit_type.MarsSelection` and + :func:`pymetkit.pymetkit.parse_mars_request`) rather than these bindings directly. + The signatures below map one-to-one onto ``metkit::mars::MarsRequest`` and are not + covered by the same value-normalization or error-translation guarantees. + +The extension is built from ``src/pymetkit_bindings/bindings.cc`` and staged into the +wheel as its own top-level package. The shared library ``libmetkit`` and its +dependencies must be loaded (via ``findlibs``) before the module is imported, and +:func:`init_bindings` must be called once before any other call — both are handled +automatically by :mod:`pymetkit._internal`. + +Module-level functions +---------------------- + +.. py:module:: pymetkit_bindings + +.. py:function:: init_bindings() + + Initialise the ``eckit`` runtime (``eckit::Main``). Must be called once, before any + other binding call, when ``metkit`` is loaded as a shared library from Python. + +.. py:function:: version_info() + + Return a list of ``(name, version, git_sha1, path)`` tuples describing every + ``eckit``-registered library currently loaded (e.g. ``eckit``, ``eccodes``, + ``metkit``). Used by the ``python -m pymetkit`` diagnostics CLI. + + :rtype: list[tuple[str, str, str, str]] + +.. py:function:: parse_marsrequest(string, strict) + + Parse a single MARS request from ``string``. + + :param str string: the MARS request text. + :param bool strict: raise an error (rather than a warning) on invalid values. + :returns: the parsed request. + :rtype: MarsRequest + +.. py:function:: parse_marsrequests(string, strict) + + Parse one or more MARS requests from ``string``. + + :param str string: text containing one or more MARS requests. + :param bool strict: raise an error (rather than a warning) on invalid values. + :returns: the parsed requests. + :rtype: list[MarsRequest] + +Classes +------- + +.. py:class:: MarsRequest + + A thin binding of ``metkit::mars::MarsRequest``. Values are passed and returned as + lists of strings; no normalization is performed at this layer. + + .. py:method:: __init__() + __init__(verb) + + Construct an empty request, or a request with the given ``verb``. + + :param str verb: the request verb (e.g. ``"retrieve"``). + + .. py:method:: verb() + + Return the request verb. + + :rtype: str + + .. py:method:: set_verb(verb) + + Set the request verb. + + :param str verb: the verb to set. + + .. py:method:: set(param, values) + + Set the values of a parameter. + + :param str param: the parameter name. + :param list[str] values: the parameter values. + + .. py:method:: has(param) + + Return whether ``param`` is present in the request. + + :param str param: the parameter name. + :rtype: bool + + .. py:method:: params() + + Return the parameter names present in the request. + + :rtype: list[str] + + .. py:method:: count_values(param) + + Return the number of values held for ``param``. + + :param str param: the parameter name. + :rtype: int + + .. py:method:: values(param) + + Return the values held for ``param``. + + :param str param: the parameter name. + :rtype: list[str] + + .. py:method:: merge(other) + + Merge ``other`` into this request in place: for each shared parameter, the values + of ``other`` that are not already present are appended (order-preserving union). + + :param MarsRequest other: the request to merge in. + + .. py:method:: expand(inherit, strict) + + Return the request expanded against the MARS language definition, using + ``metkit::mars::MarsExpansion``. + + :param bool inherit: populate the expanded request with default values. + :param bool strict: raise an error (rather than a warning) on invalid values. + :returns: the expanded request. + :rtype: MarsRequest + + .. py:method:: __repr__() + + Return the request rendered as a MARS request string (``asString()``). + + :rtype: str diff --git a/docs/pymetkit/build_docs.sh b/docs/pymetkit/build_docs.sh new file mode 100755 index 000000000..939f1bec6 --- /dev/null +++ b/docs/pymetkit/build_docs.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +# (C) Copyright 2025- ECMWF. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +# In applying this licence, ECMWF does not waive the privileges and immunities +# granted to it by virtue of its status as an intergovernmental organisation +# nor does it submit to any jurisdiction. +# +# Build the PyMetKit Sphinx documentation. +# +# Usage: docs/pymetkit/build_docs.sh +# +# The pythonic API is documented via sphinx-autoapi, which parses the sources in +# src/pymetkit statically -- the metkit library does not need to be importable. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +OUT_DIR="${1:-${SCRIPT_DIR}/doc-build/sphinx}" + +sphinx-build -j auto -E -a -T -b html "${SCRIPT_DIR}" "${OUT_DIR}" + +echo "Documentation built at: ${OUT_DIR}" diff --git a/docs/pymetkit/conf.py b/docs/pymetkit/conf.py new file mode 100644 index 000000000..0911461e3 --- /dev/null +++ b/docs/pymetkit/conf.py @@ -0,0 +1,61 @@ +# (C) Copyright 2025- ECMWF. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +# In applying this licence, ECMWF does not waive the privileges and immunities +# granted to it by virtue of its status as an intergovernmental organisation +# nor does it submit to any jurisdiction. + +import datetime + +project = "PyMetKit" +copyright = f"{datetime.datetime.today().year}, ECMWF" +author = "ECMWF" + +extensions = [ + "sphinx.ext.autosectionlabel", + "sphinxcontrib.mermaid", + "autoapi.extension", + "sphinx.ext.viewcode", + "sphinx.ext.napoleon", + "sphinx.ext.autodoc", + "sphinx.ext.doctest", + "sphinx.ext.inheritance_diagram", +] + +templates_path = ["_templates"] +exclude_patterns = ["_build", "Thumbs.db", ".DS_Store", "_internal"] + +# -- sphinx-autoapi: statically parse the pythonic pymetkit package. +# The compiled `pymetkit_bindings` layer has no Python source and is documented +# by hand in `bindings.rst`. The `_internal` glue package is hidden. +autoapi_dirs = ["../../src/pymetkit"] +autoapi_type = "python" +autoapi_generate_api_docs = True +autoapi_add_toctree_entry = False +autoapi_python_class_content = "class" +autoapi_ignore = [ + "*/_internal/*", + "*/__main__.py", +] +add_module_names = False +autoapi_keep_files = False + +# -- Napoleon settings (pymetkit docstrings are NumPy-style) +napoleon_google_docstring = False +napoleon_numpy_docstring = True + +html_theme = "pydata_sphinx_theme" +html_show_sourcelink = False +html_sidebars = {"**": []} +html_theme_options = { + "navbar_align": "left", + "navbar_start": ["navbar-logo"], + "navbar_center": ["navbar-nav"], + "navbar_end": ["navbar-icon-links", "theme-switcher", "version-switcher"], + "navbar_persistent": ["search-button"], + "primary_sidebar_end": [], + "check_switcher": False, +} +html_context = {"default_mode": "auto"} +autosectionlabel_prefix_document = True diff --git a/docs/pymetkit/conftest.py b/docs/pymetkit/conftest.py new file mode 100644 index 000000000..ff225a360 --- /dev/null +++ b/docs/pymetkit/conftest.py @@ -0,0 +1,28 @@ +# SPDX-FileCopyrightText: 2026 European Centre for Medium-Range Weather Forecasts (ECMWF) +# SPDX-License-Identifier: Apache-2.0 + +"""Sybil configuration: execute the code examples in the documentation as tests. + +Run with ``pytest`` from this directory once ``pymetkit`` is importable (i.e. the +``metkit`` library is discoverable by ``findlibs``). +""" + +from doctest import ELLIPSIS + +import pymetkit +from sybil import Sybil +from sybil.parsers.rest import DocTestParser, PythonCodeBlockParser + + +def sybil_setup(namespace): + namespace["pymetkit"] = pymetkit + + +pytest_collect_file = Sybil( + parsers=[ + DocTestParser(optionflags=ELLIPSIS), + PythonCodeBlockParser(), + ], + patterns=["*.rst", "*.py"], + setup=sybil_setup, +).pytest() diff --git a/docs/pymetkit/development.rst b/docs/pymetkit/development.rst new file mode 100644 index 000000000..824dfbbfc --- /dev/null +++ b/docs/pymetkit/development.rst @@ -0,0 +1,51 @@ +Development +########### + +Follow the guide in :ref:`installation-label`. We advise using ``uv`` for installing the +build dependencies. + +The version pinning of the ``metkitlib`` dependency is disabled by default, which is what +you want for local development. On CI it is enabled by setting the ``VERSION_SUFFIX`` +environment variable, which pins the matching ``metkitlib`` wheel version. + +To use your local ``metkit`` build, make sure ``findlibs`` is installed in your ``venv`` +and export: + +.. code-block:: sh + + export FINDLIBS_DISABLE_PACKAGE=yes + +Set the ``METKIT_HOME`` environment variable to the build folder so ``findlibs`` picks up +the correct ``metkit`` library: + +.. code-block:: sh + + export METKIT_HOME= + +You can then install the ``pymetkit`` wheel from the build folder (or the staging +directory) into your ``venv``: + +.. code-block:: sh + + uv pip install pymetkit--cp311-cp311-.whl + # or, for an editable-style install of the staged package + uv pip install -e pymetkit-python-package-staging + +Run the tests by switching to the ``pymetkit`` tests folder and executing ``pytest``: + +.. code-block:: sh + + cd /tests/pymetkit + pytest + +Building the documentation +************************** + +Install the documentation requirements and run the build script: + +.. code-block:: sh + + uv pip install -r docs/pymetkit/requirements.txt + ./docs/pymetkit/build_docs.sh + +The rendered documentation is written to ``docs/pymetkit/doc-build``. diff --git a/docs/pymetkit/examples.rst b/docs/pymetkit/examples.rst new file mode 100644 index 000000000..cbce1d98e --- /dev/null +++ b/docs/pymetkit/examples.rst @@ -0,0 +1,181 @@ +Examples +======== + +The examples below use ``PyMetKit``'s public API and are executed as sybil tests. +The building and accessing examples are self-contained; the expanding, equality, +merging and parsing examples require the MARS language definitions shipped with +``metkit`` (see :doc:`installation`). + +Building a request +------------------ + +A :class:`~pymetkit.pymetkit.MarsRequest` is a verb plus a selection of parameters +passed as a plain mapping. Values may be scalars, numbers, collections, or +``/``-separated strings. + +.. code-block:: python + + from pymetkit import MarsRequest + request = MarsRequest("retrieve", {"class": "od", "param": [151, 129]}) + assert request.verb() == "retrieve" + assert request["class"] == "od" + assert request["param"] == ['151', '129'] + +Numbers, ranges and ``/``-separated strings are normalized to lists of strings: + +.. code-block:: python + + request = MarsRequest("retrieve", {"step": range(0, 13, 6), "date": "20200101/20200102"}) + assert request["step"] == ["0", "6", "12"] + assert request["date"] == ["20200101", "20200102"] + +Accessing values +---------------- + +A request behaves like a read/write mapping. A parameter with a single value returns a +scalar, one with several values returns a list: + +.. code-block:: python + + request = MarsRequest("retrieve", {"class": "od", "param": [151, 129]}) + assert "class" in request + assert sorted(request.keys()) == ["class", "param"] + assert request.num_values("param") == 2 + request["expver"] = "0001" + assert request["expver"] == "0001" + +Iterating over a request yields ``(name, value)`` pairs, mirroring a plain dict: + +.. code-block:: python + + request = MarsRequest("retrieve", {"class": "od", "param": [151, 129]}) + assert list(request) == [("class", "od"), ("param", ["151", "129"])] + +Pass a request to :func:`dict` to get a plain Python mapping, useful for +serialisation or inspection: + +.. code-block:: python + + request = MarsRequest("retrieve", {"class": "od", "param": [151, 129]}) + assert dict(request) == {"class": "od", "param": ["151", "129"]} + +Manipulating a request +---------------------- + +Parameters can be added or overwritten after construction. All value forms accepted +at construction time are also valid for assignment — scalars, integers, ranges, lists +and ``/``-separated strings are all normalised the same way: + +.. code-block:: python + + from pymetkit import MarsRequest + + request = MarsRequest("retrieve", {"class": "od", "expver": "0001"}) + + request["date"] = "20230101/20230102" # slash-separated → list + request["step"] = range(0, 13, 6) # range → list + request["param"] = 130 # integer → scalar + + assert request["date"] == ["20230101", "20230102"] + assert request["step"] == ["0", "6", "12"] + assert request["param"] == "130" + + request["step"] = [0, 6] # overwrite with a shorter list + assert request["step"] == ["0", "6"] + +A new request can be derived from an existing one by snapshotting it with +:func:`dict`, adjusting selected values, and constructing a fresh request: + +.. code-block:: python + + from pymetkit import MarsRequest + + base = MarsRequest("retrieve", {"class": "od", "date": "-1", "param": "130", "step": "0"}) + derived = MarsRequest(base.verb(), {**dict(base), "date": "20230101"}) + + assert derived["date"] == "20230101" + assert derived["param"] == "130" + +Use ``in`` to guard access to parameters that may not be present: + +.. code-block:: python + + from pymetkit import MarsRequest + + request = MarsRequest("retrieve", {"class": "od", "param": "130"}) + step = request["step"] if "step" in request else "0" + assert step == "0" + +Equality and hashing +-------------------- + +Two requests are equal when they expand to the same result. The MARS language +defines aliases, so ``"od"`` and ``"operational"`` for ``class`` refer to the same +dataset — equality reflects that: + +.. code-block:: python + + from pymetkit import MarsRequest + + r1 = MarsRequest("retrieve", {"class": "od", "date": "20230101", "param": "130"}) + r2 = MarsRequest("retrieve", {"class": "operations", "date": "20230101", "param": "130"}) + + assert r1 == r2 + +Expanding and validating +------------------------- + +:meth:`~pymetkit.pymetkit.MarsRequest.expand` returns a new request expanded against the +MARS language definition; :meth:`~pymetkit.pymetkit.MarsRequest.validate` checks a request +without inheriting defaults and raises :class:`~pymetkit.MetKitException` on invalid input. + +.. code-block:: python + + from pymetkit import MarsRequest + + request = MarsRequest( + "retrieve", + { + "class": "od", + "domain": "g", + "date": "-1", + "expver": "0001", + "step": range(0, 13, 6), + }, + ) + + expanded = request.expand() # inherit=True: fills in default values + assert expanded.verb() == "retrieve" + + request.validate() # raises MetKitException if invalid + +Merging requests +---------------- + +:meth:`~pymetkit.pymetkit.MarsRequest.merge` combines the values of two requests that +carry the same parameters, keeping ``self``'s values first and appending only the +values of ``other`` that are not already present: + +.. code-block:: python + + from pymetkit import MarsRequest + + left = MarsRequest("retrieve", {"class": "od", "date": "-1", "levtype": "sfc"}) + right = MarsRequest("retrieve", {"class": "od", "date": "20230101", "levtype": "sfc"}) + + merged = left.merge(right) + assert merged["date"] == ["-1", "20230101"] + +Parsing requests +---------------- + +:func:`~pymetkit.pymetkit.parse_mars_request` parses one or more requests from a string +or a file-like object: + +.. code-block:: python + + from pymetkit import parse_mars_request + + requests = parse_mars_request("retrieve,class=od,date=-1,param=129,step=12") + assert len(requests) == 1 + assert requests[0].verb() == "retrieve" diff --git a/docs/pymetkit/index.rst b/docs/pymetkit/index.rst index 5afa210b4..784151763 100644 --- a/docs/pymetkit/index.rst +++ b/docs/pymetkit/index.rst @@ -13,14 +13,62 @@ It provides a thin, idiomatic Python layer over the Metkit library installed on your system, so you can parse and manipulate MARS requests directly from Python scripts and notebooks. -API Reference -------------- +It exposes the MARS request model as Pythonic objects built on a +`pybind11 `__ extension module that binds the +``metkit`` C++ library directly. The interface is organized in three layers: -The following API reference is generated automatically from the ``pymetkit`` -source. +- :doc:`api` — the Pythonic layer: :class:`~pymetkit.pymetkit.MarsRequest` + (a verb plus a :data:`~pymetkit.pymetkit_type.MarsSelection`) and + :func:`~pymetkit.pymetkit.parse_mars_request`. +- ``pymetkit._internal`` — glue that locates and loads ``libmetkit`` via + `findlibs `__, initialises the bindings, and + re-exports the raw symbols. Not intended for direct use. +- :doc:`bindings` — the compiled ``pymetkit_bindings`` module that binds + ``metkit::mars::MarsRequest`` and ``metkit::mars::MarsExpansion``. + +.. note:: + + ``pymetkit`` supersedes the legacy CFFI-based interface. If you are migrating from + the old package, see :doc:`legacy`. .. toctree:: :maxdepth: 2 :caption: Contents: + :hidden: + + installation + examples + api + +.. toctree:: + :maxdepth: 2 + :caption: Technical Insights: + :hidden: + + bindings + development + legacy + +Quick start +----------- + +.. code-block:: python + + from pymetkit import MarsRequest, parse_mars_request + + request = MarsRequest( + "retrieve", + { + "class": "od", + "domain": "g", + "date": "-1", + "expver": "0001", + "step": range(0, 13, 6), + }, + ) + + expanded = request.expand() + print(expanded.verb(), dict(expanded)) - /autoapi/pymetkit/index + for req in parse_mars_request("retrieve,class=od,date=-1,param=129,step=12"): + print(req.verb(), req["param"]) diff --git a/docs/pymetkit/installation.rst b/docs/pymetkit/installation.rst new file mode 100644 index 000000000..e3d69d2c4 --- /dev/null +++ b/docs/pymetkit/installation.rst @@ -0,0 +1,109 @@ +.. _installation-label: + +Installation +############ + +Requirements +************ + +Build Dependencies +^^^^^^^^^^^^^^^^^^^ + ++----------+---------------------------------------------+ +|Dependency|Link | ++----------+---------------------------------------------+ +|CMake |http://www.cmake.org/ | ++----------+---------------------------------------------+ +|ecbuild |https://github.com/ecmwf/ecbuild | ++----------+---------------------------------------------+ +|Pybind11 |https://pybind11.readthedocs.io | ++----------+---------------------------------------------+ + +Runtime Dependencies +^^^^^^^^^^^^^^^^^^^^^ + ++----------+---------------------------------------------+ +|Dependency|Link | ++----------+---------------------------------------------+ +|eckit |https://github.com/ecmwf/eckit | ++----------+---------------------------------------------+ +|eccodes |https://github.com/ecmwf/eccodes | ++----------+---------------------------------------------+ + +Build from sources (recommended) +******************************** + +``PyMetKit`` is built as part of ``metkit`` by enabling the +``ENABLE_PYTHON_METKIT_INTERFACE`` CMake option, which requires ``Python >= 3.11`` and +``pybind11 >= 3.0.1``. + +Configure and build ``metkit`` (with its dependencies already installed at +````): + +.. code-block:: sh + + cmake -B build -S . -G Ninja \ + -DCMAKE_PREFIX_PATH= \ + -DCMAKE_BUILD_TYPE=RelWithDebInfo \ + -DENABLE_PYTHON_METKIT_INTERFACE=ON + cmake --build build -j + +This produces the ``pymetkit`` wheel in the build directory and a ready-to-use package +layout under ``build/pymetkit-python-package-staging``. + +.. tip:: + + For local exploration you can put the staging directory on your ``PYTHONPATH`` + instead of installing the wheel: + + .. code-block:: sh + + export PYTHONPATH=/pymetkit-python-package-staging + +Run the tests to verify the build: + +.. code-block:: sh + + cd build + ctest --output-on-failure -L pymetkit + +Installation via PyPI +********************* + +.. code-block:: sh + + uv venv + source .venv/bin/activate + uv pip install pymetkit + +Set the ``METKIT_HOME`` environment variable so ``findlibs`` can locate the library: + +.. code-block:: sh + + export METKIT_HOME= + +Diagnosing Library Resolution +***************************** + +``PyMetKit`` uses `findlibs `__ to locate the +``metkit`` shared library and its runtime dependencies at import time. If you encounter +errors caused by the wrong library version being loaded, the built-in CLI can help you +inspect what ``findlibs`` resolves on your system. + +Print the installation root of the ``metkit`` library: + +.. code-block:: sh + + python -m pymetkit --print-home + +Print the resolved home directories for all runtime dependencies (``eckit``, +``eccodes``, ``metkit``), together with any active ``FINDLIBS_DISABLE_*`` environment +variables that suppress specific search paths: + +.. code-block:: sh + + python -m pymetkit --print-home-deps + +``ERROR`` lines indicate dependencies ``findlibs`` could not locate — set the +corresponding ``_HOME`` environment variable to resolve them explicitly. +``eccodes`` is optional and is reported as such. diff --git a/docs/pymetkit/legacy.rst b/docs/pymetkit/legacy.rst new file mode 100644 index 000000000..9b28baf70 --- /dev/null +++ b/docs/pymetkit/legacy.rst @@ -0,0 +1,229 @@ +Legacy CFFI interface +===================== + +.. deprecated:: 1.19.0 + The CFFI-based ``pymetkit`` package (under ``python/pymetkit``) is superseded by the + pybind11-based package documented in :doc:`api`. It is retained for reference only and + is no longer built, tested, or published. + +.. warning:: + + **Do not use the legacy CFFI interface for new code.** It wraps the ``metkit`` C API + (``metkit_c.h``) through `cffi `__ rather than binding the + C++ classes directly, and it is not wired into the build or release workflows. All new + development should target the Pythonic :doc:`api`. + +Background +---------- + +The legacy interface exposed a ``MarsRequest`` class and a ``parse_mars_request`` function +backed by ``cffi`` (``ffi.dlopen`` + ``ffi.cdef`` against a stripped copy of +``metkit_c.h``), together with a ``PatchedLib`` error-wrapping layer and the +``MetKitException`` / ``CFFIModuleLoadFailed`` exceptions. + +Installation +------------ + +.. warning:: + + The legacy package is not published on PyPI and has no build system of its own. + These steps are provided for reference only. All new code should use the + pybind11-based ``pymetkit`` documented in :doc:`installation`. + +The legacy CFFI interface requires ``metkit = 1.19.2``. The steps below build that +version from source and wire up the Python package. + +**Build dependencies** + ++----------+---------------------------------------------+ +| Tool | Link | ++----------+---------------------------------------------+ +| CMake | https://cmake.org/ | ++----------+---------------------------------------------+ +| ecbuild | https://github.com/ecmwf/ecbuild | ++----------+---------------------------------------------+ +| Ninja | https://ninja-build.org/ | ++----------+---------------------------------------------+ + +**Runtime dependencies** + ++----------+---------------------------------------------+ +| Library | Link | ++----------+---------------------------------------------+ +| eckit | https://github.com/ecmwf/eckit | ++----------+---------------------------------------------+ +| eccodes | https://github.com/ecmwf/eccodes | ++----------+---------------------------------------------+ +| metkit | https://github.com/ecmwf/metkit | ++----------+---------------------------------------------+ +| libaec | https://github.com/MathisRosenhauer/libaec | ++----------+---------------------------------------------+ + +**Python dependencies** + ++--------------+---------------------------------------------+ +| Requirement | Link | ++--------------+---------------------------------------------+ +| Python 3.11 | https://www.python.org/ | ++--------------+---------------------------------------------+ +| cffi | https://cffi.readthedocs.io | ++--------------+---------------------------------------------+ +| findlibs | https://github.com/ecmwf/findlibs | ++--------------+---------------------------------------------+ + +**1. Build metkit 1.19.2 from source** + +Create a bundle directory and switch to it: + +.. code-block:: sh + + mkdir stack && cd stack + +Place the following ``CMakeLists.txt`` in it: + +.. code-block:: cmake + + cmake_minimum_required(VERSION 3.18 FATAL_ERROR) + + find_package(ecbuild 3.8 REQUIRED HINTS ${CMAKE_CURRENT_SOURCE_DIR} $ENV{HOME}/.local/ecbuild) + + project(ecmwf_stack_bundle VERSION 0.0.1 LANGUAGES CXX) + + set(CMAKE_CXX_STANDARD 17) + set(CMAKE_CXX_STANDARD_REQUIRED ON) + + ecbuild_bundle_initialize() + + ecbuild_bundle(PROJECT eckit GIT "https://github.com/ecmwf/eckit" BRANCH develop UPDATE) + ecbuild_bundle(PROJECT eccodes GIT "https://github.com/ecmwf/eccodes" BRANCH develop UPDATE) + ecbuild_bundle(PROJECT metkit GIT "https://github.com/ecmwf/metkit" TAG 1.19.2 UPDATE) + + ecbuild_bundle_finalize() + +.. tip:: + + If ``ecbuild``, ``ninja`` or ``aec`` are not available on your ``PATH``, load them via the + environment modules system before running ``cmake``: + + .. code-block:: sh + + module load ecbuild ninja aec + + Alternatively, adjust the ``HINTS`` path in the ``find_package`` call to point at + your ``ecbuild`` installation, and drop ``-G Ninja`` to fall back to ``make``. + +Create a build directory, configure and compile: + +.. code-block:: sh + + mkdir build && cd build + cmake -DCMAKE_INSTALL_PREFIX=../install \ + -DCMAKE_BUILD_TYPE=RelWithDebInfo \ + .. -G Ninja + ninja + +**2. Set up a Python environment** + +.. note:: + + Python 3.11 is required. If it is not your system default, load it first: + + .. code-block:: sh + + module load python3/3.11 + +.. code-block:: sh + + cd ../.. # back to the stack root + python3 -m venv .venv + source .venv/bin/activate + pip install cffi findlibs + +``cffi`` provides the C-extension glue; ``findlibs`` +(``__) locates ``libmetkit`` at runtime. + +**3. Install the legacy package** + +The metkit 1.19.2 tag ships a ``pyproject.toml`` at its root that packages the +legacy CFFI interface. Install it directly from the checked-out source: + +.. code-block:: sh + + cd stack/metkit + pip install . + cd ../.. + +Then let ``findlibs`` know where ``libmetkit`` was installed: + +.. code-block:: sh + + export METKIT_DIR=stack/install + +**4. Verify** + +.. code-block:: sh + + python - <<'EOF' + from pymetkit import MarsRequest + req = MarsRequest("retrieve", class_="od", date="-1", param="130") + print(list(req.keys())) + EOF + +Migration +--------- + +The new package keeps the same core concepts, so migration is largely mechanical: + +- ``from pymetkit import MarsRequest, parse_mars_request`` — unchanged import surface. +- A request is now built from a verb and a plain mapping rather than keyword arguments: + ``MarsRequest("retrieve", {"class": "od", "date": "-1"})`` replaces + ``MarsRequest("retrieve", class_="od", date="-1")``. +- ``MarsRequest.expand``, ``validate``, ``merge``, ``keys``, ``__setitem__``, + ``__contains__`` and ``__eq__`` behaviours are preserved. +- ``num_values`` and ``__getitem__`` now raise ``KeyError`` for a missing parameter + instead of returning ``0`` / ``[]`` as the legacy C-API layer did. +- :data:`~pymetkit.pymetkit_type.MarsSelection` is now a type alias for a user-supplied + mapping; value normalisation is handled internally rather than ad hoc. +- ``MetKitException`` is still raised for MARS language errors and is importable as + ``pymetkit.MetKitException``. + +Examples +^^^^^^^^ + +**Constructing a request** + +The legacy interface accepted keyword arguments, using a trailing underscore to +escape Python reserved words such as ``class``: + +.. code-block:: python + + # Legacy CFFI + request = MarsRequest("retrieve", class_="od", date="-1", step=[0, 6, 12]) + +The new interface takes a plain mapping; no escaping is needed: + +.. code-block:: python + + from pymetkit import MarsRequest + request = MarsRequest("retrieve", {"class": "od", "date": "-1", "step": [0, 6, 12]}) + +**Missing parameters** + +The legacy C-API returned sentinel values for absent parameters. The new interface +raises :exc:`KeyError` instead. Guard with ``in`` where absence is expected: + +.. code-block:: python + + # Legacy: silent sentinel values + n = request.num_values("step") # returned 0 if not set + v = request["step"] # returned [] if not set + +.. code-block:: python + + # New: raises KeyError — guard explicitly + from pymetkit import MarsRequest + request = MarsRequest("retrieve", {"class": "od", "date": "-1"}) + n = request.num_values("step") if "step" in request else 0 + v = request["step"] if "step" in request else [] + +See :doc:`api` and :doc:`examples` for the current interface. diff --git a/docs/pymetkit/requirements.txt b/docs/pymetkit/requirements.txt new file mode 100644 index 000000000..a0ea08f22 --- /dev/null +++ b/docs/pymetkit/requirements.txt @@ -0,0 +1,7 @@ +Sphinx +sphinxcontrib-mermaid +pydata-sphinx-theme +sphinx-autoapi +autoapi +sybil +pytest diff --git a/pyproject.toml b/pyproject.toml deleted file mode 100644 index 2dca76eec..000000000 --- a/pyproject.toml +++ /dev/null @@ -1,47 +0,0 @@ -# pytest -[tool.pytest.ini_options] -minversion = "6.0" -addopts = "-vv -s" -testpaths = [ - "pymetkit/tests" -] - -# pyproject.toml - -[build-system] -requires = ["setuptools", "wheel", "cffi"] -build-backend = "setuptools.build_meta" - -[project] -name = "pymetkit" -description = "Python interface for metkit" -dynamic = ["version"] -authors = [ - { name = "European Centre for Medium-Range Weather Forecasts (ECMWF)", email = "software.support@ecmwf.int" }, -] -license = { text = "Apache License Version 2.0" } -requires-python = ">=3.10" -dependencies = [ - "cffi", - "metkitlib", - "findlibs" -] - -[tool.setuptools.dynamic] -version = { file = ["VERSION"] } - -[tool.setuptools] -packages = ["pymetkit"] -package-dir = { "pymetkit" = "./python/pymetkit/src/pymetkit" } -include-package-data = true -zip-safe = false - -[tool.setuptools.package-data] -"pymetkit" = [ - "VERSION", - "metkit_c.h" -] - -[project.optional-dependencies] -tests = ["pytest"] - diff --git a/python/metkitlib/buildconfig b/python/metkitlib/buildconfig index f176b93be..dd57739ce 100644 --- a/python/metkitlib/buildconfig +++ b/python/metkitlib/buildconfig @@ -6,11 +6,27 @@ # granted to it by virtue of its status as an intergovernmental organisation # nor does it submit to any jurisdiction. -# to be source'd by wheelmaker's compile.sh *and* wheel-linux.sh -# NOTE replace the whole thing with pyproject.toml? Less powerful, and quaint to use for sourcing ecbuild invocation -# TODO we duplicate information -- pyproject.toml's `name` and `packages` are derivable from $NAME and must stay consistent +# to be source'd by wheelmaker's compile.sh *and* wheel-linux.sh +# NOTE +# replace the whole thing with pyproject.toml? Less powerful, and quaint to use +# for sourcing ecbuild invocation +# TODO +# we duplicate information -- pyproject.toml's `name` and `packages` are +# derivable from $NAME and must stay consistent NAME="metkit" -CMAKE_PARAMS="-Deckit_ROOT=/tmp/metkit/prereqs/eckitlib -Deccodes_ROOT=/tmp/metkit/prereqs/eccodeslib -DENABLE_GRIB=1" +# NOTE zarr interface is dependent on python 3.11+ -- but we dont activate the +# venv at the time this is sourced, that's why we rely on PYVERSION. Remove +# this whole part around October 2026 when 3.10 goes EoL if [ "True" = +# "$(python -c 'import sys; print(sys.version_info[0:2] >= (3, 11))')" ] ; then +if [ "3.10" != "$PYVERSION" ] ; then + PYMETKIT_IFACE="-DENABLE_PYTHON_METKIT_INTERFACE=ON" +else + >&2 echo "Not enabling pymetkit interface because python version is $PYVERSION" + PYMETKIT_IFACE="" +fi + +CMAKE_PARAMS="-Deckit_ROOT=/tmp/metkit/prereqs/eckitlib -Deccodes_ROOT=/tmp/metkit/prereqs/eccodeslib -DENABLE_GRIB=1 $PYMETKIT_IFACE" PYPROJECT_DIR="python/metkitlib" DEPENDENCIES='["eckitlib", "eccodeslib"]' + diff --git a/python/pymetkit/README.md b/python/pymetkit/README.md deleted file mode 100644 index c0554dc08..000000000 --- a/python/pymetkit/README.md +++ /dev/null @@ -1,28 +0,0 @@ -# pymetkit - -This repository contains an Python interface to the MetKit library for parsing MARS requests. - -## Example - -The function for parsing a MARS request is `metkit.parse_mars_request` which accepts a string or file-like object -as inputs. A list of `metkit.mars.Request` instances are returned, which is a dictionary containing the keys and -values in the MARS request and the attribute `verb` for the verb in the MARS request. - -### From String -``` -from metkit import parse_mars_request - -request_str = "retrieve,class=od,date=20240124,time=12,param=129,step=12,target=test.grib" -requests = parse_mars_request(requests) - -print(requests[0]) -# verb: retrieve, request: {'class': ['od'], 'date': ['20240124'], 'time': ['1200'], 'param': ['129'], 'step': ['12'], 'target': ['test.grib'], 'domain': ['g'], 'expver': ['0001'], 'levelist': ['1000', '850', '700', '500', '400', '300'], 'levtype': ['pl'], 'stream': ['oper'], 'type': ['an']} -``` - -### From File -If the MARS request is contained inside a file, e.g. test_requests.txt: -``` -from metkit import parse_mars_request - -requests = parse_mars_request(open("test_requests.txt", "r")) -``` diff --git a/python/pymetkit/src/pymetkit/__init__.py b/python/pymetkit/src/pymetkit/__init__.py deleted file mode 100644 index f64977776..000000000 --- a/python/pymetkit/src/pymetkit/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .pymetkit import * diff --git a/python/pymetkit/src/pymetkit/_version.py b/python/pymetkit/src/pymetkit/_version.py deleted file mode 100644 index 07549de45..000000000 --- a/python/pymetkit/src/pymetkit/_version.py +++ /dev/null @@ -1,5 +0,0 @@ -from pathlib import Path -from .pymetkit import * -import importlib.metadata - -__version__ = importlib.metadata.version("pymetkit") diff --git a/python/pymetkit/src/pymetkit/metkit_c.h b/python/pymetkit/src/pymetkit/metkit_c.h deleted file mode 100644 index 42d88946b..000000000 --- a/python/pymetkit/src/pymetkit/metkit_c.h +++ /dev/null @@ -1,52 +0,0 @@ - -struct metkit_marsrequest_t; -typedef struct metkit_marsrequest_t metkit_marsrequest_t; -struct metkit_requestiterator_t; -typedef struct metkit_requestiterator_t metkit_requestiterator_t; -struct metkit_paramiterator_t; -typedef struct metkit_paramiterator_t metkit_paramiterator_t; - -typedef enum metkit_error_values_t { - METKIT_SUCCESS = 0, /* Operation succeded. */ - METKIT_ERROR = 1, /* Operation failed. */ - METKIT_ERROR_UNKNOWN = 2, /* Failed with an unknown error. */ - METKIT_ERROR_USER = 3, /* Failed with an user error. */ - METKIT_ERROR_ASSERT = 4 /* Failed with an assert() */ -} metkit_error_t; - -typedef enum metkit_iterator_status_t { - METKIT_ITERATOR_SUCCESS = 0, /* Operation succeded. */ - METKIT_ITERATOR_COMPLETE = 1, /* All elements have been returned */ - METKIT_ITERATOR_ERROR = 2 /* Operation failed. */ -} metkit_iterator_status_t; - - -const char* metkit_get_error_string(enum metkit_error_values_t err); -const char* metkit_version(); -const char* metkit_git_sha1(); -metkit_error_t metkit_initialise(); - -metkit_error_t metkit_parse_marsrequests(const char* str, metkit_requestiterator_t** requests, bool strict); -metkit_error_t metkit_marsrequest_new(metkit_marsrequest_t** request); -metkit_error_t metkit_marsrequest_delete(const metkit_marsrequest_t* request); -metkit_error_t metkit_marsrequest_set(metkit_marsrequest_t* request, const char* param, const char* values[], - int numValues); -metkit_error_t metkit_marsrequest_set_one(metkit_marsrequest_t* request, const char* param, const char* value); -metkit_error_t metkit_marsrequest_set_verb(metkit_marsrequest_t* request, const char* verb); -metkit_error_t metkit_marsrequest_verb(const metkit_marsrequest_t* request, const char** verb); -metkit_error_t metkit_marsrequest_has_param(const metkit_marsrequest_t* request, const char* param, bool* has); -metkit_error_t metkit_marsrequest_params(const metkit_marsrequest_t* request, metkit_paramiterator_t** params); -metkit_error_t metkit_marsrequest_count_values(const metkit_marsrequest_t* request, const char* param, size_t* count); -metkit_error_t metkit_marsrequest_value(const metkit_marsrequest_t* request, const char* param, int index, - const char** value); -metkit_error_t metkit_marsrequest_expand(const metkit_marsrequest_t* request, bool inherit, bool strict, - metkit_marsrequest_t* expandedRequest); -metkit_error_t metkit_marsrequest_merge(metkit_marsrequest_t* request, const metkit_marsrequest_t* otherRequest); - -metkit_error_t metkit_requestiterator_delete(const metkit_requestiterator_t* it); -metkit_iterator_status_t metkit_requestiterator_next(metkit_requestiterator_t* it); -metkit_iterator_status_t metkit_requestiterator_current(metkit_requestiterator_t* it, metkit_marsrequest_t* request); - -metkit_error_t metkit_paramiterator_delete(const metkit_paramiterator_t* it); -metkit_iterator_status_t metkit_paramiterator_next(metkit_paramiterator_t* it); -metkit_iterator_status_t metkit_paramiterator_current(const metkit_paramiterator_t* it, const char** param); diff --git a/python/pymetkit/src/pymetkit/pymetkit.py b/python/pymetkit/src/pymetkit/pymetkit.py deleted file mode 100644 index 7c0594fa8..000000000 --- a/python/pymetkit/src/pymetkit/pymetkit.py +++ /dev/null @@ -1,312 +0,0 @@ -import os -from cffi import FFI -import findlibs -from typing import IO, Iterator -import warnings -from ._version import __version__ - -ffi = FFI() - - -def ffi_encode(data) -> bytes: - if isinstance(data, bytes): - return data - - if not isinstance(data, str): - data = str(data) - - return data.encode(encoding="utf-8", errors="surrogateescape") - - -def ffi_decode(data: FFI.CData) -> str: - buf = ffi.string(data) - if isinstance(buf, str): - return buf - else: - return buf.decode(encoding="utf-8", errors="surrogateescape") - - -class MarsRequest: - def __init__(self, verb: str | None = None, **kwargs): - """ - Create MetKit MarsRequest object. Parameters and values in - the request can be specified through kwargs, noting that - reserved words in Python must be suffixed with "_" e.g. "class_" - """ - crequest = ffi.new("metkit_marsrequest_t **") - lib.metkit_marsrequest_new(crequest) - self.__request = ffi.gc(crequest[0], lib.metkit_marsrequest_delete) - if verb is not None: - lib.metkit_marsrequest_set_verb(self.__request, ffi_encode(verb)) - for param, values in kwargs.items(): - self[param.rstrip("_")] = values - - def ctype(self) -> FFI.CData: - return self.__request - - def verb(self) -> str: - cverb = ffi.new("const char **") - lib.metkit_marsrequest_verb(self.__request, cverb) - return ffi_decode(cverb[0]) - - def expand(self, inherit: bool = True, strict: bool = False) -> "MarsRequest": - """ - Return expanded request - - Params - ------ - inherit: bool, if True, populates expanded request with default values - strict: bool, if True, raise error instead of warning for invalid values - - Returns - ------- - Request, resulting from expansion - """ - expanded_request = MarsRequest() - lib.metkit_marsrequest_expand( - self.__request, inherit, strict, expanded_request.ctype() - ) - return expanded_request - - def validate(self): - """ - Check if request is valid against MARS language definition. Does not - inherit missing parameters. - - Raises - ------ - Exception if request is incompatible with MARS language definition - """ - self.expand(False, True) - - def keys(self) -> Iterator[str]: - """ - Get iterator over parameters in request - - Returns - ------- - Iterator over parameter names - """ - it_c = ffi.new("metkit_paramiterator_t **") - lib.metkit_marsrequest_params(self.__request, it_c) - it = ffi.gc(it_c[0], lib.metkit_paramiterator_delete) - - while lib.metkit_paramiterator_next(it) == lib.METKIT_ITERATOR_SUCCESS: - cparam = ffi.new("const char **") - lib.metkit_paramiterator_current(it, cparam) - param = ffi_decode(cparam[0]) - yield param - - def num_values(self, param: str) -> int: - """ - Number of values for parameter - - Params - ------ - param: parameter name - - Returns - ------- - int - """ - cparam = ffi_encode(param) - count = ffi.new("size_t *", 0) - lib.metkit_marsrequest_count_values(self.__request, cparam, count) - return count[0] - - def merge(self, other: "MarsRequest") -> "MarsRequest": - """ - Merge the values in another request with existing request and returns result as a - new Request object. Does not modify inputs to merge. Both input requests must contain - the same values and the resulting request object must be compatible with MARS language - definition - - Params - ------ - other: Request, request to merge with self - - Returns - ------- - Request, containing the result of the merge - - Raises - ------ - ValueError if parameters in the two requests do not match - MetKitException if resulting request is not compatible with MARS language definition - """ - if set(self.keys()) != set(other.keys()): - raise ValueError("Can not merge requests with different parameters.") - res = MarsRequest(self.verb(), **{k: v for k, v in self}) - lib.metkit_marsrequest_merge(res.ctype(), other.ctype()) - res.validate() - return res - - def __iter__(self) -> Iterator[tuple[str, list[str]]]: - for param in self.keys(): - yield param, self[param] - - def __getitem__(self, param: str) -> str | list[str]: - nvalues = self.num_values(param) - values = [] - for index in range(nvalues): - cvalue = ffi.new("const char **") - lib.metkit_marsrequest_value(self.__request, ffi_encode(param), index, cvalue) - value = ffi_decode(cvalue[0]) - if nvalues == 1: - return value - values.append(value) - return values - - def __contains__(self, param: str) -> bool: - has = ffi.new("bool *", False) - lib.metkit_marsrequest_has_param(self.__request, ffi_encode(param), has) - return has[0] - - def __setitem__(self, param: str, values: int | str | list[str]): - if isinstance(values, (str, int)): - values = [values] - cvals = [] - for value in values: - if isinstance(value, int): - value = str(value) - cvals.append(ffi.new("const char[]", value.encode("ascii"))) - lib.metkit_marsrequest_set( - self.__request, - ffi_encode(param), - ffi.new("const char*[]", cvals), - len(values), - ) - - def __eq__(self, other: "MarsRequest") -> bool: - if self.verb() != other.verb(): - return False - expanded = self.expand() - other_expanded = other.expand() - return dict(expanded) == dict(other_expanded) - - -def parse_mars_request(file_or_str: IO | str, strict: bool = False) -> list[MarsRequest]: - """ - Function for parsing mars request from file object or string. - - Params - ------ - file_or_str: string or file-like object, containing mars request - strict: bool, whether to raise error or warning when request is not compatible with - MARS language definition. In the case of warning, when False, the incompatible - parameters are unset from the request. - - Returns - ------- - list of Request - """ - crequest_iter = ffi.new("metkit_requestiterator_t **") - - if isinstance(file_or_str, str): - lib.metkit_parse_marsrequests(ffi_encode(file_or_str), crequest_iter, strict) - else: - lib.metkit_parse_marsrequests( - ffi_encode(file_or_str.read()), crequest_iter, strict - ) - request_iter = ffi.gc(crequest_iter[0], lib.metkit_requestiterator_delete) - - requests = [] - while lib.metkit_requestiterator_next(request_iter) == lib.METKIT_ITERATOR_SUCCESS: - new_request = MarsRequest() - lib.metkit_requestiterator_current(request_iter, new_request.ctype()) - requests.append(new_request) - - return requests - - -class MetKitException(RuntimeError): - """Raised when MetKit library throws exception""" - - pass - - -class CFFIModuleLoadFailed(ImportError): - """Raised when the shared library fails to load""" - - pass - - -class PatchedLib: - """ - Patch a CFFI library with error handling - - Finds the header file associated with the MetKit C API and parses it, - loads the shared library, and patches the accessors with - automatic python-C error handling. - """ - - def __init__(self): - libName = findlibs.find("metkit") - - if libName is None: - raise RuntimeError("MetKit library not found") - - ffi.cdef(self.__read_header()) - self.__lib = ffi.dlopen(libName) - - # All of the executable members of the CFFI-loaded library are functions in the MetKit - # C API. These should be wrapped with the correct error handling. Otherwise forward - # these on directly. - - for f in dir(self.__lib): - try: - attr = getattr(self.__lib, f) - setattr( - self, f, self.__check_error(attr, f) if callable(attr) else attr - ) - except Exception as e: - print(e) - print("Error retrieving attribute", f, "from library") - - # Initialise the library, and set it up for python-appropriate behaviour - - self.metkit_initialise() - - # Check the library version - - versionstr = ffi.string(self.metkit_version()).decode("utf-8") - if versionstr != __version__: - warnings.warn(f"Metkit library version {versionstr} does not match python version {__version__}") - - def __read_header(self): - with open(os.path.join(os.path.dirname(__file__), "metkit_c.h"), "r") as f: - return f.read() - - def __check_error(self, fn, name: str): - """ - If calls into the MetKit library return errors, ensure that they get - detected and reported by throwing an appropriate python exception. - """ - - def wrapped_fn(*args, **kwargs): - - # debug - retval = fn(*args, **kwargs) - - # Some functions dont return error codes. Ignore these. - if name in ["metkit_version", "metkit_git_sha1"]: - return retval - - # error codes: - if retval not in ( - self.__lib.METKIT_SUCCESS, - self.__lib.METKIT_ITERATOR_SUCCESS, - self.__lib.METKIT_ITERATOR_COMPLETE, - ): - err = ffi_decode(self.__lib.metkit_get_error_string(retval)) - msg = "Error in function '{}': {}".format(name, err) - raise MetKitException(msg) - return retval - - return wrapped_fn - - -try: - lib = PatchedLib() -except CFFIModuleLoadFailed as e: - raise ImportError() from e diff --git a/ruff.toml b/ruff.toml new file mode 100644 index 000000000..4fc052eee --- /dev/null +++ b/ruff.toml @@ -0,0 +1,2 @@ +target-version = "py310" +line-length = 120 diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 396ed764a..ad49e1cd9 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -4,3 +4,64 @@ add_subdirectory( tools ) if( HAVE_EXPERIMENTAL ) add_subdirectory( experimental ) endif() + +if( HAVE_PYTHON_METKIT_INTERFACE ) + # We create the complete python package layout at this location. + # This allows us to run python wheel creation at this path and + # to put this path on the PYTHONPATH to allow direct use of + # pymetkit, e.g. for testing or local exploration. + set(PYMETKIT_STAGING "${CMAKE_BINARY_DIR}/pymetkit-python-package-staging") + file(MAKE_DIRECTORY "${PYMETKIT_STAGING}") + file(CREATE_LINK + "${CMAKE_CURRENT_SOURCE_DIR}/pymetkit" + "${PYMETKIT_STAGING}/pymetkit" SYMBOLIC + ) + # Copy README.md and LICENSE at build time so changes are picked up + # without needing to re-run cmake configure + add_custom_command( + OUTPUT ${PYMETKIT_STAGING}/README.md + COMMAND ${CMAKE_COMMAND} -E copy + "${CMAKE_CURRENT_SOURCE_DIR}/pymetkit/README.md" + "${PYMETKIT_STAGING}/README.md" + DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/pymetkit/README.md" + COMMENT "Copying pymetkit README.md to staging..." + ) + add_custom_command( + OUTPUT ${PYMETKIT_STAGING}/LICENSE + COMMAND ${CMAKE_COMMAND} -E copy + "${CMAKE_CURRENT_SOURCE_DIR}/../LICENSE" + "${PYMETKIT_STAGING}/LICENSE" + DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/../LICENSE" + COMMENT "Copying LICENSE to staging..." + ) + configure_file( + ${CMAKE_CURRENT_SOURCE_DIR}/../cmake/pymetkit_setup.py.in + ${PYMETKIT_STAGING}/setup.py + @ONLY + ) + configure_file( + ${CMAKE_CURRENT_SOURCE_DIR}/../cmake/pymetkit_setup.cfg.in + ${PYMETKIT_STAGING}/setup.cfg + @ONLY + ) + add_subdirectory(pymetkit_bindings) + file(GLOB_RECURSE + _pymetkit_package_files + "pymetkit/*.py" + "pymetkit/_internal/*.py" + ) + list(APPEND _pymetkit_package_files + "${CMAKE_CURRENT_SOURCE_DIR}/pymetkit/README.md" + "${CMAKE_CURRENT_SOURCE_DIR}/../LICENSE" + ) + add_custom_command( + OUTPUT ${CMAKE_BINARY_DIR}/pymetkit.wheel.stamp + COMMAND ${Python_EXECUTABLE} -m build --wheel ${PYMETKIT_STAGING} -o . + COMMAND ${CMAKE_COMMAND} -E touch pymetkit.wheel.stamp + WORKING_DIRECTORY ${CMAKE_BINARY_DIR} + DEPENDS ${_pymetkit_package_files} pymetkit_bindings + ${PYMETKIT_STAGING}/README.md ${PYMETKIT_STAGING}/LICENSE + COMMENT "Building Python wheel for pymetkit..." + ) + add_custom_target(pymetkit-wheel ALL DEPENDS ${CMAKE_BINARY_DIR}/pymetkit.wheel.stamp) +endif() diff --git a/src/pymetkit/README.md b/src/pymetkit/README.md new file mode 100644 index 000000000..edd74b312 --- /dev/null +++ b/src/pymetkit/README.md @@ -0,0 +1,52 @@ +# pymetkit + +`pymetkit` is a Python interface to [metkit](https://github.com/ecmwf/metkit), ECMWF's +meteorological toolkit. It exposes the MARS request model in a Pythonic way, built on a +[pybind11](https://github.com/pybind/pybind11) extension module (`pymetkit_bindings`) that binds the +metkit C++ library directly. + +The native `libmetkit` shared library and its dependencies are located at runtime via +[findlibs](https://github.com/ecmwf/findlibs). + +## Architecture + +- `pymetkit_bindings` — compiled pybind11 module binding `metkit::mars::MarsRequest` and + `metkit::mars::MarsExpansion`. +- `pymetkit._internal` — loads the native library via `findlibs`, initialises the bindings, and + re-exports the raw symbols. +- `pymetkit` — the Pythonic layer: `MarsRequest` (a verb plus a `MarsSelection`), + `MarsSelection` (a type alias for the user-facing key-value mapping), + `UserInputMapper` (normalises `MarsSelection` values to and from the internal + `dict[str, list[str]]` representation), and `parse_mars_request`. + +## Usage + +```python +from pymetkit import MarsRequest, parse_mars_request + +# Build a request from a verb and a selection +request = MarsRequest( + "retrieve", + { + "class": "od", + "domain": "g", + "date": "-1", + "expver": "0001", + "step": range(0, 13, 6), + }, +) + +# Expand against the MARS language definition +expanded = request.expand() +print(expanded.verb(), dict(expanded)) + +# Parse requests from a string or a file +requests = parse_mars_request("retrieve,class=od,date=-1,param=129,step=12") +``` + +## Command line + +```bash +python -m pymetkit --print-home # metkit library home +python -m pymetkit --print-home-deps # all dependency homes and versions +``` diff --git a/src/pymetkit/__init__.py b/src/pymetkit/__init__.py new file mode 100644 index 000000000..fbcb3424c --- /dev/null +++ b/src/pymetkit/__init__.py @@ -0,0 +1,13 @@ +# SPDX-FileCopyrightText: 2026 European Centre for Medium-Range Weather Forecasts (ECMWF) +# SPDX-License-Identifier: Apache-2.0 + +from pymetkit._internal import MetKitException +from pymetkit.pymetkit import MarsRequest, parse_mars_request +from pymetkit.pymetkit_type import MarsSelection + +__all__ = [ + "MarsRequest", + "MarsSelection", + "parse_mars_request", + "MetKitException", +] diff --git a/src/pymetkit/__main__.py b/src/pymetkit/__main__.py new file mode 100644 index 000000000..318075c84 --- /dev/null +++ b/src/pymetkit/__main__.py @@ -0,0 +1,110 @@ +# SPDX-FileCopyrightText: 2026 European Centre for Medium-Range Weather Forecasts (ECMWF) +# SPDX-License-Identifier: Apache-2.0 + +import argparse +import logging +import os +import sys + +import findlibs + +import pymetkit._internal as _internal + +# INFO: This is in place because we currently can't (at runtime) +# tell which order the dependencies are in. This needs to be available in findlibs +DEPENDENCY_ORDER = ["eckit", "eccodes", "metkit"] +OPTIONAL_DEPENDENCIES = ["eccodes"] + + +def main(): + parser = argparse.ArgumentParser(description="pymetkit command line interface") + parser.add_argument( + "--print-home", + action="store_true", + help="Print the home directory of the metkit library", + ) + parser.add_argument( + "--print-home-deps", + action="store_true", + help="Print the home directories of all pymetkit dependencies", + ) + parser.add_argument( + "-v", + "--verbose", + action="store_true", + help="Enable DEBUG logging (default: INFO)", + ) + args = parser.parse_args() + + logging.basicConfig( + format="%(asctime)s | %(levelname)-6s | %(message)s", + datefmt="%Y-%m-%d %H:%M:%S", + level=logging.DEBUG if args.verbose else logging.INFO, + ) + + if not (args.print_home or args.print_home_deps): + parser.print_help() + sys.exit(2) + + def _lib_home(lib_path): + lib_dir = os.path.dirname(os.path.realpath(lib_path)) + return ( + os.path.dirname(lib_dir) + if os.path.basename(lib_dir) in ("lib", "lib64") + else lib_dir + ) + + def _print_dep_path(lib, dependency_path, optional): + missing = [] + if dependency_path is None: + msg = f"\t{lib} [Optional]" if optional else f"\t{lib}" + msg += ": not found by findlibs" + missing.append(lib) + if optional: + logging.info(msg) + else: + logging.error(msg) + else: + msg = f"\t{lib} [Optional]" if optional else f"\t{lib}" + msg += f": {_lib_home(dependency_path)}" + logging.info(msg) + + return missing + + library_info_tuple = _internal.version_info() + + if args.print_home: + dependency_path = findlibs.find("metkit") + if dependency_path is None: + logging.error("metkit library not found by findlibs") + sys.exit(1) + for name, version, gitSha, path in library_info_tuple: + if name == "metkit": + logging.info(f"\t{name} {version} ({gitSha}) {path}") + + if args.print_home_deps: + logging.info("Findlibs Environment:") + for key, value in os.environ.items(): + if key.upper().startswith("FINDLIBS_DISABLE"): + logging.info(f"\t{key: <15}: {value: <10}") + + logging.info("Findlibs Lookup") + + missing = [] + for lib in DEPENDENCY_ORDER: + dependency_path = findlibs.find(lib) + missing += _print_dep_path( + lib, dependency_path, lib in OPTIONAL_DEPENDENCIES + ) + + logging.info("Dependency Versions:") + + for name, version, gitSha, path in library_info_tuple: + logging.info(f"\t{name} {version} ({gitSha}) {path}") + + if any(lib not in OPTIONAL_DEPENDENCIES for lib in missing): + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/src/pymetkit/_internal/__init__.py b/src/pymetkit/_internal/__init__.py new file mode 100644 index 000000000..f2b33d340 --- /dev/null +++ b/src/pymetkit/_internal/__init__.py @@ -0,0 +1,30 @@ +# SPDX-FileCopyrightText: 2026 European Centre for Medium-Range Weather Forecasts (ECMWF) +# SPDX-License-Identifier: Apache-2.0 + +# libmetkit.so and dependencies have to be loaded prior to importing +# pymetkit +import findlibs + +findlibs.load("metkit") + +from pymetkit._internal.pymetkit_internal import ( + MetKitException, +) +from pymetkit_bindings.pymetkit_bindings import ( + MarsRequest as _MarsRequest, +) +from pymetkit_bindings.pymetkit_bindings import ( + init_bindings, + parse_marsrequest, + parse_marsrequests, + version_info, +) + +__all__ = [ + "init_bindings", + "version_info", + "parse_marsrequest", + "parse_marsrequests", + "_MarsRequest", + "MetKitException", +] diff --git a/src/pymetkit/_internal/pymetkit_internal.py b/src/pymetkit/_internal/pymetkit_internal.py new file mode 100644 index 000000000..0ca321ccb --- /dev/null +++ b/src/pymetkit/_internal/pymetkit_internal.py @@ -0,0 +1,13 @@ +# SPDX-FileCopyrightText: 2026 European Centre for Medium-Range Weather Forecasts (ECMWF) +# SPDX-License-Identifier: Apache-2.0 + +from pymetkit_bindings import pymetkit_bindings as pymetkit_internal + +# Initial setup of binding via eckit main +pymetkit_internal.init_bindings() + + +class MetKitException(RuntimeError): + """Raised when the MetKit library throws an exception.""" + + pass diff --git a/src/pymetkit/pymetkit.py b/src/pymetkit/pymetkit.py new file mode 100644 index 000000000..58545fbcc --- /dev/null +++ b/src/pymetkit/pymetkit.py @@ -0,0 +1,213 @@ +# SPDX-FileCopyrightText: 2026 European Centre for Medium-Range Weather Forecasts (ECMWF) +# SPDX-License-Identifier: Apache-2.0 + +from collections.abc import Mapping +from typing import IO, Iterator + +from pymetkit._internal import ( + MetKitException, + _MarsRequest, + parse_marsrequests, +) +from pymetkit.pymetkit_type import InternalMarsSelection, MarsSelection, UserInputMapper + + +class MarsRequest: + """ + A MARS request: a verb (e.g. ``retrieve``) together with a + :data:`MarsSelection` describing the parameters and their values. + + Parameters + ---------- + verb : str + The request verb, e.g. ``retrieve``. + selection : MarsSelection, optional + Initial parameter values. Scalars are wrapped in a singleton list, + collections are stringified, and ``/``-separated strings are split. + + Examples + -------- + >>> request = MarsRequest("retrieve", {"class": "od", "date": "20200101/20200102", "param": [151, 129]}) + >>> request.verb() + 'retrieve' + >>> request["class"] + 'od' + >>> request["date"] + ['20200101', '20200102'] + >>> request["param"] + ['151', '129'] + + Iterating yields ``(name, value)`` pairs: + + >>> for key, value in request: + ... print(key, value) + class od + date ['20200101', '20200102'] + param ['151', '129'] + """ + + def __init__(self, verb: str, selection: MarsSelection | None = None, /): + self._verb = verb + if selection is not None and not isinstance(selection, Mapping): + raise ValueError(f"MarsRequest: expected a mapping, got {type(selection).__name__}.") + combined: MarsSelection = selection if selection is not None else {} + self.selection: InternalMarsSelection = UserInputMapper.map_selection_to_internal(combined) + + # -- Construction / conversion helpers --------------------------------- + + def _to_internal(self) -> _MarsRequest: + internal = _MarsRequest(self._verb) + for param, values in self.selection.items(): + internal.set(param, list(values)) + return internal + + @classmethod + def _from_internal(cls, internal: _MarsRequest) -> "MarsRequest": + request = cls(internal.verb()) + for param in internal.params(): + request.selection[param] = internal.values(param) + return request + + # -- Queries ----------------------------------------------------------- + + def verb(self) -> str: + """Return the request verb.""" + return self._verb + + def keys(self) -> Iterator[str]: + """Return an iterator over the parameter names in the request.""" + return iter(self.selection.keys()) + + def num_values(self, param: str) -> int: + """Return the number of values for a parameter.""" + return len(self.selection[param]) + + # -- Operations backed by the MARS language engine -------------------- + + def expand(self, inherit: bool = True, strict: bool = False) -> "MarsRequest": + """ + Return the expanded request. + + Parameters + ---------- + inherit : bool + If True, populate the expanded request with default values. + strict : bool + If True, raise an error instead of a warning for invalid values. + + Returns + ------- + MarsRequest + The request resulting from expansion. + """ + try: + expanded = self._to_internal().expand(inherit, strict) + except RuntimeError as error: + raise MetKitException(str(error)) from error + return MarsRequest._from_internal(expanded) + + def validate(self) -> None: + """ + Check that the request is valid against the MARS language definition. + Does not inherit missing parameters. + + Raises + ------ + MetKitException + If the request is incompatible with the MARS language definition. + """ + self.expand(inherit=False, strict=True) + + def merge(self, other: "MarsRequest") -> "MarsRequest": + """ + Merge the values of another request into this one and return the result + as a new request. Does not modify either input. Both requests must + contain the same parameters and the result must be compatible with the + MARS language definition. + + Parameters + ---------- + other : MarsRequest + The request to merge with self. + + Returns + ------- + MarsRequest + The result of the merge. + + Raises + ------ + ValueError + If the parameters in the two requests do not match. + MetKitException + If the resulting request is not compatible with the MARS language definition. + """ + if set(self.keys()) != set(other.keys()): + raise ValueError("Cannot merge requests with different parameters.") + internal = self._to_internal() + try: + internal.merge(other._to_internal()) + except RuntimeError as error: + raise MetKitException(str(error)) from error + result = MarsRequest._from_internal(internal) + result.validate() + return result + + # -- Mapping-like interface ------------------------------------------- + + def __iter__(self) -> Iterator[tuple[str, str | list[str]]]: + for key in self.selection: + yield key, UserInputMapper.map_values_to_external(self.selection[key]) + + def __getitem__(self, param: str) -> str | list[str]: + return UserInputMapper.map_values_to_external(self.selection[param]) + + def __setitem__(self, param: str, values) -> None: + self.selection[param] = UserInputMapper._normalize_values(param, values) + + def __contains__(self, param: str) -> bool: + return param in self.selection + + def __eq__(self, other: object) -> bool: + if not isinstance(other, MarsRequest): + return NotImplemented + if self.verb() != other.verb(): + return False + return dict(self.expand()) == dict(other.expand()) + + def __hash__(self) -> int: + expanded = self.expand() + return hash( + ( + expanded.verb(), + frozenset((k, tuple(v)) for k, v in expanded.selection.items()), + ) + ) + + def __repr__(self) -> str: + return repr(self._to_internal()) + + +def parse_mars_request(file_or_str: IO | str, strict: bool = False) -> list[MarsRequest]: + """ + Parse one or more MARS requests from a file-like object or a string. + + Parameters + ---------- + file_or_str : str | IO + A string or file-like object containing one or more MARS requests. + strict : bool + Whether to raise an error (True) or a warning (False) when a request is + not compatible with the MARS language definition. When False, the + incompatible parameters are unset from the request. + + Returns + ------- + list[MarsRequest] + """ + text = file_or_str if isinstance(file_or_str, str) else file_or_str.read() + try: + requests = parse_marsrequests(text, strict) + except RuntimeError as error: + raise MetKitException(str(error)) from error + return [MarsRequest._from_internal(request) for request in requests] diff --git a/src/pymetkit/pymetkit_type.py b/src/pymetkit/pymetkit_type.py new file mode 100644 index 000000000..5c7fc7fb6 --- /dev/null +++ b/src/pymetkit/pymetkit_type.py @@ -0,0 +1,96 @@ +# SPDX-FileCopyrightText: 2026 European Centre for Medium-Range Weather Forecasts (ECMWF) +# SPDX-License-Identifier: Apache-2.0 + +from collections.abc import Collection, Mapping + +InternalMarsSelection = dict[str, list[str]] +""" +Internal representation of a MARS selection. + +A key-value map, mapping MARS keys to a list of string values. This is the form +handed to the ``pymetkit_bindings`` layer. +""" + +MarsSelection = Mapping[str, "str | int | float | Collection[str | int | float]"] +""" +Selection part of a MARS request: a mapping from MARS keys to user-supplied values. + +Values may be a scalar (``str``, ``int``, ``float``) or a collection of those. +A ``str`` containing ``/`` is treated as a MARS range/list expression and split +on ``/`` by :meth:`UserInputMapper.map_selection_to_internal`. +""" + + +class UserInputMapper: + """ + Normalises user-supplied MARS selections to and from the internal + ``dict[str, list[str]]`` representation used by the bindings layer. + + - :meth:`map_selection_to_internal` converts a user-facing + :data:`MarsSelection` (scalars, collections, range expressions) to an + :data:`InternalMarsSelection`. + - :meth:`map_selection_to_external` converts an :data:`InternalMarsSelection` + back to a user-friendly form, collapsing single-element lists to scalars. + """ + + @classmethod + def map_selection_to_internal(cls, selection: MarsSelection) -> InternalMarsSelection: + """Normalise a user-supplied selection to ``dict[str, list[str]]``. + + Each value is converted to a list of strings: scalars are wrapped in a + one-element list, collections are stringified element-by-element, and + ``str`` values containing ``/`` are split on ``/``. + """ + result: InternalMarsSelection = {} + + for key, values in selection.items(): + result[key] = cls._normalize_values(key, values) + + return result + + @classmethod + def map_selection_to_external(cls, selection: InternalMarsSelection) -> MarsSelection: + """Convert an internal selection back to a user-friendly form. + + Each ``list[str]`` value is collapsed to a plain ``str`` if it contains + a single element, or left as a ``list[str]`` if it contains multiple. + """ + result = {} + + for key, values in selection.items(): + result[key] = cls.map_values_to_external(values) + + return result + + @staticmethod + def map_values_to_external(values: Collection[str]) -> str | list[str]: + """Collapse a single-element list to a scalar; return multi-element lists as-is.""" + values = list(values) + if len(values) == 1: + return values[0] + return values + + @staticmethod + def _normalize_values(key: str, values) -> list[str]: + if not isinstance(values, (int, float, str, Collection)) or isinstance(values, Mapping): + raise ValueError( + f"MarsSelection: the value for key '{key}' is not valid. Values must be " + "int, float, str or a collection of those." + ) + + # Values is a collection but not a single string + if isinstance(values, Collection) and not isinstance(values, str): + return [str(value) if isinstance(value, (float, int)) else value for value in values] + # Single numeric value + if isinstance(values, (int, float)): + return [str(values)] + # Single string; '/'-separated range/list expressions are split + if isinstance(values, str): + if "/" in values: + return values.split("/") + return [values] + + raise ValueError( + f"MarsSelection: unknown type for key '{key}'. Values must be int, float, " + "str or a collection of those." + ) diff --git a/src/pymetkit_bindings/CMakeLists.txt b/src/pymetkit_bindings/CMakeLists.txt new file mode 100644 index 000000000..31043392a --- /dev/null +++ b/src/pymetkit_bindings/CMakeLists.txt @@ -0,0 +1,20 @@ +pybind11_add_module(pymetkit_bindings MODULE bindings.cc) + +target_link_libraries(pymetkit_bindings + PRIVATE pybind11::module + pybind11::lto + eckit + metkit + # We need to explicitly link to stdc++fs on gcc 8.x + $<$,$,8.0>,$,9.0>>:stdc++fs> +) + +if(NOT MSVC AND NOT ${CMAKE_BUILD_TYPE} MATCHES Debug|RelWithDebInfo) + # Strip unnecessary sections of the binary on Linux/macOS + pybind11_strip(pymetkit_bindings) +endif() + +set_target_properties(pymetkit_bindings + PROPERTIES + LIBRARY_OUTPUT_DIRECTORY ${PYMETKIT_STAGING}/pymetkit_bindings +) diff --git a/src/pymetkit_bindings/bindings.cc b/src/pymetkit_bindings/bindings.cc new file mode 100644 index 000000000..24c9a8ddf --- /dev/null +++ b/src/pymetkit_bindings/bindings.cc @@ -0,0 +1,90 @@ +/* + * (C) Copyright 2025- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation + * nor does it submit to any jurisdiction. + */ + +#include +#include + +#include +#include +#include +#include + +#include "eckit/runtime/Main.h" +#include "eckit/system/Library.h" +#include "eckit/system/LibraryManager.h" + +#include "metkit/mars/MarsExpansion.h" +#include "metkit/mars/MarsRequest.h" + +namespace py = pybind11; +namespace mars = metkit::mars; + + +PYBIND11_MODULE(pymetkit_bindings, m) { + + m.def("init_bindings", []() { + char arg0[] = "pymetkit"; + char* argv[] = {arg0, nullptr}; + eckit::Main::initialise(1, argv); + }); + + m.def("version_info", []() { + std::vector> dependencyInformation; + + for (const std::string& libname : eckit::system::LibraryManager::list()) { + const eckit::system::Library& lib = eckit::system::LibraryManager::lookup(libname); + dependencyInformation.emplace_back(lib.name(), lib.version(), lib.gitsha1(), lib.libraryPath()); + } + + return dependencyInformation; + }); + + //-------------------------------------------------- + // @brief MarsRequest + //-------------------------------------------------- + + py::class_(m, "MarsRequest") + .def(py::init()) + .def(py::init([](const std::string& verb) { + mars::MarsRequest request; + request.verb(verb); + return request; + })) + .def("verb", [](const mars::MarsRequest& request) { return request.verb(); }) + .def("set_verb", [](mars::MarsRequest& request, const std::string& verb) { request.verb(verb); }) + .def("set", [](mars::MarsRequest& request, const std::string& param, + const std::vector& values) { request.values(param, values); }) + .def("has", [](const mars::MarsRequest& request, const std::string& param) { return request.has(param); }) + .def("params", [](const mars::MarsRequest& request) { return request.params(); }) + .def("values", + [](const mars::MarsRequest& request, const std::string& param) { + const std::vector& values = request.values(param, false); + return std::vector{values.begin(), values.end()}; + }) + .def("merge", [](mars::MarsRequest& request, const mars::MarsRequest& other) { request.merge(other); }) + .def("expand", + [](const mars::MarsRequest& request, bool inherit, bool strict) { + mars::MarsExpansion expansion(inherit, strict); + return expansion.expand(request); + }) + .def("__repr__", [](const mars::MarsRequest& request) { return request.asString(); }); + + //-------------------------------------------------- + // @brief Parsing + //-------------------------------------------------- + + m.def("parse_marsrequests", [](const std::string& str, bool strict) { + std::istringstream in(str); + return mars::MarsRequest::parse(in, strict); + }); + + m.def("parse_marsrequest", + [](const std::string& str, bool strict) { return mars::MarsRequest::parse(str, strict); }); +} diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index be1a9ece6..ae9882896 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -111,3 +111,7 @@ add_subdirectory(marsgen) add_subdirectory(mars2grib) add_subdirectory(mars2mars) add_subdirectory(tools) + +if( HAVE_PYTHON_METKIT_INTERFACE ) + add_subdirectory(pymetkit) +endif() diff --git a/tests/pymetkit/CMakeLists.txt b/tests/pymetkit/CMakeLists.txt new file mode 100644 index 000000000..7aa1b152a --- /dev/null +++ b/tests/pymetkit/CMakeLists.txt @@ -0,0 +1,58 @@ +# (C) Copyright 2025- ECMWF. +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. +# In applying this licence, ECMWF does not waive the privileges and immunities +# granted to it by virtue of its status as an intergovernmental organisation +# nor does it submit to any jurisdiction. + +# Create the pytest-tmp folder for encompassing the different scenarios +file(MAKE_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/pytest-tmp") + +function(add_pymetkit_test scenario test) + get_filename_component(stem ${test} NAME_WLE) + add_test( + NAME ${scenario}_${stem} + COMMAND ${Python_EXECUTABLE} + -m pytest + --basetemp ${CMAKE_CURRENT_BINARY_DIR}/pytest-tmp/${scenario}/ + -vv -s ${CMAKE_CURRENT_SOURCE_DIR}/${test} + ) + set(_env + "PYTHONPATH=${CMAKE_BINARY_DIR}/pymetkit-python-package-staging:$ENV{PYTHONPATH}" + # prevent findlibs in pymetkit from picking up any globally installed metkit + "METKIT_DIR=${CMAKE_BINARY_DIR}" + "FINDLIBS_DISABLE_PACKAGE=yes" + # metkit_env provides METKIT_HOME and ECCODES_DEFINITION_PATH for expansion + ${metkit_env} + ) + set_tests_properties(${scenario}_${stem} + PROPERTIES + RESOURCE_LOCK ${scenario} + ENVIRONMENT "${_env}" + LABELS ${scenario} + ) +endfunction() + +set(test_files + integration/mars_request/test_construction.py + integration/mars_request/test_accessors.py + integration/mars_request/test_selection.py + integration/mars_request/test_expand.py + integration/mars_request/test_merge.py + integration/mars_request/test_equality.py + integration/mars_request/test_parse.py + cli/test_cli.py +) + +foreach(test_file ${test_files}) + add_pymetkit_test("pymetkit" ${test_file}) +endforeach() + +set(doc_test_files + ../../docs/pymetkit/examples.rst +) + +foreach(test_file ${doc_test_files}) + add_pymetkit_test("pymetkit_doc" ${test_file}) +endforeach() diff --git a/tests/pymetkit/cli/test_cli.py b/tests/pymetkit/cli/test_cli.py new file mode 100644 index 000000000..887b9bae5 --- /dev/null +++ b/tests/pymetkit/cli/test_cli.py @@ -0,0 +1,244 @@ +# SPDX-FileCopyrightText: 2026 European Centre for Medium-Range Weather Forecasts (ECMWF) +# SPDX-License-Identifier: Apache-2.0 + +import logging +import sys +from pathlib import Path +from unittest.mock import MagicMock + +import findlibs +import pymetkit._internal as _internal +import pytest + +from pymetkit.__main__ import DEPENDENCY_ORDER, OPTIONAL_DEPENDENCIES, main + +ALL_LIBS = { + "eckit": "/fake/eckit/lib/libeckit.so", + "eccodes": "/fake/eccodes/lib/libeccodes.so", + "metkit": "/fake/metkit/lib/libmetkit.so", +} + +FAKE_VERSION_INFO = [ + ("eckit", "1.32.5", "def5678", "/fake/eckit/lib/libeckit.so"), + ("eccodes", "2.46.0", "jkl3456", "/fake/eccodes/lib/libeccodes.so"), + ("metkit", "1.19.0", "ghi9012", "/fake/metkit/lib/libmetkit.so"), +] + + +def _run_cli(args, monkeypatch): + monkeypatch.setattr(sys, "argv", ["pymetkit"] + args) + try: + main() + return 0 + except SystemExit as exc: + return exc.code + + +def _expected_entry(name, path): + """Build the expected log fragment for a findlibs lookup line.""" + label = f"{name} [Optional]" if name in OPTIONAL_DEPENDENCIES else name + home = Path(path).parent.parent + return f"{label}: {home}" + + +@pytest.fixture(autouse=True) +def capture_info_logs(caplog): + caplog.set_level(logging.INFO) + + +@pytest.fixture(autouse=True) +def version_info_mock(monkeypatch): + mock = MagicMock(return_value=FAKE_VERSION_INFO) + monkeypatch.setattr(_internal, "version_info", mock) + return mock + + +@pytest.fixture +def find_mock(monkeypatch): + mock = MagicMock(side_effect=lambda name: ALL_LIBS.get(name)) + monkeypatch.setattr(findlibs, "find", mock) + return mock + + +# --------------------------------------------------------------------------- +# --print-home +# --------------------------------------------------------------------------- + + +def test_print_home_success(find_mock, monkeypatch, caplog): + exit_code = _run_cli(["--print-home"], monkeypatch) + assert exit_code == 0 + + +def test_print_home_logs_version_info(find_mock, monkeypatch, caplog): + _run_cli(["--print-home"], monkeypatch) + name, version, git_sha, path = FAKE_VERSION_INFO[-1] # metkit entry + assert name in caplog.text + assert version in caplog.text + assert git_sha in caplog.text + + +def test_print_home_not_found(monkeypatch, caplog): + monkeypatch.setattr(findlibs, "find", MagicMock(return_value=None)) + exit_code = _run_cli(["--print-home"], monkeypatch) + assert exit_code == 1 + assert "not found by findlibs" in caplog.text + + +def test_print_home_calls_find_with_metkit(find_mock, monkeypatch): + _run_cli(["--print-home"], monkeypatch) + find_mock.assert_called_once_with("metkit") + + +# --------------------------------------------------------------------------- +# --print-home-deps +# --------------------------------------------------------------------------- + + +def test_print_home_deps_all_found(find_mock, monkeypatch, caplog): + exit_code = _run_cli(["--print-home-deps"], monkeypatch) + assert exit_code == 0 + for name, path in ALL_LIBS.items(): + assert _expected_entry(name, path) in caplog.text + + +def test_print_home_deps_logs_dependency_versions(find_mock, monkeypatch, caplog): + _run_cli(["--print-home-deps"], monkeypatch) + assert "Dependency Versions:" in caplog.text + for name, version, git_sha, _ in FAKE_VERSION_INFO: + assert name in caplog.text + assert version in caplog.text + assert git_sha in caplog.text + + +def test_print_home_deps_dependency_versions_logged_before_exit(monkeypatch, caplog): + libs_without_eckit = {k: v for k, v in ALL_LIBS.items() if k != "eckit"} + monkeypatch.setattr( + findlibs, "find", MagicMock(side_effect=lambda n: libs_without_eckit.get(n)) + ) + _run_cli(["--print-home-deps"], monkeypatch) + assert "Dependency Versions:" in caplog.text + for name, version, git_sha, _ in FAKE_VERSION_INFO: + assert name in caplog.text + assert version in caplog.text + assert git_sha in caplog.text + + +def test_print_home_deps_missing_required_exits_nonzero(monkeypatch, caplog): + libs_without_eckit = {k: v for k, v in ALL_LIBS.items() if k != "eckit"} + monkeypatch.setattr( + findlibs, "find", MagicMock(side_effect=lambda n: libs_without_eckit.get(n)) + ) + exit_code = _run_cli(["--print-home-deps"], monkeypatch) + assert exit_code == 1 + assert "eckit" in caplog.text + + +def test_print_home_deps_missing_required_logs_error(monkeypatch, caplog): + libs_without_eckit = {k: v for k, v in ALL_LIBS.items() if k != "eckit"} + monkeypatch.setattr( + findlibs, "find", MagicMock(side_effect=lambda n: libs_without_eckit.get(n)) + ) + _run_cli(["--print-home-deps"], monkeypatch) + errors = [ + r for r in caplog.records + if r.levelno == logging.ERROR and "eckit" in r.message + ] + assert errors + + +def test_print_home_deps_missing_optional_exits_zero(monkeypatch): + libs_without_eccodes = {k: v for k, v in ALL_LIBS.items() if k != "eccodes"} + monkeypatch.setattr( + findlibs, "find", MagicMock(side_effect=lambda n: libs_without_eccodes.get(n)) + ) + exit_code = _run_cli(["--print-home-deps"], monkeypatch) + assert exit_code == 0 + + +def test_print_home_deps_missing_optional_logs_info_not_error(monkeypatch, caplog): + libs_without_eccodes = {k: v for k, v in ALL_LIBS.items() if k != "eccodes"} + monkeypatch.setattr( + findlibs, "find", MagicMock(side_effect=lambda n: libs_without_eccodes.get(n)) + ) + _run_cli(["--print-home-deps"], monkeypatch) + eccodes_records = [r for r in caplog.records if "eccodes" in r.message] + assert eccodes_records + assert all(r.levelno == logging.INFO for r in eccodes_records) + + +def test_print_home_deps_optional_marker_in_message(monkeypatch, caplog): + libs_without_eccodes = {k: v for k, v in ALL_LIBS.items() if k != "eccodes"} + monkeypatch.setattr( + findlibs, "find", MagicMock(side_effect=lambda n: libs_without_eccodes.get(n)) + ) + _run_cli(["--print-home-deps"], monkeypatch) + assert "[Optional]" in caplog.text + + +def test_print_home_deps_queries_all_deps(find_mock, monkeypatch): + _run_cli(["--print-home-deps"], monkeypatch) + queried = {call.args[0] for call in find_mock.call_args_list} + assert queried == set(DEPENDENCY_ORDER) + + +def test_print_home_deps_disable_vars_appear_before_homes( + find_mock, monkeypatch, caplog +): + monkeypatch.setenv("FINDLIBS_DISABLE_METKIT", "1") + _run_cli(["--print-home-deps"], monkeypatch) + lines = caplog.text.splitlines() + disable_idx = next( + i for i, line in enumerate(lines) if "FINDLIBS_DISABLE_METKIT" in line + ) + first_dep_idx = next(i for i, line in enumerate(lines) if "metkit:" in line) + assert disable_idx < first_dep_idx + + +# --------------------------------------------------------------------------- +# Output format +# --------------------------------------------------------------------------- + + +def test_logging_format(monkeypatch): + calls = [] + monkeypatch.setattr(logging, "basicConfig", lambda **kwargs: calls.append(kwargs)) + monkeypatch.setattr( + findlibs, "find", MagicMock(return_value="/fake/metkit/lib/libmetkit.so") + ) + _run_cli(["--print-home"], monkeypatch) + assert calls, "basicConfig must be called" + fmt = calls[0]["format"] + assert "%(asctime)s" in fmt + assert "%(levelname)" in fmt + assert "%(message)s" in fmt + + +def test_verbose_sets_debug_level(monkeypatch): + calls = [] + monkeypatch.setattr(logging, "basicConfig", lambda **kwargs: calls.append(kwargs)) + monkeypatch.setattr( + findlibs, "find", MagicMock(return_value="/fake/metkit/lib/libmetkit.so") + ) + _run_cli(["--print-home", "--verbose"], monkeypatch) + assert calls[0]["level"] == logging.DEBUG + + +def test_default_level_is_info(monkeypatch): + calls = [] + monkeypatch.setattr(logging, "basicConfig", lambda **kwargs: calls.append(kwargs)) + monkeypatch.setattr( + findlibs, "find", MagicMock(return_value="/fake/metkit/lib/libmetkit.so") + ) + _run_cli(["--print-home"], monkeypatch) + assert calls[0]["level"] == logging.INFO + + +# --------------------------------------------------------------------------- +# No arguments +# --------------------------------------------------------------------------- + + +def test_no_args_exits_with_code_2(monkeypatch): + exit_code = _run_cli([], monkeypatch) + assert exit_code == 2 diff --git a/tests/pymetkit/integration/mars_request/test_accessors.py b/tests/pymetkit/integration/mars_request/test_accessors.py new file mode 100644 index 000000000..aba0bc9bc --- /dev/null +++ b/tests/pymetkit/integration/mars_request/test_accessors.py @@ -0,0 +1,110 @@ +# SPDX-FileCopyrightText: 2026 European Centre for Medium-Range Weather Forecasts (ECMWF) +# SPDX-License-Identifier: Apache-2.0 + +"""The Mapping-like interface of MarsRequest: reading, mutating and iterating.""" + +import pytest + +from pymetkit import MarsRequest + + +def make_request(): + return MarsRequest("retrieve", {"class": "od", "param": [151, 129], "step": [0]}) + + +# --------------------------------------------------------------------------- +# Reading values +# --------------------------------------------------------------------------- + + +def test_single_value_returns_scalar(): + req = make_request() + assert req["class"] == "od" + assert req["step"] == "0" + + +def test_multiple_values_return_list(): + assert make_request()["param"] == ["151", "129"] + + +def test_num_values(): + req = make_request() + assert req.num_values("param") == 2 + assert req.num_values("class") == 1 + + +def test_dict_view(): + assert dict(make_request()) == { + "class": "od", + "param": ["151", "129"], + "step": "0", + } + + +# --------------------------------------------------------------------------- +# Membership and iteration +# --------------------------------------------------------------------------- + + +def test_contains(): + req = make_request() + assert "class" in req + assert "missing" not in req + + +def test_keys_yields_parameter_names(): + req = make_request() + assert set(req.keys()) == {"class", "param", "step"} + + +def test_iter_yields_key_value_pairs(): + req = make_request() + pairs = list(req) + assert {k for k, _ in pairs} == {"class", "param", "step"} + assert dict(pairs) == {"class": "od", "param": ["151", "129"], "step": "0"} + + +# --------------------------------------------------------------------------- +# Mutating values +# --------------------------------------------------------------------------- + + +def test_set_scalar_value(): + req = make_request() + req["expver"] = "0001" + assert req["expver"] == "0001" + assert "expver" in req + + +def test_set_overwrites_existing_value(): + req = make_request() + req["class"] = "ea" + assert req["class"] == "ea" + + +def test_set_list_and_range_values(): + req = make_request() + req["date"] = ["20200101", "20200102"] + req["step"] = range(0, 13, 6) + assert req["date"] == ["20200101", "20200102"] + assert req["step"] == ["0", "6", "12"] + + +# --------------------------------------------------------------------------- +# Missing keys +# --------------------------------------------------------------------------- + + +def test_getitem_missing_key_raises_keyerror(): + with pytest.raises(KeyError): + make_request()["missing"] + + +def test_num_values_missing_key_raises_keyerror(): + with pytest.raises(KeyError): + make_request().num_values("missing") + + +def test_set_invalid_value_is_rejected(): + with pytest.raises(ValueError): + make_request()["bad"] = {"nested": 1} diff --git a/tests/pymetkit/integration/mars_request/test_construction.py b/tests/pymetkit/integration/mars_request/test_construction.py new file mode 100644 index 000000000..a19f6e056 --- /dev/null +++ b/tests/pymetkit/integration/mars_request/test_construction.py @@ -0,0 +1,67 @@ +# SPDX-FileCopyrightText: 2026 European Centre for Medium-Range Weather Forecasts (ECMWF) +# SPDX-License-Identifier: Apache-2.0 + +"""Construction of MarsRequest objects: verbs and selections.""" + +import pytest + +from pymetkit import MarsRequest + +# --------------------------------------------------------------------------- +# Verb and selection sources +# --------------------------------------------------------------------------- + + +def test_verb_only(): + req = MarsRequest("retrieve") + assert req.verb() == "retrieve" + assert list(req.keys()) == [] + + +def test_selection_mapping_argument(): + req = MarsRequest("retrieve", {"class": "od", "param": [151, 129]}) + assert req["class"] == "od" + assert req["param"] == ["151", "129"] + + +# --------------------------------------------------------------------------- +# Value normalization +# --------------------------------------------------------------------------- + + +def test_numeric_values_are_stringified(): + req = MarsRequest("retrieve", {"step": [0, 6, 12], "number": 1, "threshold": 1.5}) + assert req["step"] == ["0", "6", "12"] + assert req["number"] == "1" + assert req["threshold"] == "1.5" + + +def test_range_value_is_expanded_to_strings(): + req = MarsRequest("retrieve", {"step": range(0, 13, 6)}) + assert req["step"] == ["0", "6", "12"] + + +def test_slash_separated_string_is_split(): + req = MarsRequest("retrieve", {"date": "20200101/20200102", "step": "0/to/24/by/6"}) + assert req["date"] == ["20200101", "20200102"] + assert req["step"] == ["0", "to", "24", "by", "6"] + + +# --------------------------------------------------------------------------- +# Rejected input +# --------------------------------------------------------------------------- + + +def test_mapping_value_is_rejected(): + with pytest.raises(ValueError): + MarsRequest("retrieve", {"bad": {"nested": 1}}) + + +def test_unsupported_value_object_is_rejected(): + with pytest.raises(ValueError): + MarsRequest("retrieve", {"bad": object()}) + + +def test_non_mapping_selection_is_rejected(): + with pytest.raises(ValueError): + MarsRequest("retrieve", [("class", "od")]) diff --git a/tests/pymetkit/integration/mars_request/test_equality.py b/tests/pymetkit/integration/mars_request/test_equality.py new file mode 100644 index 000000000..42471e293 --- /dev/null +++ b/tests/pymetkit/integration/mars_request/test_equality.py @@ -0,0 +1,90 @@ +# SPDX-FileCopyrightText: 2026 European Centre for Medium-Range Weather Forecasts (ECMWF) +# SPDX-License-Identifier: Apache-2.0 + +"""Equality, hashing and string representation of MarsRequest.""" + +from pymetkit import MarsRequest + + +def base_request(verb="retrieve", **overrides): + selection = { + "class": "od", + "domain": "g", + "date": "20230101", + "param": "130", + "expver": "0001", + "step": range(0, 13, 6), + } + selection.update(overrides) + return MarsRequest(verb, selection) + + +# --------------------------------------------------------------------------- +# Equal requests +# --------------------------------------------------------------------------- + + +def test_identical_requests_are_equal(): + assert base_request() == base_request() + + +def test_equality_ignores_value_representation(): + # 20230101 vs "20230101" and 130 vs "130" expand to the same request + assert base_request() == base_request(date=20230101, param=130) + + +# --------------------------------------------------------------------------- +# Unequal requests +# --------------------------------------------------------------------------- + + +def test_different_verb_is_not_equal(): + assert base_request(verb="retrieve") != base_request(verb="compute") + + +def test_different_values_are_not_equal(): + assert base_request(param="130") != base_request(param="131") + + +def test_comparison_with_non_request_is_not_equal(): + assert (base_request() == "retrieve") is False + assert base_request() != "retrieve" + + +# --------------------------------------------------------------------------- +# Representation +# --------------------------------------------------------------------------- + + +def test_repr_contains_the_verb(): + assert "retrieve" in repr(base_request()) + + +# --------------------------------------------------------------------------- +# Hashing +# --------------------------------------------------------------------------- + + +def test_equal_requests_have_same_hash(): + assert hash(base_request()) == hash(base_request()) + + +def test_hash_consistent_with_equality_across_representations(): + # date as int vs string, param as int vs string: different pre-expansion + # forms that expand to the same request and must therefore hash identically. + assert hash(base_request()) == hash(base_request(date=20230101, param=130)) + + +def test_unequal_requests_have_different_hash(): + assert hash(base_request(param="130")) != hash(base_request(param="131")) + + +def test_request_usable_as_dict_key(): + req = base_request() + d = {req: "value"} + assert d[req] == "value" + + +def test_request_usable_in_set(): + req = base_request() + assert req in {req} diff --git a/tests/pymetkit/integration/mars_request/test_expand.py b/tests/pymetkit/integration/mars_request/test_expand.py new file mode 100644 index 000000000..2b82df856 --- /dev/null +++ b/tests/pymetkit/integration/mars_request/test_expand.py @@ -0,0 +1,92 @@ +# SPDX-FileCopyrightText: 2026 European Centre for Medium-Range Weather Forecasts (ECMWF) +# SPDX-License-Identifier: Apache-2.0 + +"""Expansion and validation against the MARS language definition.""" + +from datetime import datetime, timedelta + +import pytest + +from pymetkit import MarsRequest, MetKitException + +yesterday = (datetime.today() - timedelta(days=1)).strftime("%Y%m%d") + + +def valid_request(): + return MarsRequest( + "retrieve", + { + "class": "od", + "domain": "g", + "date": "-1", + "expver": "0001", + "step": range(0, 13, 6), + }, + ) + + +# --------------------------------------------------------------------------- +# Expansion +# --------------------------------------------------------------------------- + + +def test_expand_preserves_verb(): + assert valid_request().expand().verb() == "retrieve" + + +def test_expand_normalizes_relative_date(): + assert valid_request().expand()["date"] == yesterday + + +def expansion_of_short_names(): + request = valid_request() + request_different = valid_request() + request_different["class"] = "operational" + request_different["domain"] = "global" + assert request == request_different + + +def test_expand_inherits_default_values(): + expanded = valid_request().expand() + assert "param" in expanded + + +def test_expand_without_inherit_adds_no_defaults(): + request = valid_request() + inherited = request.expand(inherit=True) + minimal = request.expand(inherit=False) + assert minimal.verb() == "retrieve" + assert set(minimal.keys()) <= set(inherited.keys()) + + +def test_expand_returns_a_new_object_and_leaves_original_untouched(): + request = valid_request() + keys_before = set(request.keys()) + expanded = request.expand() + assert expanded is not request + assert set(request.keys()) == keys_before + + +# --------------------------------------------------------------------------- +# Validation +# --------------------------------------------------------------------------- + + +def test_validate_accepts_a_valid_request(): + valid_request().validate() # must not raise + + +def test_validate_rejects_an_invalid_value(): + request = MarsRequest( + "retrieve", + { + "class": "invalid", + "domain": "g", + "date": "-1", + "expver": "0001", + "levtype": "sfc", + "step": range(0, 13, 6), + }, + ) + with pytest.raises(MetKitException): + request.validate() diff --git a/tests/pymetkit/integration/mars_request/test_merge.py b/tests/pymetkit/integration/mars_request/test_merge.py new file mode 100644 index 000000000..fdf4c5078 --- /dev/null +++ b/tests/pymetkit/integration/mars_request/test_merge.py @@ -0,0 +1,85 @@ +# SPDX-FileCopyrightText: 2026 European Centre for Medium-Range Weather Forecasts (ECMWF) +# SPDX-License-Identifier: Apache-2.0 + +"""Merging two MarsRequest objects. + +metkit's merge takes the union of the values of matching parameters, keeping the +values of ``self`` first and appending only those values of ``other`` that are +not already present (order-preserving, deduplicated). The two requests must +carry the same parameters and the merged result must validate against the MARS +language definition. +""" + +import pytest + +from pymetkit import MarsRequest, MetKitException + +# Shared parameter set so both operands carry identical keys unless a test +# deliberately diverges. +BASE = {"class": "od", "domain": "g", "expver": "0001", "step": range(0, 13, 6)} + + +def _left(): + return MarsRequest("retrieve", {**BASE, "date": "-1", "levtype": "sfc"}) + + +def _right(): + return MarsRequest("retrieve", {**BASE, "date": "20230101", "levtype": "sfc"}) + + +# --------------------------------------------------------------------------- +# Successful merge +# --------------------------------------------------------------------------- + + +def test_merge_unions_values_keeping_self_first(): + merged = _left().merge(_right()) + # date differs between the two -> union, self's value first + assert merged["date"] == ["-1", "20230101"] + + +def test_merge_deduplicates_identical_values(): + # levtype is "sfc" on both sides -> a single value, not duplicated + assert _left().merge(_right())["levtype"] == "sfc" + + +def test_merge_leaves_self_only_values_unchanged(): + merged = _left().merge(_right()) + assert merged["class"] == "od" + assert merged["step"] == ["0", "6", "12"] + + +def test_merge_returns_a_distinct_new_request(): + left, right = _left(), _right() + merged = left.merge(right) + assert isinstance(merged, MarsRequest) + assert merged is not left + assert merged is not right + + +def test_merge_does_not_mutate_the_operands(): + left, right = _left(), _right() + left.merge(right) + # values are untouched on both inputs + assert left["date"] == "-1" + assert right["date"] == "20230101" + assert set(left.keys()) == set(right.keys()) + + +# --------------------------------------------------------------------------- +# Rejected merges +# --------------------------------------------------------------------------- + + +def test_merge_with_different_parameters_raises_value_error(): + left = _left() + right = MarsRequest("retrieve", {**BASE, "date": "-1", "levtype": "sfc", "type": "em"}) + with pytest.raises(ValueError): + left.merge(right) + + +def test_merge_producing_invalid_request_raises_metkit_exception(): + left = _left() + right = MarsRequest("retrieve", {**BASE, "date": "-1", "levtype": "pl"}) + with pytest.raises(MetKitException): + left.merge(right) diff --git a/tests/pymetkit/integration/mars_request/test_parse.py b/tests/pymetkit/integration/mars_request/test_parse.py new file mode 100644 index 000000000..8e64052ca --- /dev/null +++ b/tests/pymetkit/integration/mars_request/test_parse.py @@ -0,0 +1,100 @@ +# SPDX-FileCopyrightText: 2026 European Centre for Medium-Range Weather Forecasts (ECMWF) +# SPDX-License-Identifier: Apache-2.0 + +"""Parsing MARS requests from strings and files.""" + +from datetime import datetime, timedelta + +from pymetkit import MarsRequest, parse_mars_request + +yesterday = (datetime.today() - timedelta(days=1)).strftime("%Y%m%d") + +MULTIPLE_REQUESTS = """ +retrieve, + class=od, + domain=g, + expver=0001, + levtype=sfc, + stream=enfo, + date=-1, + time=12, + param=151.128, + grid=O640, + step=0/to/24/by/6, + target=test.grib, + type=em +retrieve, + class=od, + domain=g, + expver=0001, + levtype=pl, + stream=enfo, + date=-1, + time=12, + param=129, + levelist=500, + grid=O640, + step=0/to/24/by/6, + target=test.grib, + type=em +""" + +SINGLE_REQUEST = "retrieve,class=od,date=-1,time=12,param=129,step=12,target=test.grib" + + +# --------------------------------------------------------------------------- +# Parsing requests +# --------------------------------------------------------------------------- + + +def test_parse_single_request_from_string(): + requests = parse_mars_request(SINGLE_REQUEST) + assert len(requests) == 1 + assert requests[0].verb() == "retrieve" + assert requests[0].num_values("step") == 1 + + +def test_parse_multiple_requests_from_string(): + requests = parse_mars_request(MULTIPLE_REQUESTS) + assert len(requests) == 2 + for req in requests: + assert req.verb() == "retrieve" + assert req.num_values("step") == 5 + + +def test_parse_from_file_object(tmpdir): + request_file = f"{tmpdir}/requests" + with open(request_file, "w") as handle: + handle.write(MULTIPLE_REQUESTS) + with open(request_file, "r") as handle: + requests = parse_mars_request(handle) + assert len(requests) == 2 + assert "class" in requests[0] + assert requests[1]["levelist"] == "500" + + +def test_parse_returns_marsrequest_objects(): + requests = parse_mars_request(MULTIPLE_REQUESTS) + assert all(isinstance(req, MarsRequest) for req in requests) + + +def test_parse_expands_relative_date(): + requests = parse_mars_request(MULTIPLE_REQUESTS) + for req in requests: + assert req["date"] == yesterday + + +# --------------------------------------------------------------------------- +# Empty input +# --------------------------------------------------------------------------- + + +def test_parse_empty_string_returns_no_requests(): + assert parse_mars_request("") == [] + + +def test_parse_empty_file_returns_no_requests(tmpdir): + request_file = f"{tmpdir}/requests" + with open(request_file, "w") as handle: + handle.write("") + assert parse_mars_request(open(request_file, "r")) == [] diff --git a/tests/pymetkit/integration/mars_request/test_selection.py b/tests/pymetkit/integration/mars_request/test_selection.py new file mode 100644 index 000000000..ed67558e8 --- /dev/null +++ b/tests/pymetkit/integration/mars_request/test_selection.py @@ -0,0 +1,175 @@ +# SPDX-FileCopyrightText: 2026 European Centre for Medium-Range Weather Forecasts (ECMWF) +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for internal value normalisation (pymetkit.pymetkit_type.UserInputMapper).""" + +import pytest + +from pymetkit.pymetkit_type import UserInputMapper + +# --------------------------------------------------------------------------- +# map_selection_to_internal: scalar values +# --------------------------------------------------------------------------- + + +def test_single_value_internal(): + result = UserInputMapper.map_selection_to_internal({"key-1": "value-1"}) + assert len(result) == 1 + assert "key-1" in result + assert result["key-1"] == ["value-1"] + + +def test_int_and_float_become_string_lists(): + result = UserInputMapper.map_selection_to_internal({"number": 1, "threshold": 1.5}) + assert result["number"] == ["1"] + assert result["threshold"] == ["1.5"] + + +def test_slash_separated_string_is_split(): + result = UserInputMapper.map_selection_to_internal({"date": "20200101/20200102"}) + assert result["date"] == ["20200101", "20200102"] + + +def test_string_without_slash_is_kept_whole(): + result = UserInputMapper.map_selection_to_internal({"grid": "O640"}) + assert result["grid"] == ["O640"] + + +# --------------------------------------------------------------------------- +# map_selection_to_internal: collection values +# --------------------------------------------------------------------------- + + +def test_collection_values_are_stringified(): + result = UserInputMapper.map_selection_to_internal( + { + "key-1": ["value-1", "value-2", "value-3"], + "key-2": ["value-2"], + "key-3": ["value-3", 214, 213.54], + "key-4": [120, 123, 124, 125], + } + ) + assert result["key-1"] == ["value-1", "value-2", "value-3"] + assert result["key-2"] == ["value-2"] + assert result["key-3"] == ["value-3", "214", "213.54"] + assert result["key-4"] == ["120", "123", "124", "125"] + + +def test_range_is_expanded(): + result = UserInputMapper.map_selection_to_internal({"step": range(0, 13, 6)}) + assert result["step"] == ["0", "6", "12"] + + +# --------------------------------------------------------------------------- +# map_selection_to_internal: mixed numeric types +# --------------------------------------------------------------------------- + + +def test_to_internal_mixed_numeric_types(): + result = UserInputMapper.map_selection_to_internal( + { + "key-1": ["value-2", "value-4"], + "key-2": ["0.1", 0.2], + "key-3": [0.1, 0.2], + "key-4": [1, 2], + } + ) + assert result["key-1"] == ["value-2", "value-4"] + assert result["key-2"] == ["0.1", "0.2"] + assert result["key-3"] == ["0.1", "0.2"] + assert result["key-4"] == ["1", "2"] + + +# --------------------------------------------------------------------------- +# map_selection_to_external +# --------------------------------------------------------------------------- + + +def test_single_value_external(): + result = UserInputMapper.map_selection_to_external({"key-1": ["value-1"]}) + assert result["key-1"] == "value-1" + + +def test_multi_value_external(): + result = UserInputMapper.map_selection_to_external( + { + "key-1": ["value-1", "value-2", "value-3"], + "key-2": ["value-2"], + } + ) + assert result["key-1"] == ["value-1", "value-2", "value-3"] + assert result["key-2"] == "value-2" + + +# --------------------------------------------------------------------------- +# map_values_to_external +# --------------------------------------------------------------------------- + + +def test_map_values_to_external_collapses_single_value(): + assert UserInputMapper.map_values_to_external(["od"]) == "od" + assert UserInputMapper.map_values_to_external(["151", "129"]) == ["151", "129"] + + +# --------------------------------------------------------------------------- +# Key overwrite behavior +# --------------------------------------------------------------------------- + + +def test_overwrite_key_internal(): + result = UserInputMapper.map_selection_to_internal( + { + "key-1": "value-1", + "key-1": ["value-3", "214", "213.54"], + } + ) + assert len(result) == 1 + assert result["key-1"] == ["value-3", "214", "213.54"] + + +def test_overwrite_key_external(): + result = UserInputMapper.map_selection_to_external( + { + "key-1": ["value-1"], + "key-1": ["value-3", "214", "213.54"], + } + ) + assert len(result) == 1 + assert result["key-1"] == ["value-3", "214", "213.54"] + + +# --------------------------------------------------------------------------- +# Rejected values +# --------------------------------------------------------------------------- + + +def test_mapping_value_is_rejected(): + with pytest.raises(ValueError): + UserInputMapper.map_selection_to_internal({"bad": {"nested": 1}}) + + +def test_unsupported_value_object_is_rejected(): + with pytest.raises(ValueError): + UserInputMapper.map_selection_to_internal({"bad": object()}) + + +# --------------------------------------------------------------------------- +# Pythonic interface +# --------------------------------------------------------------------------- + + +def test_pythonic_interface(): + result = UserInputMapper.map_selection_to_internal( + { + "key-1": ["value-1", "value-2", "value-3"], + "key-2": 0.1, + "key-3": list(range(1, 5)), + "key-4": [0.1, "0.2"], + "key-5": [1 + 0.5 * x for x in range(2)], + } + ) + assert result["key-1"] == ["value-1", "value-2", "value-3"] + assert result["key-2"] == ["0.1"] + assert result["key-3"] == ["1", "2", "3", "4"] + assert result["key-4"] == ["0.1", "0.2"] + assert result["key-5"] == ["1.0", "1.5"] diff --git a/python/pymetkit/tests/test_marsrequest.py b/tests/pymetkit/integration/test_marsrequest.py similarity index 67% rename from python/pymetkit/tests/test_marsrequest.py rename to tests/pymetkit/integration/test_marsrequest.py index 9d4ce6f46..1d6f4eec2 100644 --- a/python/pymetkit/tests/test_marsrequest.py +++ b/tests/pymetkit/integration/test_marsrequest.py @@ -1,8 +1,12 @@ -from datetime import datetime, timedelta +# SPDX-FileCopyrightText: 2026 European Centre for Medium-Range Weather Forecasts (ECMWF) +# SPDX-License-Identifier: Apache-2.0 + from contextlib import nullcontext as does_not_raise +from datetime import datetime, timedelta + import pytest -from pymetkit import parse_mars_request, MarsRequest, MetKitException +from pymetkit import MarsRequest, MetKitException, parse_mars_request request = """ retrieve, @@ -15,9 +19,9 @@ time=12, param=151.128, grid=O640, - step=0/to/24/by/6, - target=test.grib, - type=em + step=0/to/24/by/6, + target=test.grib, + type=em retrieve, class=od, domain=g, @@ -26,12 +30,12 @@ stream=enfo, date=-1, time=12, - param=129, + param=129, levelist=500, grid=O640, - step=0/to/24/by/6, - target=test.grib, - type=em + step=0/to/24/by/6, + target=test.grib, + type=em """ yesterday = (datetime.today() - timedelta(days=1)).strftime("%Y%m%d") @@ -50,27 +54,18 @@ def test_parse_file(tmpdir): assert "class" in requests[0] assert requests[1]["levelist"] == "500" -# @todo: [1] no longer raises an exception. Disable until METK-126 is resolved. -@pytest.mark.parametrize( - "req_str, length, steps, strict, expectation", - [ - [request, 2, 5, False, does_not_raise()], - # [request, 2, 5, True, pytest.raises(MetKitException)], - [ - "retrieve,class=od,date=-1,time=12,param=129,step=12,target=test.grib", - 1, - 1, - False, - does_not_raise(), - ], - ], -) -def test_parse_string(req_str, length, steps, strict, expectation): - with expectation: - requests = parse_mars_request(req_str, strict) - assert len(requests) == length - for req in requests: - assert req.num_values("step") == steps + +def test_parse_string(): + requests = parse_mars_request(request) + assert len(requests) == 2 + for req in requests: + assert req.num_values("step") == 5 + + requests = parse_mars_request( + "retrieve,class=od,date=-1,time=12,param=129,step=12,target=test.grib" + ) + assert len(requests) == 1 + assert requests[0].num_values("step") == 1 def test_empty_request(tmpdir): @@ -85,16 +80,22 @@ def test_new_request(): req = MarsRequest("retrieve") assert req.verb() == "retrieve" - req = MarsRequest("request", class_="od", type="pf", date=["20200101", "20200102"]) + req = MarsRequest("request", {"class": "od", "type": "pf", "date": ["20200101", "20200102"]}) assert req["class"] == "od" assert req["type"] == "pf" assert req["date"] == ["20200101", "20200102"] +def test_request_from_selection(): + req = MarsRequest("retrieve", {"class": "od", "param": [151, 129]}) + assert req["class"] == "od" + assert req["param"] == ["151", "129"] + + def test_request_from_expand(): req = MarsRequest( "retrieve", - **{ + { "class": "od", "domain": "g", "date": "-1", @@ -109,13 +110,11 @@ def test_request_from_expand(): expanded.validate() assert req == expanded -# @todo: [0] and [1] no longer raise an exception. Disable until METK-126 is resolved. + @pytest.mark.parametrize( "extra_kv", [ - # {"levelist": [500]}, - # {"type": "cf", "number": [1, 2]}, - {"class": "invalid"} + {"class": "invalid"}, ], ) def test_request_validate(extra_kv): @@ -128,7 +127,7 @@ def test_request_validate(extra_kv): "levtype": "sfc", } request.update(extra_kv) - req = MarsRequest("retrieve", **request) + req = MarsRequest("retrieve", request) with pytest.raises(MetKitException): req.validate() @@ -148,8 +147,8 @@ def test_request_merge(extra_kv, expectation): "expver": "0001", "step": range(0, 13, 6), } - req = MarsRequest("retrieve", **request, date="-1", levtype="sfc") - other_req = MarsRequest("retrieve", **request, **extra_kv) + req = MarsRequest("retrieve", {**request, "date": "-1", "levtype": "sfc"}) + other_req = MarsRequest("retrieve", {**request, **extra_kv}) with expectation: req.merge(other_req) @@ -167,10 +166,7 @@ def test_request_equality(verb, updates, expected): "expver": "0001", "step": range(0, 13, 6), } - req = MarsRequest( - "retrieve", - **init_request, - ) + req = MarsRequest("retrieve", init_request) second_request = {**init_request, **updates} - req2 = MarsRequest(verb, **second_request) + req2 = MarsRequest(verb, second_request) assert (req == req2) == expected