Feature/metk 176 port pymetkit to pybind11

.github/workflows/cd.yml

@@ -12,8 +12,3 @@ jobs:
   wheel-wrapper:
     uses: ./.github/workflows/build-wheel-wrapper.yml
     secrets: inherit
-  wheel-python:
-    uses: ecmwf/reusable-workflows/.github/workflows/cd-pypi.yml@v2
-    secrets: inherit
-    needs:
-      - wheel-wrapper

.github/workflows/pymetkit.yml

@@ -0,0 +1,143 @@
+name: Build and Test PyMetkit
+
+on:
+  # Trigger the workflow on push to master or develop, except tag creation
+  push:
+    branches:
+      - 'master'
+      - 'develop'
+    tags-ignore:
+      - '**'
+
+  # Trigger the workflow on pull request
+  pull_request: ~
+
+  # Trigger the workflow manually
+  workflow_dispatch: ~
+
+  # Trigger after public PR approved for CI
+  pull_request_target:
+    types: [labeled]
+
+jobs:
+  prepare-deps:
+    runs-on: ubuntu-latest
+    if: ${{ (success() || failure()) && (!github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci') }}
+    steps:
+      - name: Get ecbuild
+        uses: actions/checkout@v5
+        with:
+          repository: ecmwf/ecbuild
+          ref: develop
+          path: ecbuild
+      - name: Get stack-dependencies
+        uses: actions/checkout@v5
+        with:
+          repository: ecmwf/stack-dependencies
+          ref: master
+          path: stack-dependencies-src
+          token: ${{ secrets.GH_REPO_READ_TOKEN }}
+          submodules: recursive
+      - name: Install dependencies
+        run: |
+          mkdir stack-dependencies-build
+          stack-dependencies-src/build.sh --build-path stack-dependencies-build --install-path dependencies  --with-deps libaec,pybind11
+      - name: Get eccodes
+        uses: actions/checkout@v5
+        with:
+          repository: ecmwf/eccodes
+          ref: develop
+          path: eccodes-src
+      - name: Install eccodes
+        run: |
+          mkdir eccodes-build
+          cmake \
+            -B eccodes-build \
+            -S eccodes-src \
+            -GNinja \
+            -DCMAKE_INSTALL_PREFIX=dependencies \
+            -DCMAKE_PREFIX_PATH=dependencies \
+            -DCMAKE_BUILD_TYPE=RelWithDebInfo \
+            -DENABLE_MEMFS=ON \
+            -DENABLE_AEC=ON
+          cmake --build eccodes-build -j -t install
+      - name: Get eckit
+        uses: actions/checkout@v5
+        with:
+          repository: ecmwf/eckit
+          ref: develop
+          path: eckit-src
+      - name: Install eckit
+        run: |
+          mkdir eckit-build
+          cmake \
+            -B eckit-build \
+            -S eckit-src \
+            -GNinja \
+            -DCMAKE_INSTALL_PREFIX=dependencies \
+            -DCMAKE_PREFIX_PATH=dependencies \
+            -DCMAKE_BUILD_TYPE=RelWithDebInfo
+          cmake --build eckit-build -j -t install
+      - name: Archive with permissions preserved
+        run: tar --zstd -cpf files.tar.zst dependencies/ ecbuild/
+      - name: Upload dependencies
+        uses: actions/upload-artifact@v4
+        with:
+          name: deps
+          path: files.tar.zst
+          retention-days: 1
+  build-wheels:
+    needs: prepare-deps
+    runs-on: ubuntu-latest
+    if: ${{ (success() || failure()) && (!github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci') }}
+    strategy:
+      matrix:
+        python-version: ['3.11', '3.12', '3.13', '3.14']
+      fail-fast: false  # Continue running other versions if one fails
+
+    steps:
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Download dependencies
+        uses: actions/download-artifact@v4
+        with:
+          name: deps
+      - name: Extract with zstd
+        run: tar --zstd -xpf files.tar.zst
+      - name: Get metkit
+        uses: actions/checkout@v5
+        with:
+          repository: ecmwf/metkit
+          path: metkit-src
+      - name: Display Python version
+        run: python --version
+      - name: Install 'build'
+        run: pip install build pytest findlibs Sybil[pytest]
+      - name: Build metkit
+        run: |
+          export PATH=$(pwd)/dependencies/bin:$PATH
+          export ECCODES_HOME=$(pwd)/dependencies
+          export FINDLIBS_DISABLE_PACKAGE=yes
+          export FINDLIBS_DISABLE_PYTHON=yes
+          export ECCODES_PYTHON_USE_FINDLIBS=1
+          mkdir metkit-build
+          cmake \
+            -B metkit-build \
+            -S metkit-src \
+            -GNinja \
+            -DCMAKE_INSTALL_PREFIX=dependencies \
+            -DCMAKE_PREFIX_PATH=dependencies \
+            -DCMAKE_BUILD_TYPE=RelWithDebInfo \
+            -DENABLE_PYTHON_METKIT_INTERFACE=ON
+          cmake --build metkit-build -j
+          cd metkit-build
+          ctest -j $(nproc) --output-on-failure -L pymetkit
+      - name: Upload wheel
+        uses: actions/upload-artifact@v4
+        with:
+          name: pymetkit-wheel-py${{ matrix.python-version }}
+          path: "metkit-build/pymetkit-*.whl"
+          retention-days: 10
+          if-no-files-found: error # 'warn' or 'ignore' are also available, defaults to `warn`

CMakeLists.txt

@@ -96,6 +96,15 @@ ecbuild_add_option( FEATURE MARS2GRIB_PYTHON
                         "NAME Python VERSION 3.11 COMPONENTS Interpreter Development"
                         "NAME pybind11 VERSION 3.0.1" )
 
+# Pythonic pybind11 interface (pymetkit)
+
+ecbuild_add_option( FEATURE PYTHON_METKIT_INTERFACE
+                    DEFAULT OFF
+                    DESCRIPTION "Build the pybind11 pymetkit interface"
+                    REQUIRED_PACKAGES
+                        "NAME Python VERSION 3.11 COMPONENTS Interpreter Development"
+                        "NAME pybind11 VERSION 3.0.1" )
+
 # METKIT config files support
 
 ecbuild_add_option( FEATURE METKIT_CONFIG

cmake/pymetkit_setup.cfg.in

@@ -0,0 +1,38 @@
+[metadata]
+name = pymetkit
+description = Python interface to metkit
+long_description = file: README.md
+long_description_content_type = text/markdown
+author_email = European Centre for Medium-Range Weather Forecasts (ECMWF) <software.support@ecmwf.int>
+project_urls =
+    Documentation = https://github.com/ecmwf/metkit
+    Homepage = https://github.com/ecmwf/metkit
+    Issues = https://github.com/ecmwf/metkit/issues
+    Repository = https://github.com/ecmwf/metkit
+keywords = python, metkit, mars, tools
+license_expression = Apache-2.0
+license_file = LICENSE
+
+[options]
+python_requires = >=3.11
+install_requires =
+    findlibs>=0.1.2
+
+[options.extras_require]
+test =
+    pytest
+    pytest-cov
+    pytest-flakes
+    Sybil[pytest]
+docs =
+    Sphinx
+    breathe
+    sphinx-book-theme
+    requests
+    sphinxcontrib-mermaid
+    autoapi
+    sphinx-autoapi
+dev =
+    isort
+    black
+    flake8

cmake/pymetkit_setup.py.in

@@ -0,0 +1,57 @@
+import os
+import platform
+import sys
+from pathlib import Path
+
+from setuptools import setup
+from wheel.bdist_wheel import bdist_wheel
+
+# NOTE this we need to correctly link with metkitlib version on cd. For local builds feel free to ignore
+version_suffix = os.environ.get("VERSION_SUFFIX", "")
+if version_suffix:
+    version_suffix = f".{version_suffix}"
+    requires_extra = [f"metkitlib==@metkit_VERSION_STR@{version_suffix}"]
+else:
+    requires_extra = []
+
+
+# NOTE see ci-utils/wheelmaker/buildscripts/setup_utils, we need to get the right abi compat tag
+class bdist_wheel_ext(bdist_wheel):
+    def get_tag(self):
+        python, abi, plat = bdist_wheel.get_tag(self)
+        return python, abi, f"manylinux_2_28_{platform.machine()}"
+
+
+ext_kwargs = {
+    "darwin": {},
+    "linux": {"cmdclass": {"bdist_wheel": bdist_wheel_ext}},
+}
+
+setup(
+    name="pymetkit",
+    version=f"@metkit_VERSION_STR@{version_suffix}",
+    packages=["pymetkit", "pymetkit._internal", "pymetkit_bindings"],
+    package_data={
+        "pymetkit_bindings": ["*.so", "*.pyd"],
+    },
+    license="Apache 2.0",
+    license_files=["LICENSE"],
+    long_description=Path("README.md").read_text(),
+    long_description_content_type="text/markdown",
+    install_requires=["findlibs>=0.1.2"] + requires_extra,
+    python_requires=">3.10",
+    classifiers=[
+        "Development Status :: 3 - Alpha",
+        "Intended Audience :: Developers",
+        "Intended Audience :: Science/Research",
+        "Programming Language :: Python :: 3",
+        "Programming Language :: Python :: 3.11",
+        "Programming Language :: Python :: 3.12",
+        "Programming Language :: Python :: 3.13",
+        "Programming Language :: Python :: 3.14",
+        "Operating System :: OS Independent",
+        "Topic :: Software Development :: Libraries",
+    ],
+    has_ext_modules=lambda: True,
+    **ext_kwargs[sys.platform],
+)

docs/conf.py

@@ -34,7 +34,7 @@
 templates_path = ["_templates"]
 exclude_patterns = ["_build", "Thumbs.db", ".DS_Store", "_internal"]
 
-autoapi_dirs = ["../python/pymetkit/src/pymetkit"]
+autoapi_dirs = ["../src/pymetkit"]
 autoapi_type = "python"
 autoapi_generate_api_docs = True
 autoapi_add_toctree_entry = False

docs/pymetkit/api.rst

@@ -0,0 +1,32 @@
+API
+===
+
+The ``PyMetKit`` API provides a Pythonic interface to ``metkit``'s MARS request
+model. A :class:`~pymetkit.pymetkit.MarsRequest` is a verb together with a
+:data:`~pymetkit.pymetkit_type.MarsSelection` — a type alias for a user-supplied
+key-value mapping. Values are normalised automatically to the internal
+``dict[str, list[str]]`` representation used by the bindings layer. Operations
+that require the MARS language engine (expansion, validation, merging and parsing)
+are delegated to the underlying ``metkit`` library through the :doc:`bindings` layer.
+
+MarsRequest
+-----------
+.. autoapiclass:: pymetkit.pymetkit.MarsRequest
+   :members:
+
+Parsing
+-------
+.. autoapifunction:: pymetkit.pymetkit.parse_mars_request
+
+MarsSelection
+-------------
+.. autoapidata:: pymetkit.pymetkit_type.MarsSelection
+
+Exceptions
+----------
+.. py:exception:: pymetkit.MetKitException
+
+   Raised when the underlying ``metkit`` library reports an error, for example when
+   :meth:`~pymetkit.pymetkit.MarsRequest.validate` or
+   :meth:`~pymetkit.pymetkit.MarsRequest.expand` encounters a request that is
+   incompatible with the MARS language definition. Subclasses :class:`RuntimeError`.