diff --git a/README.md b/README.md index 14164d4..341b030 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,34 @@ package_name_mode: both Once users and downstream projects have migrated, switch to `new` to stop generating the compatibility packages. New ROS 1 package names use the `ros-` prefix; new ROS 2 package names use `ros2-`. +## Variants mode + +The optional `variants_mode` setting controls how rattler-build receives the +repository-level `conda_build_config.yaml`: + +- `global` (default) leaves generated recipes unchanged. Builds must pass the + root configuration explicitly, for example with + `rattler-build build ... -m ./conda_build_config.yaml`. +- `local` generates a `variants.yaml` beside every `recipe.yaml`. Vinca converts + legacy comment selectors to v1 selectors and keeps only pins used by that + package. Builds must not pass the root configuration with `-m`, because an + explicit variant file takes precedence over the recipe-local file. + +```yaml +variants_mode: local +``` + +Local mode requires `conda_build_config.yaml` next to `vinca.yaml`. Optional +per-package additions or replacements can be declared in +`pkg_additional_info.yaml`: + +```yaml +my_ros_package: + variant_overrides: + c_stdlib_version: + - '2.28' +``` + ## Configuring setup-pixi in generated workflows Generated GitHub Actions workflows use `prefix-dev/setup-pixi@v0` by default. Override the action version in `vinca.yaml` when needed: diff --git a/vinca/configuration.py b/vinca/configuration.py index 8e3ce07..08d0f77 100644 --- a/vinca/configuration.py +++ b/vinca/configuration.py @@ -22,6 +22,11 @@ from vinca.resolve import get_conda_index from vinca.utils import add_package_name_variants from vinca.v1_selectors import evaluate_selectors +from vinca.variants import ( + VariantsMode, + convert_v0_variant_selectors, + get_variants_mode, +) _PATCH_PLATFORMS = ("osx", "linux", "win", "emscripten") @@ -105,6 +110,7 @@ def read_vinca_yaml(filepath: str | Path, target_platform: str) -> dict[str, Any config_dir = filepath.parent vinca_conf = _load_selected_yaml(filepath, target_platform) vinca_conf["package_name_mode"] = get_package_name_mode(vinca_conf).value + vinca_conf["variants_mode"] = get_variants_mode(vinca_conf).value vinca_conf["conda_index"] = _normalize_conda_indexes(vinca_conf["conda_index"]) patch_dir = Path(vinca_conf["patch_dir"]).absolute() @@ -133,6 +139,17 @@ def read_vinca_yaml(filepath: str | Path, target_platform: str) -> dict[str, Any else {} ) + vinca_conf["_variant_config"] = {} + if get_variants_mode(vinca_conf) is VariantsMode.LOCAL: + variant_config_path = config_dir / "conda_build_config.yaml" + if not variant_config_path.is_file(): + raise FileNotFoundError( + "variants_mode 'local' requires conda_build_config.yaml next to vinca.yaml" + ) + vinca_conf["_variant_config"] = convert_v0_variant_selectors( + _load_yaml(variant_config_path) or {} + ) + snapshot, additional = read_snapshot(vinca_conf) vinca_conf["_snapshot"] = snapshot or {} vinca_conf["_additional_packages_snapshot"] = additional or {} diff --git a/vinca/template.py b/vinca/template.py index 3b89ea2..de10d00 100644 --- a/vinca/template.py +++ b/vinca/template.py @@ -14,6 +14,7 @@ get_pkg_additional_info, get_pkg_build_number, ) +from vinca.variants import VariantsMode, get_recipe_variants, get_variants_mode TEMPLATE = """\ # yaml-language-server: $schema=https://raw.githubusercontent.com/prefix-dev/recipe-format/main/schema.json @@ -165,6 +166,11 @@ def write_recipe(source, outputs, vinca_conf, distro, single_file=True): with open(recipe_dir / "recipe.yaml", "w") as stream: file.dump(meta, stream) + if get_variants_mode(vinca_conf) is VariantsMode.LOCAL: + variants = get_recipe_variants(meta, package_shortname, vinca_conf) + with open(recipe_dir / "variants.yaml", "w") as stream: + file.dump(variants, stream) + if meta.get("source") and meta["source"].get("patches"): for p in meta["source"]["patches"]: patch_dir, _ = os.path.split(p) diff --git a/vinca/test_configuration.py b/vinca/test_configuration.py index 5586cf2..ccf423b 100644 --- a/vinca/test_configuration.py +++ b/vinca/test_configuration.py @@ -1,3 +1,5 @@ +import pytest + from vinca.configuration import read_snapshot, read_vinca_yaml @@ -31,6 +33,59 @@ def test_read_vinca_yaml_discovers_companion_files(tmp_path, monkeypatch): assert config["_test_folders"]["demo"] == tests / "demo" assert config["_pkg_additional_info"]["demo"]["build_number"] == 2 assert config["depmods"] == {"demo": {}} + assert config["variants_mode"] == "global" + assert config["_variant_config"] == {} + + +def test_read_vinca_yaml_loads_local_variant_config(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + (tmp_path / "patches").mkdir() + (tmp_path / "vinca.yaml").write_text( + "ros_distro: humble\nconda_index: []\npatch_dir: patches\nvariants_mode: local\n" + ) + (tmp_path / "conda_build_config.yaml").write_text( + "c_compiler:\n" + " - gcc # [linux]\n" + " - clang # [osx]\n" + "c_compiler_version: # [unix]\n" + " - 14 # [linux]\n" + " - 19 # [osx]\n" + ) + + config = read_vinca_yaml(tmp_path / "vinca.yaml", "linux-64") + + assert config["variants_mode"] == "local" + assert config["_variant_config"] == { + "c_compiler": [ + {"if": "linux", "then": "gcc"}, + {"if": "osx", "then": "clang"}, + ], + "c_compiler_version": [ + {"if": "(unix) and (linux)", "then": 14}, + {"if": "(unix) and (osx)", "then": 19}, + ], + } + + +def test_read_vinca_yaml_rejects_invalid_variants_mode(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + (tmp_path / "vinca.yaml").write_text( + "ros_distro: humble\nconda_index: []\npatch_dir: patches\nvariants_mode: other\n" + ) + + with pytest.raises(ValueError, match="Invalid variants_mode 'other'"): + read_vinca_yaml(tmp_path / "vinca.yaml", "linux-64") + + +def test_read_vinca_yaml_requires_config_for_local_pinning(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + (tmp_path / "patches").mkdir() + (tmp_path / "vinca.yaml").write_text( + "ros_distro: humble\nconda_index: []\npatch_dir: patches\nvariants_mode: local\n" + ) + + with pytest.raises(FileNotFoundError, match="requires conda_build_config.yaml"): + read_vinca_yaml(tmp_path / "vinca.yaml", "linux-64") def test_read_snapshot_merges_additional_packages(tmp_path): diff --git a/vinca/test_template.py b/vinca/test_template.py new file mode 100644 index 0000000..651670f --- /dev/null +++ b/vinca/test_template.py @@ -0,0 +1,52 @@ +from pathlib import Path + +from ruamel.yaml import YAML + +from vinca.template import write_recipe + + +class FakeDistro: + def get_package_prefix(self): + return "ros2" + + def get_legacy_package_prefix(self): + return "ros-jazzy" + + +def config(variants_mode): + return { + "ros_distro": "jazzy", + "package_name_mode": "legacy", + "variants_mode": variants_mode, + "build_number": 0, + "_tests": {}, + "_test_folders": {}, + "_additional_packages_snapshot": {}, + "_pkg_additional_info": {}, + "_variant_config": {"python": ["3.12"], "unused": ["1"]}, + } + + +def output(): + return { + "package": {"name": "ros-jazzy-demo", "version": "1.0"}, + "requirements": {"host": ["python"], "run": ["python"]}, + "build": {"script": ""}, + } + + +def test_write_recipe_emits_pruned_variants_in_local_mode(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + + write_recipe({}, [output()], config("local"), FakeDistro(), single_file=False) + + variants_path = Path("recipes/ros-jazzy-demo/variants.yaml") + assert YAML(typ="safe").load(variants_path) == {"python": ["3.12"]} + + +def test_write_recipe_does_not_emit_variants_in_global_mode(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + + write_recipe({}, [output()], config("global"), FakeDistro(), single_file=False) + + assert not Path("recipes/ros-jazzy-demo/variants.yaml").exists() diff --git a/vinca/test_variants.py b/vinca/test_variants.py new file mode 100644 index 0000000..4a47f10 --- /dev/null +++ b/vinca/test_variants.py @@ -0,0 +1,111 @@ +from vinca.variants import get_recipe_variants + + +def test_get_recipe_variants_prunes_to_recipe_dependencies_and_expressions(): + recipe = { + "requirements": { + "build": ["${{ compiler('cxx') }}", "${{ stdlib('c') }}"], + "host": ["python", "libfoo >=1"], + "run": ["python"], + }, + "build": { + "script": "echo ${{ cxx_compiler_version }}", + }, + } + config = { + "_variant_config": { + "python": ["3.12.* *_cpython"], + "python_impl": ["cpython"], + "libfoo": ["2"], + "unused": ["1"], + "c_compiler": ["gcc"], + "c_compiler_version": ["14"], + "cxx_compiler": ["gxx"], + "cxx_compiler_version": ["14"], + "c_stdlib": ["sysroot"], + "c_stdlib_version": ["2.17"], + "zip_keys": [["python", "python_impl"], ["libfoo", "unused"]], + "pin_run_as_build": { + "python": {"max_pin": "x.x"}, + "unused": {"max_pin": "x"}, + }, + }, + "_pkg_additional_info": {}, + } + + assert get_recipe_variants(recipe, "demo", config) == { + "python": ["3.12.* *_cpython"], + "libfoo": ["2"], + "cxx_compiler": ["gxx"], + "cxx_compiler_version": ["14"], + "c_stdlib": ["sysroot"], + "c_stdlib_version": ["2.17"], + "pin_run_as_build": {"python": {"max_pin": "x.x"}}, + } + + +def test_get_recipe_variants_merges_and_keeps_package_overrides(): + config = { + "_variant_config": { + "python": ["3.12.* *_cpython"], + "unused": ["1"], + }, + "_pkg_additional_info": { + "demo": { + "variant_overrides": { + "python": ["3.13.* *_cpython"], + "custom_feature": [True], + } + } + }, + } + + assert get_recipe_variants( + {"requirements": {"host": ["python"]}}, "demo", config + ) == { + "python": ["3.13.* *_cpython"], + "custom_feature": [True], + } + + +def test_get_recipe_variants_trims_zip_keys_to_used_members(): + config = { + "_variant_config": { + "python": ["3.12", "3.13"], + "numpy": ["1", "2"], + "python_impl": ["cpython", "cpython"], + "zip_keys": [["python", "numpy", "python_impl"]], + }, + "_pkg_additional_info": {}, + } + + assert get_recipe_variants( + {"requirements": {"host": ["python", "numpy"]}}, "demo", config + ) == { + "python": ["3.12", "3.13"], + "numpy": ["1", "2"], + "zip_keys": [["python", "numpy"]], + } + + +def test_get_recipe_variants_uses_test_requirements_not_test_commands(): + config = { + "_variant_config": { + "python": ["3.12"], + "pytest": ["8"], + }, + "_pkg_additional_info": {}, + } + + assert get_recipe_variants( + { + "tests": [ + { + "script": ["python -c 'print(1)'"], + "requirements": {"run": ["pytest"]}, + } + ] + }, + "demo", + config, + ) == {"pytest": ["8"]} diff --git a/vinca/variants.py b/vinca/variants.py new file mode 100644 index 0000000..4f98e14 --- /dev/null +++ b/vinca/variants.py @@ -0,0 +1,211 @@ +"""Variant configuration helpers for generated rattler-build recipes.""" + +from __future__ import annotations + +import copy +import re +from enum import Enum +from typing import Any, Mapping, Sequence + + +class VariantsMode(Enum): + """How generated recipes consume the repository-wide pinning configuration.""" + + GLOBAL = "global" + LOCAL = "local" + + +def get_variants_mode(vinca_conf: Mapping[str, Any]) -> VariantsMode: + """Return the configured variants mode, defaulting to the global mode.""" + value = vinca_conf.get("variants_mode", VariantsMode.GLOBAL.value) + try: + return VariantsMode(value) + except ValueError as exc: + choices = ", ".join(mode.value for mode in VariantsMode) + raise ValueError( + f"Invalid variants_mode {value!r}; expected one of: {choices}" + ) from exc + + +_SELECTOR_RE = re.compile(r"#\s*\[(?P[^]]+)]") +_COMPILER_RE = re.compile(r"\bcompiler\(\s*['\"](?P[^'\"]+)['\"]\s*\)") +_STDLIB_RE = re.compile(r"\bstdlib\(\s*['\"](?P[^'\"]+)['\"]\s*\)") +_JINJA_NAME_RE = re.compile(r"(?[A-Za-z_][A-Za-z0-9_]*)(?![\w-])") +_SPECIAL_VARIANT_KEYS = {"zip_keys", "pin_run_as_build"} + + +def _selector_from_comment(comment: Any) -> str | None: + """Extract a v0 selector from ruamel's comment metadata.""" + if comment is None: + return None + if isinstance(comment, (list, tuple)): + for item in comment: + if selector := _selector_from_comment(item): + return selector + return None + match = _SELECTOR_RE.search(str(getattr(comment, "value", comment))) + return match.group("condition").strip() if match else None + + +def _combine_selectors(parent: str | None, child: str | None) -> str | None: + if parent and child: + return f"({parent}) and ({child})" + return parent or child + + +def convert_v0_variant_selectors(node: Any, inherited: str | None = None) -> Any: + """Convert conda-build comment selectors to rattler-build v1 conditionals. + + Variant selectors occur on mapping keys and sequence entries. V1 evaluates + conditionals in lists, so a selector attached to a key is inherited by each + entry in its value list. + """ + if isinstance(node, Mapping): + result = {} + comments = getattr(getattr(node, "ca", None), "items", {}) + for key, value in node.items(): + condition = _combine_selectors( + inherited, _selector_from_comment(comments.get(key)) + ) + result[key] = convert_v0_variant_selectors(value, condition) + return result + + if isinstance(node, Sequence) and not isinstance(node, (str, bytes)): + result = [] + comments = getattr(getattr(node, "ca", None), "items", {}) + for index, value in enumerate(node): + condition = _combine_selectors( + inherited, _selector_from_comment(comments.get(index)) + ) + converted = convert_v0_variant_selectors(value) + if condition: + converted = {"if": condition, "then": converted} + result.append(converted) + return result + + return copy.deepcopy(node) + + +def _normalized_package_name(spec: str) -> str | None: + """Extract a normalized package name from a requirement string.""" + spec = spec.strip() + if not spec or spec.startswith("${{"): + return None + match = re.match(r"[A-Za-z0-9_.-]+", spec) + return match.group(0).lower().replace("-", "_") if match else None + + +def _collect_requirement_names(node: Any, names: set[str]) -> None: + if isinstance(node, str): + if name := _normalized_package_name(node): + names.add(name) + return + if isinstance(node, Mapping): + for key, value in node.items(): + if key != "if": + _collect_requirement_names(value, names) + return + if isinstance(node, Sequence) and not isinstance(node, (str, bytes)): + for value in node: + _collect_requirement_names(value, names) + + +def _collect_recipe_requirement_names(node: Any, names: set[str]) -> None: + """Find requirement sections at the recipe top level and inside tests.""" + if isinstance(node, Mapping): + for key, value in node.items(): + if key == "requirements": + _collect_requirement_names(value, names) + else: + _collect_recipe_requirement_names(value, names) + elif isinstance(node, Sequence) and not isinstance(node, (str, bytes)): + for value in node: + _collect_recipe_requirement_names(value, names) + + +def _collect_variant_expressions(node: Any, strings: list[str]) -> None: + """Collect Jinja expressions and selector conditions, not arbitrary scripts.""" + if isinstance(node, str): + if "${{" in node: + strings.append(node) + elif isinstance(node, Mapping): + for key, value in node.items(): + if key == "if" and isinstance(value, str): + strings.append(value) + else: + _collect_variant_expressions(value, strings) + elif isinstance(node, Sequence) and not isinstance(node, (str, bytes)): + for value in node: + _collect_variant_expressions(value, strings) + + +def _used_variant_keys( + recipe: Mapping[str, Any], variants: Mapping[str, Any] +) -> set[str]: + """Find variant keys referenced by a recipe's dependencies and expressions.""" + used: set[str] = set() + _collect_recipe_requirement_names(recipe, used) + + strings: list[str] = [] + _collect_variant_expressions(recipe, strings) + variant_keys = set(variants) - _SPECIAL_VARIANT_KEYS + for value in strings: + for match in _COMPILER_RE.finditer(value): + language = match.group("language") + used.update({f"{language}_compiler", f"{language}_compiler_version"}) + for match in _STDLIB_RE.finditer(value): + language = match.group("language") + used.update({f"{language}_stdlib", f"{language}_stdlib_version"}) + used.update( + match.group("name") + for match in _JINJA_NAME_RE.finditer(value) + if match.group("name") in variant_keys + ) + return used + + +def _prune_zip_keys(groups: Any, used: set[str]) -> list[list[str]]: + if not groups: + return [] + if groups and isinstance(groups[0], str): + groups = [groups] + result = [] + for group in groups: + retained = [key for key in group if key in used] + if len(retained) > 1: + result.append(retained) + return result + + +def get_recipe_variants( + recipe: Mapping[str, Any], + package_name: str, + vinca_conf: Mapping[str, Any], +) -> dict[str, Any]: + """Return a v1, package-local subset of the repository variant config.""" + variants = copy.deepcopy(vinca_conf.get("_variant_config") or {}) + overrides = copy.deepcopy( + vinca_conf.get("_pkg_additional_info", {}) + .get(package_name, {}) + .get("variant_overrides", {}) + ) + variants.update(overrides) + + used = _used_variant_keys(recipe, variants) | set(overrides) + result = { + key: value + for key, value in variants.items() + if key in used and key not in _SPECIAL_VARIANT_KEYS + } + + if groups := _prune_zip_keys(variants.get("zip_keys"), used): + result["zip_keys"] = groups + + pin_run_as_build = { + key: value + for key, value in (variants.get("pin_run_as_build") or {}).items() + if key.lower().replace("-", "_") in used + } + if pin_run_as_build: + result["pin_run_as_build"] = pin_run_as_build + return result