diff --git a/docs/conf.py b/docs/conf.py index 7343676b..c1629367 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -28,6 +28,7 @@ "sphinxcontrib.autodoc_pydantic", "sphinx.ext.autodoc", "sphinx.ext.intersphinx", + "sphinxcontrib.spelling", ] # Enable MyST extensions to support reStructuredText directives in Markdown @@ -54,6 +55,11 @@ # "pyproject-hooks": ("https://pyproject-hooks.readthedocs.io/en/latest/", None), } +# sphinxcontrib.spelling settings +# File references a function object. Spell checker complaints about typo in +# random object id. +spelling_exclude_patterns = ["config-reference.rst"] + # -- Options for HTML output ------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output @@ -76,7 +82,7 @@ class FromagerHookDocumenter(FunctionDocumenter): objtype = "fromagerhook" - def format_name(self): + def format_name(self) -> str: name = super().format_name() if name.startswith("default_"): name = name[8:] diff --git a/docs/customization.md b/docs/customization.md index d5bead58..a6af38ee 100644 --- a/docs/customization.md +++ b/docs/customization.md @@ -97,7 +97,7 @@ support templating. The only supported template variable are: ### Resolver dist The source distribution index server used by the package resolver can -be overriden for a particular package. The resolver can also be told +be overridden for a particular package. The resolver can also be told to whether include wheels or sdist sources while trying to resolve the package. Templating is not supported here. diff --git a/docs/files.md b/docs/files.md index f48f8e41..af521ba8 100644 --- a/docs/files.md +++ b/docs/files.md @@ -157,7 +157,7 @@ wheels-repo └── simple ``` -* The `build` sub-directoy holds temporary builds. We use it as the output directory when building the wheel because we can't predict the filename, and so using an empty directory with a name we know gives us a way to find the file and move it into the `downloads` directory after it's built +* The `build` sub-directory holds temporary builds. We use it as the output directory when building the wheel because we can't predict the filename, and so using an empty directory with a name we know gives us a way to find the file and move it into the `downloads` directory after it's built * The `downloads` sub-directory contains the wheels in `.whl` format that fromager builds combined with the pre-built wheels so we can create a local package index in `simple` * The `prebuilt` sub-directory contains wheels that are being used as prebuilt * The `simple` sub-directory is laid out as a simple local wheel index. diff --git a/docs/getting-started.rst b/docs/getting-started.rst index 01583051..481f3ab0 100644 --- a/docs/getting-started.rst +++ b/docs/getting-started.rst @@ -116,5 +116,5 @@ The output below is redacted for brevity. Missing sections are replaced with ``. 11:19:56 INFO Bootstrapping typing-extensions==4.14.0 took 0:00:01 total, 0:00:00 to resolve source, 0:00:00 to download source, 0:00:00 to prepare source, 0:00:00 to build sdist, 0:00:00 to add extra metadata to wheels, 0:00:00 to build wheels As each dependency is built, fromager will show output from the build process -and progress information. At the end of the build, fromager shows the lsit of +and progress information. At the end of the build, fromager shows the list of packages that were built and how long each step took. diff --git a/docs/how-tos/containers.rst b/docs/how-tos/containers.rst index 5d6b9882..fa972147 100644 --- a/docs/how-tos/containers.rst +++ b/docs/how-tos/containers.rst @@ -227,6 +227,6 @@ followed by the wheel being built successfully. pydantic-core: built wheel for version 2.18.4: /work/bootstrap-output/wheels-repo/downloads/pydantic_core-2.18.4-0-cp311-cp311-linux_x86_64.whl The :doc:`customization` section explains other techniques for changing the -build inputs to ensure packages build properley. The collection of wheels you +build inputs to ensure packages build properly. The collection of wheels you want to build may have different build-time issues, but you can use this iterative approach to work your way though them until they all build. diff --git a/docs/index.rst b/docs/index.rst index f0bd4e5e..71597967 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -39,6 +39,7 @@ those special cases directly into fromager. cli.rst glossary.rst develop.md + proposals/index.rst What's with the name? --------------------- diff --git a/docs/proposals/index.rst b/docs/proposals/index.rst new file mode 100644 index 00000000..ad4bc012 --- /dev/null +++ b/docs/proposals/index.rst @@ -0,0 +1,7 @@ +Fromager Enhancement Proposals +============================== + +.. toctree:: + :maxdepth: 1 + + new-resolver-config diff --git a/docs/proposals/new-resolver-config.md b/docs/proposals/new-resolver-config.md new file mode 100644 index 00000000..e5dd8cc3 --- /dev/null +++ b/docs/proposals/new-resolver-config.md @@ -0,0 +1,190 @@ +# New resolver and download configuration + +- Author: Christian Heimes +- Created: 2026-02-24 +- Status: Open + +## What + +This enhancement document proposal a new approach to configure the package +resolver and source / sdist downloader. The new settings are covering a +wider range of use cases. Common patterns like building a package from a +git checkout will no longer need custom Python plugins. + +## Why + +In downstream, we are encountering an increasing amount of packages that do +not build from sdists on PyPI. Either package maintainers are not uploading +source distributions to PyPI or sdists have issues. In some cases, packages +use a midstream fork that is not on PyPI. The sources need to be build from +git. + +Because Fromager <= 0.76 does not have declarative settings for GitHub/GitLab +resolver or cloning git repositories, we have to write custom Python plugins. +The plugins are a maintenance burden. + +## Goals + +- support common use cases with package settings instead of custom plugin code +- cover most common resolver scenarios: + - resolve package on PyPI (sdist, wheel, or both) + - resolve package on GitHub or GitLab with custom tag matcher +- cover common sdist download and build scenarios: + - sdist from PyPI + - prebuilt wheel from PyPI + - download tarball from URL + - clone git repository + - download an artifact from GitHub / GitLab release or tag + - build sdist with PEP 517 hook or plain tarball +- support per-variant setting, e.g. one variant uses prebuilt wheel while the + rest uses sdist. +- gradual migration path from old system to new configuration + +## Non-goals + +- The new system will not cover all use cases. Some specific use cases will + still require custom code. +- Retrieval of additional sources is out of scope, e.g. a package `egg` that + needs `libegg-{version}.tar.gz`. +- Provide SSH transport for git. The feature can be added at a later point + when it's needed. +- Extra options for authentication. The `requests` library and `git` CLI can + use `$HOME/.netrc` for authentication. + > **NOTE:** `requests` also supports `NETRC` environment variable, + `libcurl` and `git` _only_ support `$HOME/.netrc`. + +## How + +The new system will use a new top-level configuration key `source`. The old +`download_source` and `resolver_dist` settings will stay supported for a +while. Eventually the old options will be deprecated and removed. + +The resolver and source downloader can be configuration for all variants of +a package as well as re-defined for each variant. A package can be configured +as prebuilt for all variants or a variant can have a different resolver and +sources than other. + +Each use case is handled a provider profile. The profile name acts as a tag +([discriminated union](https://docs.pydantic.dev/latest/concepts/unions/#discriminated-unions)). +Each use case has a well-defined set of mandatory and optional arguments. + +**Example:** + +```yaml +source: + # `pypi-sdist` is the default provider + provider: pypi-sdist +variants: + egg: + source: + # resolve and download prebuilt wheel + provider: pypi-prebuilt + index_url: https://custom-index.example/simple + spam: + source: + # resolve on GitLab, clone tag over https, build an sdist with PEP 517 hook + provider: gitlab-git + url: https://gitlab.example/spam/spam + matcher_factory: package_plugins.matchers:midstream_matcher_factory + build_sdist: pep517 + viking: + source: + # resolve on PyPI, git clone, and build as tarball + provider: pypi-git + clone_url: https://git.example/viking/viking.git + tag: 'v{version}' + build_sdist: tarball + camelot: + source: + # On second thought, let's not go to Camelot. It is a silly place. + provider: not-available +``` + +### Profiles + +- The `pypi-sdist` profile resolve versions on PyPI or PyPI-compatible index. + It only takes sdists into account and downloads the sdist from the index. + The profile is equivalent to the current default settings with + `include_sdists: true` and `include_wheels: false`. + +- The `pypi-prebuilt` profile resolve versions of platform-specific wheels + on PyPI and downloads the pre-built wheel. The profile is equivalent to + `include_sdists: false`, `include_wheels: true`, and variant setting + `pre_build: true`. + +- The `pypi-download` resolve versions of any package on PyPI and downloads + a tarball from an external URL (with `{version}` variable in download URL). + It takes any sdist and any wheel into account. The profile is equivalent + with `include_sdists: true`, `include_wheels: true`, `ignore_platform: true`, + and a `download_source.url`. + +- The `pypi-git` is similar to the `pypi-download` profile. Instead of + downloading a tarball, it clones a git repository at a specific tag. + +- The `gitlab-git` and `github-git` profiles use the `GitLabTagProvider` or + `GitHubTagProvider` to resolve versions. The profiles git clone a project + over `https` or `ssh` protocol. + +- The `gitlab-download` and `github-download` are similar to `gitlab-git` and + `github-git` profiles. Instead of cloning a git repository, they download + a git tarball or an release artifact. + +- The `not-available` profile raises an error. It can be used to block a + package and only enable it for a single variant. + +Like pip's VCS feature, all git clone operations automatically retrieve all +submodules recursively. + +### URL schema for git + +The resolver and `Candidate` class do not support VCS URLs, yet. Fromager can +adopt pip's [VCS support](https://pip.pypa.io/en/stable/topics/vcs-support/) +syntax. The URL `git+https://git.example/viking/viking.git@v1.1.0` clones the +git repository over HTTPS and checks out the tag `v1.1.0`. + +### Matcher factory + +The matcher factory argument is an import string. The string must resolve to +a callable that accepts a `ctx` argument and returns a `re.Pattern` +(recommended) or `MatchFunction`. If the return value is a pattern object, +then it must have exactly one match group. The pattern is matched with +`re.match`. + +The default matcher factory parsed the tag with `packaging.version.Version` +and ignores any error. + +```python +import re + +from fromager import context, resolver +from packaging.version import Version + + +def matcher_factory_pat(ctx: context.WorkContext) -> re.Pattern | resolver.MatchFunction: + # tag must must v1.2+midstream.1.cpu and results in Version("1.2+midstream.1") + variant = re.escape(ctx.variant) + pat = rf"^v(.*\+midstream\.\d+)\.{variant}$" + return re.compile(pat) + + +def matcher_factory_func(ctx: context.WorkContext) -> re.Pattern | resolver.MatchFunction: + def pep440_matcher(identifier: str, item: str) -> Version | None: + try: + return Version(item) + except ValueError: + return None + return pep440_matcher +``` + +### Deprecations + +- `download_source.url` is handled by `pypi-download` profile or + `release_artifact` parameter of `github` or `gitlab` provider +- `download_source.destination_filename` is not needed. All sdists use + standard `{dist_name}-{version}.tar.gz` file name +- `resolver_dist.sdist_server_url` is replaced by `index_url` parameter. + All `pypi-*` profile support a custom index. +- `git_options.submodules` is not needed. Like pip, Fromager will always + clone all submodules. +- variant settings `wheel_server_url` and `pre_build` are replaced by + `pypi-prebuilt` profile diff --git a/docs/spelling_wordlist.txt b/docs/spelling_wordlist.txt new file mode 100644 index 00000000..ee5b751a --- /dev/null +++ b/docs/spelling_wordlist.txt @@ -0,0 +1,81 @@ +args +backend +backends +backoff +canonicalize +canonicalized +changelog +changelogs +cheeseshop +cn +codebase +config +containerfile +cpu +csv +customizations +cython +deprecations +dir +downloader +env +environ +filesystem +fromager +frontend +frontends +graphviz +gz +installability +iterable +iteratively +json +lexicographically +linter +localhost +matcher +mypy +namespace +numpy +openssl +platlib +podman +pre +prebuilt +purelib +py +pydantic +pypi +pyproject +recurses +repo +scm +sdist +sdists +setuptools +statelessly +stderr +stdin +stdout +subcommands +subdirectory +submodule +submodules +subprocesses +templating +toml +toplevel +tos +traceback +tracebacks +txt +unshare +url +urls +vendored +vendoring +versionless +virtualenv +walkthrough +whitespace +yaml diff --git a/pyproject.toml b/pyproject.toml index 2356d2d4..2996cfcb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -85,6 +85,7 @@ docs = [ "myst-parser", "sphinx-rtd-theme", "autodoc-pydantic", + "sphinxcontrib.spelling", ] [project.urls] diff --git a/src/fromager/__main__.py b/src/fromager/__main__.py index 437fdae5..a2d4d891 100644 --- a/src/fromager/__main__.py +++ b/src/fromager/__main__.py @@ -140,7 +140,7 @@ @click.option( "--network-isolation/--no-network-isolation", default=SUPPORTS_NETWORK_ISOLATION, - help="Build sdist and wheen with network isolation (unshare -cn)", + help="Build sdist and when with network isolation (unshare -cn)", show_default=True, ) @click.pass_context diff --git a/src/fromager/commands/build_order.py b/src/fromager/commands/build_order.py index c5f2b601..88782c4e 100644 --- a/src/fromager/commands/build_order.py +++ b/src/fromager/commands/build_order.py @@ -31,7 +31,7 @@ def as_csv(build_order_file: str, output: pathlib.Path | None) -> None: Creates a file suitable for import into a spreadsheet including the distribution name, version, original requirement, dependency type, whether the package is pre-built, the build order step number, and - a full dependency chain leading to the requirment. + a full dependency chain leading to the requirement. """ fields = [