From 0173ea2092edfb87da5b1b48ef7b77f1a4f2ab8c Mon Sep 17 00:00:00 2001 From: James Rosewell Date: Sat, 29 Aug 2026 09:34:11 +0100 Subject: [PATCH] Carry the OWID source inside the published package The published package could not be imported at all. Installing fiftyone-pipeline-did 4.5.12 from PyPI into a clean environment and running "import fiftyone_pipeline_did" raised ModuleNotFoundError for "owid", because the package imported the OWID library and nothing installed it. The published metadata declared no dependencies at all. Version 4.5.11 of 8 August fails the same way, and 4.5.12 carries the new cloud client, so the new client is live and unusable. The dependency cannot come from a package registry. The 51Degrees fork of OWID is not published to PyPI, and the name "owid" there belongs to an unrelated project, so declaring a dependency on that name would install the wrong thing. The build now copies the OWID source out of the owid-python submodule into the package as the private module fiftyone_pipeline_did._owid before the source distribution and the wheel are built. The leading underscore keeps the top level name "owid" free on the consumer's machine. The Apache-2.0 licence, and a notice naming the upstream repository and the commit the copy was taken from, travel with the source into both distributions. The one third party requirement the copied source has, cryptography, is now declared. The envelope and error types the public API refers to are exported from the package itself, because a caller who has only the published package has no other supported name for them. Nothing changes in the repository, where the submodule stays the only copy of the OWID code and the copy the build makes is ignored. The same copy step runs before the project is built, so a developer checkout and the unit test job produce the package the same way the publish does. --- .gitignore | 5 ++ ci/build-package.ps1 | 6 ++ ci/build-project.ps1 | 5 ++ ci/copy-owid-source.ps1 | 76 +++++++++++++++++++ fiftyone_pipeline_did/MANIFEST.in | 5 ++ .../examples/creator_context_web/server.py | 5 +- fiftyone_pipeline_did/readme.md | 31 +++++++- fiftyone_pipeline_did/setup.py | 23 +++--- .../src/fiftyone_pipeline_did/__init__.py | 8 ++ .../src/fiftyone_pipeline_did/did_client.py | 2 +- .../src/fiftyone_pipeline_did/fod_id.py | 21 ++--- fiftyone_pipeline_did/tests/test_fodid.py | 4 +- setup.ps1 | 6 ++ 13 files changed, 171 insertions(+), 26 deletions(-) create mode 100644 ci/copy-owid-source.ps1 diff --git a/.gitignore b/.gitignore index 55d5b5a..471e767 100755 --- a/.gitignore +++ b/.gitignore @@ -255,3 +255,8 @@ dependency_links.txt top_level.txt .tox/ + +# The OWID source copied into the 51Did package at build time by +# ci/copy-owid-source.ps1. The submodule is the only copy that belongs in +# the repository. +fiftyone_pipeline_did/src/fiftyone_pipeline_did/_owid/ diff --git a/ci/build-package.ps1 b/ci/build-package.ps1 index aa54e6a..acbad7f 100644 --- a/ci/build-package.ps1 +++ b/ci/build-package.ps1 @@ -5,6 +5,12 @@ param ( [string]$Version ) +# The 51Did package carries the OWID source inside itself, because the +# dependency cannot come from a package registry. Copy it in before the +# distributions are built, so that both the source distribution and the wheel +# contain it. See ci/copy-owid-source.ps1 for the full reasoning. +& "$PSScriptRoot/copy-owid-source.ps1" -RepoRoot $RepoName + $packages = "fiftyone_pipeline_core", "fiftyone_pipeline_engines", "fiftyone_pipeline_engines_fiftyone", "fiftyone_pipeline_cloudrequestengine", "fiftyone_pipeline_translation", "fiftyone_pipeline_did" ./python/build-package-pypi.ps1 -RepoName $RepoName -Version $Version -Packages $packages diff --git a/ci/build-project.ps1 b/ci/build-project.ps1 index a7589fe..a7ae240 100644 --- a/ci/build-project.ps1 +++ b/ci/build-project.ps1 @@ -3,6 +3,11 @@ param ( [string]$RepoName ) +# The 51Did package is built from its own tox environment by the unit test +# step, and that build needs the OWID source in place, so put it there before +# anything builds. See ci/copy-owid-source.ps1 for the full reasoning. +& "$PSScriptRoot/copy-owid-source.ps1" -RepoRoot $RepoName + $packages = "fiftyone_pipeline_core", "fiftyone_pipeline_engines", "fiftyone_pipeline_engines_fiftyone", "fiftyone_pipeline_cloudrequestengine", "fiftyone_pipeline_translation" ./python/build-project.ps1 -RepoName $RepoName -Packages $packages diff --git a/ci/copy-owid-source.ps1 b/ci/copy-owid-source.ps1 new file mode 100644 index 0000000..84ffbd5 --- /dev/null +++ b/ci/copy-owid-source.ps1 @@ -0,0 +1,76 @@ +param ( + # The root of the pipeline-python working copy. Defaults to the parent of + # this script, which is right for a developer checkout, and is passed + # explicitly by the CI scripts because they run from the workspace above + # the clone. + [string]$RepoRoot = (Split-Path $PSScriptRoot -Parent) +) +$ErrorActionPreference = "Stop" + +# Copies the OWID source into the 51Did package as the private module +# fiftyone_pipeline_did._owid, so that a published wheel can be imported +# without the OWID library being installed separately. +# +# The dependency cannot come from a package registry. The 51Degrees fork of +# OWID is not published to PyPI, and the name "owid" on PyPI belongs to an +# unrelated project, so declaring install_requires=["owid"] would install the +# wrong thing. The name here carries a leading underscore so that installing +# the 51Did package never claims the top level name "owid" on a consumer's +# machine. +# +# Nothing is fetched over the network, because owid-python is a submodule and +# CI clones with --recurse-submodules. Nothing is written back to the +# repository either, as the copy is ignored by git and main keeps the +# submodule as the single source of the OWID code. + +$owidRepo = Join-Path $RepoRoot "owid-python" +$owidSource = Join-Path $owidRepo "owid" +$target = Join-Path $RepoRoot ` + "fiftyone_pipeline_did/src/fiftyone_pipeline_did/_owid" + +if (-not (Test-Path (Join-Path $owidSource "__init__.py"))) { + throw "OWID source not found at '$owidSource'. Run " + + "'git submodule update --init --recursive' first, or clone with " + + "--recurse-submodules." +} + +# The commit the copy was taken from, so the notice can say exactly which +# version of the OWID source is inside the package. +$commit = (git -C $owidRepo rev-parse HEAD 2>$null) +if (-not $commit) { + $commit = (git -C $RepoRoot rev-parse "HEAD:owid-python" 2>$null) +} +if (-not $commit) { + throw "Could not determine the owid-python commit to record in the notice." +} + +if (Test-Path $target) { + Remove-Item -Path $target -Recurse -Force +} +$null = New-Item -ItemType Directory -Path $target -Force + +Copy-Item -Path (Join-Path $owidSource "*.py") -Destination $target -Force +Copy-Item -Path (Join-Path $owidRepo "LICENSE") ` + -Destination (Join-Path $target "LICENSE") -Force + +$notice = @" +The Python modules in this directory are the OWID (Open Web Id) library. They +are copied into the fiftyone_pipeline_did package at build time and are not +part of the 51Degrees source, so they keep their own licence, which is the +Apache License 2.0 in the LICENSE file beside this notice, and not the EUPL +1.2 that covers the rest of the package. + +Copyright 2026 51 Degrees Mobile Experts Limited (51degrees.com) + +Taken from the 51Degrees fork of the OWID project, +https://github.com/51Degrees/owid-python, at commit +$commit +which follows https://github.com/SWAN-community/owid-python. + +The modules are placed under the private name _owid so that installing this +package does not claim the top level name "owid", which on PyPI belongs to an +unrelated project. Import OWID from the fork itself rather than from here. +"@ +Set-Content -Path (Join-Path $target "NOTICE") -Value $notice -Encoding utf8 + +Write-Output "Copied OWID source at $commit into '$target'" diff --git a/fiftyone_pipeline_did/MANIFEST.in b/fiftyone_pipeline_did/MANIFEST.in index 3fed507..48e2f00 100644 --- a/fiftyone_pipeline_did/MANIFEST.in +++ b/fiftyone_pipeline_did/MANIFEST.in @@ -1,2 +1,7 @@ include version.txt include readme.md +# The OWID source carried inside this package under the private name _owid is +# Apache-2.0, so its licence and the notice naming where the source came from +# have to travel with it in both the sdist and the wheel. +include src/fiftyone_pipeline_did/_owid/LICENSE +include src/fiftyone_pipeline_did/_owid/NOTICE diff --git a/fiftyone_pipeline_did/examples/creator_context_web/server.py b/fiftyone_pipeline_did/examples/creator_context_web/server.py index 9452a23..39902a8 100644 --- a/fiftyone_pipeline_did/examples/creator_context_web/server.py +++ b/fiftyone_pipeline_did/examples/creator_context_web/server.py @@ -80,8 +80,9 @@ # The package from this repository rather than a published one, so the # branch is what runs. When the package is installed the import succeeds # directly, and otherwise the package source beside this example is used, -# which is how a checkout runs the demo without an install step. The OWID -# library the package builds on must be installed either way (see the +# which is how a checkout runs the demo without an install step. Running +# from the source needs the OWID copy that ci/copy-owid-source.ps1 puts in +# place, so run pwsh ./setup.ps1 from the repository root first (see the # package readme). try: from fiftyone_pipeline_did import ( diff --git a/fiftyone_pipeline_did/readme.md b/fiftyone_pipeline_did/readme.md index e4d9fed..bb2c704 100644 --- a/fiftyone_pipeline_did/readme.md +++ b/fiftyone_pipeline_did/readme.md @@ -38,9 +38,34 @@ as `PROBABILISTIC`. `FodId` builds on the OWID envelope library ([SWAN-community/owid-python](https://github.com/SWAN-community/owid-python), -package `owid`), consumed via the `51Degrees/owid-python` fork (git submodule; -switch to upstream once published). `Owid` is composed, not subclassed: -`FodId` holds an `Owid` and delegates OWID-level concerns to it. +package `owid`), consumed via the +[51Degrees/owid-python](https://github.com/51Degrees/owid-python) fork, which +is a git submodule of this repository and will move to upstream once that is +published. `Owid` is composed, not subclassed, so `FodId` holds an `Owid` and +delegates OWID-level concerns to it. + +The published package does not take OWID from PyPI. The 51Degrees fork is not +published there, and the name `owid` on PyPI belongs to an unrelated project, +so a declared dependency would install the wrong thing. Instead +`ci/copy-owid-source.ps1` copies the fork's source into the package as the +private module `fiftyone_pipeline_did._owid` before the distribution is built, +carrying the Apache-2.0 licence and a notice naming the source commit with it. +The leading underscore keeps the top level name `owid` free on the consumer's +machine. The only third party requirement the package declares is +`cryptography`, which OWID uses for the signatures. + +Nothing changes for a developer working in this repository, because the +submodule stays where it is and the same script puts the copy in place. +Run `pwsh ./setup.ps1` from the repository root, or +`git submodule update --init --recursive` followed by +`pwsh ./ci/copy-owid-source.ps1`, before building or running the tests. The +tests and examples import the fork under its own name, `owid`, which is how +they build signed envelopes to test against. + +The two OWID types the public API refers to are re-exported from the package +itself, so a caller never has to reach into the private module. Catch +`fiftyone_pipeline_did.OwidError` for an OWID level failure, and use +`fiftyone_pipeline_did.Owid` for the envelope that `FodId.from_owid` takes. ## Usage diff --git a/fiftyone_pipeline_did/setup.py b/fiftyone_pipeline_did/setup.py index f531de7..63661d2 100644 --- a/fiftyone_pipeline_did/setup.py +++ b/fiftyone_pipeline_did/setup.py @@ -46,16 +46,21 @@ def read(file_name): long_description=read("readme.md"), long_description_content_type='text/markdown', python_requires=">=3.9", - packages=["fiftyone_pipeline_did"], + packages=["fiftyone_pipeline_did", "fiftyone_pipeline_did._owid"], package_dir={"": "src"}, - # OWID is provided by the 51Degrees owid-python fork, which is consumed via - # the git submodule / editable install (see tox.ini) rather than declared as - # a PyPI dependency. The name "owid" on PyPI belongs to an unrelated project, - # and the 51Degrees fork is not published to PyPI, so a bare - # install_requires=['owid'] would resolve to the wrong package. If this - # package is ever published standalone, depend on a 51Degrees-namespaced - # OWID distribution here instead. - install_requires=[], + # The OWID source is carried inside this package as the private module + # fiftyone_pipeline_did._owid, copied out of the 51Degrees owid-python + # fork by ci/copy-owid-source.ps1 before the distribution is built. It + # cannot + # come from a package registry, because the 51Degrees fork is not + # published to PyPI and the name "owid" there belongs to an unrelated + # project, so a bare install_requires=["owid"] would install the wrong + # thing. The module is private (leading underscore) so that installing + # this package never claims the top level name "owid" on a consumer's + # machine. Its only third party requirement is cryptography, which is + # declared below and does come from PyPI. + package_data={"fiftyone_pipeline_did._owid": ["LICENSE", "NOTICE"]}, + install_requires=["cryptography>=41"], license="EUPL-1.2", classifiers=[ "Development Status :: 5 - Production/Stable", diff --git a/fiftyone_pipeline_did/src/fiftyone_pipeline_did/__init__.py b/fiftyone_pipeline_did/src/fiftyone_pipeline_did/__init__.py index 803a5e9..d37b086 100644 --- a/fiftyone_pipeline_did/src/fiftyone_pipeline_did/__init__.py +++ b/fiftyone_pipeline_did/src/fiftyone_pipeline_did/__init__.py @@ -52,6 +52,12 @@ SignatureReason, SignatureResult, ) +# The OWID library is carried inside this package as the private module +# _owid, because it cannot be installed from a package registry (see the +# package readme). The envelope type and the error type are re-exported here +# so that callers have supported names for the two of them that the public +# API refers to, as the private module itself is not part of that API. +from ._owid import Owid, OwidError from .fod_id import DATE_EPOCH, FodId from .id_type import IdType @@ -71,4 +77,6 @@ "DidArgumentError", "DidNotSupportedError", "DEFAULT_ENDPOINT", + "Owid", + "OwidError", ] diff --git a/fiftyone_pipeline_did/src/fiftyone_pipeline_did/did_client.py b/fiftyone_pipeline_did/src/fiftyone_pipeline_did/did_client.py index 6bcd9bd..996fac3 100644 --- a/fiftyone_pipeline_did/src/fiftyone_pipeline_did/did_client.py +++ b/fiftyone_pipeline_did/src/fiftyone_pipeline_did/did_client.py @@ -54,7 +54,7 @@ from enum import Enum from typing import Any, Callable, Dict, List, Optional, Tuple, Union -from owid import Version +from ._owid import Version from .fod_id import DATE_EPOCH, FodId from .id_type import IdType diff --git a/fiftyone_pipeline_did/src/fiftyone_pipeline_did/fod_id.py b/fiftyone_pipeline_did/src/fiftyone_pipeline_did/fod_id.py index f0ff59d..ded5ad1 100644 --- a/fiftyone_pipeline_did/src/fiftyone_pipeline_did/fod_id.py +++ b/fiftyone_pipeline_did/src/fiftyone_pipeline_did/fod_id.py @@ -24,7 +24,7 @@ from datetime import datetime, timezone -from owid import Owid, Version +from ._owid import Owid, Version from .id_type import IdType @@ -39,8 +39,9 @@ class FodId: returned by the 51Degrees Cloud service. A 51Did is described at three levels. The **51Did** is the identifier as a - whole. The **envelope** is the signed :class:`owid.Owid` that carries it - (version, domain, date, payload, signature), re-issued fresh on every call. + whole. The **envelope** is the signed + :class:`~fiftyone_pipeline_did.Owid` that carries it (version, domain, + date, payload, signature), re-issued fresh on every call. The **value** is the stable, comparable part of the payload after the Flags and License Id, exposed as :attr:`hash`. Two 51Dids for the same inputs share the same value even though their envelopes differ. *Compare values, @@ -77,8 +78,8 @@ class FodId: PAYLOAD_LENGTH = HASH_OFFSET + HASH_LENGTH def __init__(self, owid: Owid) -> None: - """Promotes an already-parsed :class:`owid.Owid` into a 51Did by - unpacking its payload. + """Promotes an already-parsed :class:`~fiftyone_pipeline_did.Owid` + into a 51Did by unpacking its payload. The OWID is **copied** (round-tripped through its byte form), not aliased, so a ``FodId`` can never desync from its envelope if the caller @@ -87,8 +88,8 @@ def __init__(self, owid: Owid) -> None: Raises :class:`TypeError` if ``owid`` is ``None``, :class:`ValueError` if the payload is shorter than the minimum for its identifier type, and - :class:`owid.OwidError` if the OWID cannot be serialized (e.g. it is - unsigned). + :class:`~fiftyone_pipeline_did.OwidError` if the OWID cannot be + serialized (e.g. it is unsigned). """ if owid is None: raise TypeError("owid must not be None") @@ -141,7 +142,8 @@ def from_base64(cls, base64: str) -> "FodId": never converts an identifier it received from a link. Raises :class:`TypeError` if ``base64`` is ``None`` and - :class:`owid.OwidError` if it is not valid base64 or not a valid OWID. + :class:`~fiftyone_pipeline_did.OwidError` if it is not valid base64 + or not a valid OWID. """ if base64 is None: raise TypeError("base64 must not be None") @@ -177,7 +179,8 @@ def from_byte_array(cls, buffer: bytes) -> "FodId": """Parses a 51Did from the raw bytes of an OWID envelope. Raises :class:`TypeError` if ``buffer`` is ``None`` and - :class:`owid.OwidError` if the bytes are not a valid OWID. + :class:`~fiftyone_pipeline_did.OwidError` if the bytes are not a + valid OWID. """ if buffer is None: raise TypeError("buffer must not be None") diff --git a/fiftyone_pipeline_did/tests/test_fodid.py b/fiftyone_pipeline_did/tests/test_fodid.py index 7e1388a..517acaf 100644 --- a/fiftyone_pipeline_did/tests/test_fodid.py +++ b/fiftyone_pipeline_did/tests/test_fodid.py @@ -24,9 +24,9 @@ import unittest from datetime import datetime, timezone -from owid import Owid, Crypto, Creator, OwidError +from owid import Owid, Crypto, Creator -from fiftyone_pipeline_did import FodId, IdType +from fiftyone_pipeline_did import FodId, IdType, OwidError TEST_DOMAIN = "51degrees.com" # 0xA5: usage bits plus the HashedEmail type tag in bits 6-7. diff --git a/setup.ps1 b/setup.ps1 index 03a86e7..f986699 100644 --- a/setup.ps1 +++ b/setup.ps1 @@ -1,6 +1,12 @@ +# The 51Did package builds against the OWID source, which lives in the +# owid-python submodule and is copied into the package rather than installed +# from a package registry. See ci/copy-owid-source.ps1 for why. +& "$PSScriptRoot/ci/copy-owid-source.ps1" + python -m pip install -e fiftyone_pipeline_cloudrequestengine/ python -m pip install -e fiftyone_pipeline_core/ python -m pip install -e fiftyone_pipeline_translation/ python -m pip install -e fiftyone_pipeline_engines/ python -m pip install -e fiftyone_pipeline_engines_fiftyone/ +python -m pip install -e owid-python/ python -m pip install flask