From 4cc2c03fac980a78c05816cc6a8e1d1cb34e5686 Mon Sep 17 00:00:00 2001 From: Thibaud Perrin <93876776+thibaud-perrin@users.noreply.github.com> Date: Sat, 17 Oct 2026 22:47:11 +0200 Subject: [PATCH] ci: pin current devpi releases and monitor PyPI updates with issues --- .github/workflows/check-versions.yml | 59 ++++++++ .github/workflows/ci.yml | 13 +- Dockerfile | 7 +- README.md | 18 +++ requirements.txt | 4 + scripts/check_versions.py | 200 +++++++++++++++++++++++++++ scripts/smoke-test.sh | 6 +- tests/test_versions.py | 149 ++++++++++++++++++++ 8 files changed, 449 insertions(+), 7 deletions(-) create mode 100644 .github/workflows/check-versions.yml create mode 100644 requirements.txt create mode 100644 scripts/check_versions.py create mode 100644 tests/test_versions.py diff --git a/.github/workflows/check-versions.yml b/.github/workflows/check-versions.yml new file mode 100644 index 0000000..727be95 --- /dev/null +++ b/.github/workflows/check-versions.yml @@ -0,0 +1,59 @@ +name: Check upstream versions + +on: + schedule: + - cron: '43 5 * * *' + workflow_dispatch: + push: + branches: [main] + paths: + - '.github/workflows/check-versions.yml' + - 'scripts/check_versions.py' + - 'tests/test_versions.py' + - 'requirements.txt' + - 'Dockerfile' + pull_request: + paths: + - '.github/workflows/check-versions.yml' + - 'scripts/check_versions.py' + - 'tests/test_versions.py' + - 'requirements.txt' + - 'Dockerfile' + +permissions: + contents: read + +concurrency: + group: versions-${{ github.ref }} + cancel-in-progress: false + +jobs: + test: + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-python@v7 + with: + python-version: '3.12' + - run: python -m pip install 'packaging>=24,<27' + - run: python -m unittest discover -s tests -p 'test_versions.py' -v + + monitor: + needs: test + if: github.event_name != 'pull_request' && github.ref == format('refs/heads/{0}', github.event.repository.default_branch) + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + contents: read + issues: write + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-python@v7 + with: + python-version: '3.12' + - run: python -m pip install 'packaging>=24,<27' + - name: Check PyPI releases and maintain update issues + env: + GITHUB_TOKEN: ${{ github.token }} + run: python scripts/check_versions.py --write-issues diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d81eb17..9da11f9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,17 +5,26 @@ on: branches: [main] pull_request: branches: [main] + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ci-${{ github.ref }} + cancel-in-progress: true jobs: smoke: name: build & smoke test runs-on: ubuntu-latest + timeout-minutes: 20 steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v7 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 + uses: docker/setup-buildx-action@v4 - name: Run smoke test run: ./scripts/smoke-test.sh diff --git a/Dockerfile b/Dockerfile index 7f27a80..d488584 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,10 +12,9 @@ RUN apt-get update \ && apt-get install -y --no-install-recommends curl ca-certificates \ && rm -rf /var/lib/apt/lists/* -RUN pip install \ - "devpi-server>=6.10,<7" \ - "devpi-web>=5.0,<6" \ - "devpi-client>=7.0,<8" +COPY requirements.txt /app/requirements.txt +RUN pip install --requirement /app/requirements.txt \ + && pip check COPY theme /app/theme COPY entrypoint.sh /usr/local/bin/entrypoint.sh diff --git a/README.md b/README.md index cc685be..cb2f0dd 100644 --- a/README.md +++ b/README.md @@ -154,6 +154,24 @@ These are intentional trade-offs to keep the template simple and reliable. Fork Builds the image, boots a throwaway container, exercises the full user/index/upload/install flow, then restarts the container on the same volume to verify persistence. +## Versions and maintenance + +The three runtime dependencies are pinned explicitly in [requirements.txt](requirements.txt). As checked on 17 October 2026, these match the latest stable PyPI releases: **devpi-server 6.20.3**, **devpi-web 5.1.1**, and **devpi-client 7.3.0**. Python stays on the existing 3.12 base; the smoke test pulls the current image and checks the installed dependency graph before exercising the registry. + +[Check upstream versions](https://github.com/hallcyn/private-python-registry/actions/workflows/check-versions.yml) runs daily at **05:43 UTC**, after monitored files change on `main`, or manually via **Run workflow**. It checks published PyPI releases, ignores prereleases and yanked candidates, and flags a withdrawn installed pin or a Python compatibility change. Its unit tests also run on pull requests without permission to write issues. + +The workflow creates at most one open update issue per runtime dependency, refreshes it for newer releases, and closes it when the repository catches up. Closing as **not planned** suppresses that precise target version. Keep discussion in comments because the bot owns the issue body. An API failure fails the check and leaves issues untouched. GitHub may delay scheduled runs or disable them after 60 days without activity in a public repository; check the Actions tab if reminders stop. + +Updates and deployments stay manual. Change the relevant pins on a branch, read the upstream release notes, and require a successful container smoke test before merging. Back up the complete `/data` volume, including the server secret, before deploying an upgrade; follow any devpi storage migration instructions. The watcher checks repository pins, not the versions inside an already deployed Railway service or every transitive dependency. It does not change Railway configuration or stored packages. + +To run the watcher locally without creating issues: + +```bash +python -m pip install 'packaging>=24,<27' +python scripts/check_versions.py +python -m unittest discover -s tests -p 'test_versions.py' -v +``` + ## License MIT. See [LICENSE](./LICENSE). diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..eebabfc --- /dev/null +++ b/requirements.txt @@ -0,0 +1,4 @@ +# Runtime versions validated by the container smoke test. +devpi-server==6.20.3 +devpi-web==5.1.1 +devpi-client==7.3.0 diff --git a/scripts/check_versions.py b/scripts/check_versions.py new file mode 100644 index 0000000..d850d52 --- /dev/null +++ b/scripts/check_versions.py @@ -0,0 +1,200 @@ +#!/usr/bin/env python3 +"""Check pinned devpi versions against PyPI, without deploying or updating code.""" +import argparse +from dataclasses import dataclass +import json +import os +from pathlib import Path +import re +import sys +import time +from urllib.error import HTTPError, URLError +from urllib.parse import urlparse +from urllib.request import Request, urlopen + +from packaging.requirements import Requirement +from packaging.specifiers import SpecifierSet +from packaging.utils import canonicalize_name +from packaging.version import InvalidVersion, Version + +GITHUB = 'https://api.github.com' + + +class API: + def __init__(self, token=None): + self.token = token + + def request(self, url, method='GET', data=None): + headers = {'User-Agent': 'hallcyn-private-python-registry-version-watch', 'Accept': 'application/json'} + if urlparse(url).hostname == 'api.github.com': + headers.update({'Accept': 'application/vnd.github+json', 'X-GitHub-Api-Version': '2022-11-28'}) + if self.token: + headers['Authorization'] = f'Bearer {self.token}' + body = None if data is None else json.dumps(data).encode() + if body is not None: + headers['Content-Type'] = 'application/json' + for attempt in range(3): + try: + with urlopen(Request(url, data=body, headers=headers, method=method), timeout=30) as response: + return json.load(response) + except HTTPError as error: + if method != 'GET' or error.code not in (429, 500, 502, 503, 504) or attempt == 2: + raise + except (URLError, TimeoutError): + if method != 'GET' or attempt == 2: + raise + time.sleep(2 ** attempt) + + +def read_pins(path): + pins = {} + for line in path.read_text().splitlines(): + line = line.split('#', 1)[0].strip() + if not line: + continue + req = Requirement(line) + specs = list(req.specifier) + if req.url or req.marker or req.extras or len(specs) != 1 or specs[0].operator != '==': + raise ValueError(f'An exact runtime pin is required: {line}') + pins[canonicalize_name(req.name)] = str(Version(specs[0].version)) + if not pins: + raise ValueError('No runtime pins found') + return pins + + +def latest_stable(releases): + candidates = [] + for name, files in releases.items(): + try: + parsed = Version(name) + except InvalidVersion: + continue + if not parsed.is_prerelease and not parsed.is_devrelease and any(not f.get('yanked') for f in files): + candidates.append((parsed, name)) + if not candidates: + raise ValueError('PyPI returned no published stable releases') + return max(candidates)[1] + + +@dataclass(frozen=True) +class Report: + package: str + current: str + latest: str + compatible: bool + withdrawn: bool = False + + @property + def needed(self): + return Version(self.latest) > Version(self.current) or self.withdrawn + + @property + def marker(self): + return f'' + + @property + def target_marker(self): + return f'' + + @property + def title(self): + return f'[Mise à jour] {self.package} : {self.current} → {self.latest}' + + @property + def body(self): + compatibility = ('Les métadonnées PyPI autorisent la version Python du moniteur ; la CI Docker doit encore valider la pile complète.' + if self.compatible else 'La nouvelle version nécessite une autre version de Python : étudier cette migration avant de modifier le pin.') + withdrawn = '\n**La version actuellement fixée a été retirée (yanked) de PyPI.**\n' if self.withdrawn else '' + return f'''{self.marker} +{self.target_marker} +## Mise à jour à examiner + +| Dépendance | Version du dépôt | Dernière version stable | +| --- | --- | --- | +| {self.package} | `{self.current}` | `{self.latest}` | + +{compatibility} +{withdrawn} +Source : https://pypi.org/project/{self.package}/{self.latest}/ + +- [ ] Lire les notes de version et vérifier la compatibilité devpi-server / devpi-web / devpi-client. +- [ ] Mettre à jour `requirements.txt` sur une branche de travail. +- [ ] Attendre la réussite de la CI : construction, thème, authentification, publication, installation et persistance. +- [ ] Sauvegarder le volume `/data` avant un éventuel déploiement et vérifier les migrations requises. + +Cette issue est gérée par la CI ; conserver les échanges dans les commentaires. +Fermer comme « not planned » ignore cette version précise. Une version ultérieure +peut créer une nouvelle alerte. Aucune mise à jour ni aucun déploiement n’est automatique. +''' + + +def collect_reports(root, api): + python = '.'.join(map(str, sys.version_info[:3])) + reports = [] + for package, current in read_pins(root / 'requirements.txt').items(): + releases = api.request(f'https://pypi.org/pypi/{package}/json')['releases'] + if not releases.get(current): + raise ValueError(f'Pinned version {package}=={current} is missing from PyPI') + latest = latest_stable(releases) + files = [f for f in releases[latest] if not f.get('yanked')] + compatible = any(SpecifierSet(f.get('requires_python') or '').contains(python) for f in files) + withdrawn = all(f.get('yanked', False) for f in releases[current]) + reports.append(Report(package, current, latest, compatible, withdrawn)) + # Finish all API reads before changing any issue, including closing an old one. + return reports + + +def sync_issues(api, repository, reports): + base = f'{GITHUB}/repos/{repository}/issues' + issues = [] + for page in range(1, 21): + batch = api.request(f'{base}?state=all&per_page=100&page={page}') + issues.extend(i for i in batch if 'pull_request' not in i + and i.get('user', {}).get('login') == 'github-actions[bot]') + if len(batch) < 100: + break + else: + raise ValueError('Issue pagination limit reached; refusing possible duplicates') + for report in reports: + matching = [i for i in issues if report.marker in (i.get('body') or '')] + opened = [i for i in matching if i['state'] == 'open'] + if not report.needed: + for issue in opened: + api.request(f'{base}/{issue["number"]}', 'PATCH', {'state': 'closed', 'state_reason': 'completed'}) + continue + payload = {'title': report.title, 'body': report.body} + if opened: + issue = opened[0] + if issue['title'] != report.title or issue['body'] != report.body: + api.request(f'{base}/{issue["number"]}', 'PATCH', payload) + continue + previous = next((i for i in matching if report.target_marker in (i.get('body') or '')), None) + if previous: + if previous.get('state_reason') != 'not_planned': + api.request(f'{base}/{previous["number"]}', 'PATCH', {**payload, 'state': 'open'}) + continue + api.request(base, 'POST', payload) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--write-issues', action='store_true') + args = parser.parse_args() + api = API(os.environ.get('GITHUB_TOKEN')) + reports = collect_reports(Path(__file__).resolve().parents[1], api) + summary = '# Versions devpi\n\n| Dépendance | Dépôt | Stable publiée | Action |\n| --- | --- | --- | --- |\n' + for r in reports: + summary += f'| {r.package} | {r.current} | {r.latest} | {"À examiner" if r.needed else "À jour"} |\n' + print(summary) + if os.environ.get('GITHUB_STEP_SUMMARY'): + with open(os.environ['GITHUB_STEP_SUMMARY'], 'a') as stream: + stream.write(summary) + if args.write_issues: + repository = os.environ.get('GITHUB_REPOSITORY', '') + if not re.fullmatch(r'[\w.-]+/[\w.-]+', repository) or not api.token: + raise ValueError('GITHUB_REPOSITORY and GITHUB_TOKEN are required to write issues') + sync_issues(api, repository, reports) + + +if __name__ == '__main__': + main() diff --git a/scripts/smoke-test.sh b/scripts/smoke-test.sh index 7600bfe..2d8bae6 100755 --- a/scripts/smoke-test.sh +++ b/scripts/smoke-test.sh @@ -62,7 +62,7 @@ in_container() { docker exec "$CONTAINER_NAME" "$@"; } # --- 1. build image ------------------------------------------------------- log "building image $IMAGE_TAG" -docker build -t "$IMAGE_TAG" "$(dirname "$0")/.." +docker build --pull -t "$IMAGE_TAG" "$(dirname "$0")/.." # --- 2. prepare a minimal package on the host ---------------------------- log "preparing minimal package in $WORK_DIR" @@ -92,6 +92,10 @@ pre_clean docker volume create "$VOLUME_NAME" >/dev/null start_container +log "checking installed runtime dependencies" +in_container python -m pip check +in_container python -c 'from importlib.metadata import version; print({name: version(name) for name in ("devpi-server", "devpi-web", "devpi-client")})' + # --- 3b. verify the custom theme CSS is actually served ------------------ log "checking that the custom theme CSS is referenced and served" ROOT_HTML="$(curl -fsS "http://localhost:${HOST_PORT}/")" diff --git a/tests/test_versions.py b/tests/test_versions.py new file mode 100644 index 0000000..0efe66d --- /dev/null +++ b/tests/test_versions.py @@ -0,0 +1,149 @@ +"""Exercise PyPI release selection and issue lifecycle without network writes.""" +import importlib.util +import json +from pathlib import Path +import sys +import tempfile +import unittest +from unittest.mock import patch +from urllib.error import URLError + +spec = importlib.util.spec_from_file_location('version_watch', Path(__file__).parents[1] / 'scripts/check_versions.py') +watch = importlib.util.module_from_spec(spec) +sys.modules[spec.name] = watch +spec.loader.exec_module(watch) + + +def files(*, yanked=False, python='>=3.9'): + return [{'yanked': yanked, 'requires_python': python}] + + +class IssueAPI: + def __init__(self, issues=()): + self.issues = list(issues) + self.writes = [] + + def request(self, url, method='GET', data=None): + if method == 'GET': + return self.issues + self.writes.append((method, data)) + if method == 'POST': + self.issues.append({'number': len(self.issues) + 1, 'state': 'open', + 'user': {'login': 'github-actions[bot]'}, **data}) + else: + issue = next(i for i in self.issues if i['number'] == int(url.rsplit('/', 1)[1])) + issue.update(data) + return self.issues[-1] + + +class VersionSelection(unittest.TestCase): + def test_numeric_comparison_instead_of_lexical_sorting(self): + releases = {'6.9.0': files(), '6.20.3': files(), '6.20.10': files()} + self.assertEqual(watch.latest_stable(releases), '6.20.10') + + def test_prereleases_yanked_and_unpublished_versions_are_ignored(self): + releases = {'6.20.3': files(), '7.0.0rc1': files(), '8.0.dev1': files(), + '7.0.0': files(yanked=True), '9.0': [], 'not-a-version': files()} + self.assertEqual(watch.latest_stable(releases), '6.20.3') + + def test_post_releases_are_stable(self): + self.assertEqual(watch.latest_stable({'1.0': files(), '1.0.post1': files()}), '1.0.post1') + + def test_empty_result_is_an_error_instead_of_up_to_date(self): + with self.assertRaises(ValueError): + watch.latest_stable({}) + + def test_requirements_must_use_exact_pins(self): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / 'requirements.txt' + path.write_text('# Runtime\ndevpi-server==6.20.3\n') + self.assertEqual(watch.read_pins(path), {'devpi-server': '6.20.3'}) + path.write_text('devpi-server>=6.10,<7\n') + with self.assertRaises(ValueError): + watch.read_pins(path) + + def report(self, releases): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / 'requirements.txt').write_text('devpi-server==6.20.3\n') + api = watch.API() + with patch.object(api, 'request', return_value={'releases': releases}): + return watch.collect_reports(root, api)[0] + + def test_python_incompatible_release_is_reported_for_manual_migration(self): + report = self.report({'6.20.3': files(), '7.0.0': files(python='>=99')}) + self.assertTrue(report.needed) + self.assertFalse(report.compatible) + self.assertIn('autre version de Python', report.body) + + def test_withdrawn_current_version_opens_an_alert(self): + report = self.report({'6.20.2': files(), '6.20.3': files(yanked=True)}) + self.assertTrue(report.withdrawn) + self.assertTrue(report.needed) + + def test_missing_pin_is_an_error(self): + with self.assertRaises(ValueError): + self.report({'6.20.4': files()}) + + def test_source_failure_cannot_close_existing_issues(self): + with patch.object(watch, 'collect_reports', side_effect=OSError('PyPI unavailable')), \ + patch.object(watch, 'sync_issues') as sync, \ + patch.object(sys, 'argv', ['check_versions.py', '--write-issues']): + with self.assertRaises(OSError): + watch.main() + sync.assert_not_called() + + def test_failed_issue_creation_is_not_retried(self): + with patch.object(watch, 'urlopen', side_effect=URLError('response lost')) as request: + with self.assertRaises(URLError): + watch.API().request('https://api.github.com/repos/example/repo/issues', 'POST', {}) + self.assertEqual(request.call_count, 1) + + +class IssueLifecycle(unittest.TestCase): + def report(self, latest='6.20.4'): + return watch.Report('devpi-server', '6.20.3', latest, True) + + def test_repeated_checks_create_only_one_issue(self): + api = IssueAPI() + for _ in range(3): + watch.sync_issues(api, 'hallcyn/private-python-registry', [self.report()]) + self.assertEqual(len(api.issues), 1) + self.assertEqual(len(api.writes), 1) + + def test_new_release_updates_existing_issue(self): + api = IssueAPI() + watch.sync_issues(api, 'hallcyn/private-python-registry', [self.report()]) + watch.sync_issues(api, 'hallcyn/private-python-registry', [self.report('7.0.0')]) + self.assertEqual(len(api.issues), 1) + self.assertIn('7.0.0', api.issues[0]['title']) + + def test_updated_repository_closes_issue_and_rollback_reopens_it(self): + api = IssueAPI() + watch.sync_issues(api, 'hallcyn/private-python-registry', [self.report()]) + watch.sync_issues(api, 'hallcyn/private-python-registry', [self.report('6.20.3')]) + self.assertEqual(api.issues[0]['state'], 'closed') + watch.sync_issues(api, 'hallcyn/private-python-registry', [self.report()]) + self.assertEqual(api.issues[0]['state'], 'open') + self.assertEqual(len(api.issues), 1) + + def test_not_planned_suppresses_only_that_release(self): + api = IssueAPI() + watch.sync_issues(api, 'hallcyn/private-python-registry', [self.report()]) + api.issues[0].update(state='closed', state_reason='not_planned') + watch.sync_issues(api, 'hallcyn/private-python-registry', [self.report()]) + self.assertEqual(len(api.writes), 1) + watch.sync_issues(api, 'hallcyn/private-python-registry', [self.report('7.0.0')]) + self.assertEqual(len(api.issues), 2) + + def test_manual_issues_are_not_modified(self): + report = self.report() + api = IssueAPI([{'number': 1, 'user': {'login': 'human'}, 'state': 'open', + 'title': report.title, 'body': report.body}]) + watch.sync_issues(api, 'hallcyn/private-python-registry', [report]) + self.assertEqual(len(api.issues), 2) + self.assertEqual(api.writes[0][0], 'POST') + + +if __name__ == '__main__': + unittest.main()