fix: Added PylockTomlHandler in packagedcode/pypi.py to parse PEP 751

CHANGELOG.rst

@@ -4,6 +4,12 @@ Changelog
 Next release
 --------------
 
+- Add support for PEP 751 ``pylock.toml`` lockfiles. The new package
+  data handler parses ``pylock.toml`` and ``pylock.<name>.toml`` files,
+  reporting the locked packages together with their resolved download
+  URLs, hashes and dependency relationships.
+  https://github.com/aboutcode-org/scancode-toolkit/issues/4638
+
 - Fix the optional ``licenses`` extra dependency typo to install
   ``licensedcode-data``.
   https://github.com/aboutcode-org/scancode-toolkit/pull/5056

docs/source/reference/scancode-supported-packages.rst

@@ -812,6 +812,14 @@ parsers in scancode-toolkit during documentation builds.
      - ``pypi_poetry_pyproject_toml``
      - Python
      - https://packaging.python.org/en/latest/specifications/pyproject-toml/
+   * - Python pylock.toml lockfile
+     - ``*pylock.toml``
+       ``*pylock.*.toml``
+     - ``pypi``
+     - ``linux``, ``win``, ``mac``
+     - ``pypi_pylock_toml``
+     - Python
+     - https://peps.python.org/pep-0751/
    * - Python pyproject.toml
      - ``*pyproject.toml``
      - ``pypi``

src/packagedcode/__init__.py

@@ -176,6 +176,7 @@
     pypi.PoetryLockHandler,
     pypi.UvPyprojectTomlHandler,
     pypi.UvLockHandler,
+    pypi.PylockTomlHandler,
     pypi.PythonEditableInstallationPkgInfoFile,
     pypi.PythonEggPkgInfoFile,
     pypi.PythonInstalledWheelMetadataFile,

src/packagedcode/pypi.py

@@ -1126,6 +1126,146 @@ def parse(cls, location, package_only=False):
         yield models.PackageData.from_data(package_data, package_only)
 
 
+def is_pylock_toml(filename):
+    """
+    Return True if the ``filename`` is a valid PEP 751 lock file name, e.g.
+    ``pylock.toml`` or ``pylock.dev.toml``.
+    See https://peps.python.org/pep-0751/#file-name
+    """
+    return filename == 'pylock.toml' or bool(re.match(r'^pylock\.[^.]+\.toml$', filename))
+
+
+class PylockTomlHandler(models.DatafileHandler):
+    datasource_id = 'pypi_pylock_toml'
+    path_patterns = ('*pylock.toml', '*pylock.*.toml')
+    default_package_type = 'pypi'
+    default_primary_language = 'Python'
+    description = 'Python pylock.toml lockfile'
+    documentation_url = 'https://peps.python.org/pep-0751/'
+
+    @classmethod
+    def is_datafile(cls, location, filetypes=tuple()):
+        return (
+            super().is_datafile(location, filetypes=filetypes)
+            and is_pylock_toml(fileutils.file_name(location))
+        )
+
+    @classmethod
+    def parse(cls, location, package_only=False):
+        with open(location, "rb") as fp:
+            toml_data = tomllib.load(fp)
+
+        packages = toml_data.get('packages')
+        if not packages:
+            return
+
+        dependencies = []
+        for package in packages:
+            name = package.get('name')
+            if not name:
+                continue
+            version = package.get('version')
+
+            dependencies_for_resolved = []
+            for dep in (package.get('dependencies') or []):
+                dep_name = dep.get('name')
+                if not dep_name:
+                    continue
+                dep_purl = PackageURL(type=cls.default_package_type, name=dep_name)
+                dependencies_for_resolved.append(
+                    models.DependentPackage(
+                        purl=dep_purl.to_string(),
+                        extracted_requirement=dep.get('marker') or dep.get('version'),
+                        scope='dependencies',
+                        is_runtime=True,
+                        is_optional=False,
+                        is_direct=True,
+                        is_pinned=False,
+                    ).to_dict()
+                )
+
+            # a package can carry a single sdist and/or several wheels (one
+            # per platform/interpreter); prefer the sdist as the canonical
+            # source and fall back to the first wheel, since the resolved
+            # package purl can only point to one download.
+            source = package.get('sdist')
+            if not isinstance(source, dict):
+                wheels = package.get('wheels') or []
+                source = wheels[0] if wheels else None
+
+            download_url = None
+            sha256 = None
+            file_name = None
+            if isinstance(source, dict):
+                download_url = source.get('url')
+                hashes = source.get('hashes') or {}
+                sha256 = hashes.get('sha256')
+                file_name = source.get('name') or (
+                    download_url and posixpath.basename(download_url)
+                )
+
+            urls = get_pypi_urls(name, version)
+            if download_url:
+                urls['repository_download_url'] = download_url
+
+            qualifiers = {}
+            if file_name:
+                qualifiers['file_name'] = file_name
+
+            extra_data = {}
+            marker = package.get('marker')
+            if marker:
+                extra_data['marker'] = marker
+
+            resolved_package_data = dict(
+                datasource_id=cls.datasource_id,
+                type=cls.default_package_type,
+                primary_language='Python',
+                name=name,
+                version=version,
+                qualifiers=qualifiers,
+                sha256=sha256,
+                is_virtual=True,
+                extra_data=extra_data,
+                dependencies=dependencies_for_resolved,
+                **urls,
+            )
+            resolved_package = models.PackageData.from_data(resolved_package_data, package_only)
+
+            dependencies.append(
+                models.DependentPackage(
+                    purl=resolved_package.purl,
+                    extracted_requirement=None,
+                    scope=None,
+                    is_runtime=True,
+                    is_optional=False,
+                    is_direct=False,
+                    is_pinned=True,
+                    resolved_package=resolved_package.to_dict(),
+                ).to_dict()
+            )
+
+        extra_data = {}
+        requires_python = toml_data.get('requires-python')
+        if requires_python:
+            extra_data['python_requires'] = requires_python
+        lock_version = toml_data.get('lock-version')
+        if lock_version is not None:
+            extra_data['lock_version'] = lock_version
+        created_by = toml_data.get('created-by')
+        if created_by:
+            extra_data['created_by'] = created_by
+
+        package_data = dict(
+            datasource_id=cls.datasource_id,
+            type=cls.default_package_type,
+            primary_language='Python',
+            extra_data=extra_data,
+            dependencies=dependencies,
+        )
+        yield models.PackageData.from_data(package_data, package_only)
+
+
 class PipInspectDeplockHandler(models.DatafileHandler):
     datasource_id = 'pypi_inspect_deplock'
     path_patterns = ('*pip-inspect.deplock',)

tests/packagedcode/data/pypi/pylock/attrs-pylock.toml-expected.json

@@ -0,0 +1,175 @@
+[
+  {
+    "type": "pypi",
+    "namespace": null,
+    "name": null,
+    "version": null,
+    "qualifiers": {},
+    "subpath": null,
+    "primary_language": "Python",
+    "description": null,
+    "release_date": null,
+    "parties": [],
+    "keywords": [],
+    "homepage_url": null,
+    "download_url": null,
+    "size": null,
+    "sha1": null,
+    "md5": null,
+    "sha256": null,
+    "sha512": null,
+    "bug_tracking_url": null,
+    "code_view_url": null,
+    "vcs_url": null,
+    "copyright": null,
+    "holder": null,
+    "declared_license_expression": null,
+    "declared_license_expression_spdx": null,
+    "license_detections": [],
+    "other_license_expression": null,
+    "other_license_expression_spdx": null,
+    "other_license_detections": [],
+    "extracted_license_statement": null,
+    "notice_text": null,
+    "source_packages": [],
+    "file_references": [],
+    "is_private": false,
+    "is_virtual": false,
+    "extra_data": {
+      "python_requires": "==3.12",
+      "lock_version": "1.0",
+      "created_by": "mousebender"
+    },
+    "dependencies": [
+      {
+        "purl": "pkg:pypi/attrs@25.1.0?file_name=attrs-25.1.0-py3-none-any.whl",
+        "extracted_requirement": null,
+        "scope": null,
+        "is_runtime": true,
+        "is_optional": false,
+        "is_pinned": true,
+        "is_direct": false,
+        "resolved_package": {
+          "type": "pypi",
+          "namespace": null,
+          "name": "attrs",
+          "version": "25.1.0",
+          "qualifiers": {
+            "file_name": "attrs-25.1.0-py3-none-any.whl"
+          },
+          "subpath": null,
+          "primary_language": "Python",
+          "description": null,
+          "release_date": null,
+          "parties": [],
+          "keywords": [],
+          "homepage_url": null,
+          "download_url": null,
+          "size": null,
+          "sha1": null,
+          "md5": null,
+          "sha256": "c75a69e28a550a7e93789579c22aa26b0f5b83b75dc4e08fe092980051e1090a",
+          "sha512": null,
+          "bug_tracking_url": null,
+          "code_view_url": null,
+          "vcs_url": null,
+          "copyright": null,
+          "holder": null,
+          "declared_license_expression": null,
+          "declared_license_expression_spdx": null,
+          "license_detections": [],
+          "other_license_expression": null,
+          "other_license_expression_spdx": null,
+          "other_license_detections": [],
+          "extracted_license_statement": null,
+          "notice_text": null,
+          "source_packages": [],
+          "file_references": [],
+          "is_private": false,
+          "is_virtual": true,
+          "extra_data": {},
+          "dependencies": [],
+          "repository_homepage_url": "https://pypi.org/project/attrs",
+          "repository_download_url": "https://files.pythonhosted.org/packages/fc/30/d4986a882011f9df997a55e6becd864812ccfcd821d64aac8570ee39f719/attrs-25.1.0-py3-none-any.whl",
+          "api_data_url": "https://pypi.org/pypi/attrs/25.1.0/json",
+          "datasource_id": "pypi_pylock_toml",
+          "purl": "pkg:pypi/attrs@25.1.0?file_name=attrs-25.1.0-py3-none-any.whl"
+        },
+        "extra_data": {}
+      },
+      {
+        "purl": "pkg:pypi/cattrs@24.1.2?file_name=cattrs-24.1.2-py3-none-any.whl",
+        "extracted_requirement": null,
+        "scope": null,
+        "is_runtime": true,
+        "is_optional": false,
+        "is_pinned": true,
+        "is_direct": false,
+        "resolved_package": {
+          "type": "pypi",
+          "namespace": null,
+          "name": "cattrs",
+          "version": "24.1.2",
+          "qualifiers": {
+            "file_name": "cattrs-24.1.2-py3-none-any.whl"
+          },
+          "subpath": null,
+          "primary_language": "Python",
+          "description": null,
+          "release_date": null,
+          "parties": [],
+          "keywords": [],
+          "homepage_url": null,
+          "download_url": null,
+          "size": null,
+          "sha1": null,
+          "md5": null,
+          "sha256": "67c7495b760168d931a10233f979b28dc04daf853b30752246f4f8471c6d68d0",
+          "sha512": null,
+          "bug_tracking_url": null,
+          "code_view_url": null,
+          "vcs_url": null,
+          "copyright": null,
+          "holder": null,
+          "declared_license_expression": null,
+          "declared_license_expression_spdx": null,
+          "license_detections": [],
+          "other_license_expression": null,
+          "other_license_expression_spdx": null,
+          "other_license_detections": [],
+          "extracted_license_statement": null,
+          "notice_text": null,
+          "source_packages": [],
+          "file_references": [],
+          "is_private": false,
+          "is_virtual": true,
+          "extra_data": {},
+          "dependencies": [
+            {
+              "purl": "pkg:pypi/attrs",
+              "extracted_requirement": null,
+              "scope": "dependencies",
+              "is_runtime": true,
+              "is_optional": false,
+              "is_pinned": false,
+              "is_direct": true,
+              "resolved_package": {},
+              "extra_data": {}
+            }
+          ],
+          "repository_homepage_url": "https://pypi.org/project/cattrs",
+          "repository_download_url": "https://files.pythonhosted.org/packages/c8/d5/867e75361fc45f6de75fe277dd085627a9db5ebb511a87f27dc1396b5351/cattrs-24.1.2-py3-none-any.whl",
+          "api_data_url": "https://pypi.org/pypi/cattrs/24.1.2/json",
+          "datasource_id": "pypi_pylock_toml",
+          "purl": "pkg:pypi/cattrs@24.1.2?file_name=cattrs-24.1.2-py3-none-any.whl"
+        },
+        "extra_data": {}
+      }
+    ],
+    "repository_homepage_url": null,
+    "repository_download_url": null,
+    "api_data_url": null,
+    "datasource_id": "pypi_pylock_toml",
+    "purl": null
+  }
+]

tests/packagedcode/data/pypi/pylock/attrs/pylock.toml

@@ -0,0 +1,26 @@
+# Trimmed copy of the PEP 751 example lock file
+# https://peps.python.org/pep-0751/#example
+# used for testing pylock.toml support.
+lock-version = '1.0'
+environments = ["sys_platform == 'win32'", "sys_platform == 'linux'"]
+requires-python = '==3.12'
+created-by = 'mousebender'
+
+[[packages]]
+name = 'attrs'
+version = '25.1.0'
+requires-python = '>=3.8'
+wheels = [
+  {name = 'attrs-25.1.0-py3-none-any.whl', upload-time = 2025-01-25T11:30:10.164985+00:00, url = 'https://files.pythonhosted.org/packages/fc/30/d4986a882011f9df997a55e6becd864812ccfcd821d64aac8570ee39f719/attrs-25.1.0-py3-none-any.whl', size = 63152, hashes = {sha256 = 'c75a69e28a550a7e93789579c22aa26b0f5b83b75dc4e08fe092980051e1090a'}},
+]
+
+[[packages]]
+name = 'cattrs'
+version = '24.1.2'
+requires-python = '>=3.8'
+dependencies = [
+    {name = 'attrs'},
+]
+wheels = [
+  {name = 'cattrs-24.1.2-py3-none-any.whl', upload-time = 2024-09-22T14:58:34.812643+00:00, url = 'https://files.pythonhosted.org/packages/c8/d5/867e75361fc45f6de75fe277dd085627a9db5ebb511a87f27dc1396b5351/cattrs-24.1.2-py3-none-any.whl', size = 66446, hashes = {sha256 = '67c7495b760168d931a10233f979b28dc04daf853b30752246f4f8471c6d68d0'}},
+]