diff --git a/CHANGELOG.rst b/CHANGELOG.rst index d9a5a6b402..2bf47dea60 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -4,6 +4,12 @@ Changelog Next release -------------- +- Add support for PEP 751 ``pylock.toml`` lockfiles. The new package + data handler parses ``pylock.toml`` and ``pylock..toml`` files, + reporting the locked packages together with their resolved download + URLs, hashes and dependency relationships. + https://github.com/aboutcode-org/scancode-toolkit/issues/4638 + - Fix the optional ``licenses`` extra dependency typo to install ``licensedcode-data``. https://github.com/aboutcode-org/scancode-toolkit/pull/5056 diff --git a/docs/source/reference/scancode-supported-packages.rst b/docs/source/reference/scancode-supported-packages.rst index d2ad15803f..50fb18ed77 100644 --- a/docs/source/reference/scancode-supported-packages.rst +++ b/docs/source/reference/scancode-supported-packages.rst @@ -812,6 +812,14 @@ parsers in scancode-toolkit during documentation builds. - ``pypi_poetry_pyproject_toml`` - Python - https://packaging.python.org/en/latest/specifications/pyproject-toml/ + * - Python pylock.toml lockfile + - ``*pylock.toml`` + ``*pylock.*.toml`` + - ``pypi`` + - ``linux``, ``win``, ``mac`` + - ``pypi_pylock_toml`` + - Python + - https://peps.python.org/pep-0751/ * - Python pyproject.toml - ``*pyproject.toml`` - ``pypi`` diff --git a/src/packagedcode/__init__.py b/src/packagedcode/__init__.py index fc1e490eef..6d2b9799a2 100644 --- a/src/packagedcode/__init__.py +++ b/src/packagedcode/__init__.py @@ -176,6 +176,7 @@ pypi.PoetryLockHandler, pypi.UvPyprojectTomlHandler, pypi.UvLockHandler, + pypi.PylockTomlHandler, pypi.PythonEditableInstallationPkgInfoFile, pypi.PythonEggPkgInfoFile, pypi.PythonInstalledWheelMetadataFile, diff --git a/src/packagedcode/pypi.py b/src/packagedcode/pypi.py index dcfd237590..751e54dabb 100644 --- a/src/packagedcode/pypi.py +++ b/src/packagedcode/pypi.py @@ -1126,6 +1126,146 @@ def parse(cls, location, package_only=False): yield models.PackageData.from_data(package_data, package_only) +def is_pylock_toml(filename): + """ + Return True if the ``filename`` is a valid PEP 751 lock file name, e.g. + ``pylock.toml`` or ``pylock.dev.toml``. + See https://peps.python.org/pep-0751/#file-name + """ + return filename == 'pylock.toml' or bool(re.match(r'^pylock\.[^.]+\.toml$', filename)) + + +class PylockTomlHandler(models.DatafileHandler): + datasource_id = 'pypi_pylock_toml' + path_patterns = ('*pylock.toml', '*pylock.*.toml') + default_package_type = 'pypi' + default_primary_language = 'Python' + description = 'Python pylock.toml lockfile' + documentation_url = 'https://peps.python.org/pep-0751/' + + @classmethod + def is_datafile(cls, location, filetypes=tuple()): + return ( + super().is_datafile(location, filetypes=filetypes) + and is_pylock_toml(fileutils.file_name(location)) + ) + + @classmethod + def parse(cls, location, package_only=False): + with open(location, "rb") as fp: + toml_data = tomllib.load(fp) + + packages = toml_data.get('packages') + if not packages: + return + + dependencies = [] + for package in packages: + name = package.get('name') + if not name: + continue + version = package.get('version') + + dependencies_for_resolved = [] + for dep in (package.get('dependencies') or []): + dep_name = dep.get('name') + if not dep_name: + continue + dep_purl = PackageURL(type=cls.default_package_type, name=dep_name) + dependencies_for_resolved.append( + models.DependentPackage( + purl=dep_purl.to_string(), + extracted_requirement=dep.get('marker') or dep.get('version'), + scope='dependencies', + is_runtime=True, + is_optional=False, + is_direct=True, + is_pinned=False, + ).to_dict() + ) + + # a package can carry a single sdist and/or several wheels (one + # per platform/interpreter); prefer the sdist as the canonical + # source and fall back to the first wheel, since the resolved + # package purl can only point to one download. + source = package.get('sdist') + if not isinstance(source, dict): + wheels = package.get('wheels') or [] + source = wheels[0] if wheels else None + + download_url = None + sha256 = None + file_name = None + if isinstance(source, dict): + download_url = source.get('url') + hashes = source.get('hashes') or {} + sha256 = hashes.get('sha256') + file_name = source.get('name') or ( + download_url and posixpath.basename(download_url) + ) + + urls = get_pypi_urls(name, version) + if download_url: + urls['repository_download_url'] = download_url + + qualifiers = {} + if file_name: + qualifiers['file_name'] = file_name + + extra_data = {} + marker = package.get('marker') + if marker: + extra_data['marker'] = marker + + resolved_package_data = dict( + datasource_id=cls.datasource_id, + type=cls.default_package_type, + primary_language='Python', + name=name, + version=version, + qualifiers=qualifiers, + sha256=sha256, + is_virtual=True, + extra_data=extra_data, + dependencies=dependencies_for_resolved, + **urls, + ) + resolved_package = models.PackageData.from_data(resolved_package_data, package_only) + + dependencies.append( + models.DependentPackage( + purl=resolved_package.purl, + extracted_requirement=None, + scope=None, + is_runtime=True, + is_optional=False, + is_direct=False, + is_pinned=True, + resolved_package=resolved_package.to_dict(), + ).to_dict() + ) + + extra_data = {} + requires_python = toml_data.get('requires-python') + if requires_python: + extra_data['python_requires'] = requires_python + lock_version = toml_data.get('lock-version') + if lock_version is not None: + extra_data['lock_version'] = lock_version + created_by = toml_data.get('created-by') + if created_by: + extra_data['created_by'] = created_by + + package_data = dict( + datasource_id=cls.datasource_id, + type=cls.default_package_type, + primary_language='Python', + extra_data=extra_data, + dependencies=dependencies, + ) + yield models.PackageData.from_data(package_data, package_only) + + class PipInspectDeplockHandler(models.DatafileHandler): datasource_id = 'pypi_inspect_deplock' path_patterns = ('*pip-inspect.deplock',) diff --git a/tests/packagedcode/data/pypi/pylock/attrs-pylock.toml-expected.json b/tests/packagedcode/data/pypi/pylock/attrs-pylock.toml-expected.json new file mode 100644 index 0000000000..958ad33405 --- /dev/null +++ b/tests/packagedcode/data/pypi/pylock/attrs-pylock.toml-expected.json @@ -0,0 +1,175 @@ +[ + { + "type": "pypi", + "namespace": null, + "name": null, + "version": null, + "qualifiers": {}, + "subpath": null, + "primary_language": "Python", + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": null, + "download_url": null, + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "holder": null, + "declared_license_expression": null, + "declared_license_expression_spdx": null, + "license_detections": [], + "other_license_expression": null, + "other_license_expression_spdx": null, + "other_license_detections": [], + "extracted_license_statement": null, + "notice_text": null, + "source_packages": [], + "file_references": [], + "is_private": false, + "is_virtual": false, + "extra_data": { + "python_requires": "==3.12", + "lock_version": "1.0", + "created_by": "mousebender" + }, + "dependencies": [ + { + "purl": "pkg:pypi/attrs@25.1.0?file_name=attrs-25.1.0-py3-none-any.whl", + "extracted_requirement": null, + "scope": null, + "is_runtime": true, + "is_optional": false, + "is_pinned": true, + "is_direct": false, + "resolved_package": { + "type": "pypi", + "namespace": null, + "name": "attrs", + "version": "25.1.0", + "qualifiers": { + "file_name": "attrs-25.1.0-py3-none-any.whl" + }, + "subpath": null, + "primary_language": "Python", + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": null, + "download_url": null, + "size": null, + "sha1": null, + "md5": null, + "sha256": "c75a69e28a550a7e93789579c22aa26b0f5b83b75dc4e08fe092980051e1090a", + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "holder": null, + "declared_license_expression": null, + "declared_license_expression_spdx": null, + "license_detections": [], + "other_license_expression": null, + "other_license_expression_spdx": null, + "other_license_detections": [], + "extracted_license_statement": null, + "notice_text": null, + "source_packages": [], + "file_references": [], + "is_private": false, + "is_virtual": true, + "extra_data": {}, + "dependencies": [], + "repository_homepage_url": "https://pypi.org/project/attrs", + "repository_download_url": "https://files.pythonhosted.org/packages/fc/30/d4986a882011f9df997a55e6becd864812ccfcd821d64aac8570ee39f719/attrs-25.1.0-py3-none-any.whl", + "api_data_url": "https://pypi.org/pypi/attrs/25.1.0/json", + "datasource_id": "pypi_pylock_toml", + "purl": "pkg:pypi/attrs@25.1.0?file_name=attrs-25.1.0-py3-none-any.whl" + }, + "extra_data": {} + }, + { + "purl": "pkg:pypi/cattrs@24.1.2?file_name=cattrs-24.1.2-py3-none-any.whl", + "extracted_requirement": null, + "scope": null, + "is_runtime": true, + "is_optional": false, + "is_pinned": true, + "is_direct": false, + "resolved_package": { + "type": "pypi", + "namespace": null, + "name": "cattrs", + "version": "24.1.2", + "qualifiers": { + "file_name": "cattrs-24.1.2-py3-none-any.whl" + }, + "subpath": null, + "primary_language": "Python", + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": null, + "download_url": null, + "size": null, + "sha1": null, + "md5": null, + "sha256": "67c7495b760168d931a10233f979b28dc04daf853b30752246f4f8471c6d68d0", + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "holder": null, + "declared_license_expression": null, + "declared_license_expression_spdx": null, + "license_detections": [], + "other_license_expression": null, + "other_license_expression_spdx": null, + "other_license_detections": [], + "extracted_license_statement": null, + "notice_text": null, + "source_packages": [], + "file_references": [], + "is_private": false, + "is_virtual": true, + "extra_data": {}, + "dependencies": [ + { + "purl": "pkg:pypi/attrs", + "extracted_requirement": null, + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + } + ], + "repository_homepage_url": "https://pypi.org/project/cattrs", + "repository_download_url": "https://files.pythonhosted.org/packages/c8/d5/867e75361fc45f6de75fe277dd085627a9db5ebb511a87f27dc1396b5351/cattrs-24.1.2-py3-none-any.whl", + "api_data_url": "https://pypi.org/pypi/cattrs/24.1.2/json", + "datasource_id": "pypi_pylock_toml", + "purl": "pkg:pypi/cattrs@24.1.2?file_name=cattrs-24.1.2-py3-none-any.whl" + }, + "extra_data": {} + } + ], + "repository_homepage_url": null, + "repository_download_url": null, + "api_data_url": null, + "datasource_id": "pypi_pylock_toml", + "purl": null + } +] \ No newline at end of file diff --git a/tests/packagedcode/data/pypi/pylock/attrs/pylock.toml b/tests/packagedcode/data/pypi/pylock/attrs/pylock.toml new file mode 100644 index 0000000000..64ee3b02b6 --- /dev/null +++ b/tests/packagedcode/data/pypi/pylock/attrs/pylock.toml @@ -0,0 +1,26 @@ +# Trimmed copy of the PEP 751 example lock file +# https://peps.python.org/pep-0751/#example +# used for testing pylock.toml support. +lock-version = '1.0' +environments = ["sys_platform == 'win32'", "sys_platform == 'linux'"] +requires-python = '==3.12' +created-by = 'mousebender' + +[[packages]] +name = 'attrs' +version = '25.1.0' +requires-python = '>=3.8' +wheels = [ + {name = 'attrs-25.1.0-py3-none-any.whl', upload-time = 2025-01-25T11:30:10.164985+00:00, url = 'https://files.pythonhosted.org/packages/fc/30/d4986a882011f9df997a55e6becd864812ccfcd821d64aac8570ee39f719/attrs-25.1.0-py3-none-any.whl', size = 63152, hashes = {sha256 = 'c75a69e28a550a7e93789579c22aa26b0f5b83b75dc4e08fe092980051e1090a'}}, +] + +[[packages]] +name = 'cattrs' +version = '24.1.2' +requires-python = '>=3.8' +dependencies = [ + {name = 'attrs'}, +] +wheels = [ + {name = 'cattrs-24.1.2-py3-none-any.whl', upload-time = 2024-09-22T14:58:34.812643+00:00, url = 'https://files.pythonhosted.org/packages/c8/d5/867e75361fc45f6de75fe277dd085627a9db5ebb511a87f27dc1396b5351/cattrs-24.1.2-py3-none-any.whl', size = 66446, hashes = {sha256 = '67c7495b760168d931a10233f979b28dc04daf853b30752246f4f8471c6d68d0'}}, +] diff --git a/tests/packagedcode/data/pypi/pylock/sdist-pylock.dev.toml-expected.json b/tests/packagedcode/data/pypi/pylock/sdist-pylock.dev.toml-expected.json new file mode 100644 index 0000000000..441601e6ce --- /dev/null +++ b/tests/packagedcode/data/pypi/pylock/sdist-pylock.dev.toml-expected.json @@ -0,0 +1,163 @@ +[ + { + "type": "pypi", + "namespace": null, + "name": null, + "version": null, + "qualifiers": {}, + "subpath": null, + "primary_language": "Python", + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": null, + "download_url": null, + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "holder": null, + "declared_license_expression": null, + "declared_license_expression_spdx": null, + "license_detections": [], + "other_license_expression": null, + "other_license_expression_spdx": null, + "other_license_detections": [], + "extracted_license_statement": null, + "notice_text": null, + "source_packages": [], + "file_references": [], + "is_private": false, + "is_virtual": false, + "extra_data": { + "python_requires": ">=3.10", + "lock_version": "1.0", + "created_by": "pip" + }, + "dependencies": [ + { + "purl": "pkg:pypi/demo-sdist-pkg@1.2.3?file_name=demo-sdist-pkg-1.2.3.tar.gz", + "extracted_requirement": null, + "scope": null, + "is_runtime": true, + "is_optional": false, + "is_pinned": true, + "is_direct": false, + "resolved_package": { + "type": "pypi", + "namespace": null, + "name": "demo-sdist-pkg", + "version": "1.2.3", + "qualifiers": { + "file_name": "demo-sdist-pkg-1.2.3.tar.gz" + }, + "subpath": null, + "primary_language": "Python", + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": null, + "download_url": null, + "size": null, + "sha1": null, + "md5": null, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "holder": null, + "declared_license_expression": null, + "declared_license_expression_spdx": null, + "license_detections": [], + "other_license_expression": null, + "other_license_expression_spdx": null, + "other_license_detections": [], + "extracted_license_statement": null, + "notice_text": null, + "source_packages": [], + "file_references": [], + "is_private": false, + "is_virtual": true, + "extra_data": {}, + "dependencies": [], + "repository_homepage_url": "https://pypi.org/project/demo-sdist-pkg", + "repository_download_url": "https://files.example.com/demo-sdist-pkg-1.2.3.tar.gz", + "api_data_url": "https://pypi.org/pypi/demo-sdist-pkg/1.2.3/json", + "datasource_id": "pypi_pylock_toml", + "purl": "pkg:pypi/demo-sdist-pkg@1.2.3?file_name=demo-sdist-pkg-1.2.3.tar.gz" + }, + "extra_data": {} + }, + { + "purl": "pkg:pypi/demo-vcs-pkg", + "extracted_requirement": null, + "scope": null, + "is_runtime": true, + "is_optional": false, + "is_pinned": true, + "is_direct": false, + "resolved_package": { + "type": "pypi", + "namespace": null, + "name": "demo-vcs-pkg", + "version": null, + "qualifiers": {}, + "subpath": null, + "primary_language": "Python", + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": null, + "download_url": null, + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "holder": null, + "declared_license_expression": null, + "declared_license_expression_spdx": null, + "license_detections": [], + "other_license_expression": null, + "other_license_expression_spdx": null, + "other_license_detections": [], + "extracted_license_statement": null, + "notice_text": null, + "source_packages": [], + "file_references": [], + "is_private": false, + "is_virtual": true, + "extra_data": { + "marker": "sys_platform == 'linux'" + }, + "dependencies": [], + "repository_homepage_url": "https://pypi.org/project/demo-vcs-pkg", + "repository_download_url": null, + "api_data_url": "https://pypi.org/pypi/demo-vcs-pkg/json", + "datasource_id": "pypi_pylock_toml", + "purl": "pkg:pypi/demo-vcs-pkg" + }, + "extra_data": {} + } + ], + "repository_homepage_url": null, + "repository_download_url": null, + "api_data_url": null, + "datasource_id": "pypi_pylock_toml", + "purl": null + } +] \ No newline at end of file diff --git a/tests/packagedcode/data/pypi/pylock/sdist/pylock.dev.toml b/tests/packagedcode/data/pypi/pylock/sdist/pylock.dev.toml new file mode 100644 index 0000000000..78272818fd --- /dev/null +++ b/tests/packagedcode/data/pypi/pylock/sdist/pylock.dev.toml @@ -0,0 +1,21 @@ +# Synthetic pylock.toml test fixture (not derived from a real project) used to +# exercise the sdist and VCS-sourced package fields defined by PEP 751: +# https://peps.python.org/pep-0751/ +# This also exercises the ``pylock..toml`` naming variant. +lock-version = '1.0' +requires-python = '>=3.10' +created-by = 'pip' + +[[packages]] +name = 'demo-sdist-pkg' +version = '1.2.3' +sdist = {name = 'demo-sdist-pkg-1.2.3.tar.gz', url = 'https://files.example.com/demo-sdist-pkg-1.2.3.tar.gz', size = 1234, hashes = {sha256 = 'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855'}} + +[[packages]] +name = 'demo-vcs-pkg' +marker = "sys_platform == 'linux'" + +[packages.vcs] +type = 'git' +url = 'https://example.com/demo/demo-vcs-pkg.git' +commit-id = '1234567890abcdef1234567890abcdef12345678' diff --git a/tests/packagedcode/test_pypi.py b/tests/packagedcode/test_pypi.py index 20afae813d..c440ca9881 100644 --- a/tests/packagedcode/test_pypi.py +++ b/tests/packagedcode/test_pypi.py @@ -458,6 +458,47 @@ def test_package_scan_uv_end_to_end(self): check_json_scan(expected_file, result_file, remove_uuid=True, regen=REGEN_TEST_FIXTURES) +class TestPylockHandler(PackageTester): + # The attrs/cattrs fixture is trimmed from the PEP 751 example lock file: + # https://peps.python.org/pep-0751/#example + + test_data_dir = os.path.join(os.path.dirname(__file__), 'data') + + def test_is_pylock_toml(self): + test_file = self.get_test_loc('pypi/pylock/attrs/pylock.toml') + assert pypi.PylockTomlHandler.is_datafile(test_file) + + def test_is_pylock_toml_named_variant(self): + test_file = self.get_test_loc('pypi/pylock/sdist/pylock.dev.toml') + assert pypi.PylockTomlHandler.is_datafile(test_file) + + def test_is_pylock_toml_rejects_unrelated_toml(self): + test_file = self.get_test_loc('pypi/uv/attrs/pyproject.toml') + assert not pypi.PylockTomlHandler.is_datafile(test_file) + + def test_parse_pylock_toml_attrs(self): + test_file = self.get_test_loc('pypi/pylock/attrs/pylock.toml') + package = pypi.PylockTomlHandler.parse(test_file) + expected_loc = self.get_test_loc( + 'pypi/pylock/attrs-pylock.toml-expected.json', + must_exist=False, + ) + self.check_packages_data( + package, expected_loc, must_exist=False, regen=REGEN_TEST_FIXTURES, + ) + + def test_parse_pylock_toml_sdist_and_vcs(self): + test_file = self.get_test_loc('pypi/pylock/sdist/pylock.dev.toml') + package = pypi.PylockTomlHandler.parse(test_file) + expected_loc = self.get_test_loc( + 'pypi/pylock/sdist-pylock.dev.toml-expected.json', + must_exist=False, + ) + self.check_packages_data( + package, expected_loc, must_exist=False, regen=REGEN_TEST_FIXTURES, + ) + + class TestPipInspectDeplockHandler(PackageTester): test_data_dir = os.path.join(os.path.dirname(__file__), 'data')