From 2ab988a9a492eb0febef6d8348958aea8b464e7e Mon Sep 17 00:00:00 2001 From: Pratyksh Gupta Date: Tue, 6 Jan 2026 15:29:22 +0530 Subject: [PATCH 1/4] Add CRAN (R package) dependency analysis support Signed-off-by: Pratyksh Gupta --- .../dependency_util/dependency_calculator.py | 2 + .../dependency_util/r_deps.py | 116 ++++++++++++++++++ 2 files changed, 118 insertions(+) create mode 100644 collectoss/tasks/git/dependency_tasks/dependency_util/r_deps.py diff --git a/collectoss/tasks/git/dependency_tasks/dependency_util/dependency_calculator.py b/collectoss/tasks/git/dependency_tasks/dependency_util/dependency_calculator.py index 85aa681ab..9fcc3d90e 100644 --- a/collectoss/tasks/git/dependency_tasks/dependency_util/dependency_calculator.py +++ b/collectoss/tasks/git/dependency_tasks/dependency_util/dependency_calculator.py @@ -10,6 +10,7 @@ from collectoss.tasks.git.dependency_tasks.dependency_util import go_deps from collectoss.tasks.git.dependency_tasks.dependency_util import kotlin_deps from collectoss.tasks.git.dependency_tasks.dependency_util import rust_deps +from collectoss.tasks.git.dependency_tasks.dependency_util import r_deps from collectoss.tasks.git.dependency_tasks.dependency_util import dependency_calculator #Returns generator iterable to tuples of modules and their names @@ -26,6 +27,7 @@ def get_dependency_analysis_module_tuples(): yield go_deps, 'go' yield kotlin_deps, 'kotlin' yield rust_deps, 'rust' + yield r_deps, 'R' class Dep: def __init__(self, name, language, count): diff --git a/collectoss/tasks/git/dependency_tasks/dependency_util/r_deps.py b/collectoss/tasks/git/dependency_tasks/dependency_util/r_deps.py new file mode 100644 index 000000000..d7a126035 --- /dev/null +++ b/collectoss/tasks/git/dependency_tasks/dependency_util/r_deps.py @@ -0,0 +1,116 @@ +import re +import json +from pathlib import Path + + +def get_files(path): + """ + Scans the directory for R dependency files. + We look specifically for 'DESCRIPTION' files and 'renv.lock' files. + """ + dir_path = Path(path) + files = [] + + # We look for both standard package metadata and renv lockfiles + files.extend(list(dir_path.glob('**/DESCRIPTION'))) + files.extend(list(dir_path.glob('**/renv.lock'))) + + return files + + +def get_deps_for_file(path): + """ + Routes the file to the appropriate simple parser based on its name. + """ + path_obj = Path(path) + + if path_obj.name == 'DESCRIPTION': + return get_deps_from_description(path) + elif path_obj.name == 'renv.lock': + return get_deps_from_renv_lock(path) + + return [] + + +def get_deps_from_description(path): + """ + Extracts dependencies from a standard R DESCRIPTION file. + + We scan fields like Imports, Depends, Suggests, and others to find + external package requirements. + """ + try: + with open(path, 'r', encoding='utf-8') as f: + content = f.read() + + dependencies = set() + + # These are the standard sections where R packages list their requirements + dependency_fields = ['Imports', 'Depends', 'Suggests', 'Enhances', 'LinkingTo'] + + for field in dependency_fields: + # We use regex to grab the whole section for a field. + # It needs to handle potentially multi-line values. + pattern = rf'^{field}:\s*(.*?)(?=^\S|\Z)' + match = re.search(pattern, content, re.MULTILINE | re.DOTALL) + + if match: + field_value = match.group(1) + packages = parse_r_package_list(field_value) + dependencies.update(packages) + + # 'R' itself is often listed as a dependency, but we only care about libraries + dependencies.discard('R') + + return list(dependencies) + + except Exception: + # If something goes wrong parsing, we just return nothing for this file + return [] + + +def parse_r_package_list(field_value): + """ + Cleans up the raw text from the file to get a nice list of package names. + """ + packages = set() + + # Flatten the text to a single line to make splitting easier + field_value = ' '.join(field_value.split()) + + parts = field_value.split(',') + + for part in parts: + part = part.strip() + if not part: + continue + + # We need to grab the package name and ignore version numbers like (>= 1.0.0) + match = re.match(r'^([A-Za-z][A-Za-z0-9.]*)', part) + if match: + package_name = match.group(1).rstrip('.') + if package_name: + packages.add(package_name) + + return packages + + +def get_deps_from_renv_lock(path): + """ + Parses an renv.lock file, which is just a JSON file listing specific package versions. + """ + try: + with open(path, 'r', encoding='utf-8') as f: + lockfile = json.load(f) + + dependencies = set() + + # renv.lock keeps everything under a 'Packages' key + if 'Packages' in lockfile and isinstance(lockfile['Packages'], dict): + for package_name in lockfile['Packages'].keys(): + dependencies.add(package_name) + + return list(dependencies) + + except (json.JSONDecodeError, Exception): + return [] From f8c20c7ec91b4d5ec8f69971a6a90fe6b8236963 Mon Sep 17 00:00:00 2001 From: Pratyksh Gupta Date: Tue, 6 Jan 2026 15:33:22 +0530 Subject: [PATCH 2/4] Add tests for R dependency analysis Signed-off-by: Pratyksh Gupta --- .../test_dependency_tasks/test_r_deps.py | 149 ++++++++++++++++++ 1 file changed, 149 insertions(+) create mode 100644 tests/test_tasks/test_dependency_tasks/test_r_deps.py diff --git a/tests/test_tasks/test_dependency_tasks/test_r_deps.py b/tests/test_tasks/test_dependency_tasks/test_r_deps.py new file mode 100644 index 000000000..0e4eacb6e --- /dev/null +++ b/tests/test_tasks/test_dependency_tasks/test_r_deps.py @@ -0,0 +1,149 @@ +#!/usr/bin/env python3 +""" +Simple test script to verify that our R dependency parser is working correctly. +""" + +import os +import sys +import tempfile +import json +from pathlib import Path + +# Add project root to path so we can import augur modules +# This assumes the file is at tests/test_tasks/test_dependency_tasks/test_r_deps.py +project_root = Path(__file__).resolve().parents[3] +if str(project_root) not in sys.path: + sys.path.insert(0, str(project_root)) + +from augur.tasks.git.dependency_tasks.dependency_util import r_deps + + +def test_description_file(): + print("Checking DESCRIPTION file parsing...") + + # A standard looking DESCRIPTION file with various edge cases to test our parser + description_content = """Package: mypackage +Type: Package +Title: My Test Package +Version: 0.1.0 +Author: Test Author +Maintainer: Test Author +Description: A test package. +License: GPL-3 +Encoding: UTF-8 +LazyData: true +Depends: + R (>= 3.5.0), + dplyr (>= 1.0.0) +Imports: + ggplot2, + tidyr (>= 1.1.0), + stringr +Suggests: + testthat (>= 3.0.0), + knitr, + rmarkdown +LinkingTo: + Rcpp (>= 1.0.0) +""" + + with tempfile.TemporaryDirectory() as tmpdir: + temp_file = os.path.join(tmpdir, 'DESCRIPTION') + with open(temp_file, 'w') as f: + f.write(description_content) + + deps = r_deps.get_deps_for_file(temp_file) + + # Verify we found exactly what we expected + expected_deps = {'dplyr', 'ggplot2', 'tidyr', 'stringr', 'testthat', + 'knitr', 'rmarkdown', 'Rcpp'} + missing = expected_deps - set(deps) + extra = set(deps) - expected_deps + + if not missing and not extra: + print("Looks good! All dependencies found.") + return True + else: + print(f"Something's off. Missing: {missing}, Extra: {extra}") + return False + + +def test_renv_lock_file(): + print("\nChecking renv.lock parsing...") + + # A basic renv.lock json structure + renv_lock_content = { + "R": {"Version": "4.3.0", "Repositories": [{"Name": "CRAN", "URL": "https://cran.rstudio.com"}]}, + "Packages": { + "ggplot2": {"Package": "ggplot2", "Version": "3.4.0", "Source": "Repository", "Repository": "CRAN"}, + "dplyr": {"Package": "dplyr", "Version": "1.1.0", "Source": "Repository", "Repository": "CRAN"}, + "tidyr": {"Package": "tidyr", "Version": "1.3.0", "Source": "Repository", "Repository": "CRAN"}, + "rmarkdown": {"Package": "rmarkdown", "Version": "2.20", "Source": "Repository", "Repository": "CRAN"} + } + } + + with tempfile.TemporaryDirectory() as tmpdir: + temp_file = os.path.join(tmpdir, 'renv.lock') + with open(temp_file, 'w') as f: + json.dump(renv_lock_content, f) + + deps = r_deps.get_deps_for_file(temp_file) + + expected_deps = {'ggplot2', 'dplyr', 'tidyr', 'rmarkdown'} + missing = expected_deps - set(deps) + extra = set(deps) - expected_deps + + if not missing and not extra: + print("renv.lock parsed successfully.") + return True + else: + print(f"renv.lock parsing failed. Missing: {missing}, Extra: {extra}") + return False + + +def test_get_files(): + print("\nChecking file discovery...") + + with tempfile.TemporaryDirectory() as tmpdir: + # We set up a fake directory structure with some hidden files to find + (Path(tmpdir) / 'DESCRIPTION').touch() + (Path(tmpdir) / 'renv.lock').touch() + (Path(tmpdir) / 'subdir').mkdir() + (Path(tmpdir) / 'subdir' / 'DESCRIPTION').touch() + (Path(tmpdir) / 'another' / 'nested').mkdir(parents=True) + (Path(tmpdir) / 'another' / 'nested' / 'renv.lock').touch() + + files = r_deps.get_files(tmpdir) + file_names = [f.name for f in files] + + desc_count = file_names.count('DESCRIPTION') + lock_count = file_names.count('renv.lock') + + # We hid 2 DESCRIPTIONs and 2 lockfiles, let's make sure we found them all + if desc_count == 2 and lock_count == 2: + print("Found all the files we hid.") + return True + else: + print(f"File discovery missed something. Found {desc_count} DESCRIPTIONs and {lock_count} lockfiles.") + return False + + +def main(): + print("Starting tests...\n") + + results = [ + test_description_file(), + test_renv_lock_file(), + test_get_files() + ] + + if all(results): + print("\nAll tests passed!") + return 0 + else: + print("\nSome tests failed.") + return 1 + + +if __name__ == '__main__': + sys.exit(main()) From bf8dcb13832533d9296edbca3cccccb7ec7ab827 Mon Sep 17 00:00:00 2001 From: Pratyksh Gupta Date: Tue, 6 Jan 2026 21:10:36 +0530 Subject: [PATCH 3/4] refactor(r-deps): update tests to pytest and address review feedback - Refactor tests to use pytest syntax and fixtures. - Clarify version stripping in r_deps.py. Signed-off-by: Pratyksh Gupta --- .../dependency_util/r_deps.py | 2 + .../test_dependency_tasks/test_r_deps.py | 151 ++++++------------ 2 files changed, 48 insertions(+), 105 deletions(-) diff --git a/collectoss/tasks/git/dependency_tasks/dependency_util/r_deps.py b/collectoss/tasks/git/dependency_tasks/dependency_util/r_deps.py index d7a126035..9ec0cda11 100644 --- a/collectoss/tasks/git/dependency_tasks/dependency_util/r_deps.py +++ b/collectoss/tasks/git/dependency_tasks/dependency_util/r_deps.py @@ -86,6 +86,8 @@ def parse_r_package_list(field_value): continue # We need to grab the package name and ignore version numbers like (>= 1.0.0) + # Note: We are currently stripping version info because the Augur dependency + # model only tracks package names. If that changes, we can capture versions here. match = re.match(r'^([A-Za-z][A-Za-z0-9.]*)', part) if match: package_name = match.group(1).rstrip('.') diff --git a/tests/test_tasks/test_dependency_tasks/test_r_deps.py b/tests/test_tasks/test_dependency_tasks/test_r_deps.py index 0e4eacb6e..3c32b5d1c 100644 --- a/tests/test_tasks/test_dependency_tasks/test_r_deps.py +++ b/tests/test_tasks/test_dependency_tasks/test_r_deps.py @@ -1,27 +1,12 @@ -#!/usr/bin/env python3 -""" -Simple test script to verify that our R dependency parser is working correctly. -""" - -import os -import sys -import tempfile import json -from pathlib import Path - -# Add project root to path so we can import augur modules -# This assumes the file is at tests/test_tasks/test_dependency_tasks/test_r_deps.py -project_root = Path(__file__).resolve().parents[3] -if str(project_root) not in sys.path: - sys.path.insert(0, str(project_root)) - +import pytest from augur.tasks.git.dependency_tasks.dependency_util import r_deps - -def test_description_file(): - print("Checking DESCRIPTION file parsing...") - - # A standard looking DESCRIPTION file with various edge cases to test our parser +def test_description_file_parsing(tmp_path): + """ + Test that standard R DESCRIPTION files are parsed correctly, + extracting dependencies from Depends, Imports, Suggests, and LinkingTo. + """ description_content = """Package: mypackage Type: Package Title: My Test Package @@ -45,33 +30,23 @@ def test_description_file(): rmarkdown LinkingTo: Rcpp (>= 1.0.0) -""" + """ - with tempfile.TemporaryDirectory() as tmpdir: - temp_file = os.path.join(tmpdir, 'DESCRIPTION') - with open(temp_file, 'w') as f: - f.write(description_content) + d_file = tmp_path / "DESCRIPTION" + d_file.write_text(description_content, encoding="utf-8") - deps = r_deps.get_deps_for_file(temp_file) - - # Verify we found exactly what we expected - expected_deps = {'dplyr', 'ggplot2', 'tidyr', 'stringr', 'testthat', - 'knitr', 'rmarkdown', 'Rcpp'} - missing = expected_deps - set(deps) - extra = set(deps) - expected_deps - - if not missing and not extra: - print("Looks good! All dependencies found.") - return True - else: - print(f"Something's off. Missing: {missing}, Extra: {extra}") - return False - - -def test_renv_lock_file(): - print("\nChecking renv.lock parsing...") + deps = r_deps.get_deps_for_file(str(d_file)) - # A basic renv.lock json structure + expected_deps = {'dplyr', 'ggplot2', 'tidyr', 'stringr', 'testthat', + 'knitr', 'rmarkdown', 'Rcpp'} + + # Assert we found exactly the expected dependencies + assert set(deps) == expected_deps + +def test_renv_lock_parsing(tmp_path): + """ + Test that renv.lock JSON files are parsed correctly. + """ renv_lock_content = { "R": {"Version": "4.3.0", "Repositories": [{"Name": "CRAN", "URL": "https://cran.rstudio.com"}]}, "Packages": { @@ -82,68 +57,34 @@ def test_renv_lock_file(): } } - with tempfile.TemporaryDirectory() as tmpdir: - temp_file = os.path.join(tmpdir, 'renv.lock') - with open(temp_file, 'w') as f: - json.dump(renv_lock_content, f) + l_file = tmp_path / "renv.lock" + measure = l_file.write_text(json.dumps(renv_lock_content), encoding="utf-8") - deps = r_deps.get_deps_for_file(temp_file) - - expected_deps = {'ggplot2', 'dplyr', 'tidyr', 'rmarkdown'} - missing = expected_deps - set(deps) - extra = set(deps) - expected_deps - - if not missing and not extra: - print("renv.lock parsed successfully.") - return True - else: - print(f"renv.lock parsing failed. Missing: {missing}, Extra: {extra}") - return False - - -def test_get_files(): - print("\nChecking file discovery...") + deps = r_deps.get_deps_for_file(str(l_file)) - with tempfile.TemporaryDirectory() as tmpdir: - # We set up a fake directory structure with some hidden files to find - (Path(tmpdir) / 'DESCRIPTION').touch() - (Path(tmpdir) / 'renv.lock').touch() - (Path(tmpdir) / 'subdir').mkdir() - (Path(tmpdir) / 'subdir' / 'DESCRIPTION').touch() - (Path(tmpdir) / 'another' / 'nested').mkdir(parents=True) - (Path(tmpdir) / 'another' / 'nested' / 'renv.lock').touch() - - files = r_deps.get_files(tmpdir) - file_names = [f.name for f in files] - - desc_count = file_names.count('DESCRIPTION') - lock_count = file_names.count('renv.lock') - - # We hid 2 DESCRIPTIONs and 2 lockfiles, let's make sure we found them all - if desc_count == 2 and lock_count == 2: - print("Found all the files we hid.") - return True - else: - print(f"File discovery missed something. Found {desc_count} DESCRIPTIONs and {lock_count} lockfiles.") - return False - + expected_deps = {'ggplot2', 'dplyr', 'tidyr', 'rmarkdown'} + + assert set(deps) == expected_deps -def main(): - print("Starting tests...\n") +def test_file_discovery(tmp_path): + """ + Test that the tool finds DESCRIPTION and renv.lock files recursively. + """ + # Create a nested directory structure + (tmp_path / 'DESCRIPTION').touch() + (tmp_path / 'renv.lock').touch() - results = [ - test_description_file(), - test_renv_lock_file(), - test_get_files() - ] + subdir = tmp_path / 'subdir' + subdir.mkdir() + (subdir / 'DESCRIPTION').touch() - if all(results): - print("\nAll tests passed!") - return 0 - else: - print("\nSome tests failed.") - return 1 - - -if __name__ == '__main__': - sys.exit(main()) + nested = tmp_path / 'another' / 'nested' + nested.mkdir(parents=True) + (nested / 'renv.lock').touch() + + # Run the discovery + files = r_deps.get_files(str(tmp_path)) + file_names = [f.name for f in files] + + assert file_names.count('DESCRIPTION') == 2 + assert file_names.count('renv.lock') == 2 From f12d52cffed6db2cdf889aca1490aa9913784899 Mon Sep 17 00:00:00 2001 From: Adrian Edwards Date: Mon, 14 Sep 2026 18:28:51 -0400 Subject: [PATCH 4/4] rename one reference Signed-off-by: Adrian Edwards --- collectoss/tasks/git/dependency_tasks/dependency_util/r_deps.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/collectoss/tasks/git/dependency_tasks/dependency_util/r_deps.py b/collectoss/tasks/git/dependency_tasks/dependency_util/r_deps.py index 9ec0cda11..b1429c7d2 100644 --- a/collectoss/tasks/git/dependency_tasks/dependency_util/r_deps.py +++ b/collectoss/tasks/git/dependency_tasks/dependency_util/r_deps.py @@ -86,7 +86,7 @@ def parse_r_package_list(field_value): continue # We need to grab the package name and ignore version numbers like (>= 1.0.0) - # Note: We are currently stripping version info because the Augur dependency + # Note: We are currently stripping version info because the CollectOSS dependency # model only tracks package names. If that changes, we can capture versions here. match = re.match(r'^([A-Za-z][A-Za-z0-9.]*)', part) if match: