Fix Pipfile parsing crashes with non-string values in libyear dependency metrics

.gitignore

@@ -171,3 +171,8 @@ nohup.out
 # local db volume
 pgdata/
 postgres-data/
+
+# Helper files for PR preparation (do not commit)
+PR_DESCRIPTION.md
+PR_SUBMISSION_CHECKLIST.md
+COMMIT_MESSAGE.txt

collectoss/tasks/git/dependency_libyear_tasks/libyear_util/pypi_libyear_util.py

@@ -17,6 +17,19 @@ def get_pypi_data(name, version=None):
 
 
 def clean_version(version):
+    """
+    Clean version string by keeping only digits and dots.
+
+    Args:
+        version: Version string or value to clean
+
+    Returns:
+        str: Cleaned version string with only digits and dots
+    """
+    # Defensive check: ensure version is a string
+    if not isinstance(version, str):
+        return ''
+
     version = [v for v in version if v.isdigit() or v == '.']
     return ''.join(version)
 
@@ -47,7 +60,12 @@ def get_version(pypi_data, version, lt=False):
 
 
 def handle_upper_limit_dependency(dependency, data):
-    versions = dependency['requirement'].split(',')
+    # Defensive check: ensure requirement is a string before calling string methods
+    requirement = dependency.get('requirement')
+    if not isinstance(requirement, str):
+        return None
+
+    versions = requirement.split(',')
     upper_limit = clean_version(versions[0])
     release_list = list(data['releases'])
 
@@ -88,18 +106,27 @@ def get_release_date(data, version,logger):
 
 
 def sort_dependency_requirement(dependency,data):
-    if dependency['requirement'] == '' or dependency['requirement'] is None or dependency['requirement'] == '*':
+    # Defensive check: ensure requirement exists and is a string
+    requirement = dependency.get('requirement')
+
+    # Handle None, empty string, wildcard, or non-string types
+    if requirement is None or requirement == '' or requirement == '*':
+        return None
+
+    if not isinstance(requirement, str):
+        # Requirement is not a string (e.g., dict, list) - cannot process
         return None
 
-    elif re.search(r'<', dependency['requirement']):
+    # Now safe to use string operations on requirement
+    if re.search(r'<', requirement):
         return handle_upper_limit_dependency(dependency, data)
 
-    elif re.search(r'>=', dependency['requirement']):
+    elif re.search(r'>=', requirement):
         return None
 
     else:
-        # return get_version(data, clean_version(dependency['requirement']))
-        return clean_version(dependency['requirement'])
+        # return get_version(data, clean_version(requirement))
+        return clean_version(requirement)
 
 
 def get_libyear(current_version, current_release_date, latest_version, latest_release_date):

collectoss/tasks/git/dependency_libyear_tasks/libyear_util/pypi_parser.py

@@ -76,24 +76,125 @@ def parse_requirement_txt(file_handle):
     return deps
 
 
-def map_dependencies(info):
-    if type(info) is dict:
+def normalize_pipfile_version(dep_value):
+    """
+    Normalize Pipfile dependency values to a version string suitable for version comparison.
+
+    Handles multiple Pipfile dependency specification formats:
+    - Simple string: ">=2.0" or "*"
+    - Dict with version: {"version": ">=2.0", "markers": "..."}
+    - Dict with git ref: {"git": "https://...", "ref": "main"}
+    - Dict with only extras/markers: {"extras": ["security"]}
+    - Dict with path: {"path": "./local-package"}
+    - Dict with editable: {"editable": true, "path": "..."}
+
+    Args:
+        dep_value: The dependency value from Pipfile, can be str, dict, or other types
+
+    Returns:
+        str or None: 
+            - Version string if parseable (e.g., ">=2.0", "*")
+            - Git reference string if git dependency (e.g., "https://...#ref")
+            - None for unsupported formats (path, editable, or malformed entries)
+
+    Examples:
+        >>> normalize_pipfile_version(">=2.0")
+        ">=2.0"
+        >>> normalize_pipfile_version({"version": ">=2.0"})
+        ">=2.0"
+        >>> normalize_pipfile_version({"git": "https://github.com/...", "ref": "main"})
+        "https://github.com/...#main"
+        >>> normalize_pipfile_version({"extras": ["security"]})
+        None
+    """
+    # Case 1: Already a string (e.g., ">=2.0", "*", "==1.2.3")
+    if isinstance(dep_value, str):
+        return dep_value
+
+    # Case 2: Dictionary format - check for various keys
+    if isinstance(dep_value, dict):
+        # Case 2a: Explicit version key (most common dict format)
+        # Example: {"version": ">=2.0", "markers": "python_version >= '3.6'"}
+        if "version" in dep_value:
+            version = dep_value['version']
+            # Ensure the version value itself is a string
+            if isinstance(version, str):
+                return version
+            else:
+                logging.warning(f"Pipfile dependency has non-string version value: {dep_value}")
+                return None
+
+        # Case 2b: Git dependency
+        # Example: {"git": "https://github.com/user/repo.git", "ref": "main"}
+        elif 'git' in dep_value:
+            try:
+                git_url = dep_value['git']
+                ref = dep_value.get('ref', 'HEAD')  # Default to HEAD if no ref specified
+                return f"{git_url}#{ref}"
+            except (KeyError, TypeError) as e:
+                logging.warning(f"Malformed git dependency in Pipfile: {dep_value}, error: {e}")
+                return None
+
+        # Case 2c: Path dependencies (local packages) - not supported for version comparison
+        # Example: {"path": "./local-package"} or {"editable": true, "path": "..."}
+        elif 'path' in dep_value or 'editable' in dep_value:
+            logging.debug(f"Skipping path/editable dependency (not suitable for version tracking): {dep_value}")
+            return None
+
+        # Case 2d: Dict with only extras/markers but no version
+        # Example: {"extras": ["security"], "markers": "..."}
+        else:
+            logging.warning(f"Pipfile dependency dict has no version, git, or path key: {dep_value}")
+            return None
+
+    # Case 3: Unexpected type (not string or dict)
+    # This shouldn't happen with valid Pipfile format, but handle defensively
+    else:
+        logging.warning(f"Unexpected dependency value type in Pipfile: {type(dep_value).__name__}, value: {dep_value}")
+        return None
 
-        if "version" in info:
-            return info['version']
-        elif 'git' in info:
-            return info['git']+'#'+info['ref']
-    else:            
-        return info
+
+def map_dependencies(info):
+    """
+    Legacy wrapper for normalize_pipfile_version for backward compatibility.
+
+    Note: This function is kept for compatibility but delegates to normalize_pipfile_version.
+    Consider using normalize_pipfile_version directly in new code.
+    """
+    return normalize_pipfile_version(info)
 
 
 def map_dependencies_pipfile(packages, type):
+    """
+    Map Pipfile package dependencies to a standardized format.
+
+    Filters out dependencies that cannot be normalized (e.g., path dependencies,
+    editable installs, or malformed entries) and logs them for debugging.
+
+    Args:
+        packages: Dictionary of package names to their dependency specifications
+        type: Dependency type ('runtime' or 'develop')
+
+    Returns:
+        list: List of dependency dictionaries with normalized requirements
+    """
     deps = list()
     if not packages:
         return []
+
     for name, info in packages.items():
-        Dict = {'name': name, 'requirement': map_dependencies(info), 'type': type, 'package': 'PYPI'}
+        # Normalize the dependency value (handles strings, dicts, etc.)
+        requirement = map_dependencies(info)
+
+        # Skip dependencies that couldn't be normalized (path deps, invalid formats, etc.)
+        if requirement is None:
+            logging.debug(f"Skipping dependency '{name}' (type: {type}) - unsupported format: {info}")
+            continue
+
+        # Only add valid dependencies with parseable requirements
+        Dict = {'name': name, 'requirement': requirement, 'type': type, 'package': 'PYPI'}
         deps.append(Dict)
+
     return deps 
 
 def parse_pipfile(file_handle):

collectoss/tasks/git/dependency_libyear_tasks/libyear_util/util.py

@@ -6,12 +6,13 @@
 from collectoss.tasks.git.dependency_libyear_tasks.libyear_util.npm_libyear_utils import get_NPM_data, get_npm_release_date, get_npm_latest_version,get_npm_current_version
 
 #Files That would be parsed should be added here
+# Note: Pipfile.lock is listed before Pipfile to ensure locked versions are preferred
 file_list = [
     'Requirement.txt',
     'requirements.txt',
     'setup.py',
-    'Pipfile',
-    'Pipfile.lock',
+    'Pipfile.lock',  # Process lock file first (more reliable)
+    'Pipfile',       # Fall back to Pipfile if no lock file
     'pyproject.toml',
     'poetry.lock',
     'environment.yml',
@@ -30,6 +31,9 @@ def find(name, path):
 def get_parsed_deps(path, logger):
     import traceback
     dependency_list = []
+
+    # Track which Pipfile variant was processed to avoid duplicates
+    pipfile_processed = False
 
     for f in file_list:
         deps_file = find(f, path)
@@ -49,11 +53,26 @@ def get_parsed_deps(path, logger):
                     if short_file_name in ['Requirement.txt', 'requirements.txt']:
                         dependency_list.extend(parse_requirement_txt(file_handle))
 
-                    elif short_file_name == 'Pipfile':
-                        dependency_list.extend(parse_pipfile(file_handle))
-
                     elif short_file_name == 'Pipfile.lock':
+                        # Prefer Pipfile.lock over Pipfile (more reliable, locked versions)
+                        logger.info("Using Pipfile.lock (preferred over Pipfile for reliability)")
                         dependency_list.extend(parse_pipfile_lock(file_handle))
+                        pipfile_processed = True
+
+                    elif short_file_name == 'Pipfile':
+                        # Only parse Pipfile if Pipfile.lock hasn't been processed
+                        if pipfile_processed:
+                            logger.info("Skipping Pipfile (already processed Pipfile.lock)")
+                            continue
+
+                        # Check if Pipfile.lock exists in the same directory
+                        pipfile_lock_path = os.path.join(os.path.dirname(deps_file), 'Pipfile.lock')
+                        if os.path.exists(pipfile_lock_path):
+                            logger.info("Skipping Pipfile (Pipfile.lock exists and will be processed)")
+                            continue
+
+                        logger.info("Using Pipfile (no Pipfile.lock found)")
+                        dependency_list.extend(parse_pipfile(file_handle))
 
                     elif short_file_name == 'pyproject.toml':
                         try: