Add override mechanism for algorithm containers, transfer explicit .sif images with HTCondor

Snakefile

@@ -262,6 +262,16 @@ def collect_prepared_input(wildcards):
 
     return prepared_inputs
 
+# Look up a per-algorithm container image override from config for HTCondor transfer.
+# When a local .sif path is configured, it will be included in htcondor_transfer_input_files
+# so the HTCondor executor transfers it to the EP alongside the job.
+def get_algorithm_image(wildcards):
+    images = container_settings.images
+    override = images.get(wildcards.algorithm, "")
+    if override.endswith('.sif'):
+        return override
+    return None
+
 # Run the pathway reconstruction algorithm
 rule reconstruct:
     input: collect_prepared_input
@@ -271,6 +281,8 @@ rule reconstruct:
     # same name regardless of the inputs or parameters, and these aren't renamed until after the container command
     # terminates
     output: pathway_file = SEP.join([out_dir, '{dataset}-{algorithm}-{params}', 'raw-pathway.txt'])
+    resources:
+        htcondor_transfer_input_files=get_algorithm_image
     run:
         # Create a copy so that the updates are not written to the parameters logfile
         params = reconstruction_params(wildcards.algorithm, wildcards.params).copy()

config/config.yaml

@@ -50,6 +50,30 @@ containers:
   #       requirements = versionGE(split(Target.CondorVersion)[1], "24.8.0") && (isenforcingdiskusage =!= true)
   enable_profiling: false
 
+  # Override the default container image for specific algorithms.
+  # Keys are algorithm names (as they appear in the algorithms list below).
+  # Values are interpreted based on the container framework:
+  #
+  # Image reference (e.g., "pathlinker:v3"):
+  #   Prepends the registry prefix. Works with both Docker and Apptainer.
+  #
+  # Full image reference with registry (e.g., "ghcr.io/myorg/pathlinker:v3"):
+  #   Used as-is (prefix NOT prepended). Works with both Docker and Apptainer.
+  #
+  # Local .sif file path (e.g., "images/pathlinker_v2.sif"):
+  #   Apptainer/Singularity only. Skips pulling from registry and uses the
+  #   pre-built .sif directly. When running via HTCondor with shared-fs-usage: none (set
+  #   via the spras_profile config when running SPRAS against HTCondor), .sif paths listed
+  #   here are automatically included in htcondor_transfer_input_files.
+  #   Ignored with a warning if the framework is Docker.
+  #
+  # Example (one of each type):
+  # images:
+  #   omicsintegrator1: "images/omics-integrator-1_v2.sif"        # local .sif (Apptainer only)
+  #   pathlinker: "pathlinker:v1234"                              # image name only (base_url/owner prepended)
+  #   omicsintegrator2: "some-other-owner/oi2:latest"             # owner/image (base_url prepended)
+  #   mincostflow: "ghcr.io/reed-compbio/mincostflow:v2"          # full registry reference (used as-is)
+
 # This list of algorithms should be generated by a script which checks the filesystem for installs.
 # It shouldn't be changed by mere mortals. (alternatively, we could add a path to executable for each algorithm
 # in the list to reduce the number of assumptions of the program at the cost of making the config a little more involved)

docs/contributing/index.rst

@@ -285,14 +285,24 @@ U``.
  Step 4: Make the Local Neighborhood wrapper accessible through SPRAS
 **********************************************************************
 
-Import the new class ``LocalNeighborhood`` in ``spras/runner.py`` and
-add it to the ``algorithms`` dictionary so the wrapper functions can be
-accessed. Add an entry for Local Neighborhood to the configuration file
-``config/config.yaml`` and set ``include: true``. As a convention,
-algorithm names are written in all lowercase without special characters.
-Local Neighborhood has no other parameters. Optionally set ``include:
-false`` for the other pathway reconstruction algorithms to make testing
-faster.
+Register the new algorithm by adding a single entry to the
+``ALGORITHM_REGISTRY`` dictionary in ``spras/config/util.py``. The entry
+maps the algorithm's lowercase name to a ``(module_path, class_name)``
+tuple that ``runner.py`` will load via ``importlib`` at startup:
+
+.. code:: python
+
+   ALGORITHM_REGISTRY: dict[str, tuple[str, str]] = {
+       ...
+       "localneighborhood": ("spras.local_neighborhood", "LocalNeighborhood"),
+   }
+
+As a convention, algorithm names are written in all lowercase without
+special characters. The same name is used as the key in the config file
+``config/config.yaml``. Add an entry for Local Neighborhood there and
+set ``include: true``. Local Neighborhood has no other parameters.
+Optionally set ``include: false`` for the other pathway reconstruction
+algorithms to make testing faster.
 
 The config file has an option ``owner`` under the
 ``containers.registry`` settings that controls which Docker Hub account
@@ -448,8 +458,10 @@ of the main SPRAS repository.
    ``required_input`` files and the ``generate_inputs``, ``run``, and
    ``parse_output`` functions
 
-#. Import the new class in ``spras/runner.py`` and add it to the
-   ``algorithms`` dictionary so the wrapper functions can be accessed
+#. Register the new algorithm in ``ALGORITHM_REGISTRY`` in
+   ``spras/config/util.py`` by adding one entry ``"<name>":
+   ("spras.<module>", "<ClassName>")``; no changes to
+   ``spras/runner.py`` are needed
 
 #. Document the usage of the Docker wrapper and the assumptions made
    when implementing the wrapper

spras/config/config.py

@@ -25,6 +25,7 @@
 from spras.config.container_schema import ProcessedContainerSettings
 from spras.config.revision import attach_spras_revision, spras_revision
 from spras.config.schema import DatasetSchema, RawConfig
+from spras.config.util import AlgorithmName, get_valid_algorithm_names
 from spras.util import LoosePathLike, NpHashEncoder, hash_params_sha1_base32
 
 config = None
@@ -61,6 +62,21 @@ def __init__(self, raw_config: dict[str, Any]):
         self.hash_length = parsed_raw_config.hash_length
         # Container settings used by PRMs.
         self.container_settings = ProcessedContainerSettings.from_container_settings(parsed_raw_config.containers, self.hash_length)
+
+        # Validate that all keys in containers.images are recognized algorithm names
+        if self.container_settings.images:
+            valid_names = get_valid_algorithm_names()
+            for key in self.container_settings.images:
+                try:
+                    AlgorithmName(key)
+                except ValueError as err:
+                    raise ValueError(
+                        f"Unknown algorithm name '{key}' in configured containers.images. Is there a typo? "
+                        f"Valid algorithm names are: {sorted(valid_names)}"
+                    ) from err
+
+        # The list of algorithms to run in the workflow. Each is a dict with 'name' as an expected key.
+        self.algorithms = None
         # A nested dict mapping algorithm names to dicts that map parameter hashes to parameter combinations.
         # Only includes algorithms that are set to be run with 'include: true'.
         self.algorithm_params: dict[str, dict[str, Any]] = dict()

spras/config/container_schema.py

@@ -7,7 +7,8 @@
 """
 
 import warnings
-from dataclasses import dataclass
+from dataclasses import dataclass, field
+from typing import Optional
 
 from pydantic import BaseModel, ConfigDict
 
@@ -21,6 +22,11 @@ class ContainerFramework(CaseInsensitiveEnum):
     apptainer = 'apptainer'
     dsub = 'dsub'
 
+    @property
+    def is_singularity_family(self) -> bool:
+        """True for both 'singularity' and 'apptainer', which are treated as synonyms."""
+        return self in (ContainerFramework.singularity, ContainerFramework.apptainer)
+
 class ContainerRegistry(BaseModel):
     base_url: str = "docker.io"
     "The domain of the registry"
@@ -34,17 +40,20 @@ class ContainerSettings(BaseModel):
     framework: ContainerFramework = ContainerFramework.docker
     unpack_singularity: bool = False
 
-    model_config = ConfigDict(extra='forbid')
     enable_profiling: bool = False
     "A Boolean indicating whether to enable container runtime profiling (apptainer/singularity only)"
     registry: ContainerRegistry
 
+    images: dict[str, str] = {}
+    "Per-algorithm container image overrides. Keys are algorithm names; values are image references or local .sif file paths."
+
     model_config = ConfigDict(extra='forbid', use_attribute_docstrings=True)
 
 @dataclass
 class ProcessedContainerSettings:
     framework: ContainerFramework = ContainerFramework.docker
     unpack_singularity: bool = False
+    base_url: str = "docker.io"
     prefix: str = DEFAULT_CONTAINER_PREFIX
     enable_profiling: bool = False
     hash_length: int = 7
@@ -57,6 +66,10 @@ class ProcessedContainerSettings:
     We prefer this `hash_length` in our container-running logic to
     avoid a (future) dependency diamond.
     """
+    images: dict[str, str] = field(default_factory=dict)
+    """Per-algorithm container image overrides from config."""
+    image_override: Optional[str] = None
+    """Resolved image override for the current algorithm. Set at runtime by runner.run()."""
 
     @staticmethod
     def from_container_settings(settings: ContainerSettings, hash_length: int) -> "ProcessedContainerSettings":
@@ -65,18 +78,23 @@ def from_container_settings(settings: ContainerSettings, hash_length: int) -> "P
         container_framework = settings.framework
 
         # Unpack settings for running in singularity mode. Needed when running PRM containers if already in a container.
-        if settings.unpack_singularity and container_framework != "singularity":
-            warnings.warn("unpack_singularity is set to True, but the container framework is not singularity. This setting will have no effect.", stacklevel=2)
+        if settings.unpack_singularity and not container_framework.is_singularity_family:
+            warnings.warn("unpack_singularity is set to True, but the container framework is not singularity or apptainer. This setting will have no effect.", stacklevel=2)
         unpack_singularity = settings.unpack_singularity
 
         # Grab registry from the config, and if none is provided default to docker
+        container_base_url = "docker.io"
         container_prefix = DEFAULT_CONTAINER_PREFIX
+        if settings.registry and settings.registry.base_url != "":
+            container_base_url = settings.registry.base_url
         if settings.registry and settings.registry.base_url != "" and settings.registry.owner != "":
             container_prefix = settings.registry.base_url + "/" + settings.registry.owner
 
         return ProcessedContainerSettings(
             framework=container_framework,
             unpack_singularity=unpack_singularity,
+            base_url=container_base_url,
             prefix=container_prefix,
-            hash_length=hash_length
+            hash_length=hash_length,
+            images=dict(settings.images),
         )

spras/config/util.py

@@ -48,6 +48,33 @@ def _missing_(cls, value: Any):
     yaml.representer.SafeRepresenter.represent_str,
 )
 
+# The single source of truth for all supported algorithms.
+# Keys are algorithm names; values are (module_path, class_name) tuples for importlib loading.
+# To register a new algorithm, add an entry here.
+ALGORITHM_REGISTRY: dict[str, tuple[str, str]] = {
+    "allpairs":        ("spras.allpairs", "AllPairs"),
+    "bowtiebuilder":   ("spras.btb", "BowTieBuilder"),
+    "diamond":         ("spras.diamond", "DIAMOnD"),
+    "domino":          ("spras.domino", "DOMINO"),
+    "meo":             ("spras.meo", "MEO"),
+    "mincostflow":     ("spras.mincostflow", "MinCostFlow"),
+    "omicsintegrator1": ("spras.omicsintegrator1", "OmicsIntegrator1"),
+    "omicsintegrator2": ("spras.omicsintegrator2", "OmicsIntegrator2"),
+    "pathlinker":      ("spras.pathlinker", "PathLinker"),
+    "responsenet":     ("spras.responsenet", "ResponseNet"),
+    "rwr":             ("spras.rwr", "RWR"),
+    "strwr":           ("spras.strwr", "ST_RWR"),
+}
+
+# Auto-generated enum from the registry keys. Inherits CaseInsensitiveEnum so
+# AlgorithmName("PathLinker") resolves to AlgorithmName.pathlinker.
+AlgorithmName = CaseInsensitiveEnum("AlgorithmName", {k: k for k in ALGORITHM_REGISTRY})
+
+def get_valid_algorithm_names() -> set[str]:
+    """Return the set of valid algorithm name strings (lowercase)."""
+    return {member.value for member in AlgorithmName}
+
+
 class Empty(BaseModel):
     """
     The empty base model. Used for specifying that an algorithm takes no parameters,