diff --git a/benchmarks/test_benchmark_chain_100.py b/benchmarks/test_benchmark_chain_100.py index 223a1e3..b11b8cd 100644 --- a/benchmarks/test_benchmark_chain_100.py +++ b/benchmarks/test_benchmark_chain_100.py @@ -13,46 +13,48 @@ class ChainLink(CoSyLuigiTask, ABC): - """_summary_.""" + """An abstract class representing a chain link in an infinite chain.""" chain_link: CoSyLuigiTaskParameter | None class StartingLink(ChainLink): - """_summary_.""" + """A class that terminates the chain by needing no further chain links.""" chain_link = None class RepeatingLink(ChainLink): - """_summary_.""" + """A class that recurses the chain by requiring a further chain link.""" chain_link = CoSyLuigiTaskParameter(ChainLink) def output(self): - """_summary_. + """Assign each chain link a unique identifier. This is required because CoSy-Luigi considers tasks with + identical names, identical requirements, and identical outputs to be Singletons. This differs from Luigi, + which considers tasks with identical names and identical requirements Singletons. Returns: - _type_: _description_ + Mapping[str, MockTarget]_: The named unique target for each chain link. """ return {"counter": MockTarget(str(next(counter)))} @pytest.fixture def repo(): - """_summary_. + """Creates a CoSyLuigiRepo that contains the StartingLink and the RepeatingLink. Returns: - _type_: _description_ + CoSyLuigiRepo: The created CoSyLuigiRepo. """ return CoSyLuigiRepo(ChainLink) def create_infinite_chain(repo): - """_summary_. + """Synthesizes all pipelines up to those that carry out the same step 100 times. Args: - repo (_type_): _description_ + repo (CoSyLuigiRepo): The repository to use for synthesis. """ maestro = Maestro( repo.cls_repo, @@ -62,11 +64,11 @@ def create_infinite_chain(repo): def test_benchmark_chain_creation(repo, benchmark): - """_summary_. + """Benchmarks how long synthesizing and enumerating the pipelines takes. Args: - repo (_type_): _description_ - benchmark (_type_): _description_ + repo (CoSyLuigiRepo): The repository to use for synthesis. + benchmark (BenchmarkFixture): The benchmark fixture. """ benchmark(create_infinite_chain, repo) diff --git a/src/cosy_luigi/constraints/__init__.py b/src/cosy_luigi/constraints/__init__.py index 09e46e5..d649ab8 100644 --- a/src/cosy_luigi/constraints/__init__.py +++ b/src/cosy_luigi/constraints/__init__.py @@ -1,4 +1,4 @@ -"""_summary_.""" +"""This module contains common constraints that may be needed when modeling pipelines.""" from cosy_luigi.constraints.unique import is_unique_in_prior_tasks diff --git a/src/cosy_luigi/constraints/unique.py b/src/cosy_luigi/constraints/unique.py index 17e5b7d..aa84d82 100644 --- a/src/cosy_luigi/constraints/unique.py +++ b/src/cosy_luigi/constraints/unique.py @@ -1,4 +1,5 @@ -"""_summary_.""" +"""Contains the function for the constraint that makes a required task be unique throughout all prior tasks in a +pipeline.""" from __future__ import annotations @@ -16,14 +17,20 @@ def _is_unique_in_prior_tasks( vs: Mapping[str, CoSyLuigiTask], required_to_be_unique: Sequence[type[CoSyLuigiTask]] ) -> bool: - """_summary_. + """Examines the output of traverse_pipeline against a Sequence of CoSyLuigiTask's types that are intended to be + unique. Whenever a task that is a subclass or the required to be unique class itself is encountered in the + pipeline, remember the encountered task. If any further task that is a subclass that is not identical to the + previously encountered task is encountered, return False, else return True. + + Encountering two different subclasses within the same pipeline for given required to be unique tasks means that + it is not unique. Args: - vs (Mapping[str, CoSyLuigiTask]): _description_ - required_to_be_unique (Sequence[type[CoSyLuigiTask]]): _description_ + vs (Mapping[str, CoSyLuigiTask]): The variables passed to the function by CoSy during synthesis. Populated by the partial pipelines beginning at current tasks required tasks. + required_to_be_unique (Sequence[type[CoSyLuigiTask]]): The CoSyLuigiTasks' types that are intended to be unique. Returns: - bool: _description_ + bool: True if all types contained in required_to_be_unique are unique, False otherwise. """ classes = [pc.__class__ for pc in traverse_pipeline(vs.values())] seen_subclasses: dict[type[CoSyLuigiTask], type[CoSyLuigiTask]] = {} @@ -41,14 +48,15 @@ def _is_unique_in_prior_tasks( def is_unique_in_prior_tasks( vs: Mapping[str, CoSyLuigiTask], required_to_be_unique: type[CoSyLuigiTask] | Sequence[type[CoSyLuigiTask]] ) -> bool: - """_summary_. + """Wrapper around _is_unique_in_prior_tasks that allows passing either a single type of a CoSyLuigiTask or a + Sequence of CoSyLuigiTasks' types. Args: - vs (Mapping[str, CoSyLuigiTask]): _description_ - required_to_be_unique (type[CoSyLuigiTask] | Sequence[type[CoSyLuigiTask]]): _description_ + vs (Mapping[str, CoSyLuigiTask]): The variables passed to the function by CoSy during synthesis. Populated by the partial pipelines beginning at current tasks required tasks. + required_to_be_unique (type[CoSyLuigiTask] | Sequence[type[CoSyLuigiTask]]): The CoSyLuigiTask's type or types that are intended to be unique. Returns: - bool: _description_ + bool: True if all types contained in required_to_be_unique are unique, False otherwise. """ return _is_unique_in_prior_tasks( vs, diff --git a/src/cosy_luigi/core/combinatorics.py b/src/cosy_luigi/core/combinatorics.py index 7087714..eda8d68 100644 --- a/src/cosy_luigi/core/combinatorics.py +++ b/src/cosy_luigi/core/combinatorics.py @@ -1,4 +1,4 @@ -"""_summary_.""" +"""Contains the classes that provide the core functionality of modeling pipelines with CoSy-Luigi.""" from __future__ import annotations @@ -21,19 +21,24 @@ class CoSyLuigiTaskParameter(luigi.TaskParameter): - """_summary_. + """Serves as CoSy-specific version of luigi.TaskParameter. It has two primary uses: Providing a unique classname + for the reflection-based operations of the CoSyLuigiTask to examine, and wrapping the classes that a + CoSyLuigiTask requires to be inhabited. For instance, for a class B, which declares a CoSyLuigiTaskParameter that + wraps the Class A, the resulting CoSy combinator type would be A -> B. + Attributes: - required_task (type[CoSyLuigiTask]): _description_ - unique_across_prior_tasks (bool): _description_ + required_task (type[CoSyLuigiTask]): The type of CoSyLuigiTask this parameter wraps. + unique_across_prior_tasks (bool): Whether or not to enforce that all concrete occurrences of the potentially abstract wrapped type need to be the same. """ def __init__(self, required_task: type[CoSyLuigiTask], *, unique_across_prior_tasks: bool = False): - """_summary_. + """Initializes the CoSyLuigiTaskParameter. Setting unique_across_prior_tasks to True only makes sense if the + required_task is abstract. Args: - required_task (type[CoSyLuigiTask]): _description_ - unique_across_prior_tasks (bool): _description_ (Default value = False) + required_task (type[CoSyLuigiTask]): The type of CoSyLuigiTask this parameter wraps. + unique_across_prior_tasks (bool): Whether or not to enforce that all concrete occurrences of the potentially abstract wrapped type need to be the same. """ super().__init__() self.required_task = required_task @@ -41,17 +46,22 @@ def __init__(self, required_task: type[CoSyLuigiTask], *, unique_across_prior_ta class CoSyLuigiTask(luigi.Task): - """_summary_.""" + """Serves as a CoSy-specific version of luigi.Task. Types derived from CoSyLuigiTask can be added to a + CoSyLuigiRepo and will automatically produce typed combinators for synthesis. The main purpose of CoSyLuigiTask + is to enforce some conventions and utilize reflections to model variance through Python object inheritance. A + task that requires another task of a given type T will also find all subclasses of T as valid inputs.""" def __init__(self, *args, **kwargs): - """_summary_. + """Initializes the CoSyLuigiTask. This only happens during and after synthesis, when the combinators are + interpreted. Most functionality is instead implemented on class-level to benefit from caching, + since constraints/predicates during synthesis can cause large numbers of instances to be created. Args: - *args (_type_): _description_ - **kwargs (_type_): _description_ + *args (_type_): Passed to super().__init__(). + **kwargs (_type_): Passed to super().__init__(). Raises: - TypeError: _description_ + TypeError: The output method of a CoSyLuigiTask must return a Mapping. This is done to prevent addressing output files by index, which leads to unreadable code. """ super().__init__(*args, **kwargs) output = self.output() @@ -68,20 +78,23 @@ def __init__(self, *args, **kwargs): @classmethod @cache def get_all_variants(cls) -> set[type[CoSyLuigiTask] | Any]: - """_summary_. + """Recursively finds all subclasses of current class. Returns: - set[type[CoSyLuigiTask] | Any]: _description_ + set[type[CoSyLuigiTask] | Any]: The set of all subclasses. """ return set(cls.__subclasses__()).union([s for c in cls.__subclasses__() for s in c.get_all_variants()]) @classmethod @cache def get_all_class_attributes(cls) -> dict[str, Any]: - """_summary_. + """Collects all class attributes that will be present at runtime. Python does not store class attributes from + parent classes in the dict of the subclasses. This method traverses the __mro__, the ordered list of + superclasses to consider for looking for methods, up to the first class that no longer relates to CoSy and + copies all their attributes down to the current class. Returns: - dict[str, Any]: _description_ + dict[str, Any]: A dict containing all attributes that will be present at runtime. """ attrs: dict[str, Any] = {} for c in [cc for cc in reversed(cls.__mro__) if issubclass(cc, CoSyLuigiTask)]: @@ -89,22 +102,23 @@ def get_all_class_attributes(cls) -> dict[str, Any]: return attrs def get_all_instance_attributes(self) -> dict[str, Any]: - """_summary_. + """Collects all attributes present on an instance at runtime. This method specifically accounts for the fact + that Luigi switches out LuigiTaskParameters for the true passed objects at instantiation. Returns: - dict[str, Any]: _description_ + dict[str, Any]: A dict containing all attributes that are present at runtime. """ return {attr: getattr(self, attr) for attr in dir(self)} def requires(self) -> dict[str, CoSyLuigiTask]: - """Returns a list of other tasks required to run this task. - - This is done by retrieving all user-created attributes that are subclasses of CosyLuigiTaskParameter. - - Note that at Runtime Luigi unpacks CosyLuigiTaskParameters, so the actual check has to be for CoSyLuigiTasks. + """Returns a dict of other tasks required to run this task. This is done by retrieving all user-created + attributes that are subclasses of CosyLuigiTaskParameter. Note that at Runtime Luigi unpacks + CosyLuigiTaskParameters, so the actual check has to be for CoSyLuigiTasks. This overrides the requires method + of luigi.Task, and thus ensures that the Luigi scheduler when executing a task "sees" the requirements + allocated during synthesis. Returns: - dict[str, CoSyLuigiTask]: A list of other tasks required to run this task + dict[str, CoSyLuigiTask]: A dict of other tasks required to run this task, keyed by attribute name. """ return { k: v @@ -115,10 +129,13 @@ def requires(self) -> dict[str, CoSyLuigiTask]: @classmethod @cache def _requirements(cls) -> Mapping[str, CoSyLuigiTaskParameter]: - """_summary_. + """Filters all class attributes present at runtime to only contain CoSyLuigiTaskParameters. Its primary use + is collecting the potentially abstract requirements so that a type for the combinator can be generated. Each + entry in the dict returned by this method results in one non-rightmost entry in the arrow type of the + resulting combinator. Returns: - Mapping[str, CoSyLuigiTaskParameter]: _description_ + Mapping[str, CoSyLuigiTaskParameter]: The CoSyLuigiTaskParameters present on the class, keyed by attribute name. """ return { k: v @@ -129,20 +146,21 @@ def _requirements(cls) -> Mapping[str, CoSyLuigiTaskParameter]: @classmethod @cache def get_params(cls) -> list[tuple[str, CoSyLuigiTaskParameter]]: - """_summary_. + """Converts the output of _requirements into a list of tuples instead of a dict. Returns: - list[tuple[str, CoSyLuigiTaskParameter]]: _description_ + list[tuple[str, CoSyLuigiTaskParameter]]: A list of tuples representing the output of _requirements. """ return list(cls._requirements().items()) @classmethod @cache def requirements_unique_in_prior_tasks(cls) -> Mapping[str, CoSyLuigiTaskParameter]: - """_summary_. + """Filters the output of _requirements, returning only those dict entries where the CoSyLuigiTaskParameter + has the optional unique_across_prior_tasks flag set. Returns: - Mapping[str, CoSyLuigiTaskParameter]: _description_ + Mapping[str, CoSyLuigiTaskParameter]: The filtered output of _requirements. """ return { k: task_parameter @@ -153,38 +171,44 @@ def requirements_unique_in_prior_tasks(cls) -> Mapping[str, CoSyLuigiTaskParamet @classmethod @cache def unique_required_tasks_in_prior(cls) -> Sequence[type[CoSyLuigiTask]]: - """_summary_. + """Transforms the output of requirements_unique_in_prior_tasks into a list of classes that the collected + CoSyLuigiTaskParameter's indicate should be unique across prior tasks. Returns: - Sequence[type[CoSyLuigiTask]]: _description_ + Sequence[type[CoSyLuigiTask]]: The transformed output of requirements_unique_in_prior_tasks. """ return [task_parameter.required_task for task_parameter in cls.requirements_unique_in_prior_tasks().values()] @classmethod @cache def target(cls) -> Constructor: - """_summary_. + """The target constructed by this class. This is the right-most entry of the resulting arrow-type constructed + for a given CoSyLuigiTask. Returns: - Constructor: _description_ + Constructor: A Constructor, uniquely identified by the class name. """ return Constructor(cls.__name__) @classmethod def constraints(cls) -> Sequence[Callable[..., bool]]: - """_summary_. + """The Callables returned by this class are translated into constraints applied to the resulting combinator's + types. This method is intended to be overridden in subclasses to make use of this feature. The returned + Callables are directly passed to a SpecificationBuilder as a .constraint() call. Returns: - Sequence[Callable[..., bool]]: _description_ + Sequence[Callable[..., bool]]: A sequence of constraints. """ return [] @classmethod def __constraints(cls) -> Sequence[Callable[..., bool]]: - """_summary_. + """This method computes the auto-generated constraints that results from features that are part of the + framework itself. For instance, the unique_across_prior_tasks flag is implemented by adding a constraint, + this method creates the corresponding Callables. Returns: - Sequence[Callable[..., bool]]: _description_ + Sequence[Callable[..., bool]]: The auto-generated constraints. """ from cosy_luigi.constraints.unique import _is_unique_in_prior_tasks # noqa: PLC0415 @@ -194,10 +218,10 @@ def __constraints(cls) -> Sequence[Callable[..., bool]]: @classmethod def combinator_type(cls) -> Specification: - """_summary_. + """Computes the resulting type of the combinator represented by this class, as described in the methods referenced by this method. Returns: - Specification: _description_ + Specification: The type of the combinator. """ sp = SpecificationBuilder() for name in [v.required_task.__name__ for v in cls._requirements().values()]: @@ -210,10 +234,13 @@ def combinator_type(cls) -> Specification: @classmethod def combinator(cls) -> tuple[str, Callable[..., CoSyLuigiTask], Specification]: - """_summary_. + """Produces the typed combinator representing this class. Classes with no requirements are instantiated as + is, while classes with requirements have the resulting values for their task-parameters passed as varargs. + Note that it is not necessary to use kwargs here, as the generation of the type guarantees that the order of + passed args and CoSyLuigiTaskParameters aligns. Returns: - tuple[str, Callable[..., CoSyLuigiTask], Specification]: _description_ + tuple[str, Callable[..., CoSyLuigiTask], Specification]: The resulting combinator. """ if len(cls._requirements()) == 0: return cls.__name__, lambda: cls(), cls.combinator_type() @@ -221,12 +248,15 @@ def combinator(cls) -> tuple[str, Callable[..., CoSyLuigiTask], Specification]: class CoSyLuigiRepo: - """_summary_. + """Serves as the repository that Combinatory Logic Synthesis requires for type inhabitation. Unlike the previous + version of CoSy-Luigi, known as CLS-Luigi, having an explicit repository prevents side-effects that may occur by + collecting all CoSyLuigiTasks via reflection. Another important task of the CoSyLuigiRepo is to translate the + observed class hierarchy into a taxonomy that the CoSy framework understands during synthesis. Attributes: - luigi_repo (set[type[CoSyLuigiTask]]): _description_ - taxonomy (Mapping[str, set[str]]): _description_ - cls_repo (list[tuple[str, Callable, Specification]]): _description_ + luigi_repo (set[type[CoSyLuigiTask]]): The set of CoSyLuigiTask types that constitute the repositories' combinators. + taxonomy (Mapping[str, set[str]]): The taxonomy that describes the class hierarchy of the luigi_repo. + cls_repo (list[tuple[str, Callable, Specification]]): The final repository that can be passed to the CoSy framework. """ def __init__(self, *tasks: type[CoSyLuigiTask] | Sequence[type[CoSyLuigiTask]]): @@ -234,10 +264,16 @@ def __init__(self, *tasks: type[CoSyLuigiTask] | Sequence[type[CoSyLuigiTask]]): # This doesn't technically need to unpack as flatten could be typed to accept packed tuples # But performance is equivalent/faster because the first layer doesn't need to be checked this way - """_summary_. + """Initializes the CoSyLuigiRepo. The passed arbitrarily nested Sequence is flattened and converted into a + set. Please see the documentation of flatten, as it adds some features to the flattening. The taxonomy is + then computed by examining the method resolution order of each CoSyLuigiTask type, up to the most abstract + possible CoSyLuigiTask itself. + + Also performs rudimentary inspection of the repositories' contents, currently only checks if any set + unique_in_prior_tasks flags make logical sense. Args: - *tasks (type[CoSyLuigiTask] | Sequence[type[CoSyLuigiTask]]): _description_ + *tasks (type[CoSyLuigiTask] | Sequence[type[CoSyLuigiTask]]): An arbitrarily nested Sequence where leaves are CoSyLuigiTasks' types. """ from cosy_luigi.utils import flatten # noqa: PLC0415 @@ -253,11 +289,14 @@ def __init__(self, *tasks: type[CoSyLuigiTask] | Sequence[type[CoSyLuigiTask]]): self.taxonomy[task.__name__].add(tpe.__name__) def check_unique_in_prior_tasks_sanity(self): - """_summary_.""" + """Checks if the unique_in_prior_tasks flags set on CoSyLuigiTaskParameters make logical sense. If a required + task is set to be unique throughout pipelines, but there are no subclasses of it present, i.e. no variance is + possible, remind the user that this is nonsensical. + """ for source_task, param_name, required_type in [ - (task, k, required_unique_task.required_task) + (task, _, required_unique_task.required_task) for task in self.luigi_repo - for k, required_unique_task in task.requirements_unique_in_prior_tasks().items() + for _, required_unique_task in task.requirements_unique_in_prior_tasks().items() if not any( issubclass(task, required_unique_task.required_task) and task is not required_unique_task.required_task for task in self.luigi_repo diff --git a/src/cosy_luigi/utils/traversals.py b/src/cosy_luigi/utils/traversals.py index fbac194..6e279ff 100644 --- a/src/cosy_luigi/utils/traversals.py +++ b/src/cosy_luigi/utils/traversals.py @@ -1,4 +1,4 @@ -"""_summary_.""" +"""Contains helper methods centered around traversing collections or pipelines in ways specific to CoSy-Luigi.""" from __future__ import annotations @@ -16,13 +16,16 @@ def flatten( *heterogeneous_task_collection: type[CoSyLuigiTask] | Iterable[type[CoSyLuigiTask]], ) -> Iterable[type[CoSyLuigiTask]]: - """_summary_. + """Takes an arbitrarily nested Sequence where the leaves of the nested structure are CoSyLuigiTasks' types and + flattens it. During flattening, if an abstract Task type is encountered, it is instead expanded into the set of + its implementing subclasses and becomes part of the flattening procedure, i.e. also multiple levels of + abstraction are correctly handled. See the corresponding test_abstract_variant_expansion.py for an example. Args: - *heterogeneous_task_collection (type[CoSyLuigiTask] | Sequence[type[CoSyLuigiTask]]): _description_ + *heterogeneous_task_collection (type[CoSyLuigiTask] | Sequence[type[CoSyLuigiTask]]): An arbitrarily nested Sequence where leaves are CoSyLuigiTasks' types. Returns: - Iterable[type[CoSyLuigiTask]]: _description_ + Iterable[type[CoSyLuigiTask]]: The flattened representation of *heterogeneous_task_collection. """ return ( task # type: ignore # guaranteed by recursion to be a type[CoSyLuigiTask] instead of an Iterable itself @@ -39,30 +42,31 @@ def flatten( def _traverse_pipeline(vs: Sequence[CoSyLuigiTask] | Iterable[CoSyLuigiTask]) -> Sequence[CoSyLuigiTask]: - """_summary_. + """Recursively traverses a pipeline's structure and collect all encountered tasks. Args: - vs (Sequence[CoSyLuigiTask] | Iterable[CoSyLuigiTask]): _description_ + vs (Sequence[CoSyLuigiTask] | Iterable[CoSyLuigiTask]): The previously encountered tasks. Returns: - Sequence[CoSyLuigiTask]: _description_ + Sequence[CoSyLuigiTask]: The encountered tasks. """ result: list[CoSyLuigiTask] = [*vs] for v in vs: - result.extend(traverse_pipeline(v.requires().values())) + result.extend(_traverse_pipeline(v.requires().values())) return result def traverse_pipeline( to_traverse: CoSyLuigiTask | Sequence[CoSyLuigiTask] | Iterable[CoSyLuigiTask], ) -> Sequence[CoSyLuigiTask]: - """_summary_. + """Recursively traverses a pipeline's structure and collect all encountered tasks. Wraps _traverse_pipeline to + allow direct root task of a Pipeline to be passed. Args: - to_traverse (CoSyLuigiTask | Sequence[CoSyLuigiTask] | Iterable[CoSyLuigiTask]): _description_ + to_traverse (CoSyLuigiTask | Sequence[CoSyLuigiTask] | Iterable[CoSyLuigiTask]): The pipeline to be traversed. Returns: - Sequence[CoSyLuigiTask]: _description_ + Sequence[CoSyLuigiTask]: All tasks contained in the pipeline. """ return ( _traverse_pipeline([to_traverse]) if isinstance(to_traverse, CoSyLuigiTask) else _traverse_pipeline(to_traverse) diff --git a/tests/test_abstract_variant_expansion.py b/tests/test_abstract_variant_expansion.py index aca59a8..5d07d2b 100644 --- a/tests/test_abstract_variant_expansion.py +++ b/tests/test_abstract_variant_expansion.py @@ -1,4 +1,8 @@ -"""_summary_.""" +"""Tests if flatten correctly expands abstract classes to their concrete implementing classes for repository +construction. Note that this tests for inheriting from ABC and having abstract methods. This is due to checking for +abstractness in python checks for the presence of abstract methods, but within the modeling context of CoSy-Luigi, +classes can just be identifiers that groud their subclasses, so marking a class abstract by just inheriting from ABC +is a valid use-case.""" from abc import ABC, abstractmethod @@ -6,81 +10,83 @@ class ABCInheritedTaskWithNoInheritors(CoSyLuigiTask, ABC): - """_summary_.""" + """An abstract class that no other class inherits from. This class is abstract because it inherits from ABC.""" class ABCInheritedTask(CoSyLuigiTask, ABC): - """_summary_.""" + """An abstract class. This class is abstract because it inherits from ABC.""" class ConcreteTaskFromABCInherited(ABCInheritedTask): - """_summary_.""" + """A class that implements ABCInheritedTask.""" class DeeperConcreteTaskFromABCInherited(ConcreteTaskFromABCInherited): - """_summary_.""" + """A class that indirectly implements ABCInheritedTask by inheriting from ConcreteTaskFromABCInherited.""" # noinspection PyAbstractClass class AbstractTask(CoSyLuigiTask): - """_summary_.""" + """An abstract class. This class is abstract because it has an abstract method.""" @abstractmethod def get_class_name(self): - """_summary_. + """Gets the class name. The presence of this method makes the class abstract. Raises: - NotImplementedError: _description_ + NotImplementedError: This method should be implemented by subclasses. """ raise NotImplementedError # noinspection PyAbstractClass class AbstractTaskWithNoInheritors(CoSyLuigiTask): - """_summary_.""" + """An abstract class that no other class inherits from. This class is abstract because it has an abstract method.""" @abstractmethod def get_class_name(self): - """_summary_. + """Gets the class name. The presence of this method makes the class abstract. Raises: - NotImplementedError: _description_ + NotImplementedError: This method should be implemented by subclasses. """ raise NotImplementedError class ConcreteTaskFromAbstract(AbstractTask): - """_summary_.""" + """A class that implements AbstractTask.""" def get_class_name(self): - """_summary_. + """Overrides get_class_name to concretize the task. Returns: - _type_: _description_ + str: The task's class name. """ return "ConcreteTaskFromAbstract" class DeeperConcreteTaskFromAbstract(ConcreteTaskFromAbstract): - """_summary_.""" + """A class that implements AbstractTask by inheriting from ConcreteTaskFromAbstract.""" def get_class_name(self): - """_summary_. + """Overrides get_class_name to return the correct class name. Returns: - _type_: _description_ + str: The task's class name. """ return "DeeperConcreteTaskFromAbstract" def test_expansion_from_abc(): - """_summary_.""" + """Tests if adding a class that is abstract because it inherits from ABC expands to all of its subclasses when + added to a CoSyLuigiRepo.""" repo = CoSyLuigiRepo(ABCInheritedTask) assert repo.luigi_repo == {ConcreteTaskFromABCInherited, DeeperConcreteTaskFromABCInherited} def test_expansion_from_abstract(): - """_summary_.""" + """Tests if adding a class that is abstract because it has abstract methods expands to all of its subclasses when + added to a CoSyLuigiRepo.""" repo = CoSyLuigiRepo(AbstractTask) assert ConcreteTaskFromAbstract().get_class_name() == "ConcreteTaskFromAbstract" assert DeeperConcreteTaskFromAbstract().get_class_name() == "DeeperConcreteTaskFromAbstract" @@ -88,7 +94,8 @@ def test_expansion_from_abstract(): def test_expansion_from_abc_and_abstract(): - """_summary_.""" + """Tests if adding classes that are abstract because of different reasons expand to all of their subclasses when + added to CoSyLuigiRepo.""" repo = CoSyLuigiRepo(ABCInheritedTask, AbstractTask) assert repo.luigi_repo == { ConcreteTaskFromABCInherited, @@ -99,24 +106,26 @@ def test_expansion_from_abc_and_abstract(): def test_implementation_of_abstract_does_not_expand(): - """_summary_.""" + """Test if a tasks that concretizes an abstract class does not expand.""" repo = CoSyLuigiRepo(ConcreteTaskFromAbstract) assert repo.luigi_repo == {ConcreteTaskFromAbstract} def test_implementation_of_abc_does_not_expand(): - """_summary_.""" + """Test if a tasks that it is concrete because it does not directly inherit from ABC does not expand.""" repo = CoSyLuigiRepo(ConcreteTaskFromABCInherited) assert repo.luigi_repo == {ConcreteTaskFromABCInherited} def test_expansion_to_nothing_from_abc_with_no_inheritors(): - """_summary_.""" + """Tests if an abstract task that is abstract because it inherits from ABC but has no classes that inherit from + it expands to an empty set.""" repo = CoSyLuigiRepo(ABCInheritedTaskWithNoInheritors) assert repo.luigi_repo == set() def test_expansion_to_nothing_from_abstract_with_no_inheritors(): - """_summary_.""" + """Tests if an abstract task that is abstract because it has abstract methods but has no classes that inherit + from it expands to an empty set.""" repo = CoSyLuigiRepo(AbstractTaskWithNoInheritors) assert repo.luigi_repo == set() diff --git a/tests/test_heterogeneous_repo_input.py b/tests/test_heterogeneous_repo_input.py index 25508ff..6818b45 100644 --- a/tests/test_heterogeneous_repo_input.py +++ b/tests/test_heterogeneous_repo_input.py @@ -1,61 +1,64 @@ -"""_summary_.""" +"""Tests if flatten correctly flattens arbitrarily nested sequences.""" from cosy_luigi import CoSyLuigiRepo, CoSyLuigiTask from cosy_luigi.utils import flatten class TaskA(CoSyLuigiTask): - """_summary_.""" + """Placeholder Task A.""" class TaskB(CoSyLuigiTask): - """_summary_.""" + """Placeholder Task B.""" class TaskC(CoSyLuigiTask): - """_summary_.""" + """Placeholder Task C.""" class TaskD(CoSyLuigiTask): - """_summary_.""" + """Placeholder Task D.""" class TaskE(CoSyLuigiTask): - """_summary_.""" + """Placeholder Task E.""" class TaskF(CoSyLuigiTask): - """_summary_.""" + """Placeholder Task F.""" class TaskG(CoSyLuigiTask): - """_summary_.""" + """Placeholder Task G.""" class TaskH(CoSyLuigiTask): - """_summary_.""" + """Placeholder Task H.""" class TaskI(CoSyLuigiTask): - """_summary_.""" + """Placeholder Task I.""" class TaskJ(CoSyLuigiTask): - """_summary_.""" + """ "Placeholder Task J.""" class TaskK(CoSyLuigiTask): - """_summary_.""" + """Placeholder Task K.""" def test_heterogeneous_repo_input(): - """_summary_.""" + """Test if instantiating a CoSyLuigiRepo for a 2-times nested mixed Sequence of Sequences leads to the luigi_repo + containing a set of the types contained in the Sequence of CoSyLuigiTasks. The Sequence is passed through the + varargs of the CoSyLuigiRepo constructor.""" repo = CoSyLuigiRepo(TaskA, [TaskB, TaskC], (TaskD, TaskE), [TaskF, (TaskG, TaskH)], (TaskI, [TaskJ, TaskK])) assert repo.luigi_repo == {TaskA, TaskB, TaskC, TaskD, TaskE, TaskF, TaskG, TaskH, TaskI, TaskJ, TaskK} def test_heterogeneous_args_input(): - """_summary_.""" + """Test if calling flatten on a 2-times nested mixed Sequence of Sequences returns a set containing the types + contained in the Sequence of CoSyLuigiTasks. The Sequence is passed through the varargs of flatten.""" flattened_collection = set( flatten(TaskA, [TaskB, TaskC], (TaskD, TaskE), [TaskF, (TaskG, TaskH)], (TaskI, [TaskJ, TaskK])) ) @@ -63,7 +66,8 @@ def test_heterogeneous_args_input(): def test_heterogeneous_list_input(): - """_summary_.""" + """Test if calling flatten on a 2-times nested mixed list of Sequences returns a set containing the types + contained in the list of CoSyLuigiTasks. The list is passed as a single argument to flatten.""" flattened_collection = list( flatten([TaskA, [TaskB, TaskC], (TaskD, TaskE), [TaskF, (TaskG, TaskH)], (TaskI, [TaskJ, TaskK])]) ) @@ -71,7 +75,8 @@ def test_heterogeneous_list_input(): def test_heterogeneous_tuple_input(): - """_summary_.""" + """Test if calling flatten on a 2-times nested mixed tuple of Sequences returns a set containing the types + contained in the tuple of CoSyLuigiTasks. The tuple is passed as a single argument to flatten.""" flattened_collection = tuple( flatten((TaskA, [TaskB, TaskC], (TaskD, TaskE), [TaskF, (TaskG, TaskH)], (TaskI, [TaskJ, TaskK]))) ) diff --git a/tests/test_infinite_chain.py b/tests/test_infinite_chain.py index 471a0fb..6210df1 100644 --- a/tests/test_infinite_chain.py +++ b/tests/test_infinite_chain.py @@ -1,4 +1,5 @@ -"""_summary_.""" +"""Test if chaining the same task or sequence of tasks infinitely to each other is possible, i.e. ensure that the +caching done by Luigi is not restricting the set of results.""" import itertools from abc import ABC @@ -13,46 +14,50 @@ class ChainLink(CoSyLuigiTask, ABC): - """_summary_.""" + """An abstract class representing a chain link in an infinite chain.""" chain_link: CoSyLuigiTaskParameter | None class StartingLink(ChainLink): - """_summary_.""" + """A class that terminates the chain by needing no further chain links.""" chain_link = None class RepeatingLink(ChainLink): - """_summary_.""" + """A class that recurses the chain by requiring a further chain link.""" chain_link = CoSyLuigiTaskParameter(ChainLink) def output(self): - """_summary_. + """Assign each chain link a unique identifier. This is required because CoSy-Luigi considers tasks with + identical names, identical requirements, and identical outputs to be Singletons. This differs from Luigi, + which considers tasks with identical names and identical requirements Singletons. Returns: - _type_: _description_ + Mapping[str, MockTarget]_: The named unique target for each chain link. """ return {"counter": MockTarget(str(next(counter)))} @pytest.fixture def repo(): - """_summary_. + """Creates a CoSyLuigiRepo that contains the StartingLink and the RepeatingLink. Returns: - _type_: _description_ + CoSyLuigiRepo: The created CoSyLuigiRepo. """ return CoSyLuigiRepo(ChainLink) def test_infinite_chain(repo): - """_summary_. + """Tests if the results of pipeline synthesis are the pipelines that incrementally contain one more repeating + link. If CoSy-Luigi does not correctly override the task_id generation from Luigi, or if alternatively the + instance cache of the global Luigi Registry is not disabled, this test will fail. Args: - repo (_type_): _description_ + repo (CoSyLuigiRepo): The repository to use for synthesis. """ maestro = Maestro( repo.cls_repo, @@ -60,7 +65,7 @@ def test_infinite_chain(repo): ) results = list(maestro.query(RepeatingLink.target(), max_count=10)) - # Check for shapes of the pipelines + # Check shapes of the pipelines for i, result in enumerate(results): current_link = result for _ in range(i): diff --git a/tests/test_task_id_generation.py b/tests/test_task_id_generation.py index 0dcae7c..728073e 100644 --- a/tests/test_task_id_generation.py +++ b/tests/test_task_id_generation.py @@ -1,4 +1,13 @@ -"""_summary_.""" +"""Tests if the task_id generation for CoSyLuigiTasks is working as intended, preventing shading from happening. +Shading is defined to be a task that looks identical to another task, because it has the same required tasks and the +same name as another task, however tasks that are more than one step earlier in the pipeline still alter its outputs. +In this case, the task_id that CoSyLuigiTask sets must signal this to the Luigi scheduler, so that it knows that +these tasks are different pipelines. This can happen due to CoSy-Luigi operating on the logic that a task having an +output depending on prior tasks means introducing ad-hoc polymorphism. + +In short, CoSy-Luigi considers tasks with identical names, identical requirements, and identical outputs to be +Singletons. This differs from Luigi, which considers tasks with identical names and identical requirements +Singletons. This means Luigi allows shading, and CoSy-Luigi does not.""" from abc import ABC @@ -16,69 +25,72 @@ class Shaded(CoSyLuigiTask, ABC): - """_summary_.""" + """Abstract base class for shaded tasks.""" identifier: str class ShadedA(Shaded): - """_summary_.""" + """An example for a class that can be shaded.""" identifier = "A" class ShadedB(Shaded): - """_summary_.""" + """An example of a class that can be shaded.""" identifier = "B" class Shade(CoSyLuigiTask): - """_summary_.""" + """Shades either ShadedA or ShadedB. They are shaded because this task gives no indication of the contents of the + shaded CoSyLuigiTaskParameter as part of the signature that classes requesting this task see, i.e. a Shade( + ShadeA()) and Shade(ShadeB()) looks identical to Evaluate at Runtime.""" shaded = CoSyLuigiTaskParameter(Shaded) def complete(self): - """_summary_. + """Marks this class as always complete (avoids needing an output for testing). Returns: - _type_: _description_ + bool: True """ return True class Evaluate(CoSyLuigiTask): - """_summary_.""" + """Behaves as either a Singleton or as different Pipelines depending on how task_ids are assigned. Correct + behaviour is the latter.""" shade = CoSyLuigiTaskParameter(Shade) def run(self): - """_summary_.""" + """Write "OK". into the task's output file.""" with self.output()["output"].open("w") as f: f.write("OK.") def output(self): - """_summary_. + """This task creates a different output and thus is intended to act as a different task. Returns: - _type_: _description_ + Mapping[str, MockTarget]: An output with a different name depending on the shaded task. """ return {"output": MockTarget(self.shade.shaded.identifier)} class EvaluateWithPotentialToShade(Evaluate): - """_summary_. + """Behaves as a Singleton, showcases what happens when using the default task_id allocation of Luigi. Attributes: - task_id (_type_): _description_ + task_id (str): The task_id that Luigi would assign. """ def __init__(self, *args, **kwargs): - """_summary_. + """Initializes the task and overrides the task_id with the default allocation from Luigi. Args: - *args (_type_): _description_ - **kwargs (_type_): _description_ + *args (_type_): Passed through to super().__init__(). + **kwargs (_type_): Passed through to super().__init__(). """ super().__init__(*args, **kwargs) # Copy the behaviour of regular Luigi @@ -88,29 +100,29 @@ def __init__(self, *args, **kwargs): @pytest.fixture def repo(): - """_summary_. + """Creates a CoSyLuigiRepo for testing. Returns: - _type_: _description_ + CoSyLuigiRepo: The CoSyLuigiRepo for testing. """ return CoSyLuigiRepo(Evaluate, Shade, Shaded) @pytest.fixture def shadeable_repo(): - """_summary_. + """Creates a CoSyLuigiRepo for testing that allows shading to happen. Returns: - _type_: _description_ + CoSyLuigiRepo: The CoSyLuigiRepo for testing. """ return CoSyLuigiRepo(EvaluateWithPotentialToShade, Shade, Shaded) def test_shading_not_possible(repo): - """_summary_. + """Tests that shading does not occur when using the task_id that CoSyLuigiTask computes. Args: - repo (_type_): _description_ + repo (CoSyLuigiRepo): The CoSyLuigiRepo for testing. """ fs.clear() assert not target_a.exists() @@ -125,10 +137,10 @@ def test_shading_not_possible(repo): def test_shading_would_be_possible(shadeable_repo): - """_summary_. + """Tests that shading occurs when using the task_id default to Luigi. Args: - shadeable_repo (_type_): _description_ + repo (CoSyLuigiRepo): The CoSyLuigiRepo containing the overridden CoSyLuigiTask that allows for shading. """ fs.clear() assert not target_a.exists() @@ -142,16 +154,17 @@ def test_shading_would_be_possible(shadeable_repo): def test_output_mapping_is_enforced(): - """_summary_.""" + """Computing the task_ids for CoSyLuigiTasks assumes that the outputs are Mapping[str, Target]. Tests that this + enforced.""" class TaskWithWrongOutputA(CoSyLuigiTask): - """_summary_.""" + """Class with a wrong output mapping.""" def output(self): - """_summary_. + """Does not return a Mapping but a raw Target object. Returns: - _type_: _description_ + MockTarget: The Target object. """ return MockTarget("") @@ -159,13 +172,13 @@ def output(self): TaskWithWrongOutputA() class TaskWithWrongOutputB(CoSyLuigiTask): - """_summary_.""" + """Class with a wrong output mapping.""" def output(self): - """_summary_. + """Does not return a Mapping but a Target object wrapped in a list. Returns: - _type_: _description_ + list[MockTarget]: A list of Target objects. """ return [MockTarget("")] @@ -173,6 +186,7 @@ def output(self): TaskWithWrongOutputB() class TaskWithNoneOutput(CoSyLuigiTask): - """_summary_.""" + """A class with no outputs.""" + # This is explicitly allowed, as having no output means this class is never ad-hoc polymorphic. TaskWithNoneOutput() diff --git a/tests/test_unique_task_parameter.py b/tests/test_unique_task_parameter.py index 8704db3..bbe4a82 100644 --- a/tests/test_unique_task_parameter.py +++ b/tests/test_unique_task_parameter.py @@ -1,4 +1,4 @@ -"""_summary_.""" +"""Tests that the unique_in_prior_tasks constraint works as expected.""" import logging from abc import ABC @@ -12,67 +12,72 @@ class ScaleDataABC(CoSyLuigiTask, ABC): - """_summary_.""" + """Abstract class for Scalers.""" class ScaleData(ScaleDataABC): - """_summary_.""" + """Concrete class for Scalers.""" class ScaleDataVariantA(ScaleData): - """_summary_.""" + """A specific variant of a concrete scaler.""" class ScaleDataVariantB(ScaleData): - """_summary_.""" + """A specific variant of a concrete scaler.""" class TrainModel(CoSyLuigiTask, ABC): - """_summary_.""" + """Abstract class for training a model.""" scaled_data = CoSyLuigiTaskParameter(ScaleDataABC) class TrainModelVariantA(TrainModel): - """_summary_.""" + """A concrete model to train.""" class TrainModelVariantB(TrainModel): - """_summary_.""" + """A concrete model to train.""" class EvaluatePipelineWithUniqueScaler(CoSyLuigiTask): - """_summary_.""" + """Pipeline class that uses flag of CoSyLuigiTaskParameter to ensure that the same scaler is used throughout the + pipeline.""" train_model = CoSyLuigiTaskParameter(TrainModel) scaled_data = CoSyLuigiTaskParameter(ScaleDataABC, unique_across_prior_tasks=True) class EvaluatePipelineWithConstraintUniqueScaler(CoSyLuigiTask): - """_summary_.""" + """Pipeline class that uses constraints.is_unique_in_prior_tasks to construct a constraint that ensures that the + same scaler is used throughout the pipeline.""" train_model = CoSyLuigiTaskParameter(TrainModel) scaled_data = CoSyLuigiTaskParameter(ScaleDataABC) @classmethod def constraints(cls) -> Sequence[Callable[..., bool]]: - """_summary_. + """Overrides the constraints method with a concrete constraint that ensures that the same subclass of + ScaleDataABC is used throughout each pipeline. Returns: - Sequence[Callable[..., bool]]: _description_ + Sequence[Callable[..., bool]]: The constraints. """ return [lambda vs: is_unique_in_prior_tasks(vs, ScaleDataABC)] class EvaluatePipelineWithUniqueScalerAndNonAbstractSuper(CoSyLuigiTask): - """_summary_.""" + """Pipeline class that uses constraints.is_unique_in_prior_tasks to construct a constraint that ensures that the + same scaler is used throughout the pipeline. Instead of an abstract class for the constraint, uses a concrete + class with subclasses.""" train_model = CoSyLuigiTaskParameter(TrainModel) scaled_data = CoSyLuigiTaskParameter(ScaleData, unique_across_prior_tasks=True) class EvaluatePipeline(CoSyLuigiTask): - """_summary_.""" + """Pipeline class without constraints.""" train_model = CoSyLuigiTaskParameter(TrainModel) scaled_data = CoSyLuigiTaskParameter(ScaleDataABC) @@ -80,40 +85,40 @@ class EvaluatePipeline(CoSyLuigiTask): @pytest.fixture def repo_without_constraints() -> CoSyLuigiRepo: - """_summary_. + """Constructs a CoSyLuigiRepo with no constraints. Returns: - CoSyLuigiRepo: _description_ + CoSyLuigiRepo: The CoSyLuigiRepo with no constraints. """ return CoSyLuigiRepo(TrainModel, ScaleDataABC, EvaluatePipeline) @pytest.fixture def repo_with_constraints() -> CoSyLuigiRepo: - """_summary_. + """Constructs a CoSyLuigiRepo with constraints set on the CoSyLuigiTaskParameter. Returns: - CoSyLuigiRepo: _description_ + CoSyLuigiRepo: The CoSyLuigiRepo with constraints. """ return CoSyLuigiRepo(TrainModel, ScaleDataABC, EvaluatePipelineWithUniqueScaler) @pytest.fixture def repo_with_manual_constraints() -> CoSyLuigiRepo: - """_summary_. + """Constructs a CoSyLuigiRepo with constraints set by overriding the constraints method. Returns: - CoSyLuigiRepo: _description_ + CoSyLuigiRepo: The CoSyLuigiRepo with constraints. """ return CoSyLuigiRepo(TrainModel, ScaleDataABC, EvaluatePipelineWithConstraintUniqueScaler) @pytest.fixture def repo_with_non_abstract_super() -> CoSyLuigiRepo: - """_summary_. + """Constructs a CoSyLuigiRepo with constraints set on a non-abstract CoSyLuigiTaskParameter. Returns: - CoSyLuigiRepo: _description_ + CoSyLuigiRepo: The CoSyLuigiRepo with constraints. """ return CoSyLuigiRepo( TrainModel, ScaleData, ScaleDataVariantA, ScaleDataVariantB, EvaluatePipelineWithUniqueScalerAndNonAbstractSuper @@ -121,10 +126,11 @@ def repo_with_non_abstract_super() -> CoSyLuigiRepo: def test_implementation_is_not_unique_across_prior_tasks(repo_without_constraints: CoSyLuigiRepo): - """_summary_. + """Tests that without constraints there are unwanted pipeline variations where the only variance is using scalers + inconsistently. Args: - repo_without_constraints (CoSyLuigiRepo): _description_ + repo_without_constraints (CoSyLuigiRepo): The CoSyLuigiRepo without constraints. """ maestro = Maestro( repo_without_constraints.cls_repo, @@ -135,10 +141,11 @@ def test_implementation_is_not_unique_across_prior_tasks(repo_without_constraint def test_implementation_is_unique_across_prior_tasks(repo_with_constraints: CoSyLuigiRepo): - """_summary_. + """Test that with constraints applied to the CoSyLuigiTaskParameter there are no variants with inconsistent + scaler usage. Args: - repo_with_constraints (CoSyLuigiRepo): _description_ + repo_with_constraints (CoSyLuigiRepo): The CoSyLuigiRepo with constraints. """ maestro = Maestro( repo_with_constraints.cls_repo, @@ -153,10 +160,11 @@ def test_implementation_is_unique_across_prior_tasks(repo_with_constraints: CoSy def test_implementation_is_unique_across_prior_tasks_with_manual_constraint( repo_with_manual_constraints: CoSyLuigiRepo, ): - """_summary_. + """Test that with constraints applied by overriding the constraints method there are no variants with inconsistent + scaler usage. Args: - repo_with_manual_constraints (CoSyLuigiRepo): _description_ + repo_with_manual_constraints (CoSyLuigiRepo): The CoSyLuigiRepo with constraints. """ maestro = Maestro( repo_with_manual_constraints.cls_repo, @@ -173,10 +181,11 @@ def test_implementation_is_unique_across_prior_tasks_with_manual_constraint( def test_implementation_is_unique_across_prior_tasks_with_non_abstract_super( repo_with_non_abstract_super: CoSyLuigiRepo, ): - """_summary_. + """Test that with constraints applied to the CoSyLuigiTaskParameter for a non-abstract class there are no + variants with inconsistent scaler usage. Args: - repo_with_non_abstract_super (CoSyLuigiRepo): _description_ + repo_with_non_abstract_super (CoSyLuigiRepo): The CoSyLuigiRepo with constraints. """ maestro = Maestro( repo_with_non_abstract_super.cls_repo, @@ -191,10 +200,10 @@ def test_implementation_is_unique_across_prior_tasks_with_non_abstract_super( def test_warning_if_unique_across_prior_tasks_but_no_variance(caplog): - """_summary_. + """Tests that the inspection whether constraints are used on something that can not exhibit variance works. Args: - caplog (_type_): _description_ + caplog (Generator[LogCaptureFixture, None, None]): Captures the logging output to verify inspector message. """ caplog.set_level(logging.WARNING) repo_with_constraints_and_no_variance = CoSyLuigiRepo(