diff --git a/benchmarks/test_benchmark_chain_100.py b/benchmarks/test_benchmark_chain_100.py index 000aae7..223a1e3 100644 --- a/benchmarks/test_benchmark_chain_100.py +++ b/benchmarks/test_benchmark_chain_100.py @@ -1,3 +1,5 @@ +"""_summary_.""" + import itertools from abc import ABC @@ -11,26 +13,47 @@ class ChainLink(CoSyLuigiTask, ABC): + """_summary_.""" + chain_link: CoSyLuigiTaskParameter | None class StartingLink(ChainLink): + """_summary_.""" + chain_link = None class RepeatingLink(ChainLink): + """_summary_.""" + chain_link = CoSyLuigiTaskParameter(ChainLink) def output(self): + """_summary_. + + Returns: + _type_: _description_ + """ return {"counter": MockTarget(str(next(counter)))} @pytest.fixture def repo(): + """_summary_. + + Returns: + _type_: _description_ + """ return CoSyLuigiRepo(ChainLink) def create_infinite_chain(repo): + """_summary_. + + Args: + repo (_type_): _description_ + """ maestro = Maestro( repo.cls_repo, repo.taxonomy, @@ -39,6 +62,12 @@ def create_infinite_chain(repo): def test_benchmark_chain_creation(repo, benchmark): + """_summary_. + + Args: + repo (_type_): _description_ + benchmark (_type_): _description_ + """ benchmark(create_infinite_chain, repo) diff --git a/docs/bibliographies/bibliography.bib b/docs/bibliographies/bibliography.bib deleted file mode 100644 index f3f59a4..0000000 --- a/docs/bibliographies/bibliography.bib +++ /dev/null @@ -1,29 +0,0 @@ -@InProceedings{meyer-2025, -author="Meyer, Anne -and Kutabi, Hadi -and Bessai, Jan -and Scholtyssek, Daniel", -editor="Festa, Paola -and Ferone, Daniele -and Pastore, Tommaso -and Pisacane, Ornella", -title="CLS-Luigi: Analytics Pipeline Synthesis", -booktitle="Learning and Intelligent Optimization", -year="2025", -publisher="Springer Nature Switzerland", -address="Cham", -pages="269--284", -isbn="978-3-031-75623-8" -} - -@InProceedings{dudenhefner-2024, - author = {Dudenhefner, Andrej and Stahl, Christoph and Chaumet, Constantin and Laarmann, Felix and Rehof, Jakob}, - title = {Finite Combinatory Logic with Predicates}, - publisher = {Schloss Dagstuhl – Leibniz-Zentrum für Informatik}, - copyright = {Creative Commons Attribution 4.0 International license}, - date = {2024}, - groups = {theory}, - keywords = {combinatory logic, inhabitation, intersection types, program synthesis, Theory of computation → Type theory}, - language = {en}, -} - diff --git a/docs/bibliographies/modern-language-association.csl b/docs/bibliographies/modern-language-association.csl deleted file mode 100644 index 233c13d..0000000 --- a/docs/bibliographies/modern-language-association.csl +++ /dev/null @@ -1,1184 +0,0 @@ - - diff --git a/docs/index.md b/docs/index.md index f5203ff..da6a8cc 100644 --- a/docs/index.md +++ b/docs/index.md @@ -56,7 +56,7 @@ You can read more about that in the following papers: --- - @meyer-2025 + Meyer, Anne, et al. "_CLS-Luigi: Analytics Pipeline Synthesis._" Learning and Intelligent Optimization, edited by Paola Festa et al., Springer Nature Switzerland, 2025, pp. 269--84. @@ -64,7 +64,7 @@ You can read more about that in the following papers: --- - @dudenhefner-2024 + Dudenhefner, Andrej, et al. "_Finite Combinatory Logic with Predicates._" 2024. diff --git a/hatch.toml b/hatch.toml index 92d3236..daef2da 100644 --- a/hatch.toml +++ b/hatch.toml @@ -4,10 +4,10 @@ dependencies = [ "mkdocs-material", "mkdocs-minify-plugin", "mkdocstrings-python", + "griffe-generics", "mkdocs-gen-files", "mkdocs-literate-nav", "mkdocs-git-revision-date-localized-plugin", - "mkdocs-bibtex", "pypandoc_binary", "linkchecker", "pygments", diff --git a/properdocs.yml b/properdocs.yml index a4c42d5..79ac289 100644 --- a/properdocs.yml +++ b/properdocs.yml @@ -82,12 +82,15 @@ plugins: show_if_no_docstring: true show_signature_annotations: true show_bases: false + backlinks: tree + show_inheritance_diagram: true + extensions: + - dataclasses + - unpack_typeddict + - griffe_generics git-revision-date-localized: type: date strict: false - bibtex: - bib_dir: "docs/bibliographies" - csl_file: "docs/bibliographies/modern-language-association.csl" markdown_extensions: - attr_list: @@ -106,9 +109,9 @@ markdown_extensions: line_spans: __span pygments_lang_class: true guess_lang: false - default_lang: cosy-luigi-py + default_lang: python - pymdownx.inlinehilite: - style_plain_text: cosy-luigi-py + style_plain_text: python - pymdownx.snippets: - pymdownx.superfences: - pymdownx.blocks.tab: diff --git a/scripts/gen_example_pages.py b/scripts/gen_example_pages.py index 318b0b5..882204a 100644 --- a/scripts/gen_example_pages.py +++ b/scripts/gen_example_pages.py @@ -52,14 +52,20 @@ for path in sorted(directory.rglob("*.py")): module_path = path.relative_to(src).with_suffix("") + module_path_src = path.relative_to(src).with_suffix(".src") doc_path = path.relative_to(src).with_suffix(".md") + doc_path_src = path.relative_to(src).with_suffix(".src.md") full_doc_path = Path("examples", doc_path) + full_doc_path_src = Path("examples", doc_path_src) parts = tuple(module_path.parts) + src_parts = tuple(module_path_src.parts) if example_name: - nav[(*example_name, "Source Files", *parts[1:-1], f"{parts[-1]}.py")] = doc_path.as_posix() + nav[(*example_name, "Documentation", *parts[1:-1], f"{parts[-1]}.py")] = doc_path.as_posix() + nav[(*example_name, "Source Files", *parts[1:-1], f"{parts[-1]}.py")] = doc_path_src.as_posix() else: nav[parts] = doc_path.as_posix() + nav[src_parts] = doc_path_src.as_posix() with mkdocs_gen_files.open(full_doc_path, "w") as fd: ident = ".".join(parts) @@ -69,6 +75,9 @@ members_order: source """ ) + with mkdocs_gen_files.open(full_doc_path_src, "w") as fd: + fd.write("---\nhide:\n - toc\n---\n") + fd.write(f"```\n{path.read_text()}\n```\n") mkdocs_gen_files.set_edit_path(full_doc_path, path.relative_to(root)) diff --git a/src/cosy_luigi/__init__.py b/src/cosy_luigi/__init__.py index ac789a5..f839c5f 100644 --- a/src/cosy_luigi/__init__.py +++ b/src/cosy_luigi/__init__.py @@ -1,3 +1,5 @@ +"""_summary_.""" + from cosy_luigi.core import CoSyLuigiRepo, CoSyLuigiTask, CoSyLuigiTaskParameter __all__ = ["CoSyLuigiRepo", "CoSyLuigiTask", "CoSyLuigiTaskParameter"] diff --git a/src/cosy_luigi/constraints/__init__.py b/src/cosy_luigi/constraints/__init__.py index 8c8ee8b..09e46e5 100644 --- a/src/cosy_luigi/constraints/__init__.py +++ b/src/cosy_luigi/constraints/__init__.py @@ -1,3 +1,5 @@ +"""_summary_.""" + from cosy_luigi.constraints.unique import is_unique_in_prior_tasks __all__ = ["is_unique_in_prior_tasks"] diff --git a/src/cosy_luigi/constraints/unique.py b/src/cosy_luigi/constraints/unique.py index 8f0e9df..17e5b7d 100644 --- a/src/cosy_luigi/constraints/unique.py +++ b/src/cosy_luigi/constraints/unique.py @@ -1,3 +1,5 @@ +"""_summary_.""" + from __future__ import annotations from collections.abc import Sequence @@ -14,6 +16,15 @@ def _is_unique_in_prior_tasks( vs: Mapping[str, CoSyLuigiTask], required_to_be_unique: Sequence[type[CoSyLuigiTask]] ) -> bool: + """_summary_. + + Args: + vs (Mapping[str, CoSyLuigiTask]): _description_ + required_to_be_unique (Sequence[type[CoSyLuigiTask]]): _description_ + + Returns: + bool: _description_ + """ classes = [pc.__class__ for pc in traverse_pipeline(vs.values())] seen_subclasses: dict[type[CoSyLuigiTask], type[CoSyLuigiTask]] = {} for c in classes: @@ -30,6 +41,15 @@ def _is_unique_in_prior_tasks( def is_unique_in_prior_tasks( vs: Mapping[str, CoSyLuigiTask], required_to_be_unique: type[CoSyLuigiTask] | Sequence[type[CoSyLuigiTask]] ) -> bool: + """_summary_. + + Args: + vs (Mapping[str, CoSyLuigiTask]): _description_ + required_to_be_unique (type[CoSyLuigiTask] | Sequence[type[CoSyLuigiTask]]): _description_ + + Returns: + bool: _description_ + """ return _is_unique_in_prior_tasks( vs, required_to_be_unique if isinstance(required_to_be_unique, Sequence) else [required_to_be_unique], diff --git a/src/cosy_luigi/core/__init__.py b/src/cosy_luigi/core/__init__.py index 160decc..6ce1175 100644 --- a/src/cosy_luigi/core/__init__.py +++ b/src/cosy_luigi/core/__init__.py @@ -1,3 +1,5 @@ +"""_summary_.""" + from cosy_luigi.core.combinatorics import CoSyLuigiRepo, CoSyLuigiTask, CoSyLuigiTaskParameter __all__ = ["CoSyLuigiRepo", "CoSyLuigiTask", "CoSyLuigiTaskParameter"] diff --git a/src/cosy_luigi/core/combinatorics.py b/src/cosy_luigi/core/combinatorics.py index f3a023b..7087714 100644 --- a/src/cosy_luigi/core/combinatorics.py +++ b/src/cosy_luigi/core/combinatorics.py @@ -1,3 +1,5 @@ +"""_summary_.""" + from __future__ import annotations import logging @@ -5,7 +7,7 @@ from collections import defaultdict from collections.abc import Mapping from functools import cache, partial -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any import luigi from cosy.core import Constructor, SpecificationBuilder @@ -19,14 +21,38 @@ class CoSyLuigiTaskParameter(luigi.TaskParameter): + """_summary_. + + Attributes: + required_task (type[CoSyLuigiTask]): _description_ + unique_across_prior_tasks (bool): _description_ + """ + def __init__(self, required_task: type[CoSyLuigiTask], *, unique_across_prior_tasks: bool = False): + """_summary_. + + Args: + required_task (type[CoSyLuigiTask]): _description_ + unique_across_prior_tasks (bool): _description_ (Default value = False) + """ super().__init__() self.required_task = required_task self.unique_across_prior_tasks = unique_across_prior_tasks class CoSyLuigiTask(luigi.Task): + """_summary_.""" + def __init__(self, *args, **kwargs): + """_summary_. + + Args: + *args (_type_): _description_ + **kwargs (_type_): _description_ + + Raises: + TypeError: _description_ + """ super().__init__(*args, **kwargs) output = self.output() if not output: @@ -41,28 +67,44 @@ def __init__(self, *args, **kwargs): @classmethod @cache - def get_all_variants(cls): + def get_all_variants(cls) -> set[type[CoSyLuigiTask] | Any]: + """_summary_. + + Returns: + set[type[CoSyLuigiTask] | Any]: _description_ + """ return set(cls.__subclasses__()).union([s for c in cls.__subclasses__() for s in c.get_all_variants()]) @classmethod @cache - def get_all_class_attributes(cls): - attrs = {} + def get_all_class_attributes(cls) -> dict[str, Any]: + """_summary_. + + Returns: + dict[str, Any]: _description_ + """ + attrs: dict[str, Any] = {} for c in [cc for cc in reversed(cls.__mro__) if issubclass(cc, CoSyLuigiTask)]: attrs.update(getattr(c, "__dict__", {})) return attrs - def get_all_instance_attributes(self): - return {attr: getattr(self, attr) for attr in dir(self)} + def get_all_instance_attributes(self) -> dict[str, Any]: + """_summary_. - def requires(self): + Returns: + dict[str, Any]: _description_ """ - Returns a list of other tasks required to run this task. + return {attr: getattr(self, attr) for attr in dir(self)} + + def requires(self) -> dict[str, CoSyLuigiTask]: + """Returns a list of other tasks required to run this task. + This is done by retrieving all user-created attributes that are subclasses of CosyLuigiTaskParameter. Note that at Runtime Luigi unpacks CosyLuigiTaskParameters, so the actual check has to be for CoSyLuigiTasks. - :return: A list of other tasks required to run this task + Returns: + dict[str, CoSyLuigiTask]: A list of other tasks required to run this task """ return { k: v @@ -73,6 +115,11 @@ def requires(self): @classmethod @cache def _requirements(cls) -> Mapping[str, CoSyLuigiTaskParameter]: + """_summary_. + + Returns: + Mapping[str, CoSyLuigiTaskParameter]: _description_ + """ return { k: v for k, v in cls.get_all_class_attributes().items() @@ -81,12 +128,22 @@ def _requirements(cls) -> Mapping[str, CoSyLuigiTaskParameter]: @classmethod @cache - def get_params(cls): + def get_params(cls) -> list[tuple[str, CoSyLuigiTaskParameter]]: + """_summary_. + + Returns: + list[tuple[str, CoSyLuigiTaskParameter]]: _description_ + """ return list(cls._requirements().items()) @classmethod @cache def requirements_unique_in_prior_tasks(cls) -> Mapping[str, CoSyLuigiTaskParameter]: + """_summary_. + + Returns: + Mapping[str, CoSyLuigiTaskParameter]: _description_ + """ return { k: task_parameter for k, task_parameter in cls._requirements().items() @@ -96,19 +153,39 @@ def requirements_unique_in_prior_tasks(cls) -> Mapping[str, CoSyLuigiTaskParamet @classmethod @cache def unique_required_tasks_in_prior(cls) -> Sequence[type[CoSyLuigiTask]]: + """_summary_. + + Returns: + Sequence[type[CoSyLuigiTask]]: _description_ + """ return [task_parameter.required_task for task_parameter in cls.requirements_unique_in_prior_tasks().values()] @classmethod @cache - def target(cls): + def target(cls) -> Constructor: + """_summary_. + + Returns: + Constructor: _description_ + """ return Constructor(cls.__name__) @classmethod def constraints(cls) -> Sequence[Callable[..., bool]]: + """_summary_. + + Returns: + Sequence[Callable[..., bool]]: _description_ + """ return [] @classmethod def __constraints(cls) -> Sequence[Callable[..., bool]]: + """_summary_. + + Returns: + Sequence[Callable[..., bool]]: _description_ + """ from cosy_luigi.constraints.unique import _is_unique_in_prior_tasks # noqa: PLC0415 if cls.requirements_unique_in_prior_tasks(): @@ -116,7 +193,12 @@ def __constraints(cls) -> Sequence[Callable[..., bool]]: return [] @classmethod - def combinator_type(cls): + def combinator_type(cls) -> Specification: + """_summary_. + + Returns: + Specification: _description_ + """ sp = SpecificationBuilder() for name in [v.required_task.__name__ for v in cls._requirements().values()]: sp = sp.argument(name, Constructor(name)) @@ -127,18 +209,36 @@ def combinator_type(cls): return sp.suffix(cls.target()) @classmethod - def combinator(cls): + def combinator(cls) -> tuple[str, Callable[..., CoSyLuigiTask], Specification]: + """_summary_. + + Returns: + tuple[str, Callable[..., CoSyLuigiTask], Specification]: _description_ + """ if len(cls._requirements()) == 0: return cls.__name__, lambda: cls(), cls.combinator_type() return cls.__name__, lambda *args: cls(*args), cls.combinator_type() class CoSyLuigiRepo: + """_summary_. + + Attributes: + luigi_repo (set[type[CoSyLuigiTask]]): _description_ + taxonomy (Mapping[str, set[str]]): _description_ + cls_repo (list[tuple[str, Callable, Specification]]): _description_ + """ + def __init__(self, *tasks: type[CoSyLuigiTask] | Sequence[type[CoSyLuigiTask]]): # Accepts completely heterogeneous nested collections # This doesn't technically need to unpack as flatten could be typed to accept packed tuples # But performance is equivalent/faster because the first layer doesn't need to be checked this way + """_summary_. + + Args: + *tasks (type[CoSyLuigiTask] | Sequence[type[CoSyLuigiTask]]): _description_ + """ from cosy_luigi.utils import flatten # noqa: PLC0415 self.luigi_repo: set[type[CoSyLuigiTask]] = set(flatten(*tasks)) @@ -153,6 +253,7 @@ def __init__(self, *tasks: type[CoSyLuigiTask] | Sequence[type[CoSyLuigiTask]]): self.taxonomy[task.__name__].add(tpe.__name__) def check_unique_in_prior_tasks_sanity(self): + """_summary_.""" for source_task, param_name, required_type in [ (task, k, required_unique_task.required_task) for task in self.luigi_repo diff --git a/src/cosy_luigi/utils/__init__.py b/src/cosy_luigi/utils/__init__.py index ebd4586..60e9632 100644 --- a/src/cosy_luigi/utils/__init__.py +++ b/src/cosy_luigi/utils/__init__.py @@ -1,3 +1,5 @@ +"""_summary_.""" + from cosy_luigi.utils.traversals import flatten, traverse_pipeline __all__ = ["flatten", "traverse_pipeline"] diff --git a/src/cosy_luigi/utils/traversals.py b/src/cosy_luigi/utils/traversals.py index f90bdb7..fbac194 100644 --- a/src/cosy_luigi/utils/traversals.py +++ b/src/cosy_luigi/utils/traversals.py @@ -1,7 +1,10 @@ +"""_summary_.""" + from __future__ import annotations import inspect from abc import ABC +from collections.abc import Sequence from typing import TYPE_CHECKING, cast from cosy_luigi import CoSyLuigiTask @@ -10,12 +13,22 @@ from collections.abc import Iterable, Sequence -def flatten(*heterogeneous_task_collection: type[CoSyLuigiTask] | Sequence[type[CoSyLuigiTask]]): +def flatten( + *heterogeneous_task_collection: type[CoSyLuigiTask] | Iterable[type[CoSyLuigiTask]], +) -> Iterable[type[CoSyLuigiTask]]: + """_summary_. + + Args: + *heterogeneous_task_collection (type[CoSyLuigiTask] | Sequence[type[CoSyLuigiTask]]): _description_ + + Returns: + Iterable[type[CoSyLuigiTask]]: _description_ + """ return ( - task + task # type: ignore # guaranteed by recursion to be a type[CoSyLuigiTask] instead of an Iterable itself for task_or_task_collection in heterogeneous_task_collection for task in ( - flatten(*cast("Sequence[type[CoSyLuigiTask]]", task_or_task_collection)) + flatten(*cast("Iterable[type[CoSyLuigiTask]]", task_or_task_collection)) if isinstance(task_or_task_collection, (tuple, list)) else cast("type[CoSyLuigiTask]", task_or_task_collection).get_all_variants() if inspect.isabstract(task_or_task_collection) @@ -26,7 +39,15 @@ def flatten(*heterogeneous_task_collection: type[CoSyLuigiTask] | Sequence[type[ def _traverse_pipeline(vs: Sequence[CoSyLuigiTask] | Iterable[CoSyLuigiTask]) -> Sequence[CoSyLuigiTask]: - result = [*vs] + """_summary_. + + Args: + vs (Sequence[CoSyLuigiTask] | Iterable[CoSyLuigiTask]): _description_ + + Returns: + Sequence[CoSyLuigiTask]: _description_ + """ + result: list[CoSyLuigiTask] = [*vs] for v in vs: result.extend(traverse_pipeline(v.requires().values())) return result @@ -35,6 +56,14 @@ def _traverse_pipeline(vs: Sequence[CoSyLuigiTask] | Iterable[CoSyLuigiTask]) -> def traverse_pipeline( to_traverse: CoSyLuigiTask | Sequence[CoSyLuigiTask] | Iterable[CoSyLuigiTask], ) -> Sequence[CoSyLuigiTask]: + """_summary_. + + Args: + to_traverse (CoSyLuigiTask | Sequence[CoSyLuigiTask] | Iterable[CoSyLuigiTask]): _description_ + + Returns: + Sequence[CoSyLuigiTask]: _description_ + """ return ( _traverse_pipeline([to_traverse]) if isinstance(to_traverse, CoSyLuigiTask) else _traverse_pipeline(to_traverse) ) diff --git a/tests/__init__.py b/tests/__init__.py index e69de29..d99154d 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -0,0 +1 @@ +"""_summary_.""" diff --git a/tests/test_abstract_variant_expansion.py b/tests/test_abstract_variant_expansion.py index 6978448..aca59a8 100644 --- a/tests/test_abstract_variant_expansion.py +++ b/tests/test_abstract_variant_expansion.py @@ -1,54 +1,86 @@ +"""_summary_.""" + from abc import ABC, abstractmethod from cosy_luigi import CoSyLuigiRepo, CoSyLuigiTask class ABCInheritedTaskWithNoInheritors(CoSyLuigiTask, ABC): - pass + """_summary_.""" class ABCInheritedTask(CoSyLuigiTask, ABC): - pass + """_summary_.""" class ConcreteTaskFromABCInherited(ABCInheritedTask): - pass + """_summary_.""" class DeeperConcreteTaskFromABCInherited(ConcreteTaskFromABCInherited): - pass + """_summary_.""" # noinspection PyAbstractClass class AbstractTask(CoSyLuigiTask): + """_summary_.""" + @abstractmethod def get_class_name(self): + """_summary_. + + Raises: + NotImplementedError: _description_ + """ raise NotImplementedError # noinspection PyAbstractClass class AbstractTaskWithNoInheritors(CoSyLuigiTask): + """_summary_.""" + @abstractmethod def get_class_name(self): + """_summary_. + + Raises: + NotImplementedError: _description_ + """ raise NotImplementedError class ConcreteTaskFromAbstract(AbstractTask): + """_summary_.""" + def get_class_name(self): + """_summary_. + + Returns: + _type_: _description_ + """ return "ConcreteTaskFromAbstract" class DeeperConcreteTaskFromAbstract(ConcreteTaskFromAbstract): + """_summary_.""" + def get_class_name(self): + """_summary_. + + Returns: + _type_: _description_ + """ return "DeeperConcreteTaskFromAbstract" def test_expansion_from_abc(): + """_summary_.""" repo = CoSyLuigiRepo(ABCInheritedTask) assert repo.luigi_repo == {ConcreteTaskFromABCInherited, DeeperConcreteTaskFromABCInherited} def test_expansion_from_abstract(): + """_summary_.""" repo = CoSyLuigiRepo(AbstractTask) assert ConcreteTaskFromAbstract().get_class_name() == "ConcreteTaskFromAbstract" assert DeeperConcreteTaskFromAbstract().get_class_name() == "DeeperConcreteTaskFromAbstract" @@ -56,6 +88,7 @@ def test_expansion_from_abstract(): def test_expansion_from_abc_and_abstract(): + """_summary_.""" repo = CoSyLuigiRepo(ABCInheritedTask, AbstractTask) assert repo.luigi_repo == { ConcreteTaskFromABCInherited, @@ -66,20 +99,24 @@ def test_expansion_from_abc_and_abstract(): def test_implementation_of_abstract_does_not_expand(): + """_summary_.""" repo = CoSyLuigiRepo(ConcreteTaskFromAbstract) assert repo.luigi_repo == {ConcreteTaskFromAbstract} def test_implementation_of_abc_does_not_expand(): + """_summary_.""" repo = CoSyLuigiRepo(ConcreteTaskFromABCInherited) assert repo.luigi_repo == {ConcreteTaskFromABCInherited} def test_expansion_to_nothing_from_abc_with_no_inheritors(): + """_summary_.""" repo = CoSyLuigiRepo(ABCInheritedTaskWithNoInheritors) assert repo.luigi_repo == set() def test_expansion_to_nothing_from_abstract_with_no_inheritors(): + """_summary_.""" repo = CoSyLuigiRepo(AbstractTaskWithNoInheritors) assert repo.luigi_repo == set() diff --git a/tests/test_heterogeneous_repo_input.py b/tests/test_heterogeneous_repo_input.py index 6ffa6dd..25508ff 100644 --- a/tests/test_heterogeneous_repo_input.py +++ b/tests/test_heterogeneous_repo_input.py @@ -1,57 +1,61 @@ +"""_summary_.""" + from cosy_luigi import CoSyLuigiRepo, CoSyLuigiTask from cosy_luigi.utils import flatten class TaskA(CoSyLuigiTask): - pass + """_summary_.""" class TaskB(CoSyLuigiTask): - pass + """_summary_.""" class TaskC(CoSyLuigiTask): - pass + """_summary_.""" class TaskD(CoSyLuigiTask): - pass + """_summary_.""" class TaskE(CoSyLuigiTask): - pass + """_summary_.""" class TaskF(CoSyLuigiTask): - pass + """_summary_.""" class TaskG(CoSyLuigiTask): - pass + """_summary_.""" class TaskH(CoSyLuigiTask): - pass + """_summary_.""" class TaskI(CoSyLuigiTask): - pass + """_summary_.""" class TaskJ(CoSyLuigiTask): - pass + """_summary_.""" class TaskK(CoSyLuigiTask): - pass + """_summary_.""" def test_heterogeneous_repo_input(): + """_summary_.""" repo = CoSyLuigiRepo(TaskA, [TaskB, TaskC], (TaskD, TaskE), [TaskF, (TaskG, TaskH)], (TaskI, [TaskJ, TaskK])) assert repo.luigi_repo == {TaskA, TaskB, TaskC, TaskD, TaskE, TaskF, TaskG, TaskH, TaskI, TaskJ, TaskK} def test_heterogeneous_args_input(): + """_summary_.""" flattened_collection = set( flatten(TaskA, [TaskB, TaskC], (TaskD, TaskE), [TaskF, (TaskG, TaskH)], (TaskI, [TaskJ, TaskK])) ) @@ -59,6 +63,7 @@ def test_heterogeneous_args_input(): def test_heterogeneous_list_input(): + """_summary_.""" flattened_collection = list( flatten([TaskA, [TaskB, TaskC], (TaskD, TaskE), [TaskF, (TaskG, TaskH)], (TaskI, [TaskJ, TaskK])]) ) @@ -66,6 +71,7 @@ def test_heterogeneous_list_input(): def test_heterogeneous_tuple_input(): + """_summary_.""" flattened_collection = tuple( flatten((TaskA, [TaskB, TaskC], (TaskD, TaskE), [TaskF, (TaskG, TaskH)], (TaskI, [TaskJ, TaskK]))) ) diff --git a/tests/test_infinite_chain.py b/tests/test_infinite_chain.py index fe4fb14..471a0fb 100644 --- a/tests/test_infinite_chain.py +++ b/tests/test_infinite_chain.py @@ -1,3 +1,5 @@ +"""_summary_.""" + import itertools from abc import ABC @@ -11,26 +13,47 @@ class ChainLink(CoSyLuigiTask, ABC): + """_summary_.""" + chain_link: CoSyLuigiTaskParameter | None class StartingLink(ChainLink): + """_summary_.""" + chain_link = None class RepeatingLink(ChainLink): + """_summary_.""" + chain_link = CoSyLuigiTaskParameter(ChainLink) def output(self): + """_summary_. + + Returns: + _type_: _description_ + """ return {"counter": MockTarget(str(next(counter)))} @pytest.fixture def repo(): + """_summary_. + + Returns: + _type_: _description_ + """ return CoSyLuigiRepo(ChainLink) def test_infinite_chain(repo): + """_summary_. + + Args: + repo (_type_): _description_ + """ maestro = Maestro( repo.cls_repo, repo.taxonomy, diff --git a/tests/test_task_id_generation.py b/tests/test_task_id_generation.py index dc8936d..0dcae7c 100644 --- a/tests/test_task_id_generation.py +++ b/tests/test_task_id_generation.py @@ -1,3 +1,5 @@ +"""_summary_.""" + from abc import ABC import luigi @@ -14,37 +16,70 @@ class Shaded(CoSyLuigiTask, ABC): + """_summary_.""" + identifier: str class ShadedA(Shaded): + """_summary_.""" + identifier = "A" class ShadedB(Shaded): + """_summary_.""" + identifier = "B" class Shade(CoSyLuigiTask): + """_summary_.""" + shaded = CoSyLuigiTaskParameter(Shaded) def complete(self): + """_summary_. + + Returns: + _type_: _description_ + """ return True class Evaluate(CoSyLuigiTask): + """_summary_.""" + shade = CoSyLuigiTaskParameter(Shade) def run(self): + """_summary_.""" with self.output()["output"].open("w") as f: f.write("OK.") def output(self): + """_summary_. + + Returns: + _type_: _description_ + """ return {"output": MockTarget(self.shade.shaded.identifier)} class EvaluateWithPotentialToShade(Evaluate): + """_summary_. + + Attributes: + task_id (_type_): _description_ + """ + def __init__(self, *args, **kwargs): + """_summary_. + + Args: + *args (_type_): _description_ + **kwargs (_type_): _description_ + """ super().__init__(*args, **kwargs) # Copy the behaviour of regular Luigi self.task_id = task_id_str(self.get_task_family(), self.to_str_params(only_significant=True, only_public=True)) @@ -53,15 +88,30 @@ def __init__(self, *args, **kwargs): @pytest.fixture def repo(): + """_summary_. + + Returns: + _type_: _description_ + """ return CoSyLuigiRepo(Evaluate, Shade, Shaded) @pytest.fixture def shadeable_repo(): + """_summary_. + + Returns: + _type_: _description_ + """ return CoSyLuigiRepo(EvaluateWithPotentialToShade, Shade, Shaded) def test_shading_not_possible(repo): + """_summary_. + + Args: + repo (_type_): _description_ + """ fs.clear() assert not target_a.exists() assert not target_b.exists() @@ -75,6 +125,11 @@ def test_shading_not_possible(repo): def test_shading_would_be_possible(shadeable_repo): + """_summary_. + + Args: + shadeable_repo (_type_): _description_ + """ fs.clear() assert not target_a.exists() assert not target_b.exists() @@ -87,21 +142,37 @@ def test_shading_would_be_possible(shadeable_repo): def test_output_mapping_is_enforced(): + """_summary_.""" + class TaskWithWrongOutputA(CoSyLuigiTask): + """_summary_.""" + def output(self): + """_summary_. + + Returns: + _type_: _description_ + """ return MockTarget("") with pytest.raises(TypeError): TaskWithWrongOutputA() class TaskWithWrongOutputB(CoSyLuigiTask): + """_summary_.""" + def output(self): + """_summary_. + + Returns: + _type_: _description_ + """ return [MockTarget("")] with pytest.raises(TypeError): TaskWithWrongOutputB() class TaskWithNoneOutput(CoSyLuigiTask): - pass + """_summary_.""" TaskWithNoneOutput() diff --git a/tests/test_unique_task_parameter.py b/tests/test_unique_task_parameter.py index ef39c9f..8704db3 100644 --- a/tests/test_unique_task_parameter.py +++ b/tests/test_unique_task_parameter.py @@ -1,3 +1,5 @@ +"""_summary_.""" + import logging from abc import ABC from collections.abc import Callable, Sequence @@ -10,80 +12,120 @@ class ScaleDataABC(CoSyLuigiTask, ABC): - pass + """_summary_.""" class ScaleData(ScaleDataABC): - pass + """_summary_.""" class ScaleDataVariantA(ScaleData): - pass + """_summary_.""" class ScaleDataVariantB(ScaleData): - pass + """_summary_.""" class TrainModel(CoSyLuigiTask, ABC): + """_summary_.""" + scaled_data = CoSyLuigiTaskParameter(ScaleDataABC) class TrainModelVariantA(TrainModel): - pass + """_summary_.""" class TrainModelVariantB(TrainModel): - pass + """_summary_.""" class EvaluatePipelineWithUniqueScaler(CoSyLuigiTask): + """_summary_.""" + train_model = CoSyLuigiTaskParameter(TrainModel) scaled_data = CoSyLuigiTaskParameter(ScaleDataABC, unique_across_prior_tasks=True) class EvaluatePipelineWithConstraintUniqueScaler(CoSyLuigiTask): + """_summary_.""" + train_model = CoSyLuigiTaskParameter(TrainModel) scaled_data = CoSyLuigiTaskParameter(ScaleDataABC) @classmethod def constraints(cls) -> Sequence[Callable[..., bool]]: + """_summary_. + + Returns: + Sequence[Callable[..., bool]]: _description_ + """ return [lambda vs: is_unique_in_prior_tasks(vs, ScaleDataABC)] class EvaluatePipelineWithUniqueScalerAndNonAbstractSuper(CoSyLuigiTask): + """_summary_.""" + train_model = CoSyLuigiTaskParameter(TrainModel) scaled_data = CoSyLuigiTaskParameter(ScaleData, unique_across_prior_tasks=True) class EvaluatePipeline(CoSyLuigiTask): + """_summary_.""" + train_model = CoSyLuigiTaskParameter(TrainModel) scaled_data = CoSyLuigiTaskParameter(ScaleDataABC) @pytest.fixture def repo_without_constraints() -> CoSyLuigiRepo: + """_summary_. + + Returns: + CoSyLuigiRepo: _description_ + """ return CoSyLuigiRepo(TrainModel, ScaleDataABC, EvaluatePipeline) @pytest.fixture def repo_with_constraints() -> CoSyLuigiRepo: + """_summary_. + + Returns: + CoSyLuigiRepo: _description_ + """ return CoSyLuigiRepo(TrainModel, ScaleDataABC, EvaluatePipelineWithUniqueScaler) @pytest.fixture def repo_with_manual_constraints() -> CoSyLuigiRepo: + """_summary_. + + Returns: + CoSyLuigiRepo: _description_ + """ return CoSyLuigiRepo(TrainModel, ScaleDataABC, EvaluatePipelineWithConstraintUniqueScaler) @pytest.fixture def repo_with_non_abstract_super() -> CoSyLuigiRepo: + """_summary_. + + Returns: + CoSyLuigiRepo: _description_ + """ return CoSyLuigiRepo( TrainModel, ScaleData, ScaleDataVariantA, ScaleDataVariantB, EvaluatePipelineWithUniqueScalerAndNonAbstractSuper ) def test_implementation_is_not_unique_across_prior_tasks(repo_without_constraints: CoSyLuigiRepo): + """_summary_. + + Args: + repo_without_constraints (CoSyLuigiRepo): _description_ + """ maestro = Maestro( repo_without_constraints.cls_repo, repo_without_constraints.taxonomy, @@ -93,6 +135,11 @@ def test_implementation_is_not_unique_across_prior_tasks(repo_without_constraint def test_implementation_is_unique_across_prior_tasks(repo_with_constraints: CoSyLuigiRepo): + """_summary_. + + Args: + repo_with_constraints (CoSyLuigiRepo): _description_ + """ maestro = Maestro( repo_with_constraints.cls_repo, repo_with_constraints.taxonomy, @@ -106,6 +153,11 @@ def test_implementation_is_unique_across_prior_tasks(repo_with_constraints: CoSy def test_implementation_is_unique_across_prior_tasks_with_manual_constraint( repo_with_manual_constraints: CoSyLuigiRepo, ): + """_summary_. + + Args: + repo_with_manual_constraints (CoSyLuigiRepo): _description_ + """ maestro = Maestro( repo_with_manual_constraints.cls_repo, repo_with_manual_constraints.taxonomy, @@ -121,6 +173,11 @@ def test_implementation_is_unique_across_prior_tasks_with_manual_constraint( def test_implementation_is_unique_across_prior_tasks_with_non_abstract_super( repo_with_non_abstract_super: CoSyLuigiRepo, ): + """_summary_. + + Args: + repo_with_non_abstract_super (CoSyLuigiRepo): _description_ + """ maestro = Maestro( repo_with_non_abstract_super.cls_repo, repo_with_non_abstract_super.taxonomy, @@ -134,6 +191,11 @@ def test_implementation_is_unique_across_prior_tasks_with_non_abstract_super( def test_warning_if_unique_across_prior_tasks_but_no_variance(caplog): + """_summary_. + + Args: + caplog (_type_): _description_ + """ caplog.set_level(logging.WARNING) repo_with_constraints_and_no_variance = CoSyLuigiRepo( TrainModel, ScaleData, EvaluatePipelineWithUniqueScalerAndNonAbstractSuper