From a10f7da040c92e890b2c0087939c9a9d6836a6eb Mon Sep 17 00:00:00 2001 From: Francesco Pezzella Date: Thu, 15 Oct 2026 12:17:41 +0200 Subject: [PATCH 1/5] Allow longer titles with pymarkdown, dependency and .gitignore folder --- .gitignore | 3 +++ pyproject.toml | 4 +++- uv.lock | 2 ++ 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 931aaa556..2ed6c9d16 100644 --- a/.gitignore +++ b/.gitignore @@ -234,3 +234,6 @@ __marimo__/ **/*.dll **/.*.so **/*.dot + +# Folder containing relevant presentations made with Marp +.marp_presentations \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 14ff92f8b..a554adc75 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,6 +37,7 @@ dependencies = [ "hypothesmith>=0.3.3", "autopep8>=2.0", "flake8>=7.0", + "packaging>=24.0", ] [dependency-groups] @@ -120,6 +121,7 @@ select = [ "ANN002", "ANN003", "ANN204", "ANN205", "ANN401", ] #select = ["ALL"] # goal of issue + ignore = [ "D203", # conflicts with D211 (no blank line before class) — keep D211 "D213", # conflicts with D212 (summary on the first line) — keep D212 @@ -148,6 +150,7 @@ exclude = ["features/targets", "**/node_modules", "**/__pycache__", "**/.*", ".v extraPaths = ["src", "test", "features"] [tool.pymarkdown] +plugins.md013.heading_line_length = 140 plugins.md013.line_length = 140 plugins.md013.tables = false plugins.md013.code_blocks = false @@ -155,4 +158,3 @@ plugins.md013.code_blocks = false plugins.md007.enabled = false plugins.pml101.enabled = true extensions.markdown-tables.enabled = true - diff --git a/uv.lock b/uv.lock index 5f1fd94e8..ccf223c94 100644 --- a/uv.lock +++ b/uv.lock @@ -1165,6 +1165,7 @@ dependencies = [ { name = "libcst" }, { name = "more-itertools" }, { name = "networkx" }, + { name = "packaging" }, { name = "pyhamcrest" }, { name = "pyperclip" }, { name = "termcolor" }, @@ -1230,6 +1231,7 @@ requires-dist = [ { name = "libcst", specifier = ">=1.8.6" }, { name = "more-itertools", specifier = ">=10.0" }, { name = "networkx", specifier = ">=3.0" }, + { name = "packaging", specifier = ">=24.0" }, { name = "pyhamcrest", specifier = ">=2.1" }, { name = "pyperclip", specifier = ">=1.8" }, { name = "termcolor", specifier = ">=2.0" }, From af68756e1fe6f70675374ce0aefd6768ae796a86 Mon Sep 17 00:00:00 2001 From: Francesco Pezzella Date: Fri, 16 Oct 2026 13:46:54 +0200 Subject: [PATCH 2/5] Add the Refactoring recipes and rejected recipes docs and the Type parameter modernization feature page, define type hint, type parameter and type variable, and fix the test file name in the rewrite-semantics module doc. --- docs/TODO | 1 - docs/developer/feature-test-map/core.md | 38 +- docs/developer/modules/index.md | 3 + docs/developer/modules/recipes.md | 325 ++++++++++++++++ docs/developer/modules/rejected-recipes.md | 38 ++ docs/glossary.md | 22 ++ docs/user/concepts/index.md | 4 + docs/user/concepts/type-parameters.md | 64 ++++ docs/user/features/index.md | 4 + docs/user/features/rewrite-semantics.md | 2 +- .../features/type-parameter-modernization.md | 357 ++++++++++++++++++ mkdocs.yml | 6 +- 12 files changed, 859 insertions(+), 5 deletions(-) create mode 100644 docs/developer/modules/recipes.md create mode 100644 docs/developer/modules/rejected-recipes.md create mode 100644 docs/user/concepts/type-parameters.md create mode 100644 docs/user/features/type-parameter-modernization.md diff --git a/docs/TODO b/docs/TODO index 7705dae88..2ab4ca81a 100644 --- a/docs/TODO +++ b/docs/TODO @@ -1,4 +1,3 @@ - ## Documentation gaps: action points for docs ### Stub pages that need content diff --git a/docs/developer/feature-test-map/core.md b/docs/developer/feature-test-map/core.md index da499e348..be229dbb7 100644 --- a/docs/developer/feature-test-map/core.md +++ b/docs/developer/feature-test-map/core.md @@ -11,7 +11,16 @@ https://github.com/TNO/Renaissance.Py/blob/main/LICENSE ## 1. Pattern matching - **Feature:** [Pattern matching](../../user/features/pattern-matching.md) -- **Suggested test modules:** `tests/matching/` +- **Test file(s):** + - `test/syntax_tree/test_match_finder.py` + - `test/syntax_tree/test_match_finder_multi_assignments.py` + - `test/syntax_tree/test_match_tree.py` + - `test/syntax_tree/test_match_dict.py` + - `test/syntax_tree/test_pattern_match.py` + - `test/syntax_tree/test_pattern_kind.py` + - `test/python/ast/test_python_matcher.py` + - `test/c_cpp/test_c_match_finder.py` +- **Code file(s):** `src/renaissance/syntax_tree/match_finder.py`, `src/renaissance/syntax_tree/pattern_kind.py` ## 2. Rewrite semantics @@ -19,5 +28,30 @@ https://github.com/TNO/Renaissance.Py/blob/main/LICENSE - **Concepts:** [Rewrite semantics](../../user/concepts/rewrite-semantics.md) - **Test modules:** [Rewrite semantics test module](../../developer/modules/rewrite-semantics.md) - **BDD feature file:** `features/rewrite-semantics.feature` -- **BDD steps:** `features/steps/test-rewrite-semantics.py` +- **BDD steps:** `features/steps/test_rewrite_semantics.py` +- **Test file(s):** + - `test/common/test_rewriter.py` + - `test/syntax_tree/test_ast_rewriter.py` + - `test/syntax_tree/test_rewrite_semantics_properties.py` - **Code files:** `src/renaissance/common/rewriter.py`, `src/renaissance/syntax_tree/ast_rewriter.py` + +## 3. Type parameter modernization + +- **Feature:** [Type parameter modernization](../../user/features/type-parameter-modernization.md) +- **Concepts:** [Rewrite semantics](../../user/concepts/rewrite-semantics.md) +- **Code modules:** [Refactoring recipes](../../developer/modules/recipes.md) +- **Test file(s):** + - `test/recipes/test_type_var_check.py` + - `test/recipes/test_type_var_check_convert.py` + - `test/recipes/test_type_var_check_localize.py` + - `test/recipes/test_type_var_check_orphaned.py` + - `test/recipes/test_type_var_check_properties.py` + - `test/recipes/test_type_var_domain.py` + - `test/recipes/test_step_runner.py` + - `test/recipes/test_python_refactoring.py` + - `test/utils/test_import_resolution.py` + - `test/rejuvenation/test_migration_type_recipes.py` +- **Code file(s):** `src/renaissance/recipes/type_var_check.py`, `src/renaissance/recipes/type_var_domain.py`, + `src/renaissance/recipes/step_runner.py`, + `src/renaissance/recipes/python_refactoring.py`, + `src/renaissance/utils/import_resolution.py`, `src/rejuvenation/migration-type-recipes.py` diff --git a/docs/developer/modules/index.md b/docs/developer/modules/index.md index 39c958ad5..7c4788621 100644 --- a/docs/developer/modules/index.md +++ b/docs/developer/modules/index.md @@ -15,3 +15,6 @@ https://github.com/TNO/Renaissance.Py/blob/main/LICENSE 1. [Transformation modules](transformation.md) 1. [Observability modules](observability.md) 1. [Strategy modules](strategy.md) +1. [Rewrite semantics module](rewrite-semantics.md) +1. [Refactoring recipes](recipes.md) +1. [Rejected recipes](rejected-recipes.md) diff --git a/docs/developer/modules/recipes.md b/docs/developer/modules/recipes.md new file mode 100644 index 000000000..90f418b56 --- /dev/null +++ b/docs/developer/modules/recipes.md @@ -0,0 +1,325 @@ +# Refactoring recipes + +{ #codemod-recipes } + +**Stable ID:** `CODEMOD-RECIPES` + +## Responsibility + +Recipes are `PythonRefactoring` subclasses that rewrite one Python file at a time, for gaps `ruff` does not fully or +safely cover. This page covers `TypeVarCheck`, the recipe behind +[Type parameter modernization](../../user/features/type-parameter-modernization.md). + +## Location + +- `src/renaissance/recipes/type_var_check.py`: the `TypeVarCheck` pipeline (orchestration only). +- `src/renaissance/recipes/type_var_domain.py`: domain model and safety analysis. +- `src/renaissance/recipes/step_runner.py`: `Step`/`run_steps`, runs fix actions in order and commits only the ones + that fixed something. +- `src/renaissance/recipes/python_refactoring.py`: the base class. +- `src/renaissance/utils/import_resolution.py`: resolves `from X import Y` to a file in the project, re-anchors one + file's relative import on another file, and collects the names each project file imports from the others. + +## Public entry points + +- `TypeVarCheck.check()` (and `run()`): runs the three phases through `run_steps`, committing to disk between them. +- The phases, each returning `{name: "fixed" | "unsafe"}` and recording the `UnsafeReason` of each unsafe name: + + | Phase | Reasons attribute | + | ------------------------------------------------------------ | --------------------------- | + | `localize_imported_typevars()` | `cross_file_unsafe_reasons` | + | `convert_declared_typevars()` (only adds type parameters) | `converted_unsafe_reasons` | + | `remove_orphaned_declarations()` (only removes declarations) | `orphaned_unsafe_reasons` | + +- Attributes set by the CLI (`migration-type-recipes.py`): `min_python`, `project_root` (falls back to the file's + folder) and `project_wide_imported_names` (from `collect_project_imported_names`). +- `PythonRefactoring.process("TypeVarCheck", file)`, used by `cli.py refactor`, never sets those attributes, so every + version-gated rewrite is reported `"unsafe"` there. + +## Internal structure + +- **libcst throughout.** The recipe runs on the CST backend (`PythonRefactoring[PythonCstNode]`) and analyses the + same libcst tree it rewrites; the cross-file phase parses the origin with `libcst.parse_module`. Each edit + replaces only the node it changes, so formatting and comments elsewhere are kept. + +- **Unsafe reasons.** `is_safe_to_remove`/`is_safe_to_localize` and `_conversion_refusal` return an `UnsafeReason` + or `None`. Each of the 9 reasons has an `UnsafeRule` (message + docs anchor) in `UNSAFE_RULES`; `doc_link()` turns + it into the URL that the CLI report prints. + +- **Version gates.** `_conversion_refusal` checks `min_python` against `PEP_695_MINIMUM = (3, 12)` and, for a + `default=`, `PEP_696_MINIMUM = (3, 13)`. `None` never passes. A keyword outside `_CONVERTIBLE_KEYWORDS` is refused + (`NO_PEP695_EQUIVALENT`) instead of dropped. + +- **Uses.** `uses_of` takes a declaration's uses from libcst's scope analysis (`referenced_by`). A use inside a + string annotation or `cast("T", x)` counts; a `T` that a PEP 695 type parameter of an enclosing function or class + declares (`def f[T]`, `class C[T]`) does not. + +- **Signature edit.** `_add_type_params` replaces only the function's name (`f` → `f[T]`) or, if it has one, its + type-parameter bracket; the body is never touched. `type_param_text` writes each parameter (`T: int`, `**P`, + `*Ts`, `= default`) from the declaration's own text. + +- **One edit per function.** All type parameters of a function go into one `replace()`, in declaration order, with + the defaulted ones last (`type_params_bracket`). Two edits on one node would be rejected as conflicting rewrites. + +- **Owning function.** A legacy type parameter belongs to the function whose signature uses it. `signature_function` + finds the functions whose parameter or return annotations use the name, and `owning_function` gives each use to the + outermost of them that encloses it. A nested function whose enclosing function's signature also uses `T` gets no + `[T]` of its own, since it already sees the outer one. A use outside every such signature, e.g. `y: list[T] = []` in + a function whose signature doesn't use `T`, is given to no function. + +- **Generic classes.** Methods of a class with the name in a base (`generic_classes`) are skipped. A generic class at + the origin does not block localization. + +- **No mixed syntax.** A function using a declaration that `_conversion_refusal` rejects gets none of its type + parameters, since PEP 695 forbids mixing them with legacy ones. A name whose every function is blocked is + `SHARES_FUNCTION_WITH_UNCONVERTIBLE`. This needs two passes: all refusals are known before any parameter is added. + +- **Orphans.** A declaration with no uses left (`uses_of` is empty) is orphaned. Conversion skips such names too. + +- **Localization.** The origin's declaration is copied as written, with the comment trailing it when it is alone on + its line (`declaration_text`). `DECLARATION_NAME_UNAVAILABLE` if the origin only binds a name its arguments use + (`declaration_argument_names`) under `if TYPE_CHECKING:` or this file binds it to something else + (`from_import_sources`). All names localized from one import statement are handled together. + + A name the origin imports, the constructor or one an argument uses, is imported here from the module the origin + gets it from, with its alias (`from_import_binding`). A relative import of the origin is re-anchored on this file + by `rebase_relative_module`. A name the origin defines itself, or one whose module this file can't name, is + imported from the origin. + +- **Imports.** An import whose names are now declared locally is narrowed, keeping its comments, or dropped with its + trailing comment when no name remains. The needed imports, each once per file, and then the declarations go after + the file's last import, so no import is left below code (E402). A localization is written where its import was + when that import shares its line with other statements and is dropped, when no import of the file remains, or when + a statement between it and the last import reads one of its names. Merging and sorting the added imports, and + removing imports that became unused, is left to the CLI's `ruff check --fix --select F401,I001` pass. + +## Related features + +- [Type parameter modernization](../../user/features/type-parameter-modernization.md) + +## Related concepts + +- [Rewrite semantics](../../user/concepts/rewrite-semantics.md) +- [Python version gates](../../user/features/type-parameter-modernization.md#feature-type-parameter-modernization-version-gates) + +## Validated by test modules + +See *Verified by test modules* in [Type parameter modernization](../../user/features/type-parameter-modernization.md). + +## Extension points + +- A new recipe is a `PythonRefactoring` subclass in its own `snake_case`-named module in `src/renaissance/recipes/`; + no registration is needed. +- A new type-parameter-declaring construct: extend `_is_type_param_call` and `type_param_text` together. +- A keyword added by a later Python, or future PEP 695 syntax for variance or a bound on `**P`/`*Ts`: add it to + `_CONVERTIBLE_KEYWORDS` and carry it over in `type_param_text`. +- A new version gate uses the syntax's own minimum Python from its PEP, not a value copied from another gate. +- Re-exports through `__init__.py` or namespace packages (PEP 420): extend `resolve_project_module`. +- `Step`/`run_steps` is generic and can be reused by any recipe. + +## Non-goals + +- Make the tool more efficient. Currently the tool does not make use of any filtering or similar features, making it + exponentially slower than other tools (Ruff can run on milliseconds on big projects, while it may take upwards of 30 + minutes per project with this tool). This is a limitation of it, and a low priority fix due to time constraints. + +- The target's minimum Python is not detected (e.g. from `requires-python`); it must be passed with `--py`. A + project with: + + ```toml + [project] + requires-python = ">=3.12" + ``` + + run with `--py 3.11` is treated as below 3.12, so nothing is converted. + +- Re-exports, namespace packages, `import *` and renamed imports are not followed. In: + + ```python + # origin.py + from typing import TypeVar + + T = TypeVar("T") + + # main.py + from origin import T as U + + def first(items: list[U]) -> U: ... + ``` + + `main.py` is left unchanged. + +- Only declarations in the file itself can block a function. A legacy type parameter that stays imported, such as one + from another package, does not. In: + + ```python + from other_package import K + + V = TypeVar("V") + + def get(mapping: dict[K, V], key: K) -> V: ... + ``` + + `get` becomes `def get[V](mapping: dict[K, V], key: K) -> V: ...`. `K` is still the imported legacy type + parameter, so `get` mixes both syntaxes, which + [PEP 695](https://peps.python.org/pep-0695/#compatibility-with-traditional-typevars) forbids. Type checkers + report an error. + +- Imported names are collected once before the run, so removing a declaration whose importers were all localized + needs a second run. With `origin.py` as above and: + + ```python + # main.py + from origin import T + + def first(items: list[T]) -> T: ... + ``` + + the first run turns `main.py` into `def first[T](items: list[T]) -> T: ...`, but keeps `T` in `origin.py`, since + `main.py` imported it when the run started. A second run removes it. + +- Only a plain `__all__ = [...]` is read to detect an exported declaration. In: + + ```python + __all__: list[str] = ["T", "first"] + + T = TypeVar("T") + + def first(items: list[T]) -> T: ... + ``` + + `first` is converted and `T = TypeVar("T")` is removed, although `__all__` exports it. + +- A declaration is only treated as public API when `__all__` lists it. One that nothing in the project uses is + removed, even if other projects import it. In: + + ```python + # mylib/typing.py + from typing import TypeVar + + KeyT = TypeVar("KeyT", bytes, str) + ``` + + `KeyT = TypeVar("KeyT", bytes, str)` is removed, so `from mylib.typing import KeyT` fails elsewhere. With + `__all__ = ["KeyT"]` it is kept. + +- A name used only in the string value of a `TypeAlias` is not seen as a use, so its declaration can be removed as + orphaned. In: + + ```python + T = TypeVar("T") + Pairs: TypeAlias = "list[tuple[T, T]]" + ``` + + `T = TypeVar("T")` is removed, and `Pairs` refers to an undefined `T`. + +- `Unpack[Ts]` → `*Ts` is left to `ruff`'s `UP044`, see [Rejected recipes](rejected-recipes.md). In: + + ```python + Ts = TypeVarTuple("Ts") + + def pack(*args: Unpack[Ts]) -> tuple[Unpack[Ts]]: ... + ``` + + only `[*Ts]` is added: `def pack[*Ts](*args: Unpack[Ts]) -> tuple[Unpack[Ts]]: ...`. + +- An import that follows another statement on the same line is written back wrongly. This is rare, and `ruff`'s + `E702` flags such lines. In: + + ```python + import os; from origin import T + ``` + + the lines that replace the import get an extra indent, a syntax error. + +- Without the CLI's `ruff` pass (`--no-ruff`, or using the recipe as a library) the added imports are not merged + into the file's existing `from` imports and not sorted. They are still placed with the other imports. + +- A declaration still read by a decorator or a parameter default of a converted function can be removed. This is + rare. In: + + ```python + T = TypeVar("T") + + def first(items: list[T], default=T) -> T: ... + ``` + + `first` becomes `def first[T](items: list[T], default=T) -> T: ...`. The annotations now use the new type + parameter, but `default=T` is evaluated outside it and still reads the module-level `T`. libcst places it inside + the type parameter's scope, so the declaration looks unused and is removed: importing the module raises + `NameError`. + +- A declaration followed by `;` on the same line is removed without its `;`. This is rare, and `ruff`'s `E702` flags + such lines. In: + + ```python + T = TypeVar("T"); U = 1 + + def first(items: list[T]) -> T: ... + ``` + + the file is written as `; U = 1`, a syntax error, and reported under errors. + +- Declarations that break the naming rule of [PEP 484](https://peps.python.org/pep-0484/) (a single variable named + after the string argument) are not handled. This is rare, and `ruff`'s `PLC0132` flags a mismatched name: + + ```python + T = U = TypeVar("T") # if T is unused, the statement is removed and U is no longer defined + V = TypeVar("W") # def f(x: V) -> V becomes def f[W](x: V) -> V + X = TypeVar(name) # AttributeError: no phase is applied to the file + ``` + +- A declaration whose constructor comes from `from typing import *` is localized without importing it. This is rare, + and `ruff`'s `F403` flags the star import. In: + + ```python + # origin.py + from typing import * + + T = TypeVar("T") + + # main.py + from origin import T + + class Pair(dict[str, T]): ... + ``` + + `main.py` gets `T = TypeVar("T")` without an import of `TypeVar`, so it raises `NameError`. + +- A `default=` that uses another declaration is carried over as is. This is rare. With `--py 3.13`: + + ```python + T = TypeVar("T") + U = TypeVar("U", default=T) + + def first(items: list[U]) -> U: ... + ``` + + becomes `def first[U = T](items: list[U]) -> U: ...`, whose default still refers to the module-level `T`. + +## Need to fix + +Errors that should be fixed, not accepted. + +- The comment lines above a localized declaration are not copied. Only the comment trailing its line is, because + the rewriter handles a comment as one line + ([issue 189](https://github.com/TNO/Renaissance.Py/issues/189)). In: + + ```python + # origin.py + from typing import TypeVar + + # Placeholder until the array API types are narrower. + T = TypeVar("T") # noqa: PLC0105 + + # main.py + from origin import T + ``` + + `main.py` gets `T = TypeVar("T") # noqa: PLC0105` without the comment above it. The origin keeps its comment + when it drops the declaration. + +- Line endings are not preserved: a modified CRLF file is written back with LF. This causes no problem when the + repository stores LF and git converts on checkout, the usual case, since git commits the same content. A + repository that stores CRLF gets a diff on every line of a modified file, so a pull request may be rejected + ([issue 279](https://github.com/TNO/Renaissance.Py/issues/279)). diff --git a/docs/developer/modules/rejected-recipes.md b/docs/developer/modules/rejected-recipes.md new file mode 100644 index 000000000..5a3d2e337 --- /dev/null +++ b/docs/developer/modules/rejected-recipes.md @@ -0,0 +1,38 @@ +# Rejected recipes + +{ #codemod-rejected-recipes } + +**Stable ID:** `CODEMOD-REJECTED-RECIPES` + +## Responsibility + +Recipes that were built and later removed because an existing tool already covers them. Check that the reason still +holds before rebuilding one. + +## TypeVarTupleCheck (`Unpack[T]` → `*T`) + +{ #codemod-rejected-recipes-typevartuplecheck } + +**What it did:** rewrote `Unpack[Ts]` to `*Ts` ([PEP 646](https://peps.python.org/pep-0646/), Python 3.11+) for a +`TypeVarTuple` declared in the same file. + +**Why it was removed:** + +- `ruff`'s `UP044` covers more cases (`typing.Unpack`, imported `TypeVarTuple`s, `Unpack[tuple[...]]`) and skips + places where `*` is invalid, where the recipe wrote a `SyntaxError`. +- Its only advantage was not needing `--unsafe-fixes`. +- Legacy `Unpack[...]` is rare in real projects. + +**What to use instead**, after [Type parameter modernization](../../user/features/type-parameter-modernization.md) (which converts +the `TypeVarTuple` declaration but leaves `Unpack[Ts]` as it is): + +```shell +ruff check --select UP044 --unsafe-fixes --target-version py311 --fix +``` + +Use `py311` or higher, and review any `Unpack[tuple[...]]` rewrite: unlike `Unpack[Ts]`, it changes the runtime +object. + +## Related features + +- [Type parameter modernization](../../user/features/type-parameter-modernization.md) diff --git a/docs/glossary.md b/docs/glossary.md index fc12a9659..4625d6a56 100644 --- a/docs/glossary.md +++ b/docs/glossary.md @@ -36,6 +36,10 @@ proper ancestor. An [insertion](#insertion) that adds text immediately after the end location of an AST node. See [Rewrite semantics](user/concepts/rewrite-semantics.md). +### Around + +See [Surround](#surround). + ### AST (Abstract Syntax Tree) A hierarchical representation of source code produced by a compiler front-end, @@ -264,3 +268,21 @@ See [Transformation](user/concepts/transformation.md). ### Trivia White spaces and comments. + +### Type hint + +An annotation that states the expected type of a variable, parameter or return value, such as `x: list[int]`. +Type checkers read it; Python does not enforce it at runtime. +See [Type parameters](user/concepts/type-parameters.md). + +### Type parameter + +A name in a type hint that stands for a type chosen later, declared on a generic function, class or type alias. +Three kinds exist: [type variable](#type-variable), type variable tuple (`*Ts`) and parameter specification (`**P`). +See [Type parameters](user/concepts/type-parameters.md). + +### Type variable + +The kind of [type parameter](#type-parameter) that stands for a single type. +Also called *TypeVar*, after the legacy class that declares it (`T = TypeVar("T")`). +See [Type parameters](user/concepts/type-parameters.md). diff --git a/docs/user/concepts/index.md b/docs/user/concepts/index.md index 9019e5f98..dc4e07189 100644 --- a/docs/user/concepts/index.md +++ b/docs/user/concepts/index.md @@ -34,3 +34,7 @@ This section introduces the conceptual model of the repository. ## 5. Reusable libraries 1. [Standard analyses and transformations](standard-libraries.md) + +## 6. Python typing terms + +1. [Type parameters](type-parameters.md) diff --git a/docs/user/concepts/type-parameters.md b/docs/user/concepts/type-parameters.md new file mode 100644 index 000000000..f71851486 --- /dev/null +++ b/docs/user/concepts/type-parameters.md @@ -0,0 +1,64 @@ +# Type parameters + +{ #concept-type-parameters } + +**Stable ID:** `CONCEPT-TYPE-PARAMETERS` + +## Purpose + +Fixes the meaning of three terms that are easy to confuse: *type hint*, *type parameter* and *type variable*. + +## Scope + +The terms as used by [Type parameter modernization](../features/type-parameter-modernization.md). This page does not describe +how the tool rewrites them. + +## Definition + +A **type hint** states the type of a value. A **type parameter** is a type left open in a generic function, class or +type alias. A **type variable** is the kind of type parameter that stands for one type. + +```python +def first[T](items: list[T]) -> T: ... +``` + +| Part of the code | Term | Role | +| --------------------- | -------------- | ----------------------------------- | +| ``[T]`` | Type parameter | Declares ``T``, a type left open | +| ``items: list[T]`` | Type hint | States the type of ``items`` | +| ``T`` after the arrow | Type hint | States the type of the return value | + +Here `T` is a type variable. + +Before Python 3.12, `T` had to be declared separately. The legacy form still works on every version: + +```python +T = TypeVar("T") + +def first(items: list[T]) -> T: ... +``` + +Three kinds of type parameter exist: + +| Kind of type parameter | Written as | Legacy declaration | +| ----------------------- | ---------- | ------------------ | +| Type variable | ``T`` | ``TypeVar`` | +| Type variable tuple | ``*Ts`` | ``TypeVarTuple`` | +| Parameter specification | ``**P`` | ``ParamSpec`` | + +## Invariants / guarantees + +1. A type hint is never a type parameter: `list[T]` is a type hint, `T` is the type parameter it uses. +2. The legacy and the PEP 695 declaration define the same kind of type parameter. Only the place where it is declared + differs, and the inline form needs Python 3.12. + +## Related features + +- [Type parameter modernization](../features/type-parameter-modernization.md) + +## Notes + +- A PEP 695 type parameter is also a `TypeVar`, `TypeVarTuple` or `ParamSpec` object at runtime (see + `__type_params__`). This is why "TypeVar" is often used for all three kinds. +- The documentation says *type parameter* for the general case and *type variable* only for the single-type kind. +- The recipe class is still named `TypeVarCheck`. diff --git a/docs/user/features/index.md b/docs/user/features/index.md index 00ce2cb47..33d098163 100644 --- a/docs/user/features/index.md +++ b/docs/user/features/index.md @@ -29,3 +29,7 @@ This section documents the main user-visible features of the repository. ## 4. Reporting 1. [Observability and reporting](observability-and-reporting.md) + +## 5. Code modernization + +1. [Type parameter modernization](type-parameter-modernization.md) diff --git a/docs/user/features/rewrite-semantics.md b/docs/user/features/rewrite-semantics.md index dfb262c55..adab55663 100644 --- a/docs/user/features/rewrite-semantics.md +++ b/docs/user/features/rewrite-semantics.md @@ -26,7 +26,7 @@ which produce errors, so that transformation authors can reason about the outcom * [Rewrite semantics test module](../../developer/modules/rewrite-semantics.md) * BDD scenarios: `features/rewrite-semantics.feature` -* BDD steps: `features/steps/test-rewrite-semantics.py` +* BDD steps: `features/steps/test_rewrite_semantics.py` ## Corner case: Dominated, overlapping replacements diff --git a/docs/user/features/type-parameter-modernization.md b/docs/user/features/type-parameter-modernization.md new file mode 100644 index 000000000..990c7a7d8 --- /dev/null +++ b/docs/user/features/type-parameter-modernization.md @@ -0,0 +1,357 @@ +# Type parameter modernization { #feature-type-parameter-modernization } + +**Stable ID:** `FEATURE-TYPE-PARAMETER-MODERNIZATION` + +## User-facing summary + +Rewrites legacy `TypeVar`, `ParamSpec` and `TypeVarTuple` declarations to [PEP 695](https://peps.python.org/pep-0695/) +syntax in one command. `ruff`'s `UP047` only does part of this, as an unsafe fix: it skips type parameters imported +from another module, `default=` and nested functions, and never removes the old declaration. + +```python +# before +T = TypeVar("T") + +def first(items: list[T]) -> T: ... + +# after +def first[T](items: list[T]) -> T: ... +``` + +The tool runs three phases: + +1. **Localize cross-file imports.** A type parameter imported from another file in the project + (`from .shapes import T`) is replaced by an exact local copy of its declaration, placed after the file's imports. +2. **Convert to PEP 695.** Every function whose signature uses a declared type parameter gets its own `[T]`, with + bounds, constraints and `default=` carried over (`def f[T: str = str](...)`). A nested function gets none when an + enclosing function's signature uses `T` too. If `T` is also used outside functions (a `Generic[T]` base, an alias), + the declaration is kept. A function that also uses a type parameter that can't be + converted is left unchanged. +3. **Remove orphaned declarations.** A module-level `T = TypeVar("T")` that nothing uses is deleted, including one left + behind by `ruff`'s `UP047` or by a manual rewrite. + +## Inputs + +A Python file or directory, and the target project's minimum supported Python version (`--py`). Supported +declarations: `TypeVar` (with `bound=`, constraints or `default=`), `ParamSpec` and `TypeVarTuple` (with `default=`). + +Imports are resolved against the directory passed to the CLI (the file's own folder for a single file). These are +skipped without a report: + +- imports from outside that directory (stdlib, third-party); +- re-exports through an intermediate `__init__.py`, and namespace packages; +- a renamed import (`from .shapes import T as U`); a file whose only type parameters arrive that way counts as clean. + +### Generic classes { #feature-type-parameter-modernization-generic-class } + +A class is *generic over* `T` when it declares `T` itself (`class Box[T]:`) or has `T` in a base (`Generic[T]`, +`Protocol[T]`, `Base[int, T]`). Inside such a class, `T` belongs to the class, so the tool never adds `[T]` to its +methods. With `class Box[T]:` the methods don't use the module-level `T`, so a declaration nothing else uses is removed: + +```python +# before +T = TypeVar("T") + +class Box[T]: + def get(self) -> T: ... + +# after +class Box[T]: + def get(self) -> T: ... +``` + +## Outputs / effects + +- Every safe change is written to the file in place. +- Every candidate left unchanged for one of the reasons under *Constraints* is listed in the report as **needs manual + review**, with a link to the section that explains it. +- A file listed under `ERRORS` may already be changed: each phase is written before the next one runs, so a failure + keeps the earlier phases. Check it with `git diff`. + +### Known limitations + +None of these is a *Constraint*: the tool applies the change. Review the result with `git diff`. Most of them have an +example under *Non-goals* or *Need to fix* in [Refactoring recipes](../../developer/modules/recipes.md#non-goals). + +**Can leave code that fails or changes meaning:** + +- A decorator or a parameter default that reads `T` (`def f(x: list[T], default=T)`) is not counted as a use, so the + declaration can be removed and importing the module raises `NameError`. +- A declaration constructed from `from typing import *` is localized without importing `TypeVar`: `NameError`. +- A declaration followed by `;` on the same line is removed without its `;`, leaving invalid syntax. The file is + reported under `ERRORS`. +- A declaration that breaks the naming rule of PEP 484 (`T = U = TypeVar("T")`, `V = TypeVar("W")`, + `TypeVar(name)`) gets a wrong rewrite or is skipped with an error. +- `U = TypeVar("U", default=T)` becomes `def f[U = T]`, where `T` still means the module-level one. Fix it by hand as + `def f[T, U = T]`. +- Only `__all__` marks a declaration as public. One that nothing in the project uses is removed, even if other + projects import it. +- Only a plain `__all__ = [...]` is read. Annotated (`__all__: list[str] = ...`), `+=`, `.extend()` and `.append()` + forms are not. +- A `T` used only in the string value of a `TypeAlias` (`A: TypeAlias = "list[T]"`) is seen as unused, so its + declaration is removed. +- A function that uses a legacy type parameter that stays imported (e.g. from another package) still gets `[...]` + for the file's own declarations. It then mixes both syntaxes, which type checkers reject. +- A localized `T` is a new object: code comparing type parameters by identity (`is`) sees two different objects. + A file importing `T` from a localized file gets that copy too. + +**Cosmetic or operational:** + +- A declaration whose importers are all localized in a run is only removed by a second run. +- Removing a line can leave extra blank lines. Run your formatter afterwards. +- With `--no-ruff`, the imports added by localizing are not merged into the file's existing `from` imports and not + sorted. +- A modified CRLF file is written back with LF. +- The comment lines above a localized declaration are not copied. The comment trailing its line, such as + `# noqa: PLC0105`, is. +- `Unpack[Ts]` is left as it is. Run `ruff`'s `UP044` to rewrite it to `*Ts` (see + [Rejected recipes](../../developer/modules/rejected-recipes.md)). +- There is no filtering of files, so a large project can take tens of minutes. + +## Constraints + +Each case below stops one change and is reported as **unsafe** under *needs manual review*, with a link to its section. + +### Python version gates { #feature-type-parameter-modernization-version-gates } + +PEP 695 syntax is a `SyntaxError` on older Pythons, so the tool only writes it when `--py` (the oldest version the +target supports, not the one running the tool) is new enough. The version is never guessed: unknown counts as too +old. Removing a declaration that is already unused needs no new syntax, so it never depends on `--py`. + +| Syntax | PEP | Minimum Python | +| --------------------- | -------------------------------------------- | -------------- | +| `def f[T](...)` | [PEP 695](https://peps.python.org/pep-0695/) | 3.12 | +| `def f[T = int](...)` | [PEP 696](https://peps.python.org/pep-0696/) | 3.13 | + +#### PEP 695 version gate { #feature-type-parameter-modernization-pep695-version-gate } + +Below 3.12 nothing is converted. Localization and orphan removal still run. + +```python +T = TypeVar("T") + +def first(items: list[T]) -> T: ... +``` + +With `--py 3.11` the file is unchanged. With `--py 3.12` it becomes `def first[T](items: list[T]) -> T: ...` and the +declaration is removed. + +**To fix this yourself:** if the project supports 3.12+, run again with `--py 3.12`. Otherwise keep the old syntax. + +#### PEP 696 version gate { #feature-type-parameter-modernization-pep696-version-gate } + +Below 3.13 a declaration with `default=` is not converted, since dropping the default would change what type +checkers infer. Other declarations in the file are still converted, except in functions that also use it (see +*A function also uses a type variable that can't be converted* below). + +```python +T = TypeVar("T", default=int) + +def first(items: list[T]) -> T: ... +``` + +With `--py 3.12` the file is unchanged. With `--py 3.13` it becomes `def first[T = int](items: list[T]) -> T: ...`. + +**To fix this yourself:** run again with `--py 3.13` if the project supports it. Otherwise keep the declaration. + +### A declaration passes an argument with no PEP 695 equivalent { #feature-type-parameter-modernization-no-pep695-equivalent } + +The `[...]` syntax cannot write `covariant=`/`contravariant=`, a `bound=` on a `ParamSpec` or `TypeVarTuple`, a +`**mapping` of options, or a keyword added by a later Python version. (`infer_variance=True` is fine: a PEP 695 +type parameter always infers its variance.) + +```python +T_co = TypeVar("T_co", covariant=True) + +def first(items: Sequence[T_co]) -> T_co: ... +``` + +The file is unchanged. + +**To fix this yourself:** variance only matters for generic classes. If only functions use the type parameter, +remove `covariant=`/`contravariant=` and run again. + +### A function also uses a type variable that can't be converted { #feature-type-parameter-modernization-shares-function-with-unconvertible } + +[PEP 695](https://peps.python.org/pep-0695/#compatibility-with-traditional-typevars) forbids a function from mixing +`[...]` type parameters with legacy ones of its own. A function that uses a +declaration refused for a version gate or an argument with no PEP 695 equivalent keeps all its type parameters +legacy. A legacy type parameter of an enclosing generic class is not the method's own, so it blocks nothing. + +```python +T_co = TypeVar("T_co", covariant=True) +Ts = TypeVarTuple("Ts") + +def run(func: Callable[[*Ts], T_co], *args: *Ts) -> T_co: ... +``` + +The file is unchanged. `T_co` is reported for its own reason. `Ts` is reported for this one, because every function +using it is left unchanged. Functions that use `Ts` without `T_co` are still converted. + +**To fix this yourself:** fix the refused declaration as its section describes, then run again. + +### A declared type parameter is exported via `__all__` { #feature-type-parameter-modernization-declared-type-parameter-exported } + +`T` is in its file's plain `__all__ = [...]`, so it is public API. The functions are converted, the declaration is kept. + +```python +__all__ = ["T", "first"] + +T = TypeVar("T") + +def first(items: list[T]) -> T: ... +``` + +becomes: + +```python +__all__ = ["T", "first"] + +T = TypeVar("T") + +def first[T](items: list[T]) -> T: ... +``` + +**To remove it yourself:** remove `T` from `__all__` (a breaking change for importers), then delete the declaration. + +### A declared type parameter is only used inside a generic class { #feature-type-parameter-modernization-used-in-generic-class } + +Only methods of a class with `T` in a base (see *Generic classes* above) use `T`, so there is nothing to +convert. Standalone functions that also use `T` are still converted. + +```python +T = TypeVar("T") + +class Box(Generic[T]): + def get(self) -> T: ... +``` + +The file is unchanged. + +**To fix this yourself:** `ruff`'s `UP046` rewrites `Generic[T]` to `class Box[T]:`. Run the tool again afterwards: +it removes the now unused `T = TypeVar("T")`. + +### A name the declaration uses can't be imported here { #feature-type-parameter-modernization-declaration-name-unavailable } + +Localizing copies the declaration and imports the names it uses (`bound=Shape`) from the module the origin gets them +from, or from the origin when it defines them. That fails when the origin only imports `Shape` under +`if TYPE_CHECKING:`, or when this file already has a different `Shape`. + +```python +# shapes.py +from typing import TYPE_CHECKING, TypeVar + +if TYPE_CHECKING: + from geometry import Shape + +T = TypeVar("T", bound="Shape") + +# main.py +from shapes import T + +def biggest(items: list[T]) -> T: ... +``` + +`main.py` is unchanged and keeps importing `T` from `shapes.py`. + +**To fix this yourself:** import the name the same way the origin does, or rename one of the two, then copy the +declaration by hand. + +### The origin imports `TypeVar` conditionally { #feature-type-parameter-modernization-origin-imports-constructor-conditionally } + +The origin imports `TypeVar` inside an `if` or `try` (for example `typing` or `typing_extensions` per Python +version). A copy would use whatever `TypeVar` this file has, which may fail on some versions. + +```python +# shapes.py +try: + from typing import TypeVar +except ImportError: + from typing_extensions import TypeVar + +T = TypeVar("T") + +# main.py +from shapes import T + +def first(items: list[T]) -> T: ... +``` + +`main.py` is unchanged and keeps importing `T` from `shapes.py`. + +**To fix this yourself:** add the same conditional import to this file, then copy the declaration by hand. + +### A declared type parameter is imported by another project file { #feature-type-parameter-modernization-imported-elsewhere-in-project } + +Another file in the target directory imports `T` from this module (`from mod import T` or `mod.T`). The functions +are converted, the declaration is kept. Only checked when a directory is passed; `import *` is not checked. + +```python +# shapes.py +from typing import TypeVar + +T = TypeVar("T") + +def first(items: list[T]) -> T: ... + +# other.py +import shapes + +def second(items: list[shapes.T]) -> shapes.T: ... +``` + +`first` becomes `def first[T](items: list[T]) -> T: ...`. `T = TypeVar("T")` stays, because `other.py` reads +`shapes.T`. + +**To remove it yourself:** find that import (`from import ` or `.`), remove it, and +run the tool again. + +## Related concepts + +- [Rewrite semantics](../concepts/rewrite-semantics.md) +- [Type parameters](../concepts/type-parameters.md) + +## Verified by test modules + +- `test/recipes/test_type_var_check.py` +- `test/recipes/test_type_var_check_convert.py` +- `test/recipes/test_type_var_check_localize.py` +- `test/recipes/test_type_var_check_orphaned.py` +- `test/recipes/test_type_var_check_properties.py` +- `test/recipes/test_type_var_domain.py` +- `test/recipes/test_step_runner.py` +- `test/recipes/test_python_refactoring.py` +- `test/utils/test_import_resolution.py` +- `test/rejuvenation/test_migration_type_recipes.py` + +## Implemented by code modules + +- [Refactoring recipes](../../developer/modules/recipes.md) +- `src/renaissance/recipes/type_var_check.py`, `type_var_domain.py`, `step_runner.py` and `python_refactoring.py`. + The recipe class is still named `TypeVarCheck`. +- `src/renaissance/utils/import_resolution.py` +- `src/rejuvenation/migration-type-recipes.py` (the CLI) + +## API entry points + +```shell +python src/rejuvenation/migration-type-recipes.py --py MAJOR.MINOR [--report PATH] [--no-ruff] +``` + +- ``: a `.py` file or a directory, scanned recursively (`.git`, `__pycache__`, `.venv` and `venv` skipped). +- `--py` (required): the oldest Python version the target project supports. +- `--report`: also write the report to a file. +- `--no-ruff`: skip the final `ruff` pass. + +After the recipe, the CLI runs `ruff check --fix --select F401,I001` on the changed files. This removes **every** +unused import in them, including ones that were unused before (such as `from typing import TypeVar` after the +declaration is gone). It also merges and sorts **every** import block in them, including imports that were unsorted +before, using the `ruff` settings of the project. `ruff` must be installed in the same environment; if it cannot run, +the report still says the pass ran. Files are changed directly, so run it on a git checkout. + +The report has a summary line and the sections `MODIFIED`, `NEEDS MANUAL REVIEW` and `ERRORS`. Exit codes: 0 when +the run completes (manual review is not a failure), 2 for a usage error, 3 if any file raised an exception. + +## Change considerations + +See *Extension points* in [Refactoring recipes](../../developer/modules/recipes.md). diff --git a/mkdocs.yml b/mkdocs.yml index 6111640d8..aad3fd72d 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -125,6 +125,7 @@ nav: - 8. Transformation: user/concepts/transformation.md - 9. Composition: user/concepts/composition.md - 10. Standard analyses and transformations: user/concepts/standard-libraries.md + - 11. Type parameters: user/concepts/type-parameters.md - Features: - Overview: user/features/index.md @@ -140,6 +141,7 @@ nav: - 9. Find: user/features/find.md - 10. Modify: user/features/modify.md - 11. Rewrite semantics: user/features/rewrite-semantics.md + - 12. Type parameter modernization: user/features/type-parameter-modernization.md - Workflows: - Overview: user/workflows/index.md @@ -207,6 +209,8 @@ nav: - Observability modules: developer/modules/observability.md - Strategy modules: developer/modules/strategy.md - Rewrite semantics module: developer/modules/rewrite-semantics.md + - Refactoring recipes: developer/modules/recipes.md + - Rejected recipes: developer/modules/rejected-recipes.md - API reference: - Overview: developer/api/index.md @@ -244,4 +248,4 @@ nav: - Linking policy enforcement: documentation-system/enforcement/linking-policy-enforcement.md - CI overview: documentation-system/enforcement/ci-overview.md - - Shared images: assets/images/README.md \ No newline at end of file + - Shared images: assets/images/README.md From dd3fe96f1ce7fb2c062d71ad7952a7c3f256bc30 Mon Sep 17 00:00:00 2001 From: Francesco Pezzella Date: Fri, 16 Oct 2026 13:56:15 +0200 Subject: [PATCH 3/5] Add missing copyright --- docs/developer/modules/recipes.md | 8 ++++++++ docs/developer/modules/rejected-recipes.md | 8 ++++++++ docs/user/concepts/type-parameters.md | 8 ++++++++ docs/user/features/type-parameter-modernization.md | 8 ++++++++ 4 files changed, 32 insertions(+) diff --git a/docs/developer/modules/recipes.md b/docs/developer/modules/recipes.md index 90f418b56..fbcc6386d 100644 --- a/docs/developer/modules/recipes.md +++ b/docs/developer/modules/recipes.md @@ -1,3 +1,11 @@ + + # Refactoring recipes { #codemod-recipes } diff --git a/docs/developer/modules/rejected-recipes.md b/docs/developer/modules/rejected-recipes.md index 5a3d2e337..c09511d75 100644 --- a/docs/developer/modules/rejected-recipes.md +++ b/docs/developer/modules/rejected-recipes.md @@ -1,3 +1,11 @@ + + # Rejected recipes { #codemod-rejected-recipes } diff --git a/docs/user/concepts/type-parameters.md b/docs/user/concepts/type-parameters.md index f71851486..1644902d5 100644 --- a/docs/user/concepts/type-parameters.md +++ b/docs/user/concepts/type-parameters.md @@ -1,3 +1,11 @@ + + # Type parameters { #concept-type-parameters } diff --git a/docs/user/features/type-parameter-modernization.md b/docs/user/features/type-parameter-modernization.md index 990c7a7d8..39f99affd 100644 --- a/docs/user/features/type-parameter-modernization.md +++ b/docs/user/features/type-parameter-modernization.md @@ -1,3 +1,11 @@ + + # Type parameter modernization { #feature-type-parameter-modernization } **Stable ID:** `FEATURE-TYPE-PARAMETER-MODERNIZATION` From 31e0764e7e56f4b57f9b6b0b2d5038116cde0275 Mon Sep 17 00:00:00 2001 From: Francesco Pezzella Date: Mon, 19 Oct 2026 11:11:18 +0200 Subject: [PATCH 4/5] Remove unused module added --- pyproject.toml | 2 -- uv.lock | 2 -- 2 files changed, 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index a554adc75..8e29481a6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,7 +37,6 @@ dependencies = [ "hypothesmith>=0.3.3", "autopep8>=2.0", "flake8>=7.0", - "packaging>=24.0", ] [dependency-groups] @@ -121,7 +120,6 @@ select = [ "ANN002", "ANN003", "ANN204", "ANN205", "ANN401", ] #select = ["ALL"] # goal of issue - ignore = [ "D203", # conflicts with D211 (no blank line before class) — keep D211 "D213", # conflicts with D212 (summary on the first line) — keep D212 diff --git a/uv.lock b/uv.lock index ccf223c94..5f1fd94e8 100644 --- a/uv.lock +++ b/uv.lock @@ -1165,7 +1165,6 @@ dependencies = [ { name = "libcst" }, { name = "more-itertools" }, { name = "networkx" }, - { name = "packaging" }, { name = "pyhamcrest" }, { name = "pyperclip" }, { name = "termcolor" }, @@ -1231,7 +1230,6 @@ requires-dist = [ { name = "libcst", specifier = ">=1.8.6" }, { name = "more-itertools", specifier = ">=10.0" }, { name = "networkx", specifier = ">=3.0" }, - { name = "packaging", specifier = ">=24.0" }, { name = "pyhamcrest", specifier = ">=2.1" }, { name = "pyperclip", specifier = ">=1.8" }, { name = "termcolor", specifier = ">=2.0" }, From 8190772bb756d5a9a4851da634778eadb176e2e6 Mon Sep 17 00:00:00 2001 From: Francesco Pezzella Date: Mon, 26 Oct 2026 13:10:18 +0100 Subject: [PATCH 5/5] Sync the type parameter modernization and recipes docs with the TypeParameterCheck documentation --- .../adr/15_camel_snake_case_conversion.md | 2 +- .../architecture/position-consistency.md | 7 +- docs/developer/feature-test-map/core.md | 23 +- docs/developer/modules/parser-and-ast.md | 7 +- docs/developer/modules/recipes.md | 158 +++++++++++--- docs/user/concepts/type-parameters.md | 1 - .../features/type-parameter-modernization.md | 198 +++++++++++------- 7 files changed, 279 insertions(+), 117 deletions(-) diff --git a/docs/developer/architecture/adr/15_camel_snake_case_conversion.md b/docs/developer/architecture/adr/15_camel_snake_case_conversion.md index 883caf22a..b908827ca 100644 --- a/docs/developer/architecture/adr/15_camel_snake_case_conversion.md +++ b/docs/developer/architecture/adr/15_camel_snake_case_conversion.md @@ -55,7 +55,7 @@ review thread, to spell `Pyunit` out fully as well (`TautToPythonUnittest`). ## Implementation notes - `src/renaissance/utils/text_utils.py`: `snake_case()` (Rule A), `camel_case()` removed. -- `src/renaissance/recipes/python_refactoring.py:39`: `PythonRefactoring.process()` resolves +- `src/renaissance/recipes/python_refactoring.py`: `PythonRefactoring.process()` resolves `class_name` -> `renaissance.recipes.`; names following the guideline above resolve correctly by construction (`snake_case("TautToPythonUnittest")` -> `taut_to_python_unittest`). diff --git a/docs/developer/architecture/position-consistency.md b/docs/developer/architecture/position-consistency.md index d93a4999f..bf934ef6e 100644 --- a/docs/developer/architecture/position-consistency.md +++ b/docs/developer/architecture/position-consistency.md @@ -159,9 +159,10 @@ the two sides can silently disagree, producing incorrect or corrupted output. * `src/renaissance/syntax_tree/ast_rewriter.py` - derives the buffer a rewrite batch is applied to. * `src/renaissance/common/rewriter.py` - splices replacement text into that buffer, in bytes. * `src/renaissance/syntax_tree/ast_node.py` - `binary_file_content`, the root-owned buffer accessor. -* `src/renaissance/integrations/python/ast/rst_node.py`, `src/renaissance/integrations/python/ast/cst_node.py` - and `src/renaissance/integrations/python/ast/util.py` - the Python adapters and the line/column to - offset conversion they share. +* `src/renaissance/integrations/python/ast/rst_node.py` and `src/renaissance/integrations/python/ast/util.py` - + the RST adapter and its line/column to offset conversion. +* `src/renaissance/integrations/python/ast/cst_node.py` - the CST adapter, which converts libcst's line/column + positions to byte offsets itself. * `src/renaissance/integrations/tree_sitter/lst.py` - the tree-sitter adapter. * `src/renaissance/integrations/clang/clang_ast_node.py` and `src/renaissance/integrations/clang/clang_json_ast_node.py` - the clang adapters. diff --git a/docs/developer/feature-test-map/core.md b/docs/developer/feature-test-map/core.md index be229dbb7..427db3487 100644 --- a/docs/developer/feature-test-map/core.md +++ b/docs/developer/feature-test-map/core.md @@ -41,17 +41,24 @@ https://github.com/TNO/Renaissance.Py/blob/main/LICENSE - **Concepts:** [Rewrite semantics](../../user/concepts/rewrite-semantics.md) - **Code modules:** [Refactoring recipes](../../developer/modules/recipes.md) - **Test file(s):** - - `test/recipes/test_type_var_check.py` - - `test/recipes/test_type_var_check_convert.py` - - `test/recipes/test_type_var_check_localize.py` - - `test/recipes/test_type_var_check_orphaned.py` - - `test/recipes/test_type_var_check_properties.py` - - `test/recipes/test_type_var_domain.py` + - `test/recipes/test_type_parameter_check.py` + - `test/recipes/test_type_parameter_check_convert.py` + - `test/recipes/test_type_parameter_check_localize.py` + - `test/recipes/test_type_parameter_check_orphaned.py` + - `test/recipes/test_type_parameter_check_properties.py` + - `test/recipes/test_type_parameter_check_variance.py` + - `test/recipes/test_type_parameter_declarations.py` + - `test/recipes/test_type_parameter_localization.py` + - `test/recipes/test_type_parameter_rules.py` - `test/recipes/test_step_runner.py` - `test/recipes/test_python_refactoring.py` + - `test/utils/test_cst_utils.py` - `test/utils/test_import_resolution.py` - `test/rejuvenation/test_migration_type_recipes.py` -- **Code file(s):** `src/renaissance/recipes/type_var_check.py`, `src/renaissance/recipes/type_var_domain.py`, +- **Code file(s):** `src/renaissance/recipes/type_parameter_check.py`, `src/renaissance/recipes/type_parameter_rules.py`, + `src/renaissance/recipes/type_parameter_declarations.py`, `src/renaissance/recipes/type_parameter_localization.py`, + `src/renaissance/recipes/type_parameter_uses.py`, `src/renaissance/recipes/step_runner.py`, `src/renaissance/recipes/python_refactoring.py`, - `src/renaissance/utils/import_resolution.py`, `src/rejuvenation/migration-type-recipes.py` + `src/renaissance/utils/import_resolution.py`, `src/renaissance/utils/cst_utils.py`, + `src/rejuvenation/migration-type-recipes.py` diff --git a/docs/developer/modules/parser-and-ast.md b/docs/developer/modules/parser-and-ast.md index 5bc9acf75..d6f1a19f1 100644 --- a/docs/developer/modules/parser-and-ast.md +++ b/docs/developer/modules/parser-and-ast.md @@ -107,11 +107,16 @@ preserves comments and whitespace, or reparses source. Each adapter supplies those behaviors individually. Reference and data-flow collections (`references`, `referenced_by`) are also parser-specific because not every parser can provide the same analysis. Keep the original parser node available -when callers need parser-specific data. +when callers need parser-specific data. The Python CST adapter resolves them +with libcst's scope analysis, which also finds names used inside string +annotations. An annotation or type hint written above a definition in the +same scope still refers to it, since it is evaluated lazily. The native Python `ast.AST` adapter is a lightweight structural adapter and does not preserve source trivia in the same way as the lossless Python RST/CST adapters. Use the lossless adapter when source-preserving rewriting is needed. +`PythonRefactoring` parses with the RST adapter by default; pass +`node_class=PythonCstNode` to use the CST adapter instead. ## Classify nodes diff --git a/docs/developer/modules/recipes.md b/docs/developer/modules/recipes.md index fbcc6386d..da5b731ae 100644 --- a/docs/developer/modules/recipes.md +++ b/docs/developer/modules/recipes.md @@ -15,22 +15,29 @@ https://github.com/TNO/Renaissance.Py/blob/main/LICENSE ## Responsibility Recipes are `PythonRefactoring` subclasses that rewrite one Python file at a time, for gaps `ruff` does not fully or -safely cover. This page covers `TypeVarCheck`, the recipe behind +safely cover. This page covers `TypeParameterCheck`, the recipe behind [Type parameter modernization](../../user/features/type-parameter-modernization.md). ## Location -- `src/renaissance/recipes/type_var_check.py`: the `TypeVarCheck` pipeline (orchestration only). -- `src/renaissance/recipes/type_var_domain.py`: domain model and safety analysis. +- `src/renaissance/recipes/type_parameter_check.py`: the `TypeParameterCheck` pipeline and the edits its phases write. +- `src/renaissance/recipes/type_parameter_rules.py`: the reasons a type parameter is left unconverted, and the refusal rules. +- `src/renaissance/recipes/type_parameter_declarations.py`: reads and rewrites a declaration (`T = TypeVar(...)`). +- `src/renaissance/recipes/type_parameter_localization.py`: decides which imported type parameters get a local + declaration, and the imports and text that replace them. +- `src/renaissance/recipes/type_parameter_uses.py`: finds where a declared name is used. - `src/renaissance/recipes/step_runner.py`: `Step`/`run_steps`, runs fix actions in order and commits only the ones that fixed something. - `src/renaissance/recipes/python_refactoring.py`: the base class. -- `src/renaissance/utils/import_resolution.py`: resolves `from X import Y` to a file in the project, re-anchors one - file's relative import on another file, and collects the names each project file imports from the others. +- `src/renaissance/utils/import_resolution.py`: the import statements of a module and what they bind, the resolution of + `from X import Y` to a file in the project, the re-anchoring of one file's relative import on another file, and the + names each project file imports from the others. +- `src/renaissance/utils/cst_utils.py`: a module's top-level statements, the names they read or bind and the comment + trailing a statement's line. ## Public entry points -- `TypeVarCheck.check()` (and `run()`): runs the three phases through `run_steps`, committing to disk between them. +- `TypeParameterCheck.check()` (and `run()`): runs the three phases through `run_steps`, committing to disk between them. - The phases, each returning `{name: "fixed" | "unsafe"}` and recording the `UnsafeReason` of each unsafe name: | Phase | Reasons attribute | @@ -39,10 +46,14 @@ safely cover. This page covers `TypeVarCheck`, the recipe behind | `convert_declared_typevars()` (only adds type parameters) | `converted_unsafe_reasons` | | `remove_orphaned_declarations()` (only removes declarations) | `orphaned_unsafe_reasons` | +- `type_param_kinds`: maps each type parameter name the phases see to `"TypeVar"`, `"ParamSpec"` or `"TypeVarTuple"`. - Attributes set by the CLI (`migration-type-recipes.py`): `min_python`, `project_root` (falls back to the file's - folder) and `project_wide_imported_names` (from `collect_project_imported_names`). -- `PythonRefactoring.process("TypeVarCheck", file)`, used by `cli.py refactor`, never sets those attributes, so every - version-gated rewrite is reported `"unsafe"` there. + folder), `project_wide_imported_names` (from `collect_project_imported_names`), `ignore_localize_origins` (absolute + paths of the files `--ignore` names) and `localize_imports` (`False` with `--no-localization`). `ignored_names` holds + the type parameter names this file imports from those files, or from any project file when `localize_imports` is + `False`. +- `PythonRefactoring.process("TypeParameterCheck", file)`, used by `cli.py refactor`, never sets those attributes, so + `min_python` stays `MIN_SUPPORTED_PYTHON` there and a `default=` declaration is reported `"unsafe"`. ## Internal structure @@ -50,13 +61,23 @@ safely cover. This page covers `TypeVarCheck`, the recipe behind same libcst tree it rewrites; the cross-file phase parses the origin with `libcst.parse_module`. Each edit replaces only the node it changes, so formatting and comments elsewhere are kept. -- **Unsafe reasons.** `is_safe_to_remove`/`is_safe_to_localize` and `_conversion_refusal` return an `UnsafeReason` - or `None`. Each of the 9 reasons has an `UnsafeRule` (message + docs anchor) in `UNSAFE_RULES`; `doc_link()` turns - it into the URL that the CLI report prints. +- **Unsafe reasons.** `is_safe_to_remove`, `is_safe_to_localize` and `conversion_refusal` return an `UnsafeReason` + or `None`. Each of the 8 reasons has an `UnsafeRule` (message + docs anchor) in `UNSAFE_RULES`. The CLI report prints + each message and links `DOCS_BASE_URL` once. -- **Version gates.** `_conversion_refusal` checks `min_python` against `PEP_695_MINIMUM = (3, 12)` and, for a - `default=`, `PEP_696_MINIMUM = (3, 13)`. `None` never passes. A keyword outside `_CONVERTIBLE_KEYWORDS` is refused - (`NO_PEP695_EQUIVALENT`) instead of dropped. +- **Refusal rules.** Five of the reasons are independent checks, listed as `RefusalRule` entries in + `CONVERSION_RULES`, `REMOVAL_RULES` and `LOCALIZATION_RULES`; the first rule of a stage that refuses decides the + reason. The other three (`USED_IN_GENERIC_CLASS`, `SHARES_FUNCTION_WITH_UNCONVERTIBLE`, `NAME_NOT_AT_MODULE_LEVEL`) + follow from how the phases work and are not listed. Commenting out an entry switches its rule off. + `NO_PEP695_EQUIVALENT`, `PEP696_VERSION_GATE` and `ORIGIN_IMPORTS_CONSTRUCTOR_CONDITIONALLY` then let the recipe write + code that fails or binds another object; `DECLARED_TYPEVAR_EXPORTED` and `IMPORTED_ELSEWHERE_IN_PROJECT` then let it + break the files that import the name. + +- **Version gates.** The CLI rejects `--py` below `MIN_SUPPORTED_PYTHON = (3, 12)`, so PEP 695 syntax is always + available there; the recipe itself does not check it. `conversion_refusal` compares `min_python` with + `PEP_696_MINIMUM = (3, 13)` for a `default=`. A keyword outside `_CONVERTIBLE_KEYWORDS` is refused + (`NO_PEP695_EQUIVALENT`) instead of dropped. `covariant` and `contravariant` are in the set and are dropped, because + a function's type parameter has no variance; converting classes would have to refuse them. - **Uses.** `uses_of` takes a declaration's uses from libcst's scope analysis (`referenced_by`). A use inside a string annotation or `cast("T", x)` counts; a `T` that a PEP 695 type parameter of an enclosing function or class @@ -78,14 +99,14 @@ safely cover. This page covers `TypeVarCheck`, the recipe behind - **Generic classes.** Methods of a class with the name in a base (`generic_classes`) are skipped. A generic class at the origin does not block localization. -- **No mixed syntax.** A function using a declaration that `_conversion_refusal` rejects gets none of its type +- **No mixed syntax.** A function using a declaration that `conversion_refusal` rejects gets none of its type parameters, since PEP 695 forbids mixing them with legacy ones. A name whose every function is blocked is `SHARES_FUNCTION_WITH_UNCONVERTIBLE`. This needs two passes: all refusals are known before any parameter is added. - **Orphans.** A declaration with no uses left (`uses_of` is empty) is orphaned. Conversion skips such names too. - **Localization.** The origin's declaration is copied as written, with the comment trailing it when it is alone on - its line (`declaration_text`). `DECLARATION_NAME_UNAVAILABLE` if the origin only binds a name its arguments use + its line (`declaration_text`). `NAME_NOT_AT_MODULE_LEVEL` if the origin only binds a name its arguments use (`declaration_argument_names`) under `if TYPE_CHECKING:` or this file binds it to something else (`from_import_sources`). All names localized from one import statement are handled together. @@ -101,6 +122,9 @@ safely cover. This page covers `TypeVarCheck`, the recipe behind a statement between it and the last import reads one of its names. Merging and sorting the added imports, and removing imports that became unused, is left to the CLI's `ruff check --fix --select F401,I001` pass. +- **Line endings.** Files are read with universal newlines and a modified file is written back with LF, also on the + lines the recipe did not change. A file the recipe does not modify is not rewritten. + ## Related features - [Type parameter modernization](../../user/features/type-parameter-modernization.md) @@ -118,11 +142,15 @@ See *Verified by test modules* in [Type parameter modernization](../../user/feat - A new recipe is a `PythonRefactoring` subclass in its own `snake_case`-named module in `src/renaissance/recipes/`; no registration is needed. -- A new type-parameter-declaring construct: extend `_is_type_param_call` and `type_param_text` together. +- A new type-parameter-declaring construct: extend `_is_type_param_call` and `type_param_text` together, both in + `type_parameter_declarations.py`. - A keyword added by a later Python, or future PEP 695 syntax for variance or a bound on `**P`/`*Ts`: add it to `_CONVERTIBLE_KEYWORDS` and carry it over in `type_param_text`. +- A new refusal rule: add a check function and a `RefusalRule` entry to the stage's tuple in `type_parameter_rules.py`, + a message and anchor to `UNSAFE_RULES`, and a section to the feature page. - A new version gate uses the syntax's own minimum Python from its PEP, not a value copied from another gate. -- Re-exports through `__init__.py` or namespace packages (PEP 420): extend `resolve_project_module`. +- Re-exports through `__init__.py`, and a namespace package (PEP 420) as an import's origin: extend + `resolve_project_module`. - `Step`/`run_steps` is generic and can be reused by any recipe. ## Non-goals @@ -136,12 +164,15 @@ See *Verified by test modules* in [Type parameter modernization](../../user/feat ```toml [project] - requires-python = ">=3.12" + requires-python = ">=3.13" ``` - run with `--py 3.11` is treated as below 3.12, so nothing is converted. + run with `--py 3.12` is treated as below 3.13, so a `default=` declaration is not converted. -- Re-exports, namespace packages, `import *` and renamed imports are not followed. In: +- Only the file an import names is read. A name it re-exports from another module (`pkg/__init__.py` with + `from .mod import T`) is not followed, so the result depends on whether that file was already localized earlier in + the run. Renamed imports and `import *` are skipped, and so is a namespace package (a directory without + `__init__.py`) as the origin; the modules inside it are followed. In: ```python # origin.py @@ -157,6 +188,22 @@ See *Verified by test modules* in [Type parameter modernization](../../user/feat `main.py` is left unchanged. +- A file that imports a declaration with `from origin import *` does not count as an importer, so the declaration is + removed from its origin once nothing else in the origin uses it. In: + + ```python + # origin.py + T = TypeVar("T") + + def identity(x: T) -> T: ... + + # main.py + from origin import * + ``` + + `T = TypeVar("T")` is removed from `origin.py`, and `main.py` raises `NameError` where it uses `T`. The run exits + with 0 and gives no warning. + - Only declarations in the file itself can block a function. A legacy type parameter that stays imported, such as one from another package, does not. In: @@ -174,7 +221,7 @@ See *Verified by test modules* in [Type parameter modernization](../../user/feat report an error. - Imported names are collected once before the run, so removing a declaration whose importers were all localized - needs a second run. With `origin.py` as above and: + needs a second run, which `--second-pass` does for the files concerned. With `origin.py` as above and: ```python # main.py @@ -186,10 +233,11 @@ See *Verified by test modules* in [Type parameter modernization](../../user/feat the first run turns `main.py` into `def first[T](items: list[T]) -> T: ...`, but keeps `T` in `origin.py`, since `main.py` imported it when the run started. A second run removes it. -- Only a plain `__all__ = [...]` is read to detect an exported declaration. In: +- `__all__.extend(...)` and `__all__.append(...)` are not read to detect an exported declaration. In: ```python - __all__: list[str] = ["T", "first"] + __all__ = ["first"] + __all__.extend(["T"]) T = TypeVar("T") @@ -198,8 +246,9 @@ See *Verified by test modules* in [Type parameter modernization](../../user/feat `first` is converted and `T = TypeVar("T")` is removed, although `__all__` exports it. -- A declaration is only treated as public API when `__all__` lists it. One that nothing in the project uses is - removed, even if other projects import it. In: +- A declaration is only treated as public API when `__all__` lists it, since the tool sees nothing outside the scanned + project. One that nothing in the project uses is removed, even if other projects import it. `--ignore` leaves + such a file untouched. In: ```python # mylib/typing.py @@ -277,6 +326,9 @@ See *Verified by test modules* in [Type parameter modernization](../../user/feat X = TypeVar(name) # AttributeError: no phase is applied to the file ``` + If both `T` and `U` are unused, the run fails with a `ValueError` ("Conflicting rewrites") reported under errors, + and the declaration stays. + - A declaration whose constructor comes from `from typing import *` is localized without importing it. This is rare, and `ruff`'s `F403` flags the star import. In: @@ -305,6 +357,31 @@ See *Verified by test modules* in [Type parameter modernization](../../user/feat becomes `def first[U = T](items: list[U]) -> U: ...`, whose default still refers to the module-level `T`. +- A constructor written with its module, such as `typing.TypeVar`, is not recognised. This is rare. In: + + ```python + T = typing.TypeVar("T") + + def first(items: list[T]) -> T: ... + ``` + + `first` is left unchanged and the file is reported clean. + +- Absolute imports are resolved from the given path, so in a `src/` layout run on the repository root, + `from pkg.typing import T` does not find `src/pkg/typing.py`. This is rare. In: + + ```python + # src/pkg/typing.py + T = TypeVar("T") + + # src/pkg/other.py + from pkg.typing import T + + def first(items: list[T]) -> T: ... + ``` + + `T = TypeVar("T")` is removed and `other.py` fails with `ImportError`. + ## Need to fix Errors that should be fixed, not accepted. @@ -327,7 +404,28 @@ Errors that should be fixed, not accepted. `main.py` gets `T = TypeVar("T") # noqa: PLC0105` without the comment above it. The origin keeps its comment when it drops the declaration. -- Line endings are not preserved: a modified CRLF file is written back with LF. This causes no problem when the - repository stores LF and git converts on checkout, the usual case, since git commits the same content. A - repository that stores CRLF gets a diff on every line of a modified file, so a pull request may be rejected +- An existing type-parameter bracket on a converted function is rebuilt on one line, so its comments and line breaks + are lost ([issue 279](https://github.com/TNO/Renaissance.Py/issues/279)). In: + + ```python + T = TypeVar("T") + + def f[ + A, # first + B, + ](x: A, y: B, z: T) -> T: ... + ``` + + `f` becomes `def f[A, B, T](x: A, y: B, z: T) -> T: ...`. + +- A localized declaration whose constructor is already imported from another module, such as `TypeVar` from `typing` + in this file and from `typing_extensions` in the origin, adds a second import of the name that shadows the first ([issue 279](https://github.com/TNO/Renaissance.Py/issues/279)). + +- A file that starts with a UTF-8 byte order mark is rewritten with invalid syntax and reported under `ERRORS`: a + converted `def identity(x: T) -> T:` becomes `def ideidentity[T] T) -> T:`. The file should keep its encoding and + its byte order mark ([issue 153](https://github.com/TNO/Renaissance.Py/issues/153)). + +- A file that is not valid UTF-8 stops the whole run with an uncaught `UnicodeDecodeError`, before any file is + changed. The file should be reported under `ERRORS` and the others processed + ([issue 188](https://github.com/TNO/Renaissance.Py/issues/188)). diff --git a/docs/user/concepts/type-parameters.md b/docs/user/concepts/type-parameters.md index 1644902d5..b4d029be4 100644 --- a/docs/user/concepts/type-parameters.md +++ b/docs/user/concepts/type-parameters.md @@ -69,4 +69,3 @@ Three kinds of type parameter exist: - A PEP 695 type parameter is also a `TypeVar`, `TypeVarTuple` or `ParamSpec` object at runtime (see `__type_params__`). This is why "TypeVar" is often used for all three kinds. - The documentation says *type parameter* for the general case and *type variable* only for the single-type kind. -- The recipe class is still named `TypeVarCheck`. diff --git a/docs/user/features/type-parameter-modernization.md b/docs/user/features/type-parameter-modernization.md index 39f99affd..67ee433eb 100644 --- a/docs/user/features/type-parameter-modernization.md +++ b/docs/user/features/type-parameter-modernization.md @@ -29,9 +29,11 @@ def first[T](items: list[T]) -> T: ... The tool runs three phases: 1. **Localize cross-file imports.** A type parameter imported from another file in the project - (`from .shapes import T`) is replaced by an exact local copy of its declaration, placed after the file's imports. + (`from .shapes import T`) is replaced by an exact local copy of its declaration, placed after the file's imports, + unless the origin is named by `--ignore` (see *Shared declaration files* below) or `--no-localization` is given. 2. **Convert to PEP 695.** Every function whose signature uses a declared type parameter gets its own `[T]`, with - bounds, constraints and `default=` carried over (`def f[T: str = str](...)`). A nested function gets none when an + bounds, constraints and `default=` carried over (`def f[T: str = str](...)`). `covariant=` and `contravariant=` are + dropped, since only generic classes have a variance. A nested function gets none when an enclosing function's signature uses `T` too. If `T` is also used outside functions (a `Generic[T]` base, an alias), the declaration is kept. A function that also uses a type parameter that can't be converted is left unchanged. @@ -47,9 +49,40 @@ Imports are resolved against the directory passed to the CLI (the file's own fol skipped without a report: - imports from outside that directory (stdlib, third-party); -- re-exports through an intermediate `__init__.py`, and namespace packages; +- re-exports through an intermediate `__init__.py`, unless that file was already changed earlier in the run, so the + result depends on the order of the files; +- a namespace package itself as the origin of an import (the modules inside it are not skipped); - a renamed import (`from .shapes import T as U`); a file whose only type parameters arrive that way counts as clean. +### Shared declaration files { #feature-type-parameter-modernization-ignore } + +`--ignore FILE...` names files, relative to `` as the report lists them, that are left untouched: no phase runs +on them, and their type parameters are never copied into the files importing them. Those files keep their imports, so +the functions using the imported names are not converted. Use it for a file that other projects may import, such as a +shared `types.py`. + +```shell +python src/rejuvenation/migration-type-recipes.py ./some_repo --py 3.12 --ignore pkg/types.py pkg/aliases.py +``` + +Give `` first, since the option takes every value after it. A file that does not exist is a usage error. The +report lists the files under `IGNORED FILES` and adds an `Ignored` column to the table, counting each name imported +from an ignored file once per importing file. The ignored files are not in the processed count. + +### Disabling localization { #feature-type-parameter-modernization-no-localization } + +Converting a function and removing an unused declaration only improve the file. Localizing does not always: the copy is a +second declaration, and a localized `T` is a different object from the original. `--no-localization` turns the first +phase off for the whole project. Every project import of a type parameter stays, so the functions using the imported +names are not converted. The other phases run as usual, on every file. + +```shell +python src/rejuvenation/migration-type-recipes.py ./some_repo --py 3.12 --no-localization +``` + +The report adds the `Ignored` column, counting each imported name once per importing file. `--ignore` differs in that +it leaves chosen files unprocessed, so their declarations are never removed. + ### Generic classes { #feature-type-parameter-modernization-generic-class } A class is *generic over* `T` when it declares `T` itself (`class Box[T]:`) or has `T` in a base (`Generic[T]`, @@ -72,7 +105,7 @@ class Box[T]: - Every safe change is written to the file in place. - Every candidate left unchanged for one of the reasons under *Constraints* is listed in the report as **needs manual - review**, with a link to the section that explains it. + review**, with its reason. The report links this page once, at the top of that section. - A file listed under `ERRORS` may already be changed: each phase is written before the next one runs, so a failure keeps the earlier phases. Check it with `git diff`. @@ -86,16 +119,19 @@ example under *Non-goals* or *Need to fix* in [Refactoring recipes](../../develo - A decorator or a parameter default that reads `T` (`def f(x: list[T], default=T)`) is not counted as a use, so the declaration can be removed and importing the module raises `NameError`. - A declaration constructed from `from typing import *` is localized without importing `TypeVar`: `NameError`. +- A file that imports a declaration with `from module import *` is not seen as an importer. The declaration is removed + from `module` once nothing in it uses it, and the importing file raises `NameError`. No warning is given. +- A file that starts with a UTF-8 byte order mark is rewritten with invalid syntax and reported under `ERRORS`. - A declaration followed by `;` on the same line is removed without its `;`, leaving invalid syntax. The file is reported under `ERRORS`. - A declaration that breaks the naming rule of PEP 484 (`T = U = TypeVar("T")`, `V = TypeVar("W")`, `TypeVar(name)`) gets a wrong rewrite or is skipped with an error. - `U = TypeVar("U", default=T)` becomes `def f[U = T]`, where `T` still means the module-level one. Fix it by hand as `def f[T, U = T]`. -- Only `__all__` marks a declaration as public. One that nothing in the project uses is removed, even if other - projects import it. -- Only a plain `__all__ = [...]` is read. Annotated (`__all__: list[str] = ...`), `+=`, `.extend()` and `.append()` - forms are not. +- Only `__all__` marks a declaration as public, since the tool sees nothing outside the scanned project. One that + nothing in the project uses is removed, even if other projects import it, as with a shared `types.py`. Name such a + file with `--ignore` (see *Shared declaration files*). +- `__all__.extend(...)` and `__all__.append(...)` are not read. - A `T` used only in the string value of a `TypeAlias` (`A: TypeAlias = "list[T]"`) is seen as unused, so its declaration is removed. - A function that uses a legacy type parameter that stays imported (e.g. from another package) still gets `[...]` @@ -105,47 +141,37 @@ example under *Non-goals* or *Need to fix* in [Refactoring recipes](../../develo **Cosmetic or operational:** -- A declaration whose importers are all localized in a run is only removed by a second run. +- A declaration whose importers are all localized in a run is only removed by a second run, or by `--second-pass`. + A chain of three files, each importing from the previous one, can still need another run. - Removing a line can leave extra blank lines. Run your formatter afterwards. - With `--no-ruff`, the imports added by localizing are not merged into the file's existing `from` imports and not sorted. -- A modified CRLF file is written back with LF. -- The comment lines above a localized declaration are not copied. The comment trailing its line, such as - `# noqa: PLC0105`, is. +- A modified CRLF file is written back with LF, so `git diff` shows all its lines changed in a repository that stores CRLF. +- The comment lines above a localized declaration are not copied, and the ones above a removed declaration stay. The + comment trailing a declaration's line, such as `# noqa: PLC0105`, is copied or removed with it. +- An existing type-parameter bracket on a converted function is rebuilt on one line, losing its comments and line breaks. +- A localized declaration whose constructor is already imported from another module gets a second import of the name, + which shadows the first. - `Unpack[Ts]` is left as it is. Run `ruff`'s `UP044` to rewrite it to `*Ts` (see [Rejected recipes](../../developer/modules/rejected-recipes.md)). - There is no filtering of files, so a large project can take tens of minutes. +- A file that is not valid UTF-8 stops the whole run with a `UnicodeDecodeError`, before any file is changed. Re-save + it as UTF-8 or move it out of the target. ## Constraints -Each case below stops one change and is reported as **unsafe** under *needs manual review*, with a link to its section. +Each case below stops one change and is reported under *needs manual review*. ### Python version gates { #feature-type-parameter-modernization-version-gates } -PEP 695 syntax is a `SyntaxError` on older Pythons, so the tool only writes it when `--py` (the oldest version the -target supports, not the one running the tool) is new enough. The version is never guessed: unknown counts as too -old. Removing a declaration that is already unused needs no new syntax, so it never depends on `--py`. +PEP 695 syntax (`def f[T](...)`) is a `SyntaxError` before Python 3.12, so `--py` (the oldest version the target +supports, not the one running the tool) must be 3.12 or higher. A lower value is a usage error and nothing is +changed. The version is never guessed. Newer syntax is written only when `--py` is new enough: | Syntax | PEP | Minimum Python | | --------------------- | -------------------------------------------- | -------------- | -| `def f[T](...)` | [PEP 695](https://peps.python.org/pep-0695/) | 3.12 | | `def f[T = int](...)` | [PEP 696](https://peps.python.org/pep-0696/) | 3.13 | -#### PEP 695 version gate { #feature-type-parameter-modernization-pep695-version-gate } - -Below 3.12 nothing is converted. Localization and orphan removal still run. - -```python -T = TypeVar("T") - -def first(items: list[T]) -> T: ... -``` - -With `--py 3.11` the file is unchanged. With `--py 3.12` it becomes `def first[T](items: list[T]) -> T: ...` and the -declaration is removed. - -**To fix this yourself:** if the project supports 3.12+, run again with `--py 3.12`. Otherwise keep the old syntax. - #### PEP 696 version gate { #feature-type-parameter-modernization-pep696-version-gate } Below 3.13 a declaration with `default=` is not converted, since dropping the default would change what type @@ -164,20 +190,19 @@ With `--py 3.12` the file is unchanged. With `--py 3.13` it becomes `def first[T ### A declaration passes an argument with no PEP 695 equivalent { #feature-type-parameter-modernization-no-pep695-equivalent } -The `[...]` syntax cannot write `covariant=`/`contravariant=`, a `bound=` on a `ParamSpec` or `TypeVarTuple`, a -`**mapping` of options, or a keyword added by a later Python version. (`infer_variance=True` is fine: a PEP 695 -type parameter always infers its variance.) +The `[...]` syntax cannot write a `bound=` on a `ParamSpec` or `TypeVarTuple`, a `**mapping` of options, or a keyword +added by a later Python version. (`covariant=`, `contravariant=` and `infer_variance=` are fine: a function's type +parameter has no variance.) ```python -T_co = TypeVar("T_co", covariant=True) +P = ParamSpec("P", bound=int) -def first(items: Sequence[T_co]) -> T_co: ... +def run(func: Callable[P, int]) -> Callable[P, int]: ... ``` The file is unchanged. -**To fix this yourself:** variance only matters for generic classes. If only functions use the type parameter, -remove `covariant=`/`contravariant=` and run again. +**To fix this yourself:** keep the declaration. ### A function also uses a type variable that can't be converted { #feature-type-parameter-modernization-shares-function-with-unconvertible } @@ -187,20 +212,20 @@ declaration refused for a version gate or an argument with no PEP 695 equivalent legacy. A legacy type parameter of an enclosing generic class is not the method's own, so it blocks nothing. ```python -T_co = TypeVar("T_co", covariant=True) +P = ParamSpec("P", bound=int) Ts = TypeVarTuple("Ts") -def run(func: Callable[[*Ts], T_co], *args: *Ts) -> T_co: ... +def run(func: Callable[P, tuple[*Ts]]) -> Callable[P, tuple[*Ts]]: ... ``` -The file is unchanged. `T_co` is reported for its own reason. `Ts` is reported for this one, because every function -using it is left unchanged. Functions that use `Ts` without `T_co` are still converted. +The file is unchanged. `P` is reported for its own reason. `Ts` is reported for this one, because every function +using it is left unchanged. Functions that use `Ts` without `P` are still converted. **To fix this yourself:** fix the refused declaration as its section describes, then run again. ### A declared type parameter is exported via `__all__` { #feature-type-parameter-modernization-declared-type-parameter-exported } -`T` is in its file's plain `__all__ = [...]`, so it is public API. The functions are converted, the declaration is kept. +`T` is in its file's `__all__`, so it is public API. The functions are converted, the declaration is kept. ```python __all__ = ["T", "first"] @@ -225,7 +250,8 @@ def first[T](items: list[T]) -> T: ... ### A declared type parameter is only used inside a generic class { #feature-type-parameter-modernization-used-in-generic-class } Only methods of a class with `T` in a base (see *Generic classes* above) use `T`, so there is nothing to -convert. Standalone functions that also use `T` are still converted. +convert. Standalone functions that also use `T` are still converted, and `T` is then listed both as modified and as +needing review, because its declaration stays. ```python T = TypeVar("T") @@ -237,13 +263,17 @@ class Box(Generic[T]): The file is unchanged. **To fix this yourself:** `ruff`'s `UP046` rewrites `Generic[T]` to `class Box[T]:`. Run the tool again afterwards: -it removes the now unused `T = TypeVar("T")`. +it removes the now unused `T = TypeVar("T")`. `UP046` does not touch `Protocol[T]` or a `T` in any other base, so +write `class Box[T](MutableSet[T]):` by hand. + +### The declaration uses a name that isn't available at module level { #feature-type-parameter-modernization-name-not-at-module-level } -### A name the declaration uses can't be imported here { #feature-type-parameter-modernization-declaration-name-unavailable } +Localizing copies the declaration into this file, so every name it uses (`bound=Shape`) must be available at module +level, as the same object, in both files. The file is left unchanged when: -Localizing copies the declaration and imports the names it uses (`bound=Shape`) from the module the origin gets them -from, or from the origin when it defines them. That fails when the origin only imports `Shape` under -`if TYPE_CHECKING:`, or when this file already has a different `Shape`. +- **The origin binds the name only under `if TYPE_CHECKING:`**, or inside a function or class. The copy could not + import it at runtime. +- **This file binds the name to something else.** It already has a different `Shape`. ```python # shapes.py @@ -262,20 +292,23 @@ def biggest(items: list[T]) -> T: ... `main.py` is unchanged and keeps importing `T` from `shapes.py`. -**To fix this yourself:** import the name the same way the origin does, or rename one of the two, then copy the -declaration by hand. +**To fix this yourself:** import the name at module level the same way the origin does, or rename one of the two, +then copy the declaration by hand. ### The origin imports `TypeVar` conditionally { #feature-type-parameter-modernization-origin-imports-constructor-conditionally } The origin imports `TypeVar` inside an `if` or `try` (for example `typing` or `typing_extensions` per Python -version). A copy would use whatever `TypeVar` this file has, which may fail on some versions. +version, or `typing_extensions` under `TYPE_CHECKING` and `typing` otherwise). A copy would use whatever `TypeVar` +this file has, which may fail on some versions or differ for the type checker. ```python # shapes.py -try: - from typing import TypeVar -except ImportError: +from typing import TYPE_CHECKING + +if TYPE_CHECKING: from typing_extensions import TypeVar +else: + from typing import TypeVar T = TypeVar("T") @@ -321,44 +354,63 @@ run the tool again. ## Verified by test modules -- `test/recipes/test_type_var_check.py` -- `test/recipes/test_type_var_check_convert.py` -- `test/recipes/test_type_var_check_localize.py` -- `test/recipes/test_type_var_check_orphaned.py` -- `test/recipes/test_type_var_check_properties.py` -- `test/recipes/test_type_var_domain.py` +- `test/recipes/test_type_parameter_check.py` +- `test/recipes/test_type_parameter_check_convert.py` +- `test/recipes/test_type_parameter_check_localize.py` +- `test/recipes/test_type_parameter_check_orphaned.py` +- `test/recipes/test_type_parameter_check_properties.py` +- `test/recipes/test_type_parameter_check_variance.py` +- `test/recipes/test_type_parameter_declarations.py` +- `test/recipes/test_type_parameter_localization.py` +- `test/recipes/test_type_parameter_rules.py` - `test/recipes/test_step_runner.py` - `test/recipes/test_python_refactoring.py` +- `test/utils/test_cst_utils.py` - `test/utils/test_import_resolution.py` - `test/rejuvenation/test_migration_type_recipes.py` ## Implemented by code modules - [Refactoring recipes](../../developer/modules/recipes.md) -- `src/renaissance/recipes/type_var_check.py`, `type_var_domain.py`, `step_runner.py` and `python_refactoring.py`. - The recipe class is still named `TypeVarCheck`. -- `src/renaissance/utils/import_resolution.py` +- `src/renaissance/recipes/type_parameter_check.py`, `type_parameter_rules.py`, `type_parameter_declarations.py`, + `type_parameter_localization.py`, `type_parameter_uses.py`, `step_runner.py` and `python_refactoring.py`. +- `src/renaissance/utils/import_resolution.py` and `cst_utils.py` - `src/rejuvenation/migration-type-recipes.py` (the CLI) ## API entry points ```shell -python src/rejuvenation/migration-type-recipes.py --py MAJOR.MINOR [--report PATH] [--no-ruff] +python src/rejuvenation/migration-type-recipes.py --py MAJOR.MINOR \ + [--report PATH] [--no-ruff] [--ignore FILE...] [--no-localization] [--second-pass] ``` -- ``: a `.py` file or a directory, scanned recursively (`.git`, `__pycache__`, `.venv` and `venv` skipped). -- `--py` (required): the oldest Python version the target project supports. +- ``: a `.py` file or a directory, scanned recursively except `.git`, `__pycache__`, `.pixi`, `.venv` and `venv`. +- `--py` (required): the oldest Python version the target project supports, 3.12 or higher. - `--report`: also write the report to a file. - `--no-ruff`: skip the final `ruff` pass. +- `--ignore`: files left untouched, whose type parameters are never copied into the files importing them (see + *Shared declaration files*). +- `--no-localization`: keep every project import of a type parameter (see *Disabling localization*). +- `--second-pass`: process again only the files that kept a declaration because another file imported it when the run + started, and merge the result into the same report. After the recipe, the CLI runs `ruff check --fix --select F401,I001` on the changed files. This removes **every** unused import in them, including ones that were unused before (such as `from typing import TypeVar` after the declaration is gone). It also merges and sorts **every** import block in them, including imports that were unsorted -before, using the `ruff` settings of the project. `ruff` must be installed in the same environment; if it cannot run, -the report still says the pass ran. Files are changed directly, so run it on a git checkout. - -The report has a summary line and the sections `MODIFIED`, `NEEDS MANUAL REVIEW` and `ERRORS`. Exit codes: 0 when -the run completes (manual review is not a failure), 2 for a usage error, 3 if any file raised an exception. +before, using the `ruff` settings of the project. `ruff` must be installed in the same environment. Files are changed +directly, so run it on a git checkout. + +The report is Markdown, with a summary line, up to two summary tables and the sections `MODIFIED`, `NEEDS MANUAL REVIEW` and +`ERRORS`. The tables count the type parameters per kind (`TypeVar`, `ParamSpec`, `TypeVarTuple`) and rank the reasons for +manual review from most to least common; the reasons table is left out when no name needs review. A name counts once +per file, as needing review if it has a reason to review. +`MODIFIED` says what was done to each name: converted in functions, import replaced by a local declaration, or unused +declaration removed; a name localized and then removed is listed only as removed. `NEEDS MANUAL REVIEW` groups the files +and names by reason, most common reason first, and a name can be in both sections. With `--ignore` the report also has +the `IGNORED FILES` section, and with `--ignore` or `--no-localization` the `Ignored` column. Files are listed relative +to ``. When files were modified, the run ends with a commit message: the names grouped by what was done to them +(with the number of files if above one, at most ten per group), then the files left for manual review by reason. Exit +codes: 0 when the run completes (manual review is not a failure), 2 for a usage error, 3 if any file raised an exception. ## Change considerations