Make the libcst (CST) backend usable for recipes

src/rejuvenation/cli.py

@@ -30,7 +30,7 @@ def refactor() -> None:
         extractor = PythonExtractor()
         for file in PythonScanner().find_sources():
             filename = sys.argv[2]
-            extractor.process(file)
+            extractor.process(Path(file))
         extractor.save_graph(filename)
     if sys.argv[1] == "inspect":
         print(f"inspect {Path().cwd()}")

src/renaissance/integrations/python/ast/cst_node.py

@@ -6,42 +6,129 @@
 
 """AI: ASTNode implementation backed by libcst's concrete syntax tree."""
 
+import io
+import itertools
+import sys
+from collections import defaultdict
 from pathlib import Path
 from typing import Self
 
 import libcst
 from libcst import BaseCompoundStatement, BaseSmallStatement, ClassDef, CSTNode, FunctionDef, MetadataWrapper
-from libcst.metadata import WhitespaceInclusivePositionProvider
+from libcst.metadata import Access, Assignment, CodePosition, PositionProvider, ScopeProvider
 
 from renaissance.integrations.python.ast.kinds import PYTHON_KIND_MAP
-from renaissance.integrations.python.ast.util import convert
+from renaissance.syntax_tree.ast_node import ASTReference
 from renaissance.syntax_tree.semantic_kind import SemanticKind
 from renaissance.utils.ast_utils import next_sibling, preceding_sibling
 
+type _Link = tuple[CSTNode, str, Access]
+
 
 class PythonCstTranslationUnit:
     """AI: Parse Python source into a libcst tree with position lookups for AST-node wrapping."""
 
     def __init__(self, content, file_name: str) -> None:
         """AI: Parse Python source into a libcst tree with position lookups for AST-node wrapping."""
-        # TODO: see docs/developer/architecture/position-consistency.md - exposes no buffer, so the rewriter derives its own.
+        # TODO(pjljvandelaar): see docs/developer/architecture/position-consistency.md - exposes no buffer, so the rewriter
+        #  derives its own.
+        #  https://github.com/TNO/Renaissance.Py/issues/153
+        # TODO(FrancescoPezzella): content is to become the file's UTF-8 bytes, with every offset a UTF-8 byte offset into
+        #  them (decided for AST and regular expression matching; no str). Until then content is the str decoded by load
+        #  and offsets count bytes of self.encoding, the filesystem encoding, so never slice content with them (#153).
+        #  https://github.com/TNO/Renaissance.Py/issues/188
         self.content = content
-        self.lines = content.splitlines()
+        self.encoding = sys.getfilesystemencoding()
+        # newline="" splits on \n, \r\n and \r only, like libcst, and keeps each line ending.
+        self._lines = io.StringIO(content, newline="").readlines()
+        self._line_starts = list(itertools.accumulate((len(line.encode(self.encoding)) for line in self._lines), initial=0))
         self.file_name = file_name
-        self.references_initialized = False
         self.wrapper = MetadataWrapper(libcst.parse_module(content))
         self.atu = self.wrapper.module
-        self.spans = self.wrapper.resolve(WhitespaceInclusivePositionProvider)
+        self.spans = self.wrapper.resolve(PositionProvider)
+        self.wrappers: dict[CSTNode, PythonCstNode] = {}
+        self._uses_of: dict[CSTNode, list[_Link]] | None = None
+        self._definitions_of: dict[CSTNode, list[_Link]] = {}
+
+    def _index_references(self) -> dict[CSTNode, list[_Link]]:
+        """Resolve libcst's scopes once, linking every definition to its uses; return the definition-to-uses map.
+
+        Also fills self._definitions_of, the use-to-definitions map. Builtins, which have no defining node, are skipped.
+        An annotation or type hint above a definition in the same scope is linked to it too, since it is evaluated
+        lazily (PEP 563, PEP 649); a value read there at once is not.
+        """
+        if self._uses_of is not None:
+            return self._uses_of
+        uses_of: defaultdict[CSTNode, list[_Link]] = defaultdict(list)
+        definitions_of: defaultdict[CSTNode, list[_Link]] = defaultdict(list)
+        scopes = {scope for scope in self.wrapper.resolve(ScopeProvider).values() if scope is not None}
+        for scope in scopes:
+            for assignment in scope.assignments:
+                if not isinstance(assignment, Assignment):
+                    continue
+                lazy_accesses_above = {
+                    access
+                    for access in scope.accesses[assignment.name]
+                    if not access.referents and (access.is_annotation or access.is_type_hint)
+                }
+                # libcst only links an access to the definitions above it in the same scope.
+                for access in [*assignment.references, *lazy_accesses_above]:
+                    uses_of[assignment.node].append((access.node, assignment.name, access))
+                    definitions_of[access.node].append((assignment.node, assignment.name, access))
+        self._uses_of = dict(uses_of)
+        self._definitions_of = dict(definitions_of)
+        return self._uses_of
+
+    def _to_references(self, links: list[_Link], ref_kind: str) -> list[ASTReference[PythonCstNode]]:
+        """Wrap links as references, ordered by the position of the node each points at.
+
+        Each reference's properties hold the bound "name" and whether the use sits in an annotation ("is_annotation")
+        or another type hint such as `cast("T", x)` ("is_type_hint").
+        """
+        references = [
+            ASTReference(
+                self.wrappers[node],
+                ref_kind,
+                {"name": name, "is_annotation": access.is_annotation, "is_type_hint": access.is_type_hint},
+            )
+            for node, name, access in links
+            if node in self.wrappers
+        ]
+        return sorted(references, key=lambda reference: reference.node.offset)
+
+    def uses_of(self, node: CSTNode) -> list[ASTReference[PythonCstNode]]:
+        """Return the uses of the name(s) that node defines; empty if node defines nothing."""
+        return self._to_references(self._index_references().get(node, []), "use")
+
+    def definitions_of(self, node: CSTNode) -> list[ASTReference[PythonCstNode]]:
+        """Return the definitions that node, a use of a name, resolves to; empty if node is not a resolved use."""
+        self._index_references()
+        return self._to_references(self._definitions_of.get(node, []), "definition")
+
+    def _byte_offset(self, position: CodePosition) -> int:
+        """Return the offset, in bytes of self.encoding, of a libcst position whose column counts characters.
+
+        A position on the line after the last one (the end of a node that ends with the file) maps to the file's length.
+        """
+        line_index = position.line - 1
+        if line_index >= len(self._lines):
+            return self._line_starts[-1]
+        return self._line_starts[line_index] + len(self._lines[line_index][: position.column].encode(self.encoding))
 
     def start_of(self, node: CSTNode) -> int:
-        """AI: Return the character offset where node begins in the source text."""
+        """Return the byte offset where node begins in the source text, or 0 if it has no position.
+
+        A decorated function or class begins at its first decorator.
+        """
+        if isinstance(node, (ClassDef, FunctionDef)) and node.decorators:
+            node = node.decorators[0]
         span = self.spans.get(node)
-        return convert(self.lines, span.start.line, span.start.column) if span else 0
+        return self._byte_offset(span.start) if span else 0
 
     def end_of(self, node: CSTNode) -> int:
-        """AI: Return the character offset where node ends in the source text."""
+        """Return the byte offset where node's code ends in the source text, or 0 if it has no position."""
         span = self.spans.get(node)
-        return convert(self.lines, span.end.line, span.end.column) if span else 0
+        return self._byte_offset(span.end) if span else 0
 
     def signature_of(self, node: CSTNode) -> str:
         """AI: Return the source code text corresponding to node, or an empty string on failure."""
@@ -54,21 +141,19 @@ def signature_of(self, node: CSTNode) -> str:
 class PythonCstNode:
     """AI: ASTNode implementation backed by libcst's concrete syntax tree."""
 
-    def __init__(self, node: CSTNode, translation_unit: PythonCstTranslationUnit, parent=None) -> None:
-        """AI: Wrap a libcst node as an AST node within the given translation unit."""
+    def __init__(self, node: CSTNode, translation_unit: PythonCstTranslationUnit, parent: Self | None = None) -> None:
+        """Wrap node and, recursively, its children as nodes of this class within the given translation unit."""
         self.parent = parent
-        if parent and parent.root:
-            self.root = parent.root
-        else:
-            self.root = self
+        self.root: Self = parent.root if parent and parent.root else self
         self.translation_unit = translation_unit
         self.node = node
+        translation_unit.wrappers[node] = self
 
         self.is_statement = isinstance(self.node, (BaseSmallStatement, BaseCompoundStatement))
         self.parser_kind = type(node).__name__
         self.semantic_kind = PYTHON_KIND_MAP.get(self.parser_kind, SemanticKind.NODE)
 
-        self.children: list[Self] = [PythonCstNode(node, translation_unit, self) for node in node.children]
+        self.children: list[Self] = [type(self)(child, translation_unit, self) for child in node.children]
         self.properties = {}
 
         # for shower
@@ -93,22 +178,22 @@ def signature(self):
 
     @property
     def offset(self):
-        """AI: Return the character offset where this node begins in the source text."""
+        """Return the offset, in bytes of translation_unit.encoding, where this node begins in the source text."""
         return self.translation_unit.start_of(self.node)
 
     @property
     def length(self):
-        """AI: Return the length in characters of this node's source text."""
+        """Return the length, in bytes of translation_unit.encoding, of this node's source text."""
         return self.end_offset - self.offset
 
     @property
     def end_offset(self):
-        """AI: Return the character offset where this node ends in the source text."""
+        """Return the offset, in bytes of translation_unit.encoding, where this node ends in the source text."""
         return self.translation_unit.end_of(self.node)
 
     @property
     def extended_end_offset(self) -> int:
-        """AI: Return the end offset; libcst attaches trailing trivia to the node itself, so there is nothing to extend."""
+        """Return the end offset; trailing comments are found by the rewriter, as for the RST backend."""
         return self.end_offset
 
     @property
@@ -124,6 +209,16 @@ def name(self):
         return ""
         self.name = ""  # self._derive_name()
 
+    @property
+    def references(self) -> list[ASTReference[PythonCstNode]]:
+        """Return the definitions this node uses, resolved with libcst's scope analysis (empty if it uses none)."""
+        return self.translation_unit.definitions_of(self.node)
+
+    @property
+    def referenced_by(self) -> list[ASTReference[PythonCstNode]]:
+        """Return the uses of the name(s) this node defines, resolved with libcst's scope analysis (empty if none)."""
+        return self.translation_unit.uses_of(self.node)
+
     @property
     def next_sibling(self) -> Self | None:
         """AI: Return the sibling node immediately following this one, or None."""
@@ -134,19 +229,21 @@ def preceding_sibling(self) -> Self | None:
         """AI: Return the sibling node immediately preceding this one, or None."""
         return preceding_sibling(self)
 
-    @staticmethod
-    def load(file_path: Path) -> PythonCstNode:
-        """AI: Parse the Python source file at file_path into a PythonCstNode tree."""
-        with Path(file_path).open() as file:
+    @classmethod
+    def load(cls, file_path: Path) -> Self:
+        """Parse the Python source file at file_path into a tree of nodes of this class."""
+        # TODO(FrancescoPezzella): drop the explicit encoding with Python 3.15, where UTF-8 is the default.
+        #  https://github.com/TNO/Renaissance.Py/issues/188
+        with Path(file_path).open(encoding="utf-8") as file:
             content = file.read()
-            return PythonCstNode.load_from_text(content, str(file_path))
+            return cls.load_from_text(content, str(file_path))
 
-    @staticmethod
+    @classmethod
     def load_from_text(
+        cls,
         text: str,
         file_name: str = "cst_snippet.py",
-    ) -> PythonCstNode:
-        """AI: Parse Python source text into a PythonCstNode tree, attributed to file_name."""
+    ) -> Self:
+        """Parse Python source text into a tree of nodes of this class, attributed to file_name."""
         translation_unit = PythonCstTranslationUnit(text, file_name=str(file_name))
-        root_node = PythonCstNode(translation_unit.atu, translation_unit)
-        return root_node
+        return cls(translation_unit.atu, translation_unit)

src/renaissance/integrations/python/ast/rst_node.py

@@ -356,7 +356,9 @@ def load(
         # Keep a uniform loader signature across AST node implementations.
         # Python's AST parser does not need extra arguments or a working dir.
         _ = extra_args, working_dir
-        with Path(file_path).open() as file:
+        # TODO(FrancescoPezzella): drop the explicit encoding with Python 3.15, where UTF-8 is the default.
+        #  https://github.com/TNO/Renaissance.Py/issues/188
+        with Path(file_path).open(encoding="utf-8") as file:
             content = file.read()
             return PythonRstNode.load_from_text(content, str(file_path))
 

src/renaissance/recipes/python_refactoring.py

@@ -13,6 +13,7 @@
 
 from termcolor import colored
 
+from renaissance.integrations.python.ast.cst_node import PythonCstNode
 from renaissance.integrations.python.ast.factory import PythonFactory, PythonPatternFactory
 from renaissance.integrations.python.ast.rst_node import PythonRstNode
 from renaissance.integrations.python.ast.util import to_str
@@ -22,12 +23,18 @@
 from renaissance.utils.text_utils import snake_case
 
 
-class PythonRefactoring(ASTProcessor):
-    """AI: Base processor for Python-specific source refactoring recipes."""
+class PythonRefactoring[NodeT: PythonRstNode | PythonCstNode = PythonRstNode](ASTProcessor):
+    """Base processor for Python-specific source refactoring recipes.
 
-    def __init__(self, file) -> None:
-        """AI: Prepare a Python-specific refactoring processor for the given source file."""
-        factory = PythonFactory(PythonRstNode)
+    NodeT is the node class the file is parsed into: PythonRstNode by default, or PythonCstNode. A subclass picks the
+    CST backend with PythonRefactoring[PythonCstNode] and passes the same class as node_class; NodeT then types body.
+    See PEP 695 (https://peps.python.org/pep-0695/) for the bound and PEP 696 (https://peps.python.org/pep-0696/)
+    for the default.
+    """
+
+    def __init__(self, file: Path, node_class: type[NodeT] = PythonRstNode) -> None:
+        """Prepare a refactoring processor for file, parsed with node_class (the RST backend by default)."""
+        factory = PythonFactory(node_class)
         atu = factory.create(file)
         super().__init__(atu, factory, False)
         self.pattern_factory = PythonPatternFactory(self.factory)
@@ -111,9 +118,13 @@ def class_base_arguments(self, class_node: PythonRstNode) -> list[str]:
         return [child.signature for child in bases_implicit.children]
 
     @property
-    def body(self) -> Sequence[PythonRstNode]:
-        """AI: Return the root node's body statements."""
-        return cast("PythonRstNode", cast("object", self.root)).body
+    def body(self) -> Sequence[NodeT]:
+        """Return the module's top-level statements; with the CST backend, a simple statement comes as its whole line."""
+        root = cast("NodeT", cast("object", self.root))
+        if isinstance(root, PythonCstNode):
+            wrappers = root.translation_unit.wrappers
+            return cast("Sequence[NodeT]", [wrappers[statement] for statement in root.translation_unit.atu.body])
+        return cast("Sequence[NodeT]", root.body)
 
     def run(self) -> None:
         """AI: Run this refactoring recipe. Subclasses override this to perform the refactoring."""

src/renaissance/syntax_tree/ast_refactor_actions.py

@@ -16,6 +16,8 @@
 from renaissance.syntax_tree.match_finder import MatchFinder, PatternMatch
 from renaissance.syntax_tree.semantic_kind import SemanticKind
 
+# TODO(FrancescoPezzella): Replace with a lazy import in Python 3.15.
+#  https://github.com/TNO/Renaissance.Py/issues/275
 if TYPE_CHECKING:
     from renaissance.integrations.clang.c_pattern_factory import CPPPatternFactory
 

src/renaissance/syntax_tree/ast_rewriter.py

@@ -303,6 +303,16 @@ def __check_for_conflicting_rewrites(self) -> None:
                         "in the same file - applying both would corrupt the output."
                     )
 
+    @staticmethod
+    def __is_nested(node: Rewritable, maybe_ancestor: Rewritable) -> bool:
+        """Return True if maybe_ancestor is an ancestor of node, walking .parent (works for every Rewritable)."""
+        parent = node.parent
+        while parent:
+            if parent is maybe_ancestor:
+                return True
+            parent = parent.parent
+        return False
+
     def __is_ancestor_in_nodes(self, node: Rewritable) -> bool:
         """Check if the given node is a descendant of any nodes in the rewrite list.
 
@@ -327,6 +337,11 @@ def no_conflict(node1, rew) -> bool:
 
         result = any(no_conflict(node, rew) for rew in rewrite_nodes)
 
+        # TODO(FrancescoPezzella): an outer rewrite should suppress the rewrites nested inside it, but both are applied.
+        #  Dropping `and False` is not enough: no_conflict also matches each node with its own rewrite, so result is almost
+        #  always True and apply() would skip nearly every rewrite. A fix has to leave out the node's own rewrite and use a
+        #  true ancestor check such as __is_nested.
+        #  https://github.com/TNO/Renaissance.Py/issues/286
         return result and False  # TODO: Why `and False`
 
     def __replace(
@@ -441,6 +456,8 @@ def __insert(
             else spaces
         )
         # indent the new content except the first line
+        # TODO(FrancescoPezzella): unlike __replace, this ignores correct_indent, so the re-indentation can't be turned off.
+        #  https://github.com/TNO/Renaissance.Py/issues/283
         new_content = TextUtils.shift_right(new_content, indent, start_line=1)
 
         if before:
@@ -489,6 +506,8 @@ def __compose_replacement(self, replacement: str, matches: Sequence[PatternMatch
                         if regex_match:
                             raw_signature = "".join(regex_match.groups())
                         place_holder_length = end_index - index + 1
+                    # TODO(FrancescoPezzella): this also indents the lines inside a multi-line string in the captured text.
+                    #  https://github.com/TNO/Renaissance.Py/issues/283
                     indent_replacement = raw_signature.replace("\n", "\n" + spaces)
                     if (
                         placeholder.startswith("$$")