Collect docnames of analyzed source in advance (#2)

* Remove container_of and support "of {key: value}" syntax

This syntax is used by Pandas [1]. I'm not entirely sold yet, but let's
add it for now. To avoid confusion, literals should probably be made to
only be accepted on the top-level. Otherwise something like
`dict of {{"a", "b"}: int}` becomes possible.

Co-authored-by: Oriol Abril-Pla <oriol.abril.pla@gmail.com>

* Only allow literals on top-level

which avoids potentially confusing constructs like
`dict of {{"a", "b"}: int}`

Co-authored-by: Oriol Abril-Pla <oriol.abril.pla@gmail.com>

* Make "= | :" optional in default syntax

since this is what NumPyDoc recommends as well [1].

[1] https://numpydoc.readthedocs.io/en/latest/format.html#parameters

Co-authored-by: Oriol Abril-Pla <oriol.abril.pla@gmail.com>

* Add ipython as dev dependency

* Move matching to imports to qualname level

Doing this on the NAME level meant that stuff like "np.int16" would be
turned into "np.np.int16" because both "np" and "int16" where matched.

* Split docname into KnownImport and replace

Reduce responsibility of the former DocName class. Replacing docstring
specific type description should be handled separately.

* Test correctness when nesting classes in classes

* Rework analysis and other major changes

While not perfect or anywhere near to finished the refactored code
uses clearer separation of responsiblities and is one step closer to an
architecture that feels right. :)

---------

Co-authored-by: Oriol Abril-Pla <oriol.abril.pla@gmail.com>

Author: lagru

examples/docstub.toml

@@ -5,14 +5,16 @@ extend_grammar = """
 
 """
 
-# A mapping of docnames to import information. Each item maps a docname on the
-# left side to a dictionary on the right side, which supports the following
-# fields:
-#     use : A string to replace the docname with, defaults to the docname.
-#     from : Indicate that the docname can be imported from this path.
-#     import : Import this object, defaults to the docname.
+# Import information for type annotations, declared ahead of time.
+#
+# Each item maps an annotation name on the left side to a dictionary on the
+# right side.
+#
+# Import information can be declared with the following fields:
+#     from : Indicate that the DocType can be imported from this path.
+#     import : Import this object, defaults to the DocType.
 #     as : Use this alias for the imported object
-#     is_builtin : Indicate that this docname doesn't need to be imported,
+#     is_builtin : Indicate that this DocType doesn't need to be imported,
 #         defaults to "false"
-[tool.docstub.docnames]
+[tool.docstub.known_imports]
 configparser = {import = "configparser"}

examples/example_pkg-stubs/__init__.pyi

@@ -1,8 +1,10 @@
 import _numpy as np_
-from _basic import func_comment, func_contains
+from _basic import func_contains
 
 __all__ = [
-    "func_comment",
     "func_contains",
     "np_",
 ]
+
+class CustomException(Exception):
+    pass

examples/example_pkg-stubs/_basic.pyi

@@ -3,31 +3,48 @@ import logging
 from collections.abc import Sequence
 from typing import Any, Literal, Self, Union
 
-logger = logging.getLogger(__name__)
+from . import CustomException
+
+logger = ...
 
 __all__ = [
     "func_empty",
     "ExampleClass",
 ]
 
-def func_empty(a1: Any, a2: Any, a3: Any) -> None: ...
+def func_empty(a1, a2, a3) -> None: ...
 def func_contains(
-    self,
     a1: list[float],
     a2: dict[str, Union[int, str]],
     a3: Sequence[int | float],
     a4: frozenset[bytes],
-) -> tuple[tuple[int, ...], list[int]]: ...
+    a5: tuple[int],
+    a6: list[int, str],
+    a7: dict[str, int],
+) -> None: ...
 def func_literals(
     a1: Literal[1, 3, "foo"], a2: Literal["uno", 2, "drei", "four"] = ...
 ) -> None: ...
+def func_use_from_elsewhere(
+    a1: CustomException,
+    a2: ExampleClass,
+    a3: CustomException.NestedClass,
+    a4: ExampleClass.NestedClass,
+) -> tuple[CustomException, ExampleClass.NestedClass]: ...
 
 class ExampleClass:
-    def __init__(self, a1: int, a2: float | None = ...) -> None: ...
+    class NestedClass:
+        def method_in_nested_class(self, a1: complex) -> None: ...
+
+    def __init__(self, a1: str, a2: float = ...) -> None: ...
     def method(self, a1: float, a2: float | None) -> list[float]: ...
     @staticmethod
     def some_staticmethod(a1: float, a2: float | None = ...) -> dict[str, Any]: ...
     @property
     def some_property(self) -> str: ...
+    @some_property.setter
+    def some_property(self, value: str) -> None: ...
     @classmethod
     def method_returning_cls(cls, config: configparser.ConfigParser) -> Self: ...
+    @classmethod
+    def method_returning_cls2(cls, config: configparser.ConfigParser) -> Self: ...

examples/example_pkg-stubs/_numpy.pyi

@@ -2,7 +2,7 @@ import numpy as np
 from numpy.typing import ArrayLike, NDArray
 
 def func_object_with_numpy_objects(
-    a1: np.np.int8, a2: np.np.int16, a3: np.typing.DTypeLike, a4: np.typing.DTypeLike
+    a1: np.int8, a2: np.int16, a3: np.typing.DTypeLike, a4: np.typing.DTypeLike
 ) -> None: ...
 def func_ndarray(
     a1: NDArray, a2: np.NDArray, a3: NDArray[float], a4: NDArray[np.uint8] = ...

examples/example_pkg/__init__.py

@@ -1,10 +1,13 @@
 """Example of an init file."""
 
 import _numpy as np_
-from _basic import func_comment, func_contains
+from _basic import func_contains
 
 __all__ = [
-    "func_comment",
     "func_contains",
     "np_",
 ]
+
+
+class CustomException(Exception):
+    pass

examples/example_pkg/_basic.py

@@ -26,7 +26,7 @@ def func_empty(a1, a2, a3):
     """
 
 
-def func_contains(self, a1, a2, a3, a4):
+def func_contains(a1, a2, a3, a4, a5, a6, a7):
     """Dummy.
 
     Parameters
@@ -35,11 +35,9 @@ def func_contains(self, a1, a2, a3, a4):
     a2 : dict[str, Union[int, str]]
     a3 : Sequence[int | float]
     a4 : frozenset[bytes]
-
-    Returns
-    -------
-    r1 : tuple of int
-    r2 : list of int
+    a5 : tuple of int
+    a6 : list of (int, str)
+    a7 : dict of {str: int}
     """
 
 
@@ -53,16 +51,44 @@ def func_literals(a1, a2="uno"):
     """
 
 
+def func_use_from_elsewhere(a1, a2, a3, a4):
+    """Check if types with full import names are matched.
+
+    Parameters
+    ----------
+    a1 : example_pkg.CustomException
+    a2 : ExampleClass
+    a3 : example_pkg.CustomException.NestedClass
+    a4 : ExampleClass.NestedClass
+
+    Returns
+    -------
+    r1 : ~.CustomException
+    r2 : ~.NestedClass
+    """
+
+
 class ExampleClass:
-    # TODO also take into account class level docstring
+    """Dummy.
 
-    def __init__(self, a1, a2=None):
-        """
-        Parameters
-        ----------
-        a1 : int
-        a2 : float, optional
-        """
+    Parameters
+    ----------
+    a1 : str
+    a2 : float, default 0
+    """
+
+    class NestedClass:
+
+        def method_in_nested_class(self, a1):
+            """
+
+            Parameters
+            ----------
+            a1 : complex
+            """
+
+    def __init__(self, a1, a2=0):
+        pass
 
     def method(self, a1, a2):
         """Dummy.
@@ -101,6 +127,15 @@ def some_property(self):
         """
         return str(self)
 
+    @some_property.setter
+    def some_property(self, value):
+        """Dummy
+
+        Parameters
+        ----------
+        value : str
+        """
+
     @classmethod
     def method_returning_cls(cls, config):
         """Using `Self` in context of classmethods is supported.
@@ -115,3 +150,18 @@ def method_returning_cls(cls, config):
         out : Self
             New class.
         """
+
+    @classmethod
+    def method_returning_cls2(cls, config):
+        """Using `Self` in context of classmethods is supported.
+
+        Parameters
+        ----------
+        config : configparser.ConfigParser
+            Configuation.
+
+        Returns
+        -------
+        out : Self
+            New class.
+        """

examples/example_pkg/_numpy.py

@@ -19,7 +19,7 @@ def func_ndarray(a1, a2, a3, a4=None):
     Parameters
     ----------
     a1 : ndarray
-    a2 : np.ndarray
+    a2 : np.NDArray
     a3 : (N, 3) ndarray of float
     a4 : ndarray of shape (1,) and dtype uint8
 

pyproject.toml

@@ -40,6 +40,7 @@ optional = [
 ]
 dev = [
   "pre-commit >=3.7",
+  "ipython",
 ]
 test = [
   "pytest >=5.0.0",
@@ -87,3 +88,8 @@ ignore = [
   "RET504",   # Assignment before `return` statement facilitates debugging
   "PTH123",   # Using builtin open() instead of Path.open() is fine
 ]
+
+
+[tool.docstub.docnames]
+cst = {import = "libcst", as="cst"}
+lark = {import = "lark"}