[docutils] Improve and complete stubs

stubs/docutils/@tests/stubtest_allowlist.txt

@@ -1,25 +1,61 @@
-docutils.nodes.Element.__iter__ # doesn't exist at runtime, but the class is iterable due to __getitem__
-docutils.nodes.Element.tagname  # class variable is overridden in __init__ method
-docutils.nodes.NodeVisitor.depart_\w+ # Methods are discovered dynamically on commonly-used subclasses
-docutils.nodes.NodeVisitor.visit_\w+  # Methods are discovered dynamically on commonly-used subclasses
-docutils.nodes.NodeVisitor.__init__   # Argument "document" should be positional-only, but subclasses are not
+# This class variable is overridden in __init__.
+docutils.nodes.Element.tagname
+# The `document` parameter should be positional-only, but there are subclasses where it is not.
+docutils.nodes.NodeVisitor.__init__
 
-# these methods take a rawsource parameter that has been deprecated and is completely ignored, so we omit it from the stub
+# This constructor technically still takes a `rawsource` parameter;
+# however, it is deprecated on Text nodes. Passing 'None' prints a warning, and passing
+# anything else will raise.
+# It is omitted for simplicity (e.g. not requiring matching overloads in child classes).
 docutils.nodes.Text.__new__
-docutils.parsers.rst.directives.admonitions.BaseAdmonition.node_class  # must be overridden by base classes (pseudo-abstract)
-docutils.statemachine.State.nested_sm  # is initialised in __init__
-docutils.statemachine.State.nested_sm_kwargs  # is initialised in __init__
-docutils.statemachine.ViewList.__iter__ # doesn't exist at runtime, but the class is iterable due to __getitem__
-docutils.transforms.Transform.apply # method apply is not implemented
-docutils.transforms.Transform.__getattr__
-docutils.TransformSpec.unknown_reference_resolvers
+
+# This attribute is required to be overridden by base classes (pseudo-abstractmethod).
+docutils.parsers.rst.directives.admonitions.BaseAdmonition.node_class
+
+# These attributes are initialized to None in the classs-body, then initialized in __init__.
+# It is never None in practice.
+docutils.statemachine.State.nested_sm
+docutils.statemachine.State.nested_sm_kwargs
+
+# These attribute names contain spaces and are set using setattr from docutils.sty.
 docutils.writers.latex2e.PreambleCmds... contents
-docutils.writers.latex2e.PreambleCmds.inline role \w+  # attribute names with spaces, set with setattr() from docutils.sty
+docutils.writers.latex2e.PreambleCmds.inline role \w+
 
-# Files that don't exist at runtime of stubtests, raises ImportError:
+# These modules have optional dependencies that likely are not available at runtime.
 docutils.parsers.commonmark_wrapper
 docutils.parsers.recommonmark_wrapper
-docutils.writers.odf_odt.pygmentsformatter # import `pygments` third-party library
 
-# `TYPE_CHECKING` variable is for internal use:
+# Imports `pygments`; using types-Pygments caused ts_utils to have a recuesion error.
+docutils.writers.odf_odt.pygmentsformatter
+
+# Docutils defines `TYPE_CHECKING` instead of importing it.
 docutils.*\.TYPE_CHECKING
+
+# stubtest infers `dict[Any, Any]` for the optional keyword arguments.
+# That obviously fails to match `_OptionKwargs`.
+docutils\..*\.settings_spec
+
+# Not generic at runtime, but it is runtime subscriptable via `__class_getitem__()`.
+# Take advantage by treating it like a proper generic.
+docutils.languages.LanguageImporter.__class_getitem__
+
+# These are functions defined in the class bodies without `self`.
+# They are only meant to be accessed via the class (e.g. in `option_spec`);
+# When docutils drops 3.9 support, they should theoretically be real staticmethods.
+docutils.parsers.rst.directives.images.Figure.align
+docutils.parsers.rst.directives.images.Figure.figwidth_value
+docutils.parsers.rst.directives.images.Image.align
+docutils.parsers.rst.directives.images.Image.loading
+docutils.parsers.rst.directives.parts.Contents.backlinks
+
+# These are aliased to `invalid_input(), with defaulted parameters, at runtime.
+# Subclasses must explicitly re-enable. This causes unavoidable signature conflicts.
+# The stub keeps the expected transition signatures so that subclasses may override without errors.
+docutils.parsers.rst.states.SpecializedBody.(anonymous|bullet|doctest|enumerator|explicit_markup|field_marker|grid_table_top|indent|line|line_block|option_marker|simple_table_top|text)
+docutils.parsers.rst.states.SpecializedText.(blank|indent|text|underline)
+
+# This is mutated at runtime. It is `None` at class level, and
+# once `Reader.read()` is called, it is no longer None.
+# `Reader.document` has no real good reason to be accessed before calling `read`, and
+# annotating as "| None" is an unnecessary nuissance.
+docutils.readers.standalone.Reader.document

stubs/docutils/@tests/test_cases/check_core.py

@@ -0,0 +1,26 @@
+from __future__ import annotations
+
+from typing_extensions import assert_type
+
+from docutils.core import publish_doctree, publish_parts
+
+# `publish_parts()` returns writer-specific parts for the HTML and LaTeX writers.
+# It should be possible to deduce key types with this knowledge.
+html_parts = publish_parts("Hello", writer="html5")
+assert_type(html_parts["body"], str)
+assert_type(html_parts["whole"], str)
+latex_parts = publish_parts("Hello", writer="latex")
+assert_type(latex_parts["titledata"], str)
+other_parts = publish_parts("Hello", writer="pseudoxml")
+assert_type(other_parts["whole"], "str | bytes")
+assert_type(other_parts.get("body"), "str | None")
+
+# `settings_overrides` should accept any copyable mapping.
+# This should include `dict`s of narrower value types.
+overrides: dict[str, int] = {"report_level": 5, "halt_level": 5}
+publish_doctree("x", settings_overrides=overrides)
+publish_doctree("x", settings_overrides={"report_level": 5, "input_encoding": "utf-8"})
+
+
+def not_a_mapping() -> None:
+    publish_doctree("x", settings_overrides=[("report_level", 5)])  # type: ignore[arg-type]

stubs/docutils/@tests/test_cases/check_directives.py

@@ -0,0 +1,53 @@
+from __future__ import annotations
+
+from collections.abc import Callable, Sequence
+from typing import ClassVar
+
+from docutils import nodes
+from docutils.parsers.rst import Directive, directives
+from docutils.parsers.rst.directives.body import Rubric, Topic
+from docutils.parsers.rst.directives.images import Figure
+from docutils.parsers.rst.directives.tables import CSVTable
+
+
+# `option_spec` should accept any mapping.
+# This should include `dict` subclasses with narrower converters.
+class DummyOptionSpec(dict[str, Callable[[str], str]]):
+    def __bool__(self) -> bool:
+        return True
+
+
+class AnyOptions(Directive):
+    option_spec = DummyOptionSpec()
+
+
+# Concrete directives have a `dict` option spec that should be extendable.
+class Exercise(Topic):
+    option_spec: ClassVar[dict[str, Callable[[str], object]]] = {**Topic.option_spec, "difficulty": directives.nonnegative_int}
+
+
+class MyFigure(Figure):
+    option_spec: ClassVar[dict[str, Callable[[str], object]]] = Figure.option_spec.copy()
+    option_spec["caption"] = directives.unchanged
+
+    # Overrides should be able to return a (narrower) sequence of nodes.
+    def run(self) -> Sequence[nodes.Node]:
+        return super().run()
+
+
+class MyRubric(Rubric):
+    def run(self) -> list[nodes.rubric | nodes.system_message]:
+        return []
+
+
+class MyCSVTable(CSVTable):
+    def run(self) -> Sequence[nodes.table | nodes.system_message]:
+        return super().run()
+
+
+# A directive's own `option_spec` literal should be acceptable.
+class MyDirective(Directive):
+    option_spec = {"class": directives.class_option, "flag": directives.flag, "count": directives.nonnegative_int}
+
+    def run(self) -> list[nodes.Node]:
+        return []

stubs/docutils/@tests/test_cases/check_languages.py

@@ -0,0 +1,20 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+from typing_extensions import assert_type
+
+from docutils.languages import LanguageImporter, get_language
+from docutils.parsers.rst.languages import get_language as get_rst_language
+
+if TYPE_CHECKING:
+    from docutils.languages import LanguageModule
+    from docutils.parsers.rst.languages import RSTLanguageModule
+
+
+assert_type(get_language("de"), "LanguageModule")
+
+# The rST importer has should have no fallback language.
+assert_type(get_rst_language("de"), "RSTLanguageModule | None")
+
+# The module type should default to `LanguageModule`.
+assert_type(LanguageImporter()("de"), "LanguageModule")

stubs/docutils/@tests/test_cases/check_nodes.py

@@ -0,0 +1,52 @@
+from __future__ import annotations
+
+from typing import Any
+from typing_extensions import assert_type
+
+from docutils import nodes
+
+
+# Well-known attributes with stable types have `Literal`-keyed overloads.
+# So, it should be able to deduce the types of certain keys.
+def attributes(node: nodes.Element) -> None:
+    assert_type(node["classes"], list[str])
+    assert_type(node.get("classes"), list[str])
+    assert_type(node.get("backrefs"), "list[str] | None")
+    assert_type(node["refuri"], str)
+    assert_type(node.get("refuri"), "str | None")
+    assert_type(node.get("refuri", ""), str)
+
+    # Arbitrary attributes are `Any`.
+    assert_type(node.get("highlight_args", {}), Any)
+
+
+# Visitor methods should be overridable with any return type.
+class Visitor(nodes.SparseNodeVisitor):
+    def visit_paragraph(self, node: nodes.paragraph) -> None:
+        raise nodes.SkipNode
+
+    def visit_section(self, node: nodes.Element) -> bool:
+        return True
+
+
+# A node's parent may be `None`; the root document never has one.
+def parents(document: nodes.document, paragraph: nodes.paragraph) -> None:
+    assert_type(document.parent, None)
+    assert_type(paragraph.parent, "nodes.Element | None")
+    if paragraph.parent is not None:
+        paragraph.parent.remove(paragraph)
+
+
+# Visit/depart methods do not exist on the base `NodeVisitor` at runtime.
+# The methods only exist when a subclass defines them.
+class DirectVisitor(nodes.NodeVisitor):
+    def visit_paragraph(self, node: nodes.paragraph) -> None:
+        # Pyright correctly flags this as not existing NodeVisitor.
+        super().visit_paragraph(node)  # pyright: ignore[reportGeneralTypeIssues]
+
+
+class SparseVisitor(nodes.SparseNodeVisitor):
+    # Pyright correctly allows this because it is concretely defined
+    # in SparseNodeVisitor.
+    def visit_paragraph(self, node: nodes.paragraph) -> None:
+        super().visit_paragraph(node)

stubs/docutils/@tests/test_cases/check_roles.py

@@ -0,0 +1,45 @@
+from __future__ import annotations
+
+from collections.abc import Mapping, Sequence
+from typing import Any
+from typing_extensions import assert_type
+
+from docutils import nodes
+from docutils.parsers.rst import roles
+from docutils.parsers.rst.states import Inliner
+
+# Docutils calls Role functions with five positional arguments, so the names do not matter.
+# `options` and `content` are passed as keyword-only by `CustomRole`, so they must have defaults.
+
+
+def role_with_defaults(
+    name: str,
+    rawtext: str,
+    text: str,
+    lineno: int,
+    inliner: Inliner,
+    options: Mapping[str, Any] | None = None,
+    content: Sequence[str] | None = None,
+) -> tuple[list[nodes.Node], list[nodes.system_message]]:
+    return [], []
+
+
+def role_with_dict_defaults(
+    typ: str, rawtext: str, text: str, lineno: int, inliner: Inliner, options: dict[str, Any] = {}, content: list[str] = []
+) -> tuple[list[nodes.reference], list[nodes.system_message]]:
+    return [], []
+
+
+def role_without_defaults(
+    name: str, rawtext: str, text: str, lineno: int, inliner: Inliner, options: dict[str, Any], content: list[str]
+) -> tuple[list[nodes.Node], list[nodes.system_message]]:
+    return [], []
+
+
+roles.register_local_role("a", role_with_defaults)
+roles.register_local_role("b", role_with_dict_defaults)
+roles.register_local_role("c", role_without_defaults)  # type: ignore[arg-type]
+roles.register_canonical_role("d", roles.GenericRole("d", nodes.emphasis))
+
+# `normalize_options()` should preserve the value type of the options.
+assert_type(roles.normalize_options({"class": ["a"]}), dict[str, list[str]])

stubs/docutils/@tests/test_cases/check_transforms.py

@@ -0,0 +1,25 @@
+from __future__ import annotations
+
+from typing import Any
+
+from docutils.transforms import Transform
+
+# `apply()` overrides should be accepted with and without kwargs,
+# True keyword arguments (and positional) should error.
+
+
+class DocutilsStyle(Transform):
+    def apply(self) -> None: ...
+
+
+class SphinxStyle(Transform):
+    def apply(self, **kwargs: Any) -> None: ...
+
+
+class RequiredKeyword(Transform):
+    def apply(self, *, level: int) -> None: ...  # type: ignore[override]
+
+
+def apply(transform: Transform) -> None:
+    transform.apply()
+    transform.apply(level=1)  # type: ignore[call-arg]