From 50f2e1112a1e0054e553db8c4071efba92e79761 Mon Sep 17 00:00:00 2001 From: Anders Kaseorg Date: Mon, 5 Oct 2026 23:45:26 +0000 Subject: [PATCH 1/2] Describe the whole _rustlib module in its type stub _rustlib.pyi described only the options, QualName, and AttrsListItems, so type checkers reported every use of the rest of the module as an error: mypy found 54 errors in the package, mostly "Module has no attribute". Add TreeDom, the node classes, AttrsList, Parser, Select, serialize, _is_node_impl, and __all__. Give the __new__ methods of HtmlOptions and XmlOptions parameter types and fix their return types, which mypy rejected. Mark the classes final, since they can't be subclassed, and declare those that can't be constructed with a __new__ that doesn't return. mypy's stubtest now finds no differences from the runtime module, except that the stub doesn't declare the iter submodule. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Anders Kaseorg --- python/markupever/_rustlib.pyi | 210 +++++++++++++++++++++++++++++---- 1 file changed, 190 insertions(+), 20 deletions(-) diff --git a/python/markupever/_rustlib.pyi b/python/markupever/_rustlib.pyi index e40ee0c..82b2157 100644 --- a/python/markupever/_rustlib.pyi +++ b/python/markupever/_rustlib.pyi @@ -1,5 +1,29 @@ import typing +__all__ = [ + "QUIRKS_MODE_FULL", + "QUIRKS_MODE_LIMITED", + "QUIRKS_MODE_OFF", + "AttrsList", + "AttrsListItems", + "Comment", + "Doctype", + "Document", + "Element", + "HtmlOptions", + "Parser", + "ProcessingInstruction", + "QualName", + "Select", + "Text", + "TreeDom", + "XmlOptions", + "__author__", + "__version__", + "_is_node_impl", + "serialize", +] + __version__: str __author__: str @@ -7,6 +31,7 @@ QUIRKS_MODE_FULL: typing.Literal[0] QUIRKS_MODE_LIMITED: typing.Literal[1] QUIRKS_MODE_OFF: typing.Literal[2] +@typing.final class HtmlOptions: """ These are options for HTML parsing. @@ -15,16 +40,16 @@ class HtmlOptions: """ def __new__( - cls: type, - full_document=..., - exact_errors=..., - discard_bom=..., - profile=..., - iframe_srcdoc=..., - drop_doctype=..., - quirks_mode=..., + cls, + full_document: bool | None = ..., + exact_errors: bool = ..., + discard_bom: bool = ..., + profile: bool = ..., + iframe_srcdoc: bool = ..., + drop_doctype: bool = ..., + quirks_mode: int = ..., fragment_context: QualName | str | None = ..., - ) -> typing.Self: + ) -> HtmlOptions: """ Creates a new `HtmlOptions` @@ -58,6 +83,7 @@ class HtmlOptions: @property def quirks_mode(self) -> int: ... +@typing.final class XmlOptions: """ These are options for XML parsing. @@ -66,11 +92,11 @@ class XmlOptions: """ def __new__( - cls: type, - exact_errors=..., - discard_bom=..., - profile=..., - ) -> typing.Self: + cls, + exact_errors: bool = ..., + discard_bom: bool = ..., + profile: bool = ..., + ) -> XmlOptions: """ Creates a new `XmlOptions` @@ -86,6 +112,7 @@ class XmlOptions: @property def profile(self) -> bool: ... +@typing.final class QualName: """ A fully qualified name (with a namespace), used to depict names of tags and attributes. @@ -166,14 +193,157 @@ class QualName: Returns a new QualName instance with the same local name, namespace, and prefix. """ - def __eq__(self, value) -> bool: ... - def __ne__(self, value) -> bool: ... - def __gt__(self, value) -> bool: ... - def __ge__(self, value) -> bool: ... - def __lt__(self, value) -> bool: ... - def __le__(self, value) -> bool: ... + def __eq__(self, value: object, /) -> bool: ... + def __ne__(self, value: object, /) -> bool: ... + def __gt__(self, value: QualName, /) -> bool: ... + def __ge__(self, value: QualName, /) -> bool: ... + def __lt__(self, value: QualName, /) -> bool: ... + def __le__(self, value: QualName, /) -> bool: ... def __hash__(self) -> int: ... +_QualNameOrStr: typing.TypeAlias = QualName | str +_Node: typing.TypeAlias = ( + Document | Doctype | Comment | Text | Element | ProcessingInstruction +) + +@typing.final +class TreeDom: + """A tree of nodes, which owns all of its nodes.""" + + def __new__(cls, *, namespaces: dict[str, str] | None = ...) -> TreeDom: ... + @classmethod + def with_capacity( + cls, capacity: int, *, namespaces: dict[str, str] | None = ... + ) -> TreeDom: ... + def namespaces(self) -> dict[str, str]: ... + def root(self) -> Document: ... + def append(self, parent: _Node, child: _Node) -> None: ... + def prepend(self, parent: _Node, child: _Node) -> None: ... + def insert_before(self, sibling: _Node, new_sibling: _Node) -> None: ... + def insert_after(self, sibling: _Node, new_sibling: _Node) -> None: ... + def detach(self, node: _Node) -> None: ... + def reparent_append(self, parent: _Node, child: _Node) -> None: ... + def reparent_prepend(self, parent: _Node, child: _Node) -> None: ... + def __eq__(self, value: object, /) -> bool: ... + def __ne__(self, value: object, /) -> bool: ... + __hash__: typing.ClassVar[None] # type: ignore[assignment] + def __len__(self) -> int: ... + +class _NodeMethods: + """Methods that all node types have (not a real class).""" + + def tree(self) -> TreeDom: ... + def parent(self) -> _Node | None: ... + def prev_sibling(self) -> _Node | None: ... + def next_sibling(self) -> _Node | None: ... + def first_child(self) -> _Node | None: ... + def last_child(self) -> _Node | None: ... + def has_children(self) -> bool: ... + def has_siblings(self) -> bool: ... + def __eq__(self, value: object, /) -> bool: ... + def __ne__(self, value: object, /) -> bool: ... + __hash__: typing.ClassVar[None] # type: ignore[assignment] + +@typing.final +class Document(_NodeMethods): + """A document node, the root of a tree. It has no constructor.""" + + def __new__(cls, treedom: object) -> typing.NoReturn: ... + +@typing.final +class Doctype(_NodeMethods): + def __new__( + cls, treedom: TreeDom, name: str, public_id: str, system_id: str + ) -> Doctype: ... + name: str + public_id: str + system_id: str + +@typing.final +class Comment(_NodeMethods): + def __new__(cls, treedom: TreeDom, content: str) -> Comment: ... + content: str + +@typing.final +class Text(_NodeMethods): + def __new__(cls, treedom: TreeDom, content: str) -> Text: ... + content: str + +@typing.final +class Element(_NodeMethods): + def __new__( + cls, + treedom: TreeDom, + name: _QualNameOrStr, + attrs: typing.Iterable[tuple[_QualNameOrStr, str]], + template: bool, + mathml_annotation_xml_integration_point: bool, + ) -> Element: ... + @property + def name(self) -> QualName: ... + @name.setter + def name(self, value: _QualNameOrStr) -> None: ... + @property + def attrs(self) -> AttrsList: ... + @attrs.setter + def attrs(self, value: typing.Iterable[tuple[_QualNameOrStr, str]]) -> None: ... + template: bool + mathml_annotation_xml_integration_point: bool + def id(self) -> str | None: ... + def class_list(self) -> list[str]: ... + +@typing.final +class ProcessingInstruction(_NodeMethods): + def __new__( + cls, treedom: TreeDom, data: str, target: str + ) -> ProcessingInstruction: ... + data: str + target: str + +@typing.final +class AttrsList: + """The attributes of an element. It has no constructor.""" + + def __new__(cls, *args: object, **kwargs: object) -> typing.NoReturn: ... + def clear(self) -> None: ... + def insert(self, index: int, key: _QualNameOrStr, value: str) -> None: ... + def push(self, key: _QualNameOrStr, value: str) -> None: ... + def items(self) -> AttrsListItems: ... + def update_item(self, index: int, key: _QualNameOrStr, value: str) -> None: ... + def update_value(self, index: int, value: str) -> None: ... + def get_by_index(self, index: int) -> tuple[QualName, str]: ... + def remove(self, index: int) -> tuple[QualName, str]: ... + def swap_remove(self, index: int) -> tuple[QualName, str]: ... + def dedup(self) -> None: ... + def reverse(self) -> None: ... + def __len__(self) -> int: ... + +@typing.final class AttrsListItems: + def __new__(cls, *args: object, **kwargs: object) -> typing.NoReturn: ... def __iter__(self) -> AttrsListItems: ... def __next__(self) -> tuple[QualName, str]: ... + def __len__(self) -> int: ... + +@typing.final +class Parser: + def __new__(cls, options: HtmlOptions | XmlOptions) -> Parser: ... + def process(self, content: str | bytes) -> None: ... + def finish(self) -> None: ... + def into_dom(self) -> TreeDom: ... + def errors(self) -> list[str]: ... + def quirks_mode(self) -> int: ... + def lineno(self) -> int: ... + +@typing.final +class Select: + """An iterator over the elements matching a CSS selector, in document order.""" + + def __new__(cls, node: _Node, expression: str) -> Select: ... + def __iter__(self) -> Select: ... + def __next__(self) -> Element: ... + +def serialize( + node: _Node, indent: int = ..., include_self: bool = ..., is_html: bool | None = ... +) -> bytes: ... +def _is_node_impl(object: object) -> bool: ... From e0a926e4b3dca4f44aa8df0ca7355b16aef003c0 Mon Sep 17 00:00:00 2001 From: Anders Kaseorg Date: Mon, 5 Oct 2026 23:45:36 +0000 Subject: [PATCH 2/2] Make _rustlib.iter a submodule with its own type stub The iter submodule was only an attribute of _rustlib, so import markupever._rustlib.iter failed because _rustlib isn't a package, and its classes claimed to be in markupever._rustlib. Register iter in sys.modules as markupever._rustlib.iter, as for a submodule of a package, and set its classes' module to match. Replace _rustlib.pyi with a stub package, _rustlib/__init__.pyi and _rustlib/iter.pyi, since a stub file can't declare a submodule. At runtime, Python still imports the extension module rather than the directory, which has no __init__.py. Wheels include the stub package, and mypy and pyright find it there. mypy's stubtest now finds no differences from the runtime module or the iter submodule. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Anders Kaseorg --- .../{_rustlib.pyi => _rustlib/__init__.pyi} | 3 + python/markupever/_rustlib/iter.pyi | 75 +++++++++++++++++++ python/tests/test_rustlib.py | 10 +++ src/iter/iterator.rs | 6 +- src/iter/mod.rs | 13 +++- src/iter/traverse.rs | 4 +- 6 files changed, 104 insertions(+), 7 deletions(-) rename python/markupever/{_rustlib.pyi => _rustlib/__init__.pyi} (99%) create mode 100644 python/markupever/_rustlib/iter.pyi diff --git a/python/markupever/_rustlib.pyi b/python/markupever/_rustlib/__init__.pyi similarity index 99% rename from python/markupever/_rustlib.pyi rename to python/markupever/_rustlib/__init__.pyi index 82b2157..26a8dbe 100644 --- a/python/markupever/_rustlib.pyi +++ b/python/markupever/_rustlib/__init__.pyi @@ -1,5 +1,7 @@ import typing +from . import iter as iter + __all__ = [ "QUIRKS_MODE_FULL", "QUIRKS_MODE_LIMITED", @@ -21,6 +23,7 @@ __all__ = [ "__author__", "__version__", "_is_node_impl", + "iter", "serialize", ] diff --git a/python/markupever/_rustlib/iter.pyi b/python/markupever/_rustlib/iter.pyi new file mode 100644 index 0000000..12e6cfc --- /dev/null +++ b/python/markupever/_rustlib/iter.pyi @@ -0,0 +1,75 @@ +"""Iterators over nodes.""" + +import typing + +from . import TreeDom, _Node + +__all__ = [ + "Ancestors", + "Children", + "Descendants", + "FirstChildren", + "Iterator", + "LastChildren", + "NextSiblings", + "PrevSiblings", + "Traverse", +] + +@typing.final +class Iterator: + """Iterates over all of the nodes of a tree in the order they were created, including orphans.""" + + def __new__(cls, dom: TreeDom) -> Iterator: ... + def __iter__(self) -> Iterator: ... + def __next__(self) -> _Node: ... + +@typing.final +class Ancestors: + def __new__(cls, node: _Node) -> Ancestors: ... + def __iter__(self) -> Ancestors: ... + def __next__(self) -> _Node: ... + +@typing.final +class PrevSiblings: + def __new__(cls, node: _Node) -> PrevSiblings: ... + def __iter__(self) -> PrevSiblings: ... + def __next__(self) -> _Node: ... + +@typing.final +class NextSiblings: + def __new__(cls, node: _Node) -> NextSiblings: ... + def __iter__(self) -> NextSiblings: ... + def __next__(self) -> _Node: ... + +@typing.final +class FirstChildren: + def __new__(cls, node: _Node) -> FirstChildren: ... + def __iter__(self) -> FirstChildren: ... + def __next__(self) -> _Node: ... + +@typing.final +class LastChildren: + def __new__(cls, node: _Node) -> LastChildren: ... + def __iter__(self) -> LastChildren: ... + def __next__(self) -> _Node: ... + +@typing.final +class Children: + def __new__(cls, node: _Node) -> Children: ... + def __iter__(self) -> Children: ... + def __next__(self) -> _Node: ... + +@typing.final +class Traverse: + """Iterates over the edges of a subtree: each node with `False` when opened and `True` when closed.""" + + def __new__(cls, node: _Node) -> Traverse: ... + def __iter__(self) -> Traverse: ... + def __next__(self) -> tuple[_Node, bool]: ... + +@typing.final +class Descendants: + def __new__(cls, node: _Node) -> Descendants: ... + def __iter__(self) -> Descendants: ... + def __next__(self) -> _Node: ... diff --git a/python/tests/test_rustlib.py b/python/tests/test_rustlib.py index 6545e6e..32824f7 100644 --- a/python/tests/test_rustlib.py +++ b/python/tests/test_rustlib.py @@ -624,3 +624,13 @@ def test_serialize(): assert rl.serialize(dom.root(), 0) == b"Ali" assert rl.serialize(dom.root(), 0, is_html=False) == b"Ali" + + +def test_iter_submodule(): + import markupever._rustlib.iter + + assert markupever._rustlib.iter is rl.iter + assert rl.iter.__name__ == "markupever._rustlib.iter" + + for name in rl.iter.__all__: + assert getattr(rl.iter, name).__module__ == "markupever._rustlib.iter", name diff --git a/src/iter/iterator.rs b/src/iter/iterator.rs index ca89da2..aebaea8 100644 --- a/src/iter/iterator.rs +++ b/src/iter/iterator.rs @@ -2,7 +2,7 @@ use std::sync::atomic; use std::sync::Arc; /// Iterates the nodes in insert order - don't matter which are orphan which not -#[pyo3::pyclass(name = "Iterator", module = "markupever._rustlib", frozen)] +#[pyo3::pyclass(name = "Iterator", module = "markupever._rustlib.iter", frozen)] pub struct PyIterator { dom: Arc>, index: atomic::AtomicUsize, @@ -48,7 +48,7 @@ macro_rules! axis_iterators { ) => { $( #[$m] - #[pyo3::pyclass(name = $pyname, module = "markupever._rustlib")] + #[pyo3::pyclass(name = $pyname, module = "markupever._rustlib.iter")] pub struct $name { guard: Option, } @@ -94,7 +94,7 @@ axis_iterators! { PyLastChildren(crate::nodes::NodeGuard::last_child) as "LastChildren"; } -#[pyo3::pyclass(name = "Children", module = "markupever._rustlib")] +#[pyo3::pyclass(name = "Children", module = "markupever._rustlib.iter")] pub struct PyChildren { front: Option, back: Option, diff --git a/src/iter/mod.rs b/src/iter/mod.rs index ac41d1c..a5f8c2f 100644 --- a/src/iter/mod.rs +++ b/src/iter/mod.rs @@ -1,4 +1,4 @@ -use pyo3::types::PyModuleMethods; +use pyo3::types::{PyAnyMethods, PyModuleMethods}; mod iterator; mod traverse; @@ -20,5 +20,14 @@ pub fn register_iter_module(m: &pyo3::Bound<'_, pyo3::types::PyModule>) -> pyo3: iter_module.add_class::()?; iter_module.add_class::()?; - m.add_submodule(&iter_module) + m.add_submodule(&iter_module)?; + + // Register it like a submodule of a package, so that `import markupever._rustlib.iter` + // works. + let name = format!("{}.iter", m.name()?); + iter_module.setattr("__name__", &name)?; + m.py() + .import("sys")? + .getattr("modules")? + .set_item(name, iter_module) } diff --git a/src/iter/traverse.rs b/src/iter/traverse.rs index 47a6829..bf74589 100644 --- a/src/iter/traverse.rs +++ b/src/iter/traverse.rs @@ -4,7 +4,7 @@ enum EdgeSign { } /// An iterator which traverses a subtree. -#[pyo3::pyclass(name = "Traverse", module = "markupever._rustlib")] +#[pyo3::pyclass(name = "Traverse", module = "markupever._rustlib.iter")] pub struct PyTraverse { root: Option, edge: Option, @@ -73,7 +73,7 @@ impl PyTraverse { } /// An iterator over a node and its descendants. -#[pyo3::pyclass(name = "Descendants", module = "markupever._rustlib")] +#[pyo3::pyclass(name = "Descendants", module = "markupever._rustlib.iter")] pub struct PyDescendants(PyTraverse); #[pyo3::pymethods]