Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions docs/conf.py
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@

intersphinx_mapping = {
"python": ("https://docs.python.org/3/", None),
"numpy": ("https://numpy.org/doc/stable/", None),
"typing": ("https://typing.python.org/en/latest/", None),
"lark": ("https://lark-parser.readthedocs.io/en/latest/", None),
"libcst": ("https://libcst.readthedocs.io/en/latest/", None),
Expand Down
62 changes: 4 additions & 58 deletions docs/create_plugin.md
Original file line number Diff line number Diff line change
Expand Up @@ -26,9 +26,7 @@ from docstub.doctype import DoctypeTransformer, Expr, Term

my_grammar = """
%import doctype (start, WS_INLINE, ARRAY_NAME)

%extend ARRAY_NAME: "my_array"

%ignore WS_INLINE
"""

Expand Down Expand Up @@ -62,13 +60,11 @@ from itertools import product

import lark
from docstub.plugins import hookimpl
from docstub.doctype import DoctypeTransformer, Expr, Term
from docstub.doctype import DoctypeTransformer, Expr, Term, expand_expression

my_grammar = """
%import doctype (start, WS_INLINE, qualname)

%extend qualname: "{" qualname ("," qualname)+ "}" -> expansion

%ignore WS_INLINE
"""

Expand All @@ -86,59 +82,9 @@ class ExpansionTransformer(DoctypeTransformer):
-------
expr : Expr
"""
children = map(self._expand_expression, tree.children)
return Expr(rule="start", children=tuple(children))

def expansion(self, tree):
"""Transform "expansion" rule.

Actual expansion happens in `start` when the complete expression with
all potential expansions can be seen.

Parameters
----------
tree : lark.Tree

Returns
-------
expr : Expr
"""
return Expr(rule="expansion", children=tuple(tree.children))

def _expand_expression(self, expr):
"""Join combination of all expansions in a new union.

Parameters
----------
expr : Expr

Returns
-------
expanded : Expr
"""
expansions = expr.filter(rule="expansion")
if not expansions:
return expr

# Create Cartesian product of all expansions where each
# value is prefixed by the expansion it will replace
expansions = [
[(expansion, value) for value in expansion.children]
for expansion in expansions
]
combinations = list(product(*expansions))

# Create a new expression for each combination
interpolated = []
for combination in combinations:
new_expr = expr
for expansion, value in combination:
new_expr = new_expr.replace(expansion, value)
interpolated.append(new_expr)

# Join new expressions as union
expanded = self.union(lark.Tree("union", children=interpolated))
return expanded
assert len(tree.children) == 1
child = expand_expression(tree.children[0], expand_rule="expansion")
return Expr(rule="start", children=(child,))


@hookimpl
Expand Down
8 changes: 4 additions & 4 deletions docs/introduction.md
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ def example_metric(image, *, mask=None, sigma=1.0, method='standard'):
----------
image : array-like
First image.
mask : array of dtype uint8, optional
mask : array of dtype uint8 and shape (M, N), optional
Second image.
sigma : float or Iterable of float, default: 1.0
Sigma value for each dimension in `image`. A single value is broadcast
Expand Down Expand Up @@ -51,18 +51,18 @@ will create the [stub file](https://typing.python.org/en/latest/spec/distributin
# File generated with docstub

from collections.abc import Iterable
from typing import Literal
from typing import Any, Literal

import numpy as np
from numpy.typing import ArrayLike, NDArray

def example_metric(
image: ArrayLike,
*,
mask: NDArray[np.uint8] | None = ...,
mask: NDArray[tuple[int, int], np.dtype[np.uint8]] | None = ...,
sigma: float | Iterable[float] = ...,
method: Literal["standard", "modified"] = ...
) -> NDArray[float]: ...
) -> NDArray[tuple[Any, ...], np.dtype[float]]: ...
```

<!--- end example.pyi --->
Expand Down
119 changes: 65 additions & 54 deletions docs/typing_syntax.md
Original file line number Diff line number Diff line change
@@ -1,9 +1,8 @@
# Typing syntax in docstrings
# Extended typing syntax

Docstub defines its own [grammar](../src/docstub/doctype.lark) to parse and transform type information in docstrings into valid Python type expressions.
This grammar fully supports [Python's conventional typing syntax](https://typing.python.org/en/latest/index.html).
So any {term}`annotation expression` that is valid in Python, can be used in a docstrings as is.
In addition, docstub extends this syntax with several "natural language" expressions that are commonly used in the scientific Python ecosystem.
Any valid Python {term}`annotation expression` can be used in a docstrings as is.
However, Docstub extends [Python's conventional typing syntax](https://typing.python.org/en/latest/index.html) with several "natural language" expressions that are commonly used in the scientific Python ecosystem.
This extended syntax is defined in its own [grammar](#appendix-grammar).

Docstrings should follow a form that is inspired by the [NumPyDoc style](https://numpydoc.readthedocs.io/en/latest/format.html):
```none
Expand All @@ -28,83 +27,87 @@ Combining multiple names that share a doctype and description is supported.

In addition to Python's conventional shorthand `|` syntax for [union types](https://typing.python.org/en/latest/spec/concepts.html#union-types), you can use `or` to join types.

| Docstring type | Python type annotation |
|----------------|------------------------|
| `X or Y` | `X \| Y` |
| `int or float` | `int \| float` |
| doctype | Python annotation |
|----------------|-------------------|
| `X or Y` | `X \| Y` |
| `int or float` | `int \| float` |


## Containers

The content of containers can be typed using a `CONTAINER of X` like form.
This extends the basic subscription syntax for [generics](https://typing.python.org/en/latest/spec/generics.html#generics).

| Docstring type | Python type annotation |
|-------------------------|------------------------|
| `CONTAINER of X` | `CONTAINER[X]` |
| `CONTAINER of (X or Y)` | `CONTAINER[X \| Y]` |
| doctype | Python annotation |
|-------------------------|---------------------|
| `CONTAINER of X` | `CONTAINER[X]` |
| `CONTAINER of (X or Y)` | `CONTAINER[X \| Y]` |

For the simple case `CONTAINER of X`, where `X` is a name, you can append `(s)` to indicate the plural form.
For example, `list of float(s)`.

Variants of for [**tuples**](https://typing.python.org/en/latest/spec/tuples.html)

| Docstring type | Python type annotation |
|---------------------|------------------------|
| `tuple of (X, Y)` | `tuple[X, Y]` |
| `tuple of (X, ...)` | `tuple[X, ...]` |
| doctype | Python annotation |
|---------------------|-------------------|
| `tuple of (X, Y)` | `tuple[X, Y]` |
| `tuple of (X, ...)` | `tuple[X, ...]` |

and **mappings** exist.

| Docstring type | Python type annotation |
|----------------------|------------------------|
| `MAPPING of {X: Y}` | `MAPPING[X, Y]` |
| `dict of {str: int}` | `dict[str, int]` |
| doctype | Python annotation |
|----------------------|-------------------|
| `MAPPING of {X: Y}` | `MAPPING[X, Y]` |
| `dict of {str: int}` | `dict[str, int]` |


:::{tip}
While it is possible to nest these variants repeatedly, it decreases the readability.
While it is possible to nest these variants, it decreases the readability.
For complex nested annotations with nested containers, consider using Python's conventional syntax.
In the future, docstub may warn against or disallow nesting these natural language variants.
:::


(natlang-array)=
## Shape and dtype syntax for arrays
## NDArrays with shape and dtype

This expression allows adding shape and datatype information for data structures like [NumPy arrays](https://numpy.org/doc/stable/reference/generated/numpy.ndarray.html).
This "natural language" expression describes shape and dtype information for [NumPy arrays](https://numpy.org/doc/stable/reference/generated/numpy.ndarray.html).
The basic form is as follows:

`array` and `ndarray`, and `array-like` and `array_like` can be used interchange-ably for the variable `ARRAY` below.
| doctype | Python annotation |
|----------------------------------------|-------------------------------------------|
| `ARRAY of shape SHAPE` | `ARRAY[SHAPE]` |
| `ARRAY of dtype DTYPE` | `ARRAY[tuple[Any, ...], np.dtype[DTYPE]]` |
| `ARRAY of dtype DTYPE and shape SHAPE` | `ARRAY[SHAPE, np.dtype[DTYPE]]` |
| `ARRAY of shape SHAPE and dtype DTYPE` | `ARRAY[SHAPE, np.dtype[DTYPE]]` |

| Docstring type | Python type annotation |
|----------------------------------------|------------------------|
| `ARRAY of dtype DTYPE` | `ARRAY[DTYPE]` |
| `ARRAY of dtype DTYPE and shape SHAPE` | `ARRAY[DTYPE]` |
| `ARRAY of shape SHAPE` | `ARRAY[DTYPE]` |
| `ARRAY of shape SHAPE and dtype DTYPE` | `ARRAY[DTYPE]` |
Only `array`, `ndarray`, `array-like` and `array_like` can be used in place of `ARRAY` above.
`array-like` and `array_like` map to {obj}`numpy.typing.ArrayLike` which does not support adding shape and dtype information.
In this case, Docstub omits the dtype and shape automatically.

For example
`SHAPE` can be given in forms like `(n,)`, `(M, N)`, `(x, y, 4)`, or `(..., 2)`.
Optional dimensions are supported by enclosing them in square brackets like `(x, y[, z])`.

| Docstring type | Python type annotation |
|------------------------------------------|------------------------|
| `array of dtype int` | `ndarray[int]` |
| `ndarray of dtype bool and shape (4, 4)` | `ndarray[bool]` |
| `array-like of dtype float` | `ArrayLike[float]` |
| `array_like of shape (M, 2)` | `ArrayLike` |
`DTYPE` can be any valid name or a union of names.

For example:

:::{note}
Noting the **shape** of an array in the docstring is supported.
However, [support for including shapes in generated stubs](https://github.com/scientific-python/docstub/issues/76) is not yet included in docstub.
:::
| doctype | Python annotation |
|------------------------------------------|--------------------------------------------|
| `array of dtype int` | `NDArray[tuple[Any, ...], np.dtype[int]]` |
| `array of shape (..., 2)` | `NDArray[tuple[Any, ...]]` |
| `array of shape (x, y[, z])` | `NDArray[tuple[int, int]]`<br>`\| NDArray[tuple[int, int, int]]` |
| `ndarray of dtype bool and shape (4, 4)` | `NDArray[tuple[int, int], np.dtype[bool]]` |
| `array-like of dtype float` | `ArrayLike` |
| `array_like of shape (M, 2)` | `ArrayLike` |


## Literals

[Literals](https://typing.python.org/en/latest/spec/literal.html#literals) indicate a concrete value instead of type.
Instead of using [`typing.Literal`](https://docs.python.org/3/library/typing.html#typing.Literal), you can enclose literal values in `{...}` in docstrings.

| Docstring type | Python type annotation |
| doctype | Python annotation |
|---------------------------|----------------------------------|
| `{-1, 0, 3, True, False}` | `Literal[-1, 0, 3, True, False]` |
| `{"red", "blue", None}` | `Literal["red", "blue", None]` |
Expand All @@ -117,19 +120,27 @@ However, `Literal[X]` is more explicit.
:::{warning}
Python's `typing.Literal` only supports a restricted set of parameters.
For example, `float` literals are not yet supported by the type system but are allowed by docstub.
Addressing this use case is on the roadmap.
See [issue 47](https://github.com/scientific-python/docstub/issues/47) for more details.
Use a broader inlined type such as `Literal[0, 1] | float` if you want to use something like `{0, .5, 1}` in the docstring.
:::

## reStructuredText role

Since docstrings are also used to generate documentation with Sphinx, you may want to use [restructuredText roles](https://docutils.sourceforge.io/docs/ref/rst/roles.html).
This is supported anywhere in where a {term}`type name` is used in a {term}`doctype`.

| Docstring type | Python type annotation |
|-------------------------|------------------------|
| `` `X` `` | `X` |
| ``:ref:`X` `` | `X` |
| ``:class:`X`[Y, ...] `` | `X[Y, ...]` |
| ``:class:`Y.X` `` | `Y.X` |
| ``:py:class:`Y.X` `` | `Y.X` |
This is supported anywhere in where a {term}`type name` used in a {term}`doctype`.

| doctype | Python annotation |
|-------------------------|-------------------|
| `` `X` `` | `X` |
| ``:ref:`X` `` | `X` |
| ``:class:`X`[Y, ...] `` | `X[Y, ...]` |
| ``:class:`Y.X` `` | `Y.X` |
| ``:py:class:`Y.X` `` | `Y.X` |


---

## Appendix: Grammar

:::{literalinclude} ../src/docstub/doctype.lark
:caption: doctype.lark
:::
16 changes: 12 additions & 4 deletions examples/example_pkg-stubs/_numpy.pyi
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
# File generated with docstub

from typing import Any

import numpy
import numpy as np
from numpy.typing import ArrayLike, NDArray
Expand All @@ -8,8 +10,14 @@ def func_object_with_numpy_objects(
a1: numpy.int8, a2: np.int16, a3: numpy.typing.DTypeLike, a4: np.typing.DTypeLike
) -> None: ...
def func_ndarray(
a1: NDArray, a2: np.NDArray, a3: NDArray[float], a4: NDArray[np.uint8] | None = ...
) -> tuple[NDArray[np.uint8], NDArray[complex]]: ...
a1: NDArray,
a2: np.NDArray,
a3: NDArray[tuple[int, int], np.dtype[float]],
a4: NDArray[tuple[int], np.dtype[np.uint8]] | None = ...,
) -> tuple[
NDArray[tuple[Any, ...], np.dtype[np.uint8]],
NDArray[tuple[Any, ...], np.dtype[complex]],
]: ...
def func_array_like(
a1: ArrayLike, a2: ArrayLike, a3: ArrayLike[float], a4: ArrayLike[np.uint8]
) -> tuple[ArrayLike[np.uint8], ArrayLike[complex]]: ...
a1: ArrayLike, a2: ArrayLike, a3: ArrayLike, a4: ArrayLike
) -> tuple[ArrayLike, ArrayLike]: ...
2 changes: 1 addition & 1 deletion src/docstub-stubs/_docstrings.pyi
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ import numpydoc.docscrape as npds
from ._analysis import PyImport, TypeMatcher
from ._report import ContextReporter, Stats
from ._utils import escape_qualname
from .doctype import BlacklistedQualname, DoctypeTranspiler, TermKind
from .doctype import DoctypeTranspiler, QualnameIsForbidden, TermKind

logger: logging.Logger

Expand Down
Loading
Loading