diff --git a/.github/ISSUE_TEMPLATE/bugreport.yml b/.github/ISSUE_TEMPLATE/bugreport.yml index f5276925ab5..40313ea81b1 100644 --- a/.github/ISSUE_TEMPLATE/bugreport.yml +++ b/.github/ISSUE_TEMPLATE/bugreport.yml @@ -92,7 +92,8 @@ body: attributes: label: Environment description: | - Paste the output of `xr.show_versions()` between the `
` tags, leaving an empty line following the opening tag. + Replace the `
` block below with the output of `xr.show_versions(markdown=True)`, which already includes the `
` tags. + Older versions of xarray do not support `markdown=True`; then paste the output of `xr.show_versions()` between the `
` tags, leaving an empty line following the opening tag. value: |
diff --git a/doc/api/top-level.rst b/doc/api/top-level.rst index c6f45c1863c..3494ca8a765 100644 --- a/doc/api/top-level.rst +++ b/doc/api/top-level.rst @@ -58,6 +58,7 @@ Miscellaneous decode_cf infer_freq + show_backends show_versions set_options get_options diff --git a/doc/whats-new.rst b/doc/whats-new.rst index 0ebce396662..cb0f2d6800f 100644 --- a/doc/whats-new.rst +++ b/doc/whats-new.rst @@ -42,6 +42,16 @@ New Features type defaults to ``Hashable``, so a bare ``Variable`` annotation means ``Variable[Hashable]`` and keeps its previous meaning (:pull:`11677`). By `Michael Niklas `_. +- Add :py:func:`~xarray.show_backends` to print the available IO backends, + including those of other packages, and in which order they are tried + (:issue:`6577`, :pull:`11700`). + By `Nick Hodgskin `_ and + `Michael Niklas `_. +- :py:func:`~xarray.show_versions` displays formatted tables in Jupyter + notebooks and gained a ``markdown`` option to print markdown that can be pasted + as is into GitHub issues. Package versions are read from the + package metadata, so the packages are no longer imported (:pull:`11700`). + By `Michael Niklas `_. - The ``in`` operator on :py:class:`DataTree` now accepts the same unix-like paths as indexing, so ``key in tree`` is ``True`` exactly when ``tree[key]`` succeeds, e.g. ``"group/subgroup" in tree`` or ``"/group/var" in tree``. diff --git a/pyproject.toml b/pyproject.toml index ac03a2e2d58..98f522bd26a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -151,6 +151,7 @@ module = [ "fsspec.*", "h5netcdf.*", "h5py.*", + "IPython.*", "iris.*", "mpl_toolkits.*", "nc_time_axis.*", diff --git a/xarray/__init__.py b/xarray/__init__.py index 7d56816cdfe..b646f00d46f 100644 --- a/xarray/__init__.py +++ b/xarray/__init__.py @@ -11,6 +11,7 @@ open_groups, open_mfdataset, ) +from xarray.backends.plugins import show_backends from xarray.backends.writers import save_mfdataset from xarray.backends.zarr import open_zarr from xarray.coding.cftime_offsets import cftime_range, date_range, date_range_like @@ -114,6 +115,7 @@ "register_datatree_accessor", "save_mfdataset", "set_options", + "show_backends", "show_versions", "unify_chunks", "where", diff --git a/xarray/backends/plugins.py b/xarray/backends/plugins.py index 62c79c880bf..d36a5732f3b 100644 --- a/xarray/backends/plugins.py +++ b/xarray/backends/plugins.py @@ -6,12 +6,14 @@ import os import warnings from collections.abc import Callable +from html import escape from importlib.metadata import entry_points -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, TextIO from xarray.backends.common import BACKEND_ENTRYPOINTS, BackendEntrypoint from xarray.core.options import OPTIONS from xarray.core.utils import is_remote_uri, module_available +from xarray.util._report import package_version, show_report if TYPE_CHECKING: from importlib.metadata import EntryPoint, EntryPoints @@ -135,6 +137,83 @@ def refresh_engines() -> None: list_engines.cache_clear() +def _engine_sources() -> dict[str, tuple[str, str | None]]: + """Map each engine name to its providing package and version.""" + sources: dict[str, tuple[str, str | None]] = dict.fromkeys( + BACKEND_ENTRYPOINTS, ("xarray", None) + ) + for ep in entry_points(group="xarray.backends"): + if ep.dist is not None: + sources[ep.name] = (ep.dist.name, ep.dist.version) + else: + sources[ep.name] = (ep.module.partition(".")[0], None) + return sources + + +def show_backends(file: TextIO | None = None, markdown: bool = False) -> None: + """Print the available backends (engines) and information about them. + + The backends are listed in the order in which they are tried when opening + a file without specifying the ``engine``. + Use :py:func:`xarray.backends.list_engines` to get the backend objects. + In Jupyter notebooks the backends are displayed formatted. + + Parameters + ---------- + file : file-like, optional + print to the given file-like object. Defaults to sys.stdout. + markdown : bool, default: False + print a markdown list wrapped in a ``
`` block, which can be + pasted as is into a GitHub issue. + """ + engines = list_engines() + sources = _engine_sources() + + text_lines = [] + md_lines = [] + html_items = [] + for name, backend in engines.items(): + package, version = sources.get(name, ("unknown", None)) + summary = f"from {package}" + if version is not None: + summary += f" {version}" + details = [] + if package == "xarray" and name in BACKEND_ENTRYPOINTS: + dependency = BACKEND_ENTRYPOINTS[name][0] + if dependency is not None: + details.append(f"using {dependency} {package_version(dependency)}") + if backend.supports_groups: + details.append("supports groups") + if details: + summary += f" ({', '.join(details)})" + cls_name = type(backend).__name__ + text_lines.append(f"{name}: {cls_name} {summary}") + md_lines.append(f"- **{name}**: `{cls_name}` {summary}") + + extra = [] + html_extra = [] + if backend.description: + description = " ".join(backend.description.split()) + extra.append(description) + html_extra.append(escape(description)) + if backend.url: + extra.append(backend.url) + url = escape(backend.url) + html_extra.append(f'{url}') + text_lines.extend(f" {line}" for line in extra) + md_lines.extend(f" - {line}" for line in extra) + html_items.append( + f"
  • {escape(name)}: {escape(cls_name)} " + f"{escape(summary)}
      {''.join(f'
    • {x}
      • ' for x in html_extra)}
    • " + ) + + empty = "No backends available." + text = "\n".join(text_lines) or empty + md = "\n".join(md_lines) or empty + html = f"
      {''.join(html_items)}
    " if html_items else f"

    {empty}

    " + show_report("AVAILABLE BACKENDS", text, md, html, file=file, as_markdown=markdown) + + def guess_engine( store_spec: str | os.PathLike[Any] diff --git a/xarray/tests/backends/test_plugins.py b/xarray/tests/backends/test_plugins.py index 6ad8379e6bc..8cdbc44300c 100644 --- a/xarray/tests/backends/test_plugins.py +++ b/xarray/tests/backends/test_plugins.py @@ -1,5 +1,6 @@ from __future__ import annotations +import io import subprocess import sys from importlib.metadata import EntryPoint, EntryPoints @@ -8,6 +9,7 @@ import pytest +import xarray as xr from xarray.backends import common, plugins from xarray.core.options import OPTIONS from xarray.tests import ( @@ -372,6 +374,61 @@ def test_refresh_engines() -> None: refresh_engines() +class DummyDescribedBackend(common.BackendEntrypoint): + description = "Open dummy files" + url = "https://example.com/dummy" + supports_groups = True + + def open_dataset(self, filename_or_obj, *, decoder): # type: ignore[override] + pass + + +def test_show_backends() -> None: + from xarray.backends import list_engines, refresh_engines + + dist = mock.MagicMock() + dist.name = "dummy-pkg" + dist.version = "1.2.3" + ep = mock.MagicMock() + ep.name = "dummy" + ep.dist = dist + ep.load.return_value = DummyDescribedBackend + + text = io.StringIO() + md = io.StringIO() + with mock.patch( + "xarray.backends.plugins.entry_points", return_value=EntryPoints([ep]) + ): + list_engines.cache_clear() + xr.show_backends(file=text) + xr.show_backends(file=md, markdown=True) + refresh_engines() + + out = text.getvalue() + assert out.startswith("AVAILABLE BACKENDS\n------------------\n") + expected_dummy = ( + "dummy: DummyDescribedBackend from dummy-pkg 1.2.3 (supports groups)\n" + " Open dummy files\n" + " https://example.com/dummy\n" + ) + assert expected_dummy in out + assert "store: StoreBackendEntrypoint from xarray\n" in out + # backends that are not installed are not listed + assert ("\nscipy: " in out) == has_scipy + if has_scipy: + assert "scipy: ScipyBackendEntrypoint from xarray (using scipy " in out + + out = md.getvalue() + assert out.startswith("
    AVAILABLE BACKENDS\n\n") + expected_dummy = ( + "- **dummy**: `DummyDescribedBackend` from dummy-pkg 1.2.3 (supports groups)\n" + " - Open dummy files\n" + " - https://example.com/dummy\n" + ) + assert expected_dummy in out + assert "- **store**: `StoreBackendEntrypoint` from xarray\n" in out + + @requires_h5netcdf @requires_netCDF4 @requires_zarr diff --git a/xarray/tests/test_print_versions.py b/xarray/tests/test_print_versions.py index 23b6350fb57..a026072a9d6 100644 --- a/xarray/tests/test_print_versions.py +++ b/xarray/tests/test_print_versions.py @@ -1,10 +1,16 @@ from __future__ import annotations +import importlib.metadata import io +import sys +import types +from html import escape +from unittest import mock import pytest import xarray +from xarray.util import _report # importing setuptools after distutils (done by some dependencies) warns @@ -12,4 +18,89 @@ def test_show_versions() -> None: f = io.StringIO() xarray.show_versions(file=f) - assert "INSTALLED VERSIONS" in f.getvalue() + out = f.getvalue() + assert out.startswith("INSTALLED VERSIONS\n------------------\n") + assert f"\nxarray: {importlib.metadata.version('xarray')}\n" in out + assert "Not installed: " in out + + +@pytest.mark.filterwarnings("ignore:Setuptools is replacing distutils:UserWarning") +def test_show_versions_markdown() -> None: + f = io.StringIO() + xarray.show_versions(file=f, markdown=True) + out = f.getvalue() + assert out.startswith("
    INSTALLED VERSIONS\n\n") + assert out.endswith("\n
    \n") + assert "| package " in out + assert "| xarray " in out + + +def test_package_version() -> None: + assert _report.package_version("xarray") == importlib.metadata.version("xarray") + assert _report.package_version("not_a_real_package_xyz") is None + + +def test_package_version_dist_name_differs() -> None: + with mock.patch.object( + _report, "_packages_distributions", return_value={"fake_mod": ["pytest"]} + ): + assert _report.package_version("fake_mod") == pytest.__version__ + + +def test_markdown_table() -> None: + actual = _report.markdown_table(("a", "bb"), [("x", "1 | 2"), ("long", None)]) + expected = ( + "| a | bb |\n| ---- | ------ |\n| x | 1 \\| 2 |\n| long | None |" + ) + assert actual == expected + + +def test_show_report(monkeypatch) -> None: + displayed: list[dict[str, str]] = [] + + def display(bundle, raw=False): + assert raw + displayed.append(bundle) + + display_mod = types.ModuleType("IPython.display") + display_mod.display = display # type: ignore[attr-defined] + monkeypatch.setitem(sys.modules, "IPython.display", display_mod) + + md = "
    TITLE\n\n*md*\n\n
    " + + def show(**kwargs) -> str: + f = io.StringIO() + _report.show_report("TITLE", "text", "*md*", "

    x

    ", file=f, **kwargs) + return f.getvalue() + + assert show() == "TITLE\n-----\ntext\n" + assert show(as_markdown=True) == md + "\n" + + # in notebooks the html is displayed, unless printing is requested + monkeypatch.setattr(_report, "_in_rich_frontend", lambda: True) + _report.show_report("TITLE", "text", "*md*", "

    x

    ") + (bundle,) = displayed + assert bundle["text/plain"] == "TITLE\n-----\ntext" + # copying the output copies the markdown + assert bundle["text/html"].startswith(f'

    TITLE

    x

    ") + + assert show() == "TITLE\n-----\ntext\n" + stdout = io.StringIO() + monkeypatch.setattr(sys, "stdout", stdout) + _report.show_report("TITLE", "text", "*md*", "

    x

    ", as_markdown=True) + assert stdout.getvalue() == md + "\n" + assert len(displayed) == 1 + + +def test_in_rich_frontend_without_ipython(monkeypatch) -> None: + monkeypatch.delitem(sys.modules, "IPython", raising=False) + assert not _report._in_rich_frontend() + + +def test_html_table() -> None: + actual = _report.html_table("T<", [("a", "")]) + assert actual == ( + '
    T<' + '
    a<b>
    ' + ) diff --git a/xarray/util/_report.py b/xarray/util/_report.py new file mode 100644 index 00000000000..897aaa77112 --- /dev/null +++ b/xarray/util/_report.py @@ -0,0 +1,124 @@ +"""Helpers to render diagnostic reports (show_versions, show_backends). + +Reports are printed as plain text in terminals, displayed as formatted markdown +in notebooks and can be printed as markdown to paste them into GitHub issues. +""" + +from __future__ import annotations + +import functools +import importlib.metadata +import sys +from collections.abc import Iterable, Mapping, Sequence +from html import escape +from typing import TextIO + + +@functools.cache +def _packages_distributions() -> Mapping[str, list[str]]: + return importlib.metadata.packages_distributions() + + +def package_version(module_name: str) -> str | None: + """Return the installed version of the package providing ``module_name``. + + Uses the package metadata, so the module is not imported. + Returns None if the package is not installed. + """ + try: + return importlib.metadata.version(module_name) + except importlib.metadata.PackageNotFoundError: + pass + # the distribution name differs from the module name, e.g. scitools-iris + for dist_name in _packages_distributions().get(module_name, []): + try: + return importlib.metadata.version(dist_name) + except importlib.metadata.PackageNotFoundError: + continue + return None + + +def _escape_cell(value: object) -> str: + return " ".join(str(value).split()).replace("|", r"\|") + + +def markdown_table(header: Sequence[str], rows: Iterable[Sequence[object]]) -> str: + """Render an aligned markdown table that is also readable as plain text.""" + cells = [[_escape_cell(v) for v in row] for row in [header, *rows]] + widths = [max(len(row[i]) for row in cells) for i in range(len(header))] + widths = [max(w, 3) for w in widths] + + def fmt(row: Sequence[str]) -> str: + return ( + "| " + + " | ".join(c.ljust(w) for c, w in zip(row, widths, strict=True)) + + " |" + ) + + lines = [fmt(cells[0]), fmt(["-" * w for w in widths])] + lines.extend(fmt(row) for row in cells[1:]) + return "\n".join(lines) + + +def html_table(title: str, rows: Iterable[Sequence[object]]) -> str: + """Render a titled HTML table for the notebook display.""" + body = "".join( + "" + + "".join(f'{escape(str(c))}' for c in row) + + "" + for row in rows + ) + return f"
    {escape(title)}{body}
    " + + +def html_columns(*blocks: str) -> str: + """Lay out HTML blocks side by side, wrapping when there is not enough space.""" + style = "display: flex; flex-wrap: wrap; gap: 0 3em; align-items: flex-start" + return f'
    {"".join(blocks)}
    ' + + +def _in_rich_frontend() -> bool: + """Whether we run in a frontend that can display markdown, e.g. Jupyter.""" + ipython = sys.modules.get("IPython") + if ipython is None: + return False + shell = ipython.get_ipython() + return shell is not None and type(shell).__name__ == "ZMQInteractiveShell" + + +# replace the copied content of a report with its markdown, so that copying the +# notebook output gives something to paste into a GitHub issue +_ON_COPY = ( + "event.clipboardData.setData('text/plain', this.dataset.md); event.preventDefault()" +) + + +def show_report( + title: str, + text: str, + markdown: str, + html: str, + file: TextIO | None = None, + as_markdown: bool = False, +) -> None: + """Print a report as plain text or markdown, or display it in notebooks. + + In Jupyter notebooks the ``html`` is displayed, unless a ``file`` is given or + ``as_markdown`` is requested. Copying the displayed report copies the + markdown instead. The markdown is wrapped in a ``
    `` block, so it + can be pasted as is into a GitHub issue. + """ + md = f"
    {title}\n\n{markdown}\n\n
    " + plain = f"{title}\n{'-' * len(title)}\n{text}" + + if file is None and not as_markdown and _in_rich_frontend(): + from IPython.display import display + + out = ( + f'
    ' + f"

    {escape(title)}

    {html}
    " + ) + display({"text/html": out, "text/plain": plain}, raw=True) + return + + print(md if as_markdown else plain, file=sys.stdout if file is None else file) diff --git a/xarray/util/print_versions.py b/xarray/util/print_versions.py index 0226956d9b9..59bc4edfb1e 100644 --- a/xarray/util/print_versions.py +++ b/xarray/util/print_versions.py @@ -1,13 +1,22 @@ """Utility functions for printing version information.""" import contextlib -import importlib import locale import os import platform import struct import subprocess import sys +from html import escape +from typing import TextIO + +from xarray.util._report import ( + html_columns, + html_table, + markdown_table, + package_version, + show_report, +) def get_sys_info(): @@ -76,85 +85,93 @@ def netcdf_and_hdf5_versions(): return [("libhdf5", libhdf5_version), ("libnetcdf", libnetcdf_version)] -def show_versions(file=sys.stdout): +DEPENDENCIES = [ + "xarray", + "pandas", + "numpy", + "scipy", + # xarray optionals + "netCDF4", + "pydap", + "h5netcdf", + "h5py", + "zarr", + "cftime", + "nc_time_axis", + "iris", + "bottleneck", + "dask", + "distributed", + "matplotlib", + "cartopy", + "seaborn", + "numbagg", + "fsspec", + "cupy", + "pint", + "sparse", + "flox", + "numpy_groupies", + # xarray setup/test + "setuptools", + "pip", + "conda", + "pytest", + "mypy", + # Misc. + "IPython", + "sphinx", +] + + +def show_versions(file: TextIO | None = None, markdown: bool = False) -> None: """Print the versions of xarray and its dependencies. + In Jupyter notebooks the versions are displayed as formatted tables. + Parameters ---------- file : file-like, optional print to the given file-like object. Defaults to sys.stdout. + markdown : bool, default: False + print markdown tables wrapped in a ``
    `` block, which can be + pasted as is into a GitHub issue. """ sys_info = get_sys_info() try: sys_info.extend(netcdf_and_hdf5_versions()) except Exception as e: - print(f"Error collecting netcdf / hdf5 version: {e}") - - deps = [ - # (MODULE_NAME, f(mod) -> mod version) - ("xarray", lambda mod: mod.__version__), - ("pandas", lambda mod: mod.__version__), - ("numpy", lambda mod: mod.__version__), - ("scipy", lambda mod: mod.__version__), - # xarray optionals - ("netCDF4", lambda mod: mod.__version__), - ("pydap", lambda mod: mod.__version__), - ("h5netcdf", lambda mod: mod.__version__), - ("h5py", lambda mod: mod.__version__), - ("zarr", lambda mod: mod.__version__), - ("cftime", lambda mod: mod.__version__), - ("nc_time_axis", lambda mod: mod.__version__), - ("iris", lambda mod: mod.__version__), - ("bottleneck", lambda mod: mod.__version__), - ("dask", lambda mod: mod.__version__), - ("distributed", lambda mod: mod.__version__), - ("matplotlib", lambda mod: mod.__version__), - ("cartopy", lambda mod: mod.__version__), - ("seaborn", lambda mod: mod.__version__), - ("numbagg", lambda mod: mod.__version__), - ("fsspec", lambda mod: mod.__version__), - ("cupy", lambda mod: mod.__version__), - ("pint", lambda mod: mod.__version__), - ("sparse", lambda mod: mod.__version__), - ("flox", lambda mod: mod.__version__), - ("numpy_groupies", lambda mod: mod.__version__), - # xarray setup/test - ("setuptools", lambda mod: mod.__version__), - ("pip", lambda mod: mod.__version__), - ("conda", lambda mod: mod.__version__), - ("pytest", lambda mod: mod.__version__), - ("mypy", lambda mod: importlib.metadata.version(mod.__name__)), - # Misc. - ("IPython", lambda mod: mod.__version__), - ("sphinx", lambda mod: mod.__version__), - ] - - deps_blob = [] - for modname, ver_f in deps: - try: - if modname in sys.modules: - mod = sys.modules[modname] - else: - mod = importlib.import_module(modname) - except Exception: - deps_blob.append((modname, None)) - else: - try: - ver = ver_f(mod) - deps_blob.append((modname, ver)) - except Exception: - deps_blob.append((modname, "installed")) - - print("\nINSTALLED VERSIONS", file=file) - print("------------------", file=file) - - for k, stat in sys_info: - print(f"{k}: {stat}", file=file) - - print("", file=file) - for k, stat in deps_blob: - print(f"{k}: {stat}", file=file) + sys_info.append(("libhdf5 / libnetcdf", f"error: {e}")) + + sys_info = [(k, v) for k, v in sys_info if v not in (None, "")] + versions = {name: package_version(name) for name in DEPENDENCIES} + installed = [(name, ver) for name, ver in versions.items() if ver is not None] + missing = ", ".join(name for name, ver in versions.items() if ver is None) + not_installed = f"Not installed: {missing or 'none'}" + + text = "\n\n".join( + [ + "\n".join(f"{k}: {v}" for k, v in sys_info), + "\n".join(f"{k}: {v}" for k, v in installed), + not_installed, + ] + ) + md = "\n\n".join( + [ + markdown_table(("system", "info"), sys_info), + markdown_table(("package", "version"), installed), + not_installed, + ] + ) + html = ( + html_columns( + html_table("System", sys_info), + html_table("Packages", installed), + ) + + f"

    {escape(not_installed)}

    " + ) + show_report("INSTALLED VERSIONS", text, md, html, file=file, as_markdown=markdown) if __name__ == "__main__":