From 2b986848da5e0957651d2139ce0c740eca4b1f5c Mon Sep 17 00:00:00 2001 From: erogluorhan Date: Fri, 16 Oct 2026 16:59:30 -0600 Subject: [PATCH] Add Sphinx + myst-nb community site deployed to GitHub Pages Entry pages and a catalog are generated from entries/ at build time, reusing the metadata parsing in scripts/build_index.py. Notebooks are never executed (nb_execution_mode = "off"): most need HPC-only data, so the committed outputs, including interactive HoloViews/Bokeh plots, are rendered as-is. PRs build the site; pushes to main deploy it. Also skip hidden folders (e.g. .ipynb_checkpoints) when collecting entries. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/docs.yml | 58 +++++++++++++++++++ .gitignore | 4 ++ CONTRIBUTING.md | 8 +++ README.md | 4 +- docs/conf.py | 42 ++++++++++++++ docs/generate.py | 112 +++++++++++++++++++++++++++++++++++++ docs/index.md | 21 +++++++ docs/requirements.txt | 5 ++ scripts/build_index.py | 2 +- 9 files changed, 254 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/docs.yml create mode 100644 docs/conf.py create mode 100644 docs/generate.py create mode 100644 docs/index.md create mode 100644 docs/requirements.txt diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 0000000..edde4aa --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,58 @@ +name: Docs + +on: + push: + branches: [main] + paths: + - "entries/**" + - "docs/**" + - "scripts/build_index.py" + - ".github/workflows/docs.yml" + pull_request: + paths: + - "entries/**" + - "docs/**" + - "scripts/build_index.py" + - ".github/workflows/docs.yml" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: docs-${{ github.ref }} + # Cancel superseded PR builds, but never interrupt a Pages deployment. + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + cache: pip + cache-dependency-path: docs/requirements.txt + - run: pip install -r docs/requirements.txt + # Notebooks aren't executed (nb_execution_mode = "off"), so no data or + # UXarray install is needed; this only renders committed outputs. + - run: sphinx-build -b html --keep-going docs docs/_build/html + - uses: actions/upload-pages-artifact@v3 + if: github.event_name != 'pull_request' && github.ref == 'refs/heads/main' + with: + path: docs/_build/html + + deploy: + if: github.event_name != 'pull_request' && github.ref == 'refs/heads/main' + needs: build + runs-on: ubuntu-latest + permissions: + pages: write + id-token: write + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + steps: + - id: deployment + uses: actions/deploy-pages@v4 diff --git a/.gitignore b/.gitignore index 38dc770..bba9935 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,7 @@ __pycache__/ *.zarr/ *.h5 *.hdf5 + +# Docs build output (docs/entries/ is generated from entries/) +docs/_build/ +docs/entries/ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 730fb4a..5835c7a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -51,3 +51,11 @@ You can validate your entry locally: pip install pyyaml python scripts/build_index.py --check ``` + +To preview how your entry looks on the community site (notebooks are rendered from their +committed outputs, never executed): + +```bash +pip install -r docs/requirements.txt +sphinx-build -b html docs docs/_build/html # then open docs/_build/html/index.html +``` diff --git a/README.md b/README.md index 368b1f6..b21fc95 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,9 @@ user community. Here you'll find demonstration notebooks from the team, notebook together to ask questions or report behavior, workshop and tutorial material, and anything else that comes out of working together. -Everything is organized by date. Browse the full catalog in **[INDEX.md](INDEX.md)**. +Everything is organized by date. Browse the full catalog in **[INDEX.md](INDEX.md)**, or read +the notebooks with their rendered outputs on the +**[community site](https://uxarray.github.io/uxarray-community/)**. ## Layout diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 0000000..f124847 --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,42 @@ +"""Sphinx configuration for the UXarray Community site. + +Entry pages are generated from entries/ at build time (see generate.py). Notebooks are +never executed: most entries need data that only exists on HPC systems, so the outputs +committed by their authors are what gets rendered. +""" + +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import generate # noqa: E402 + +generate.main() + +project = "UXarray Community" +author = "UXarray developers and community" +copyright = "UXarray developers" + +extensions = ["myst_nb", "sphinx_copybutton"] + +# README content is folded into each entry's generated index.md. +exclude_patterns = ["_build", "**.ipynb_checkpoints", "entries/*/README.md"] + +# Render committed outputs as-is; never run notebooks during the build. +nb_execution_mode = "off" + +myst_enable_extensions = ["colon_fence", "dollarmath", "amsmath"] +myst_heading_anchors = 3 + +# Contributor notebooks often skip heading levels, and HoloViews adds helper outputs +# (application/vnd.holoviews_*) next to the HTML/JS that actually renders the plot. +suppress_warnings = ["myst.header", "mystnb.unknown_mime_type"] + +html_theme = "sphinx_book_theme" +html_title = "UXarray Community" +html_theme_options = { + "repository_url": "https://github.com/UXARRAY/uxarray-community", + "use_repository_button": True, + "use_issues_button": True, + "home_page_in_toc": True, +} diff --git a/docs/generate.py b/docs/generate.py new file mode 100644 index 0000000..462e5b5 --- /dev/null +++ b/docs/generate.py @@ -0,0 +1,112 @@ +"""Generate the site's entry pages from entries/ before Sphinx reads the sources. + +Each entry folder is copied to docs/entries// and given an index.md built from +its README metadata, README body, and notebooks. docs/entries/index.md is the catalog. +Everything under docs/entries/ is regenerated on every build and is git-ignored. +""" + +from __future__ import annotations + +import shutil +import sys +from pathlib import Path + +DOCS = Path(__file__).resolve().parent +ROOT = DOCS.parent +OUT = DOCS / "entries" + +sys.path.insert(0, str(ROOT / "scripts")) +from build_index import ENTRIES, FRONT_MATTER_RE, cell, collect # noqa: E402 + +IGNORE = shutil.ignore_patterns(".ipynb_checkpoints", "__pycache__", ".DS_Store") + + +def entry_page(entry: dict, folder: Path) -> str: + readme = (folder / "README.md").read_text(encoding="utf-8") + body = FRONT_MATTER_RE.sub("", readme, count=1).strip() + + authors = ", ".join( + f"[@{a.lstrip('@')}](https://github.com/{a.lstrip('@')})" for a in entry["authors"] + ) + facts = [ + f"**Date:** {entry['date'].isoformat()}", + f"**Type:** {entry['type']}", + f"**Authors:** {authors}", + f"**UXarray:** {entry['uxarray_version']}", + ] + if entry["tags"]: + facts.append("**Tags:** " + " ".join(f"`{t}`" for t in entry["tags"])) + + lines = [f"# {entry['title']}", ""] + if entry["status"] == "superseded": + lines += [ + ":::{warning}", + "This entry is superseded. See its description for the newer approach.", + ":::", + "", + ] + lines += [" ยท ".join(facts), "", body, ""] + + notebooks = sorted( + p.relative_to(folder).with_suffix("").as_posix() + for p in folder.rglob("*.ipynb") + if ".ipynb_checkpoints" not in p.parts + ) + if notebooks: + lines += ["```{toctree}", ":maxdepth: 1", ":titlesonly:", ":caption: Notebooks", "", *notebooks, "```", ""] + + # Other Markdown files (e.g. data.md) are pages too; keep them out of the sidebar. + others = sorted( + p.relative_to(folder).with_suffix("").as_posix() + for p in folder.rglob("*.md") + if p.name != "README.md" and ".ipynb_checkpoints" not in p.parts + ) + if others: + lines += ["```{toctree}", ":hidden:", "", *others, "```", ""] + return "\n".join(lines) + + +def catalog(entries: list[dict]) -> str: + lines = ["# Catalog", "", f"{len(entries)} entries, newest first.", ""] + entries = sorted(entries, key=lambda e: (e["date"], e["path"]), reverse=True) + for year in sorted({e["date"].year for e in entries}, reverse=True): + lines += [ + f"## {year}", + "", + "| Date | Title | Type | Authors | UXarray |", + "|------|-------|------|---------|---------|", + ] + for e in (e for e in entries if e["date"].year == year): + slug = Path(e["path"]).name + title = f"[{cell(e['title'])}]({slug}/index.md)" + if e["status"] == "superseded": + title = f"~~{title}~~ (superseded)" + authors = ", ".join(f"@{a.lstrip('@')}" for a in e["authors"]) + lines.append( + f"| {e['date'].isoformat()} | {title} | {e['type']} | {authors} " + f"| {cell(e['uxarray_version'])} |" + ) + lines.append("") + slugs = [f"{Path(e['path']).name}/index" for e in entries] + lines += ["```{toctree}", ":hidden:", "", *slugs, "```", ""] + return "\n".join(lines) + + +def main() -> None: + entries, errors = collect() + for err in errors: + print(f"warning: {err}", file=sys.stderr) + + if OUT.exists(): + shutil.rmtree(OUT) + OUT.mkdir() + for entry in entries: + src = ENTRIES / Path(entry["path"]).name + dst = OUT / src.name + shutil.copytree(src, dst, ignore=IGNORE) + (dst / "index.md").write_text(entry_page(entry, src), encoding="utf-8") + (OUT / "index.md").write_text(catalog(entries), encoding="utf-8") + + +if __name__ == "__main__": + main() diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 0000000..e01f1f4 --- /dev/null +++ b/docs/index.md @@ -0,0 +1,21 @@ +# UXarray Community + +A shared space between the [UXarray](https://uxarray.readthedocs.io) team and its user +community: demonstration notebooks from the team, notebooks users put together to ask +questions or report behavior, workshop and tutorial material, and anything else that comes +out of working together. + +Browse everything in the **[Catalog](entries/index.md)**, organized by date. + +Notebooks are shown with the outputs their authors committed. Many of them run only on HPC +systems where the data lives, so each entry describes its data and pins its own +environment if you want to run it yourself. + +Want to add an entry or ask a question with a notebook? See the +[contributing guide](https://github.com/UXARRAY/uxarray-community/blob/main/CONTRIBUTING.md). + +```{toctree} +:hidden: + +entries/index +``` diff --git a/docs/requirements.txt b/docs/requirements.txt new file mode 100644 index 0000000..75991f2 --- /dev/null +++ b/docs/requirements.txt @@ -0,0 +1,5 @@ +sphinx +myst-nb +sphinx-book-theme +sphinx-copybutton +pyyaml diff --git a/scripts/build_index.py b/scripts/build_index.py index 334ea85..0c76ac0 100755 --- a/scripts/build_index.py +++ b/scripts/build_index.py @@ -91,7 +91,7 @@ def collect() -> tuple[list[dict], list[str]]: entries, errors = [], [] if not ENTRIES.is_dir(): return entries, errors - for path in sorted(p for p in ENTRIES.iterdir() if p.is_dir()): + for path in sorted(p for p in ENTRIES.iterdir() if p.is_dir() and not p.name.startswith(".")): entry, errs = load_entry(path) errors.extend(errs) if entry: