docs: publish versioned documentation to GitHub Pages

.github/workflows/ci.yaml

@@ -81,6 +81,41 @@ jobs:
       - name: Run `make test-tutorials`
         run: make test-tutorials
 
+  # Build the docs and fail on any error, mirroring the internal `docs-build-check`
+  # pipeline so a PR can't break the Sphinx build. `make test-docs` builds .venv-docs
+  # itself through use_env, so no separate env step is needed.
+  #
+  # The build shells out to mermaid-cli (headless Chrome) and pandoc on top of the
+  # usual venv sync, which is why this is the slowest of the Stage 1 jobs. Caching
+  # the puppeteer download is what keeps it off the critical path — with a warm
+  # cache scripts/ensure_mmdc_chrome.sh probes for the real binary and returns
+  # immediately.
+  docs:
+    name: Linux / make test-docs
+    # Pull requests only. This workflow also runs on push to main, where
+    # docs.yml already builds the same tree to publish it — running here too
+    # would build the docs twice per merge on two runners. Nothing in Stage 2
+    # needs this job, so skipping it on main gates nothing.
+    if: github.event_name == 'pull_request'
+    runs-on: ubuntu-latest
+    timeout-minutes: 45
+    env:
+      INSTALL_PRECOMMIT: "false"
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
+      - name: Install uv
+        uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0
+        with:
+          enable-cache: false
+      - name: Cache headless Chrome for mermaid-cli
+        uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+        with:
+          path: ~/.cache/puppeteer
+          key: puppeteer-${{ runner.os }}-${{ hashFiles('scripts/ensure_mmdc_chrome.sh') }}
+      - name: Run `make test-docs`
+        run: make test-docs
+
   # ── Stage 2: full test matrix, gated on Stage 1. ──
   macos-tests:
     name: macOS / make test-highest-pytorch / markers=not-slow

.github/workflows/docs.yml

@@ -0,0 +1,193 @@
+name: Docs
+
+run-name: >-
+  ${{ github.event_name == 'release'
+  && format('Docs · publish {0} · @{1}', github.ref_name, github.actor)
+  || format('Docs · publish main · @{0}', github.actor) }}
+
+# Publishes versioned Sphinx documentation to GitHub Pages.
+#
+# The site holds one independent Sphinx build per version, so publishing one never
+# rebuilds or disturbs another:
+#
+#   /            redirects to main/
+#   /main/       rebuilt on every push to main
+#   /vX.Y.Z/     built once, when the GitHub Release for that tag is published
+#
+# The `gh-pages` branch is the durable copy of the assembled site; Pages itself is
+# served from an Actions artifact (Settings -> Pages -> Source -> GitHub Actions),
+# not from the branch. Those are separate on purpose: a push made with
+# GITHUB_TOKEN does not trigger a branch-source Pages build, so a branch-served
+# site would silently go stale. Keeping the branch as state and deploying through
+# the Pages API gets both a full history of the published site and a reliable
+# deploy.
+#
+# Because each run assembles the whole site from that branch and `deploy-pages`
+# replaces everything it is given, docs/scripts/assemble_versioned_site.py refuses
+# to publish a tree that lost a version rather than taking previously published
+# docs offline.
+on:
+  push:
+    branches: [main]
+  # `published` rather than `created` so a release promoted out of draft still
+  # publishes its docs. This fires independently of release.yml's `v*` tag-push
+  # trigger, so docs for a new version can go live while the PyPI publish is still
+  # waiting on the `pypi` environment's manual approval — accepted, since the docs
+  # describe the tagged source either way, and a rejected release is re-cut.
+  release:
+    types: [published]
+  # Manual re-publish, e.g. after fixing the site shell. Builds whatever ref it is
+  # dispatched from and publishes it as `main`, so it cannot be used to overwrite a
+  # release's frozen docs.
+  workflow_dispatch:
+
+# Least privilege by default; each job opts into exactly what it needs.
+permissions:
+  contents: read
+
+concurrency:
+  # One global group, deliberately NOT keyed on github.ref: every run rewrites the
+  # same gh-pages branch and the same Pages deployment, so two publishes on
+  # different refs would race over shared state rather than run in parallel.
+  group: docs-publish
+  # Never interrupt a publish — a cancelled run can leave gh-pages half written.
+  cancel-in-progress: false
+
+jobs:
+  # ── Build the doc set for this ref and assemble the full site tree. ──
+  build:
+    name: Build and assemble site
+    # Skip pre-releases: they are not the current docs for any version, and
+    # v0.2.0 shipped with prerelease=true, so this is a real case.
+    if: github.event_name != 'release' || github.event.release.prerelease == false
+    runs-on: ubuntu-latest
+    timeout-minutes: 45
+    permissions:
+      contents: write # push the assembled site to gh-pages
+    env:
+      INSTALL_PRECOMMIT: "false"
+      # Both assembler invocations share these. Passed through the environment
+      # rather than interpolated into each `run:` string so the values never
+      # become part of the shell command text.
+      ASSEMBLER: source/docs/scripts/assemble_versioned_site.py
+      SITE_DIR: site
+      HTML_DIR: source/docs/build/html
+      URL_PREFIX: /${{ github.event.repository.name }}
+    steps:
+      # No `ref:` — the default is the ref that triggered the run, which is
+      # already what we want: `main` for a push, refs/tags/vX.Y.Z for a release.
+      - name: Check out the source to build
+        uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
+        with:
+          path: source
+          persist-credentials: false
+
+      - name: Check out the published site
+        uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
+        with:
+          ref: gh-pages
+          path: site
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0
+        with:
+          enable-cache: false
+
+      - name: Cache headless Chrome for mermaid-cli
+        uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
+        with:
+          path: ~/.cache/puppeteer
+          key: puppeteer-${{ runner.os }}-${{ hashFiles('source/scripts/ensure_mmdc_chrome.sh') }}
+
+      - name: Resolve the version being published
+        id: version
+        run: |
+          if [ "${GITHUB_EVENT_NAME}" = "release" ]; then
+            echo "name=${GITHUB_REF_NAME}" >> "${GITHUB_OUTPUT}"
+          else
+            echo "name=main" >> "${GITHUB_OUTPUT}"
+          fi
+
+      # The dropdown is rendered into every page at build time, so the version
+      # list has to be known before Sphinx runs. `plan` reports the site as it
+      # will look *after* this publish, including the version being built.
+      - name: Plan the version list
+        id: plan
+        env:
+          DOCS_VERSION: ${{ steps.version.outputs.name }}
+        run: |
+          versions="$(python3 "$ASSEMBLER" --site "$SITE_DIR" --version "$DOCS_VERSION" \
+            --url-prefix "$URL_PREFIX" plan)"
+          echo "Publishing with versions: ${versions}"
+          echo "versions=${versions}" >> "${GITHUB_OUTPUT}"
+
+      # `make docs` rather than sphinx-build: the target owns the mermaid-cli,
+      # headless Chrome, pandoc and .venv-docs setup, and its DOCS_DIR indirection
+      # is what keeps this build identical to the internal one. Calling Sphinx
+      # directly would let the two drift apart silently.
+      - name: Build the documentation
+        working-directory: source
+        env:
+          DOCS_VERSION: ${{ steps.version.outputs.name }}
+          DOCS_VERSIONS: ${{ steps.plan.outputs.versions }}
+        run: make docs
+
+      - name: Assemble the site
+        env:
+          DOCS_VERSION: ${{ steps.version.outputs.name }}
+        run: |
+          python3 "$ASSEMBLER" --site "$SITE_DIR" --version "$DOCS_VERSION" \
+            --url-prefix "$URL_PREFIX" assemble --html "$HTML_DIR"
+
+      - name: Commit the assembled site to gh-pages
+        working-directory: site
+        env:
+          DOCS_VERSION: ${{ steps.version.outputs.name }}
+        run: |
+          git config user.name 'github-actions[bot]'
+          git config user.email '41898282+github-actions[bot]@users.noreply.github.com'
+          git add -A
+          if git diff --cached --quiet; then
+            echo "Site unchanged; nothing to commit."
+            exit 0
+          fi
+          git commit -m "docs: publish ${DOCS_VERSION}"
+          # Retry once against a concurrent publish. The concurrency group above
+          # makes this near-impossible, so a second failure is a real problem and
+          # should fail the run rather than be papered over.
+          git push origin HEAD:gh-pages || {
+            git pull --rebase origin gh-pages
+            git push origin HEAD:gh-pages
+          }
+
+      - name: Configure Pages
+        uses: actions/configure-pages@45bfe0192ca1faeb007ade9deae92b16b8254a0d # v6.0.0
+
+      - name: Upload the Pages artifact
+        uses: actions/upload-pages-artifact@fc324d3547104276b827a68afc52ff2a11cc49c9 # v5.0.0
+        with:
+          # Uploads the gh-pages checkout directly. The action's tar step always
+          # passes `--exclude=.git --exclude=.github`, so the checkout's own git
+          # directory is dropped without staging a copy of the site first.
+          path: site
+          # Required, and not the default: the action otherwise tars with
+          # `--exclude=.[^/]*`, which would drop every .nojekyll and let Jekyll
+          # strip the _static and _images directories from the published site.
+          include-hidden-files: true
+
+  # ── Deploy the artifact. The only job holding Pages credentials. ──
+  deploy:
+    name: Deploy to GitHub Pages
+    needs: build
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    permissions:
+      pages: write # create the Pages deployment
+      id-token: write # mint the OIDC token deploy-pages exchanges for it
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    steps:
+      - name: Deploy
+        id: deployment
+        uses: actions/deploy-pages@cd2ce8fcbc39b97be8ca5fce6e763baed58fa128 # v5.0.0

.pre-commit-config.yaml

@@ -334,6 +334,29 @@ repos:
         files: (^|/)_about\.py$
         pass_filenames: false
 
+  - repo: local
+    hooks:
+      - id: check-api-doc-coverage
+        name: Check API doc coverage
+        description: |
+          Ensure the committed docs/src/api/index.md matches what
+          docs/scripts/generate_api_index.py produces from the package tree, so a
+          new or renamed public symbol can't ship undocumented. Verify-only: it
+          reports the drift but does not edit the file — run
+          `make render-api-index`, then stage the result. Triggers on the inputs
+          the generator reads plus the generated file itself. Runs only the
+          coverage test; the heavier autodoc-filter test in the same module runs
+          under `make test-docs`.
+        entry: env USE_LOCAL_COREAI=1 uv run --no-sync --active pytest -s docs/tests/test_api_doc_coverage.py::test_api_doc_coverage
+        language: system
+        pass_filenames: false
+        files: |
+          (?x)^(
+            src/coreai_opt/.*\.py|
+            docs/src/api/index\.md|
+            docs/scripts/generate_api_index\.py
+          )$
+
   - repo: local
     hooks:
       - id: towncrier-check

CONTRIBUTING.md

@@ -30,10 +30,17 @@ make docs
 
 # Build the documentation and open it in a browser.
 make docs-open
+
+# Regenerate docs/src/api/index.md from the package tree.
+make render-api-index
 ```
 
 All make targets and their flags are listed in the [Makefile](Makefile).
 
+`docs/src/api/index.md` is generated from the public API and committed, so a reviewer can see
+API-surface changes in the diff. Don't edit it by hand — run `make render-api-index` and stage
+the result. The `check-api-doc-coverage` pre-commit hook fails the commit if it drifts.
+
 ## Submitting issues
 
 Before opening an issue:
@@ -70,6 +77,7 @@ Before pushing your changes, run these locally:
 - `make test` — full test suite (parallelized with `pytest-xdist`)
 - `make test-fast` — excludes tests marked `@pytest.mark.slow` for quicker iteration
 - `make test-smoke` — builds the package, installs it into a clean environment, and verifies that imports plus basic quantization and palettization work end to end
+- `make test-docs` — builds the documentation and checks the output, including that the committed API index is current (CI runs this on every pull request)
 
 A clean `make check` and `make test` are required before a pull request will be reviewed.
 

Makefile

@@ -3,7 +3,7 @@
 # Use of this source code is governed by a BSD-3-Clause license that can
 # be found in the LICENSE file or at https://opensource.org/licenses/BSD-3-Clause
 
-.PHONY: _maybe_patch_pyproject all api-list build build-dev check clean distclean distclean-all docs docs-clean docs-open env env-all env-docs env-highest-torch env-lowest-torch env-tutorial render-api-index set-auto-venv test test-cov test-fast test-highest-pytorch test-lowest-pytorch test-slow test-smoke test-tutorials version version-dev
+.PHONY: _maybe_patch_pyproject all api-list build build-dev check clean distclean distclean-all docs docs-clean docs-open docs-open-multiversions env env-all env-docs env-highest-torch env-lowest-torch env-tutorial render-api-index set-auto-venv test test-cov test-docs test-fast test-highest-pytorch test-lowest-pytorch test-slow test-smoke test-tutorials version version-dev
 
 SHELL := /bin/bash
 
@@ -31,6 +31,20 @@ SCRIPTS := $(MAKEFILE_DIR)scripts
 # ini option. In the OSS mirror the second entry is just the repo root ($(CURDIR)/./).
 export PYTHONPATH := $(CURDIR):$(CURDIR)/$(MAKEFILE_DIR)
 
+# Repo-local install prefix for tools that aren't pip- or npm-installable and
+# would otherwise need a system prefix. `scripts/ensure_pandoc.sh` downloads the
+# upstream pandoc binary here on Linux, since CI runners and other unprivileged
+# environments can't write to /usr/local and a docs build shouldn't need root.
+#
+# Exported (like PYTHONPATH above) so every recipe subshell inherits it: the
+# `docs` target ensures pandoc in one recipe line and runs sphinx-build in
+# another, so a PATH exported inside the ensure script would be gone by the time
+# nbsphinx shells out to pandoc. Prepended, so a repo-local copy takes precedence
+# over a system one — which only matters when both exist, since the ensure
+# scripts download nothing when the binary is already on PATH.
+LOCAL_BIN := $(CURDIR)/$(MAKEFILE_DIR).local/bin
+export PATH := $(LOCAL_BIN):$(PATH)
+
 # Tell coreai's runtime to skip the symbol-version check against the host's
 # installed /System/Library/Frameworks/CoreAI.framework. Required when the
 # precompiled coreai wheel was built against a newer SDK than what's on the
@@ -325,6 +339,22 @@ test-highest-pytorch:
 	$(RUN_TESTS) $(PYTEST_ARGS) && \
 	echo "All tests passed!"
 
+# Run docs tests only: builds the site with `make docs` and checks the output
+# (index.html, llms.txt, mermaid SVGs) plus API-reference coverage. Excludes the
+# tutorial notebook tests, which need the tutorial env — see test-tutorials.
+# This is the target CI runs as the PR docs gate.
+#
+# The internal Makefile currently defines its own `test-docs` after including this
+# file, which shadows this recipe (Make prints an "overriding commands" warning and
+# keeps the later definition). The two are equivalent once $(DOCS_DIR) expands, so
+# nothing breaks today, but the internal copy should be deleted so this one is the
+# single definition — the same arrangement `docs` and `test-tutorials` already use.
+test-docs:
+	@$(call use_env,VENV_DOCS,--with-docs) && \
+	echo "Running docs tests..." && \
+	uv run --no-sync --active pytest -s $(DOCS_DIR)/tests/ --ignore=$(DOCS_DIR)/tests/test_tutorials.py && \
+	echo "All docs tests passed!"
+
 # Run tutorial notebook tests
 test-tutorials:
 	@$(call use_env,VENV_TUTORIAL,--with-tutorial --with-test) && \
@@ -417,5 +447,52 @@ render-api-index:
 # Build and open documentation in browser
 # Uses --serve so the docs are loaded over HTTP, not file:// — required for
 # the Copy page button (and any other feature using fetch()/clipboard APIs).
+#
+# Opens a single doc set with no version picker, which is what a local build
+# produces: DOCS_VERSION/DOCS_VERSIONS are unset, so conf.py skips html_context
+# and the theme renders no dropdown. For the published multi-version layout, see
+# docs-open-multiversions.
 docs-open: docs
 	@$(DOCS_DIR)/scripts/open_in_browser.py --serve $(DOCS_DIR)/build/html/index.html
+
+# Build the docs and open them inside a local copy of the published site layout.
+#
+# `docs-open` serves the doc set itself as the web root, but the real site nests
+# each version one level down (/main/, /v0.2.1/) with a redirect and versions.json
+# at the root. So the version picker, the root redirect, and the switcher script's
+# versions.json fetch cannot be exercised by `docs-open` at all — its dropdown
+# links would resolve above the server root and 404. This target assembles the
+# same tree the docs workflow publishes and serves that instead.
+#
+# Each version is built from its real source, using the current tree's tooling.
+# A tag contributes source only — its docstrings, prose, and notebooks are staged
+# under .local/docs/<version>/ and built with this conf.py, these docs/scripts/,
+# and this docs environment. Nothing about the build comes from the tag, so a tag
+# cut before versioned docs existed still renders with the version picker and
+# labels itself correctly.
+#
+# Tags without a docs/src + src/coreai_opt to build from are skipped.
+#
+#   make docs-open-multiversions                            # main only
+#   make docs-open-multiversions TAGS="v0.2.1 v0.3.0"       # main + those tags
+#   make docs-open-multiversions ALL_TAGS=1                 # main + every vX.Y.Z tag
+#
+# Tags must be present locally; run `git fetch --tags origin` first.
+TAGS ?=
+ALL_TAGS ?=
+PREVIEW_SITE = $(DOCS_DIR)/build/site
+docs-open-multiversions:
+	@echo ""
+	@echo "════════════════════════════════════════════════════════════════════"
+	@echo "▶ Building a local preview of the published multi-version site"
+	@echo "════════════════════════════════════════════════════════════════════"
+	@rm -rf $(PREVIEW_SITE)
+	@$(MAKEFILE_DIR)docs/scripts/build_versioned_preview.sh \
+		--site $(PREVIEW_SITE) \
+		$(if $(TAGS),--tags "$(TAGS)",) \
+		$(if $(ALL_TAGS),--all-tags,)
+	@echo ""
+	@echo "════════════════════════════════════════════════════════════════════"
+	@echo "Serving $(PREVIEW_SITE) — the root redirect lands on /main/"
+	@echo "════════════════════════════════════════════════════════════════════"
+	@$(DOCS_DIR)/scripts/open_in_browser.py --serve $(PREVIEW_SITE)/index.html