From b39c19b9c55e9b0e3192508375acbc56c49662bf Mon Sep 17 00:00:00 2001 From: "Qiaoyu (Joey) Deng" Date: Thu, 20 Aug 2026 17:04:29 -0700 Subject: [PATCH] docs: publish versioned documentation to GitHub Pages - Serve one doc set per version: `/main/` rebuilt on every push to `main`, `/vX.Y.Z/` published when a GitHub Release is created, and `/` redirecting to `/main/`. Publishing one version rewrites only its own directory, so each release keeps the docs it shipped with. - `docs/scripts/assemble_versioned_site.py` assembles the site in two phases because the version picker is rendered into every page at build time: `plan` computes the version list before Sphinx runs, `assemble` installs the built HTML afterwards. - Fail closed rather than silently dropping docs. `actions/deploy-pages` replaces the *entire* artifact each run, so a truncated `gh-pages` checkout would take published releases offline; `assemble` aborts when the tree lacks a version its own `versions.json` lists, or holds a directory the picker cannot reach. - Track `docs/src/api/index.md` and guard it with a verify-only `check-api-doc-coverage` hook, mirroring the internal repo. Run `make render-api-index` to regenerate. - Add `make test-docs` as a pull-request gate (mirroring internal `docs-build-check`) and `make docs-open-multiversions` to preview the published layout locally. - Install pandoc into the repo's gitignored `.local/bin` instead of `/usr/local`, so the docs build needs no root. Unprivileged CI runners cannot write a system prefix, and extracting the upstream tarball's root-owned entries there failed the build. `PATH` is exported from the Makefile because `make docs` ensures pandoc and runs `sphinx-build` in separate recipe lines. --- .github/workflows/ci.yaml | 35 +++ .github/workflows/docs.yml | 193 ++++++++++++ .pre-commit-config.yaml | 23 ++ CONTRIBUTING.md | 8 + Makefile | 79 ++++- README.md | 2 + RELEASE.md | 26 ++ changelog.d/185435382.added | 1 + docs/contributing/pre-commit-hook-notes.md | 20 ++ docs/scripts/assemble_versioned_site.py | 296 ++++++++++++++++++ docs/scripts/build_versioned_preview.sh | 192 ++++++++++++ docs/scripts/puppeteer-config.json | 3 + docs/src/_static/custom.css | 171 ++++++++++ docs/src/_static/version-switcher.js | 253 +++++++++++++++ .../_templates/components/nav-versions.html | 59 ++++ docs/src/_templates/partials/site-head.html | 74 +++++ docs/src/api/index.md | 255 +++++++++++++++ docs/src/conf.py | 66 +++- docs/tests/conftest.py | 27 ++ docs/tests/test_api_doc_coverage.py | 79 +++++ docs/tests/test_docs.py | 92 ++++++ scripts/ensure_pandoc.sh | 40 ++- .../devtools/test_assemble_versioned_site.py | 197 ++++++++++++ 23 files changed, 2181 insertions(+), 10 deletions(-) create mode 100644 .github/workflows/docs.yml create mode 100644 changelog.d/185435382.added create mode 100644 docs/scripts/assemble_versioned_site.py create mode 100755 docs/scripts/build_versioned_preview.sh create mode 100644 docs/scripts/puppeteer-config.json create mode 100644 docs/src/_static/version-switcher.js create mode 100644 docs/src/_templates/components/nav-versions.html create mode 100644 docs/src/_templates/partials/site-head.html create mode 100644 docs/src/api/index.md create mode 100644 docs/tests/conftest.py create mode 100644 docs/tests/test_api_doc_coverage.py create mode 100644 docs/tests/test_docs.py create mode 100644 tests/devtools/test_assemble_versioned_site.py diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index a0c6542..d889216 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -81,6 +81,41 @@ jobs: - name: Run `make test-tutorials` run: make test-tutorials + # Build the docs and fail on any error, mirroring the internal `docs-build-check` + # pipeline so a PR can't break the Sphinx build. `make test-docs` builds .venv-docs + # itself through use_env, so no separate env step is needed. + # + # The build shells out to mermaid-cli (headless Chrome) and pandoc on top of the + # usual venv sync, which is why this is the slowest of the Stage 1 jobs. Caching + # the puppeteer download is what keeps it off the critical path — with a warm + # cache scripts/ensure_mmdc_chrome.sh probes for the real binary and returns + # immediately. + docs: + name: Linux / make test-docs + # Pull requests only. This workflow also runs on push to main, where + # docs.yml already builds the same tree to publish it — running here too + # would build the docs twice per merge on two runners. Nothing in Stage 2 + # needs this job, so skipping it on main gates nothing. + if: github.event_name == 'pull_request' + runs-on: ubuntu-latest + timeout-minutes: 45 + env: + INSTALL_PRECOMMIT: "false" + steps: + - name: Check out repository + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - name: Install uv + uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 + with: + enable-cache: false + - name: Cache headless Chrome for mermaid-cli + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ~/.cache/puppeteer + key: puppeteer-${{ runner.os }}-${{ hashFiles('scripts/ensure_mmdc_chrome.sh') }} + - name: Run `make test-docs` + run: make test-docs + # ── Stage 2: full test matrix, gated on Stage 1. ── macos-tests: name: macOS / make test-highest-pytorch / markers=not-slow diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml new file mode 100644 index 0000000..e5213df --- /dev/null +++ b/.github/workflows/docs.yml @@ -0,0 +1,193 @@ +name: Docs + +run-name: >- + ${{ github.event_name == 'release' + && format('Docs · publish {0} · @{1}', github.ref_name, github.actor) + || format('Docs · publish main · @{0}', github.actor) }} + +# Publishes versioned Sphinx documentation to GitHub Pages. +# +# The site holds one independent Sphinx build per version, so publishing one never +# rebuilds or disturbs another: +# +# / redirects to main/ +# /main/ rebuilt on every push to main +# /vX.Y.Z/ built once, when the GitHub Release for that tag is published +# +# The `gh-pages` branch is the durable copy of the assembled site; Pages itself is +# served from an Actions artifact (Settings -> Pages -> Source -> GitHub Actions), +# not from the branch. Those are separate on purpose: a push made with +# GITHUB_TOKEN does not trigger a branch-source Pages build, so a branch-served +# site would silently go stale. Keeping the branch as state and deploying through +# the Pages API gets both a full history of the published site and a reliable +# deploy. +# +# Because each run assembles the whole site from that branch and `deploy-pages` +# replaces everything it is given, docs/scripts/assemble_versioned_site.py refuses +# to publish a tree that lost a version rather than taking previously published +# docs offline. +on: + push: + branches: [main] + # `published` rather than `created` so a release promoted out of draft still + # publishes its docs. This fires independently of release.yml's `v*` tag-push + # trigger, so docs for a new version can go live while the PyPI publish is still + # waiting on the `pypi` environment's manual approval — accepted, since the docs + # describe the tagged source either way, and a rejected release is re-cut. + release: + types: [published] + # Manual re-publish, e.g. after fixing the site shell. Builds whatever ref it is + # dispatched from and publishes it as `main`, so it cannot be used to overwrite a + # release's frozen docs. + workflow_dispatch: + +# Least privilege by default; each job opts into exactly what it needs. +permissions: + contents: read + +concurrency: + # One global group, deliberately NOT keyed on github.ref: every run rewrites the + # same gh-pages branch and the same Pages deployment, so two publishes on + # different refs would race over shared state rather than run in parallel. + group: docs-publish + # Never interrupt a publish — a cancelled run can leave gh-pages half written. + cancel-in-progress: false + +jobs: + # ── Build the doc set for this ref and assemble the full site tree. ── + build: + name: Build and assemble site + # Skip pre-releases: they are not the current docs for any version, and + # v0.2.0 shipped with prerelease=true, so this is a real case. + if: github.event_name != 'release' || github.event.release.prerelease == false + runs-on: ubuntu-latest + timeout-minutes: 45 + permissions: + contents: write # push the assembled site to gh-pages + env: + INSTALL_PRECOMMIT: "false" + # Both assembler invocations share these. Passed through the environment + # rather than interpolated into each `run:` string so the values never + # become part of the shell command text. + ASSEMBLER: source/docs/scripts/assemble_versioned_site.py + SITE_DIR: site + HTML_DIR: source/docs/build/html + URL_PREFIX: /${{ github.event.repository.name }} + steps: + # No `ref:` — the default is the ref that triggered the run, which is + # already what we want: `main` for a push, refs/tags/vX.Y.Z for a release. + - name: Check out the source to build + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + path: source + persist-credentials: false + + - name: Check out the published site + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + ref: gh-pages + path: site + + - name: Install uv + uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 + with: + enable-cache: false + + - name: Cache headless Chrome for mermaid-cli + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ~/.cache/puppeteer + key: puppeteer-${{ runner.os }}-${{ hashFiles('source/scripts/ensure_mmdc_chrome.sh') }} + + - name: Resolve the version being published + id: version + run: | + if [ "${GITHUB_EVENT_NAME}" = "release" ]; then + echo "name=${GITHUB_REF_NAME}" >> "${GITHUB_OUTPUT}" + else + echo "name=main" >> "${GITHUB_OUTPUT}" + fi + + # The dropdown is rendered into every page at build time, so the version + # list has to be known before Sphinx runs. `plan` reports the site as it + # will look *after* this publish, including the version being built. + - name: Plan the version list + id: plan + env: + DOCS_VERSION: ${{ steps.version.outputs.name }} + run: | + versions="$(python3 "$ASSEMBLER" --site "$SITE_DIR" --version "$DOCS_VERSION" \ + --url-prefix "$URL_PREFIX" plan)" + echo "Publishing with versions: ${versions}" + echo "versions=${versions}" >> "${GITHUB_OUTPUT}" + + # `make docs` rather than sphinx-build: the target owns the mermaid-cli, + # headless Chrome, pandoc and .venv-docs setup, and its DOCS_DIR indirection + # is what keeps this build identical to the internal one. Calling Sphinx + # directly would let the two drift apart silently. + - name: Build the documentation + working-directory: source + env: + DOCS_VERSION: ${{ steps.version.outputs.name }} + DOCS_VERSIONS: ${{ steps.plan.outputs.versions }} + run: make docs + + - name: Assemble the site + env: + DOCS_VERSION: ${{ steps.version.outputs.name }} + run: | + python3 "$ASSEMBLER" --site "$SITE_DIR" --version "$DOCS_VERSION" \ + --url-prefix "$URL_PREFIX" assemble --html "$HTML_DIR" + + - name: Commit the assembled site to gh-pages + working-directory: site + env: + DOCS_VERSION: ${{ steps.version.outputs.name }} + run: | + git config user.name 'github-actions[bot]' + git config user.email '41898282+github-actions[bot]@users.noreply.github.com' + git add -A + if git diff --cached --quiet; then + echo "Site unchanged; nothing to commit." + exit 0 + fi + git commit -m "docs: publish ${DOCS_VERSION}" + # Retry once against a concurrent publish. The concurrency group above + # makes this near-impossible, so a second failure is a real problem and + # should fail the run rather than be papered over. + git push origin HEAD:gh-pages || { + git pull --rebase origin gh-pages + git push origin HEAD:gh-pages + } + + - name: Configure Pages + uses: actions/configure-pages@45bfe0192ca1faeb007ade9deae92b16b8254a0d # v6.0.0 + + - name: Upload the Pages artifact + uses: actions/upload-pages-artifact@fc324d3547104276b827a68afc52ff2a11cc49c9 # v5.0.0 + with: + # Uploads the gh-pages checkout directly. The action's tar step always + # passes `--exclude=.git --exclude=.github`, so the checkout's own git + # directory is dropped without staging a copy of the site first. + path: site + # Required, and not the default: the action otherwise tars with + # `--exclude=.[^/]*`, which would drop every .nojekyll and let Jekyll + # strip the _static and _images directories from the published site. + include-hidden-files: true + + # ── Deploy the artifact. The only job holding Pages credentials. ── + deploy: + name: Deploy to GitHub Pages + needs: build + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + pages: write # create the Pages deployment + id-token: write # mint the OIDC token deploy-pages exchanges for it + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + steps: + - name: Deploy + id: deployment + uses: actions/deploy-pages@cd2ce8fcbc39b97be8ca5fce6e763baed58fa128 # v5.0.0 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 62245a0..1426c27 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -334,6 +334,29 @@ repos: files: (^|/)_about\.py$ pass_filenames: false + - repo: local + hooks: + - id: check-api-doc-coverage + name: Check API doc coverage + description: | + Ensure the committed docs/src/api/index.md matches what + docs/scripts/generate_api_index.py produces from the package tree, so a + new or renamed public symbol can't ship undocumented. Verify-only: it + reports the drift but does not edit the file — run + `make render-api-index`, then stage the result. Triggers on the inputs + the generator reads plus the generated file itself. Runs only the + coverage test; the heavier autodoc-filter test in the same module runs + under `make test-docs`. + entry: env USE_LOCAL_COREAI=1 uv run --no-sync --active pytest -s docs/tests/test_api_doc_coverage.py::test_api_doc_coverage + language: system + pass_filenames: false + files: | + (?x)^( + src/coreai_opt/.*\.py| + docs/src/api/index\.md| + docs/scripts/generate_api_index\.py + )$ + - repo: local hooks: - id: towncrier-check diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9fb80ad..d980b7b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -30,10 +30,17 @@ make docs # Build the documentation and open it in a browser. make docs-open + +# Regenerate docs/src/api/index.md from the package tree. +make render-api-index ``` All make targets and their flags are listed in the [Makefile](Makefile). +`docs/src/api/index.md` is generated from the public API and committed, so a reviewer can see +API-surface changes in the diff. Don't edit it by hand — run `make render-api-index` and stage +the result. The `check-api-doc-coverage` pre-commit hook fails the commit if it drifts. + ## Submitting issues Before opening an issue: @@ -70,6 +77,7 @@ Before pushing your changes, run these locally: - `make test` — full test suite (parallelized with `pytest-xdist`) - `make test-fast` — excludes tests marked `@pytest.mark.slow` for quicker iteration - `make test-smoke` — builds the package, installs it into a clean environment, and verifies that imports plus basic quantization and palettization work end to end +- `make test-docs` — builds the documentation and checks the output, including that the committed API index is current (CI runs this on every pull request) A clean `make check` and `make test` are required before a pull request will be reviewed. diff --git a/Makefile b/Makefile index 376d024..d2fdefd 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ # Use of this source code is governed by a BSD-3-Clause license that can # be found in the LICENSE file or at https://opensource.org/licenses/BSD-3-Clause -.PHONY: _maybe_patch_pyproject all api-list build build-dev check clean distclean distclean-all docs docs-clean docs-open env env-all env-docs env-highest-torch env-lowest-torch env-tutorial render-api-index set-auto-venv test test-cov test-fast test-highest-pytorch test-lowest-pytorch test-slow test-smoke test-tutorials version version-dev +.PHONY: _maybe_patch_pyproject all api-list build build-dev check clean distclean distclean-all docs docs-clean docs-open docs-open-multiversions env env-all env-docs env-highest-torch env-lowest-torch env-tutorial render-api-index set-auto-venv test test-cov test-docs test-fast test-highest-pytorch test-lowest-pytorch test-slow test-smoke test-tutorials version version-dev SHELL := /bin/bash @@ -31,6 +31,20 @@ SCRIPTS := $(MAKEFILE_DIR)scripts # ini option. In the OSS mirror the second entry is just the repo root ($(CURDIR)/./). export PYTHONPATH := $(CURDIR):$(CURDIR)/$(MAKEFILE_DIR) +# Repo-local install prefix for tools that aren't pip- or npm-installable and +# would otherwise need a system prefix. `scripts/ensure_pandoc.sh` downloads the +# upstream pandoc binary here on Linux, since CI runners and other unprivileged +# environments can't write to /usr/local and a docs build shouldn't need root. +# +# Exported (like PYTHONPATH above) so every recipe subshell inherits it: the +# `docs` target ensures pandoc in one recipe line and runs sphinx-build in +# another, so a PATH exported inside the ensure script would be gone by the time +# nbsphinx shells out to pandoc. Prepended, so a repo-local copy takes precedence +# over a system one — which only matters when both exist, since the ensure +# scripts download nothing when the binary is already on PATH. +LOCAL_BIN := $(CURDIR)/$(MAKEFILE_DIR).local/bin +export PATH := $(LOCAL_BIN):$(PATH) + # Tell coreai's runtime to skip the symbol-version check against the host's # installed /System/Library/Frameworks/CoreAI.framework. Required when the # precompiled coreai wheel was built against a newer SDK than what's on the @@ -325,6 +339,22 @@ test-highest-pytorch: $(RUN_TESTS) $(PYTEST_ARGS) && \ echo "All tests passed!" +# Run docs tests only: builds the site with `make docs` and checks the output +# (index.html, llms.txt, mermaid SVGs) plus API-reference coverage. Excludes the +# tutorial notebook tests, which need the tutorial env — see test-tutorials. +# This is the target CI runs as the PR docs gate. +# +# The internal Makefile currently defines its own `test-docs` after including this +# file, which shadows this recipe (Make prints an "overriding commands" warning and +# keeps the later definition). The two are equivalent once $(DOCS_DIR) expands, so +# nothing breaks today, but the internal copy should be deleted so this one is the +# single definition — the same arrangement `docs` and `test-tutorials` already use. +test-docs: + @$(call use_env,VENV_DOCS,--with-docs) && \ + echo "Running docs tests..." && \ + uv run --no-sync --active pytest -s $(DOCS_DIR)/tests/ --ignore=$(DOCS_DIR)/tests/test_tutorials.py && \ + echo "All docs tests passed!" + # Run tutorial notebook tests test-tutorials: @$(call use_env,VENV_TUTORIAL,--with-tutorial --with-test) && \ @@ -417,5 +447,52 @@ render-api-index: # Build and open documentation in browser # Uses --serve so the docs are loaded over HTTP, not file:// — required for # the Copy page button (and any other feature using fetch()/clipboard APIs). +# +# Opens a single doc set with no version picker, which is what a local build +# produces: DOCS_VERSION/DOCS_VERSIONS are unset, so conf.py skips html_context +# and the theme renders no dropdown. For the published multi-version layout, see +# docs-open-multiversions. docs-open: docs @$(DOCS_DIR)/scripts/open_in_browser.py --serve $(DOCS_DIR)/build/html/index.html + +# Build the docs and open them inside a local copy of the published site layout. +# +# `docs-open` serves the doc set itself as the web root, but the real site nests +# each version one level down (/main/, /v0.2.1/) with a redirect and versions.json +# at the root. So the version picker, the root redirect, and the switcher script's +# versions.json fetch cannot be exercised by `docs-open` at all — its dropdown +# links would resolve above the server root and 404. This target assembles the +# same tree the docs workflow publishes and serves that instead. +# +# Each version is built from its real source, using the current tree's tooling. +# A tag contributes source only — its docstrings, prose, and notebooks are staged +# under .local/docs// and built with this conf.py, these docs/scripts/, +# and this docs environment. Nothing about the build comes from the tag, so a tag +# cut before versioned docs existed still renders with the version picker and +# labels itself correctly. +# +# Tags without a docs/src + src/coreai_opt to build from are skipped. +# +# make docs-open-multiversions # main only +# make docs-open-multiversions TAGS="v0.2.1 v0.3.0" # main + those tags +# make docs-open-multiversions ALL_TAGS=1 # main + every vX.Y.Z tag +# +# Tags must be present locally; run `git fetch --tags origin` first. +TAGS ?= +ALL_TAGS ?= +PREVIEW_SITE = $(DOCS_DIR)/build/site +docs-open-multiversions: + @echo "" + @echo "════════════════════════════════════════════════════════════════════" + @echo "▶ Building a local preview of the published multi-version site" + @echo "════════════════════════════════════════════════════════════════════" + @rm -rf $(PREVIEW_SITE) + @$(MAKEFILE_DIR)docs/scripts/build_versioned_preview.sh \ + --site $(PREVIEW_SITE) \ + $(if $(TAGS),--tags "$(TAGS)",) \ + $(if $(ALL_TAGS),--all-tags,) + @echo "" + @echo "════════════════════════════════════════════════════════════════════" + @echo "Serving $(PREVIEW_SITE) — the root redirect lands on /main/" + @echo "════════════════════════════════════════════════════════════════════" + @$(DOCS_DIR)/scripts/open_in_browser.py --serve $(PREVIEW_SITE)/index.html diff --git a/README.md b/README.md index 9b8a0ee..35eeec1 100644 --- a/README.md +++ b/README.md @@ -60,6 +60,8 @@ finalized_model = quantizer.finalize() For APIs, options, and detailed workflows, see the hosted documentation at [apple.github.io/coreai-optimization](https://apple.github.io/coreai-optimization/). +The site is published one doc set per version. `/main/` tracks the `main` branch and may document work that no release includes yet; `/vX.Y.Z/` is the documentation for that release, frozen when it shipped. The version picker in the site header switches between them. + ## Contributing Contributions are welcome within a defined scope. Please read [CONTRIBUTING.md](CONTRIBUTING.md) before opening a pull request or issue, particularly the section on contribution scope. diff --git a/RELEASE.md b/RELEASE.md index dce550c..cc970b7 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -77,6 +77,32 @@ A release branch is the one place where the two match: it sets `latest_released_ Cut the branch before moving `main` to the next dev release. +### Documentation + +The documentation site carries one doc set per version, published by `.github/workflows/docs.yml`: + +| Trigger | What it publishes | +| -------------------------- | ----------------------------------------- | +| Push (merge) to `main` | `/main/`, rebuilt from the `main` branch | +| A GitHub Release published | `/vX.Y.Z/`, built from that release's tag | + +The site root redirects to `/main/`, so that is the default view. Publishing one version +rewrites only its own directory — every already-published release keeps the docs it shipped +with. There is nothing to do by hand. + +Two things to know when cutting a release: + +- **Publish the GitHub Release, don't just push the tag.** Pushing a `vX.Y.Z` tag starts + `release.yml` (build, smoke test, PyPI publish); the docs are published by the *release* + being created. A tag with no release gets no versioned docs. +- **The docs publish does not wait for PyPI.** The two triggers fire in parallel, and the PyPI + upload is gated behind the `pypi` environment's approval, so `/vX.Y.Z/` can appear while + that approval is still pending. The docs describe the tagged source either way. If a release + is abandoned after its docs went live, delete the directory from the `gh-pages` branch. + +Marking a release as a pre-release skips the docs publish, since a pre-release is not the +current documentation for any version. + ### Extending the scheme downstream A repo that uses this one as a submodule and includes this `Makefile` — building one combined wheel from both trees — can add its own 4th number. Set `COREAI_OPT_VERSION_EXTENSION` to the number it's about to release next, then call `make build`, `make build-dev`, or `make version` unchanged. diff --git a/changelog.d/185435382.added b/changelog.d/185435382.added new file mode 100644 index 0000000..8527e89 --- /dev/null +++ b/changelog.d/185435382.added @@ -0,0 +1 @@ +Publish versioned documentation to GitHub Pages: `/main/` tracks the `main` branch and `/vX.Y.Z/` is published when a GitHub Release is created, so each release keeps the documentation it shipped with and the site root defaults to `/main/`. A version picker in the site header switches between them. `docs/src/api/index.md` is now committed and checked by the `check-api-doc-coverage` pre-commit hook (`make render-api-index` regenerates it), and `make test-docs` builds the docs and validates the output as a pull-request gate. diff --git a/docs/contributing/pre-commit-hook-notes.md b/docs/contributing/pre-commit-hook-notes.md index 97f03dd..b5c130f 100644 --- a/docs/contributing/pre-commit-hook-notes.md +++ b/docs/contributing/pre-commit-hook-notes.md @@ -16,3 +16,23 @@ When you add a source file whose extension or name isn't already covered by the ### Why an inclusion list, not an exclusion list New file types stay silently ignored until someone deliberately adds them, rather than getting `# header` prepended (which would break parsers like JSON). + +## `check-api-doc-coverage` + +Fails the commit when the committed `docs/src/api/index.md` no longer matches what `docs/scripts/generate_api_index.py` produces from the package tree. The point is that a new or renamed public symbol can't ship undocumented, and that an API-surface change shows up in the diff a reviewer reads. + +The hook only reports the drift — run `make render-api-index` and stage the result. + +### `docs/src/api/index.md` is generated; don't edit it + +The file is committed but not hand-written. Two things write it: `make render-api-index` (the cheap path, base venv) and `make docs`, whose `setup()` in `docs/src/conf.py` regenerates it before Sphinx reads sources. Any manual edit is silently overwritten by the next build, and the hook rejects it in the meantime. + +`conf.py` writes it through `_write_if_changed` rather than `write_text`, so a build that changes nothing leaves the file's mtime alone. Keep it that way: an unconditional write would make every docs build touch a tracked file, so the tree looks dirty to git's stat cache even when the generated content is identical. + +### Adding a public symbol + +Nothing extra to do — add the symbol to its module's `__all__`, run `make render-api-index`, and commit the regenerated index alongside the code. The hook's `files:` regex already covers `src/coreai_opt/**.py`, so it runs whenever the API surface can have moved. + +### Why the hook runs one test, not the module + +The hook entry names `docs/tests/test_api_doc_coverage.py::test_api_doc_coverage` specifically. The other test in that module, `test_autodoc_skip_filters_external_methods`, imports `conf.py` and therefore needs Sphinx, which lives only in `.venv-docs` — it runs under `make test-docs` in CI instead. Widening the hook to the whole module would break commits for anyone who hasn't built the docs environment. diff --git a/docs/scripts/assemble_versioned_site.py b/docs/scripts/assemble_versioned_site.py new file mode 100644 index 0000000..2bc5002 --- /dev/null +++ b/docs/scripts/assemble_versioned_site.py @@ -0,0 +1,296 @@ +#!/usr/bin/env python3 + +# Copyright 2026 Apple Inc. +# +# Use of this source code is governed by a BSD-3-Clause license that can +# be found in the LICENSE file or at https://opensource.org/licenses/BSD-3-Clause + +"""Assemble the multi-version documentation site published to GitHub Pages. + +The published site is a plain directory tree, one independent Sphinx build per +version, with a redirect at the root:: + + / -> redirects to main/ + /main/ docs built from the main branch + /v0.2.1/ docs built from the v0.2.1 release + /versions.json the manifest that drives the version dropdown + +Sphinx never needs to know about the other versions: each directory is a separate +build, so publishing one version cannot disturb another. That is what lets a +release's docs stay frozen after it ships while ``main`` keeps moving. + +This script runs twice per publish, because the version list has to exist +*before* Sphinx runs (the dropdown is rendered into every page at build time) but +the built HTML only exists after: + +``plan`` + Read the versions already present in the site tree, add the one being + published, and print the manifest. The workflow feeds this to ``sphinx-build`` + via ``DOCS_VERSIONS`` and, on success, writes it to ``versions.json``. + +``assemble`` + Copy the freshly built HTML into its version directory, write + ``versions.json``, the root redirect, and ``.nojekyll``, after checking that + every version the previous publish recorded is still present. + +That last check is the reason this is a script rather than a few shell lines. +``actions/deploy-pages`` republishes the *entire* artifact every run, so it has no +notion of touching one version: if the site tree handed to it were missing a +version — a truncated checkout, a wrong ref — that version would be deleted from +the live site even though the ``gh-pages`` branch still holds it. Comparing the +tree against the ``versions.json`` the last publish wrote catches that before +anything is deployed. +""" + +from __future__ import annotations + +import argparse +import json +import re +import shutil +import sys +from pathlib import Path, PurePosixPath + +# Directory name of the doc set built from the default branch. Sorts first in the +# dropdown and is what the site root redirects to. +MAIN_VERSION = "main" + +# A published release's directory name: `v` + the release tag's version numbers. +# Two or more segments are accepted, not exactly three, so the 4-segment scheme a +# downstream repo can opt into via COREAI_OPT_VERSION_EXTENSION (see RELEASE.md) +# is listed rather than silently omitted from the dropdown. +_RELEASE_DIR = re.compile(r"^v(\d+(?:\.\d+)+)$") + +# Top-level entries that are part of the site shell rather than a doc set. Any +# other unexpected directory is reported instead of ignored — see _is_version_dir. +_SITE_SHELL = frozenset({".git", ".github"}) + +_REDIRECT_HTML = """ + + + + Redirecting to the CoreAI-Opt documentation + + + + +

Redirecting to {target}/…

+ + +""" + + +def _is_version_name(name: str) -> bool: + """Return True if ``name`` is a published doc set's directory name. + + Takes the name rather than the path so the same rule validates ``--version`` + (a string, before any directory exists) and filters the site listing. + """ + return name == MAIN_VERSION or bool(_RELEASE_DIR.match(name)) + + +def _release_sort_key(name: str) -> tuple[int, ...]: + """Return the numeric sort key for a release directory name. + + Compares segment by segment as integers, so v0.10.0 outranks v0.9.0 — which a + lexicographic sort gets backwards. Shorter versions sort below longer ones with + the same prefix (v0.2.1 below v0.2.1.1), matching how the extension scheme in + RELEASE.md layers an extra segment onto a published release. + """ + return tuple(int(part) for part in _RELEASE_DIR.match(name).group(1).split(".")) + + +def sort_versions(versions: set[str] | list[str]) -> list[str]: + """Return ``versions`` ordered newest first. + + ``main`` sorts ahead of every release, since it documents unreleased work and + is therefore the most current thing on the site. + """ + releases = sorted( + (v for v in versions if v != MAIN_VERSION), key=_release_sort_key, reverse=True + ) + return ([MAIN_VERSION] if MAIN_VERSION in versions else []) + releases + + +def discover_versions(site: Path) -> list[str]: + """Return the version directories present in ``site``, newest first.""" + return sort_versions( + [child.name for child in site.iterdir() if child.is_dir() and _is_version_name(child.name)] + ) + + +def versions_after_publish(site: Path, version: str) -> list[str]: + """Return the site's versions as they will be once ``version`` is published. + + Both subcommands go through this so they cannot disagree: ``plan`` bakes this + list into every page's dropdown, while ``assemble`` writes it to versions.json + for the switcher script to re-read. A divergence would leave a page whose + dropdown contradicts the manifest. + """ + return sort_versions({*discover_versions(site), version}) + + +def unrecognized_dirs(site: Path) -> list[str]: + """Return top-level directories that are neither a doc set nor the site shell. + + Reported rather than ignored: a directory this script does not recognize is + still served by Pages but is absent from the dropdown, so silently skipping it + would hide a published doc set from every reader. + """ + return sorted( + child.name + for child in site.iterdir() + if child.is_dir() and not _is_version_name(child.name) and child.name not in _SITE_SHELL + ) + + +def build_manifest(versions: list[str], url_prefix: str) -> list[list[str]]: + """Return ``[label, href]`` pairs for the version dropdown. + + Hrefs are absolute from the server root rather than relative, because a + relative ``../main/`` would resolve against the current page's directory and + so would break on any nested page (e.g. ``/v0.2.1/quantization/api.html``). + """ + prefix = url_prefix.strip("/") + prefix = f"/{prefix}" if prefix else "" + return [[version_label(v), f"{prefix}/{v}/"] for v in versions] + + +def version_label(version: str) -> str: + """Return the human-facing label for ``version`` in the version dropdown. + + ``conf.py`` imports this so the label on the dropdown button matches the one + in the list the workflow generates; the two are rendered from separate + template variables, so a divergence would show a button that disagrees with + its own menu. + """ + return f"{version} (development)" if version == MAIN_VERSION else version + + +def _write_manifest(site: Path, manifest: list[list[str]]) -> None: + (site / "versions.json").write_text(json.dumps(manifest, indent=2) + "\n") + + +def _published_versions(site: Path) -> set[str]: + """Return the versions the last publish recorded in ``versions.json``. + + Read back from the manifest rather than the directory listing, so it reflects + what the previous run *said* it published. Comparing the two is what catches a + site tree that arrived incomplete. + """ + manifest = site / "versions.json" + if not manifest.is_file(): + return set() + try: + entries = json.loads(manifest.read_text()) + except json.JSONDecodeError: + return set() + # Recover the version from each href's trailing path segment ("/prefix/v0.2.1/"). + return {PurePosixPath(href).name for _label, href in entries} + + +def _check_site_is_complete(site: Path, present: set[str]) -> str | None: + """Return an error message if the site tree looks wrong to publish. + + ``deploy-pages`` replaces the entire site with whatever this script assembles, + so a tree that arrived incomplete — a truncated or wrong-ref checkout — would + silently take live documentation offline. Fail before that happens. + """ + missing = _published_versions(site) - present + if missing: + return ( + f"Site tree is missing version(s) that versions.json says are " + f"published: {', '.join(sorted(missing))}. Refusing to publish, because " + f"deploy-pages replaces the whole site and these would go offline. " + f"Check that the gh-pages checkout completed." + ) + + unknown = unrecognized_dirs(site) + if unknown: + return ( + f"Unrecognized top-level director(ies) on the site: " + f"{', '.join(unknown)}. These are served but cannot appear in the " + f"version dropdown, so readers would have no way to reach them. Either " + f"rename them to 'main' or 'vX.Y.Z', or remove them from the gh-pages " + f"branch." + ) + + return None + + +def cmd_plan(args: argparse.Namespace) -> int: + """Print the version manifest for the site as it will look after publishing. + + Includes the version being published even though its directory does not exist + yet, so the dropdown Sphinx bakes into the pages lists the doc set the reader + is currently looking at. + """ + versions = versions_after_publish(args.site, args.version) + print(json.dumps(build_manifest(versions, args.url_prefix))) + return 0 + + +def cmd_assemble(args: argparse.Namespace) -> int: + """Install the built HTML as ``//`` and refresh the site shell.""" + if not (args.html / "index.html").is_file(): + print(f"::error::No index.html under {args.html} — refusing to publish an empty doc set.") + return 1 + + versions = versions_after_publish(args.site, args.version) + error = _check_site_is_complete(args.site, set(versions)) + if error: + print(f"::error::{error}") + return 1 + + target = args.site / args.version + # Replace rather than merge: a stale file from the previous build of this same + # version (a page that was renamed or removed) would otherwise linger forever. + if target.exists(): + shutil.rmtree(target) + shutil.copytree(args.html, target) + + _write_manifest(args.site, build_manifest(versions, args.url_prefix)) + + # The root redirect stands in for a symlink, which GitHub Pages artifacts + # reject. MAIN_VERSION is the default view: it documents the current source. + (args.site / "index.html").write_text(_REDIRECT_HTML.format(target=MAIN_VERSION)) + + # sphinx.ext.githubpages writes .nojekyll into each build, so every version + # directory has one — but not the assembled root, which needs its own or + # Jekyll strips the underscore-prefixed directories (_static, _images). + (args.site / ".nojekyll").touch() + + print(f"Assembled site with versions: {', '.join(versions)}") + return 0 + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--site", type=Path, required=True, help="checkout of the gh-pages branch") + parser.add_argument( + "--version", required=True, help=f"version being published ('{MAIN_VERSION}' or 'vX.Y.Z')" + ) + parser.add_argument( + "--url-prefix", + default="", + help="path the site is served under (e.g. /coreai-optimization)", + ) + sub = parser.add_subparsers(dest="command", required=True) + sub.add_parser("plan", help="print the post-publish version manifest as JSON") + assemble = sub.add_parser("assemble", help="install built HTML and refresh the site shell") + assemble.add_argument("--html", type=Path, required=True, help="Sphinx HTML output directory") + + args = parser.parse_args(argv) + if not _is_version_name(args.version): + parser.error( + f"--version must be '{MAIN_VERSION}' or a version tag like 'v1.2.3', " + f"got {args.version!r}" + ) + if not args.site.is_dir(): + parser.error(f"--site is not a directory: {args.site}") + + return cmd_plan(args) if args.command == "plan" else cmd_assemble(args) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/docs/scripts/build_versioned_preview.sh b/docs/scripts/build_versioned_preview.sh new file mode 100755 index 0000000..3b0c2ff --- /dev/null +++ b/docs/scripts/build_versioned_preview.sh @@ -0,0 +1,192 @@ +#!/usr/bin/env bash + +# Copyright 2026 Apple Inc. +# +# Use of this source code is governed by a BSD-3-Clause license that can +# be found in the LICENSE file or at https://opensource.org/licenses/BSD-3-Clause + +# +# Build a local copy of the published multi-version documentation site. +# +# `make docs-open` serves one doc set as the web root, but the real site nests each +# version a level down (/main/, /v0.2.1/) with a redirect and versions.json at the +# root — so it cannot exercise the version picker, the root redirect, or the +# switcher script's versions.json fetch. This builds that layout locally: +# +# /main/ built from the working tree +# /vX.Y.Z/ built from that tag's documented source +# +# A tag supplies *source only* — its docstrings, prose pages, notebooks and +# tutorials. The build itself always uses the current tree's tooling: this +# conf.py, these docs/scripts/, this Makefile, this docs environment. So a tag +# cut before versioned docs existed still renders with the version picker, +# because the picker comes from the current conf.py rather than the tag's. +# +# That is what the layout below buys. Each tag's `src/` and `docs/src/` are staged +# under .local/docs// and the current `conf.py` is pointed at them, so no +# tag ever contributes build logic: +# +# .local/docs/vX.Y.Z/ +# ├── src/ <- from the tag (autodoc reads docstrings here) +# ├── docs/src/ <- from the tag (prose, notebooks, images) +# └── docs/scripts/<- from the CURRENT tree (extensions, api index generator) +# +# Usage: +# build_versioned_preview.sh --site [--tags "v0.2.1 v0.3.0"] [--all-tags] +# +# With neither --tags nor --all-tags, only `main` is built. +# +# Environment: +# MAKE_FLAGS extra flags for the `main` build's `make docs` (e.g. QUIET=0) + +set -euo pipefail + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(git -C "$SCRIPT_DIR" rev-parse --show-toplevel)" +ASSEMBLER="$SCRIPT_DIR/assemble_versioned_site.py" +STAGE_ROOT="$REPO_ROOT/.local/docs" + +SITE="" +TAGS=() +ALL_TAGS=false + +while [[ $# -gt 0 ]]; do + case "$1" in + --site) + SITE="$2" + shift 2 + ;; + --tags) + # shellcheck disable=SC2206 # deliberate word-split of a space-separated list + TAGS=($2) + shift 2 + ;; + --all-tags) + ALL_TAGS=true + shift + ;; + *) + echo "Error: unknown argument '$1'" >&2 + exit 1 + ;; + esac +done + +if [[ -z "$SITE" ]]; then + echo "Error: --site is required" >&2 + exit 1 +fi + +# Absolute, because the staged builds run from other directories. +mkdir -p "$SITE" +SITE="$(cd -- "$SITE" && pwd)" + +if [[ "$ALL_TAGS" == true ]]; then + # Sorted oldest-first only for readable progress output; the assembler owns the + # order the dropdown actually uses. + mapfile -t TAGS < <(git -C "$REPO_ROOT" tag --list 'v[0-9]*.[0-9]*.[0-9]*' --sort=v:refname) + if [[ ${#TAGS[@]} -eq 0 ]]; then + echo "==> No vX.Y.Z tags found locally. Fetch them first:" + echo " git fetch --tags origin" + echo "==> Building 'main' only." + fi +fi + +# A tag can only be previewed if it carries the two source trees the current +# conf.py reads: docs/src/ for prose and src/coreai_opt/ for docstrings. Checked +# with `git cat-file` so an unusable tag is skipped without any checkout. +tag_has_docs_source() { + git -C "$REPO_ROOT" cat-file -e "$1:docs/src/index.md" 2>/dev/null && + git -C "$REPO_ROOT" cat-file -e "$1:src/coreai_opt/__init__.py" 2>/dev/null +} + +BUILDABLE=() +for tag in ${TAGS[@]+"${TAGS[@]}"}; do + if tag_has_docs_source "$tag"; then + BUILDABLE+=("$tag") + else + echo "==> Skipping $tag: no docs/src + src/coreai_opt to build from" + fi +done + +# Stand up the version directories before planning. The dropdown is rendered into +# every page at build time, so the full list has to exist before any Sphinx run. +for tag in ${BUILDABLE[@]+"${BUILDABLE[@]}"}; do + mkdir -p "$SITE/$tag" +done + +# Locally the site root *is* the server root, so no URL prefix — on GitHub Pages +# the site is served under // and the workflow passes that instead. +VERSIONS="$(python3 "$ASSEMBLER" --site "$SITE" --version main --url-prefix '' plan)" +echo "==> Previewing versions: $VERSIONS" + +# Stage one tag's source under .local/docs// and build it with the current +# tooling. Layout mirrors the repo (src/ beside docs/) because conf.py locates the +# package by walking up from itself looking for src/coreai_opt — the same search +# that makes it work in both the OSS and internal trees. +stage_tag() { + local tag="$1" + local stage="$STAGE_ROOT/$tag" + + rm -rf "$stage" + mkdir -p "$stage/docs" + + # Source from the tag: docstrings and prose. + git -C "$REPO_ROOT" archive "$tag" src | tar -x -C "$stage" + git -C "$REPO_ROOT" archive "$tag" docs/src | tar -x -C "$stage" + + # Tooling from the current tree: conf.py, the templates and static assets it + # references, and the api-index generator. Applied after the tag's docs/src so + # the current versions win. + # + # The trailing `/.` matters. The tag already ships _templates/, _static/ and + # docs/scripts/, and `cp -a src dest` copies *into* an existing dest — which + # would produce _templates/_templates/ and leave the tag's own templates in + # place, so the version picker would render from the tag's markup instead of + # ours. Removing the destination first makes the copy a replacement. + for dir in docs/scripts docs/src/_templates docs/src/_static; do + rm -rf "$stage/$dir" + cp -a "$REPO_ROOT/$dir" "$stage/$dir" + done + cp -a "$REPO_ROOT/docs/src/conf.py" "$stage/docs/src/conf.py" +} + +build_version() { + local version="$1" src_dir="$2" + + echo "" + echo "==> Building $version" + # PYTHONPATH is what makes this a build of the *tag's* source. The docs venv + # installs coreai_opt editable, pointing at the working tree's src/, so without + # this autodoc would read current docstrings while the prose came from the tag. + # PYTHONPATH precedes site-packages, so the staged copy wins. + # + # Reuses the repo's .venv-docs rather than syncing a per-version env: the + # staged tree is built by the current conf.py, so the current environment is + # the correct one by definition. `sphinx-build` directly rather than + # `make docs` because the staged tree has no Makefile — and the mermaid-cli / + # Chrome / pandoc setup `make docs` performs has already run for `main`. + PYTHONPATH="$src_dir/src" DOCS_VERSION="$version" DOCS_VERSIONS="$VERSIONS" \ + uv run --no-sync --active sphinx-build -E -b html \ + "$src_dir/docs/src" "$src_dir/docs/build/html" + + cp -a "$src_dir/docs/build/html/." "$SITE/$version/" +} + +# Build `main` first, through `make docs`, so its mermaid-cli / Chrome / pandoc / +# .venv-docs setup runs once and the staged tag builds can reuse all of it. +echo "" +echo "==> Building main from the working tree" +DOCS_VERSION=main DOCS_VERSIONS="$VERSIONS" \ + make -C "$REPO_ROOT" docs _QUIET_HEADER=1 _DOCS_ALL=1 ${MAKE_FLAGS:-} + +# shellcheck source=/dev/null +source "$REPO_ROOT/.venv-docs/bin/activate" + +for tag in ${BUILDABLE[@]+"${BUILDABLE[@]}"}; do + stage_tag "$tag" + build_version "$tag" "$STAGE_ROOT/$tag" +done + +python3 "$ASSEMBLER" --site "$SITE" --version main --url-prefix '' \ + assemble --html "$REPO_ROOT/docs/build/html" diff --git a/docs/scripts/puppeteer-config.json b/docs/scripts/puppeteer-config.json new file mode 100644 index 0000000..251b509 --- /dev/null +++ b/docs/scripts/puppeteer-config.json @@ -0,0 +1,3 @@ +{ + "args": ["--no-sandbox", "--disable-setuid-sandbox", "--disable-dev-shm-usage"] +} diff --git a/docs/src/_static/custom.css b/docs/src/_static/custom.css index 694ed91..a8b04a0 100644 --- a/docs/src/_static/custom.css +++ b/docs/src/_static/custom.css @@ -247,3 +247,174 @@ html.dark #copy-page-content img { .nboutput.nblast.container { margin-bottom: 1.5em; } + +/* =========================================================================== + * Documentation version picker + * + * Markup: _templates/components/nav-versions.html (overrides shibuya's), placed + * in the brand cluster by _templates/partials/site-head.html. Behaviour: + * _static/version-switcher.js. + * + * The theme's own .nav-versions styling still applies — these rules cover what + * moving it next to the project name and swapping hover for click require. + * =========================================================================== */ + +/* Separate the picker from the project name the way a breadcrumb divider would, + * so it reads as metadata about the title rather than another nav item. The + * theme's stock rule adds a right-hand border for its old position between the + * search box and the socials; drop that and put the divider on the left. + */ +#version-picker { + margin-left: 0.75rem; + padding-left: 0.75rem; + border-left: 1px solid var(--sy-c-divider); +} + +#version-picker > button { + gap: 0.4rem; + border-right: 0; + padding: 0.3rem 0.5rem; + border-radius: 6px; + display: inline-flex; + align-items: center; + color: var(--sy-c-text); + font-size: 0.85rem; + font-weight: 500; + transition: background-color 0.2s; +} + +#version-picker > button:hover { + background-color: var(--sy-c-surface); +} + +/* The milestone glyph is decorative next to the version string, so mute it to + * match the chevron rather than competing with the label. + */ +#version-picker .milestone { + color: var(--sy-c-light); + font-size: 0.95em; +} + +/* Click-to-open, not hover: the theme reveals .nav-versions-choices on :hover, + * which pops the menu open in passing and offers nothing to a keyboard or touch + * user. Neutralise that and drive visibility from aria-expanded instead. + */ +#version-picker:hover .nav-versions-choices { + visibility: hidden; +} + +#version-picker > button[aria-expanded="true"] + .nav-versions-choices { + visibility: visible; +} + +#version-picker > button[aria-expanded="true"] { + background-color: var(--sy-c-surface); +} + +/* Anchor the menu under the trigger (the theme's rule pins it to the right edge, + * correct for its old far-right position, wrong here) and match the Copy page + * dropdown's border so the two read as the same component. + */ +#version-picker .nav-versions-choices { + top: calc(var(--sy-s-navbar-height) - 0.5rem); + right: auto; + left: 0; + min-width: 13rem; + border: 1px solid var(--sy-c-divider); + padding: 0.5rem; +} + +#version-picker .nav-versions-choices ul { + padding: 0; +} + +#version-picker .nav-versions-choices a { + border-radius: 6px; + padding: 0.4rem 0.6rem; + font-size: 0.9rem; +} + +/* Mark the version being read. Weight rather than colour, so it stays legible + * against the accent-tinted background and in both colour modes. + */ +#version-picker .nav-versions-choices a[aria-current="true"] { + background-color: var(--sy-c-surface); + font-weight: 600; +} + +/* --------------------------------------------------------------------------- + * Version banners + * + * Built by version-switcher.js from the theme's own .announcement element, so + * they inherit the sticky positioning and the --sy-s-banner-height offset the + * theme already accounts for. Only the palette changes: the stock banner uses the + * accent colour, which reads as a feature announcement rather than a note about + * which version the reader has landed on. + * + * Two severities, because the two situations are not equally alarming: `main` + * documenting unreleased work is a neutral heads-up, while a superseded release + * is a caution. + * --------------------------------------------------------------------------- */ +.version-banner { + font-size: 0.9rem; +} + +.version-banner .announcement-inner { + text-align: center; +} + +/* Development: informational blue, matching how the theme's admonitions signal a + * note rather than a warning. + */ +.version-banner-dev { + --sy-c-banner: var(--blue-12); + --sy-c-banner-bg: var(--blue-3); +} + +html.dark .version-banner-dev { + --sy-c-banner: var(--blue-12); + --sy-c-banner-bg: var(--blue-2); +} + +/* Superseded release: caution red. */ +.version-banner-old { + --sy-c-banner: var(--red-12); + --sy-c-banner-bg: var(--red-3); +} + +html.dark .version-banner-old { + --sy-c-banner: var(--red-12); + --sy-c-banner-bg: var(--red-2); +} + +/* The call to action is the point of the banner, so give it a button shape + * instead of the theme's underlined-link treatment for banner anchors. Colour is + * inherited per severity from the rules below. + */ +.version-banner a.version-banner-action { + margin-left: 0.5rem; + border-radius: 6px; + padding: 0.25rem 0.7rem; + color: #fff; + font-weight: 500; + text-decoration: none; + white-space: nowrap; + display: inline-block; + transition: background-color 0.2s; +} + +.version-banner-dev a.version-banner-action { + background-color: var(--blue-9); +} + +.version-banner-dev a.version-banner-action:hover { + background-color: var(--blue-10); +} + +.version-banner-old a.version-banner-action { + background-color: var(--red-9); +} + +.version-banner-old a.version-banner-action:hover { + background-color: var(--red-10); +} diff --git a/docs/src/_static/version-switcher.js b/docs/src/_static/version-switcher.js new file mode 100644 index 0000000..3ce6dcc --- /dev/null +++ b/docs/src/_static/version-switcher.js @@ -0,0 +1,253 @@ +// Copyright 2026 Apple Inc. +// +// Use of this source code is governed by a BSD-3-Clause license that can +// be found in the LICENSE file or at https://opensource.org/licenses/BSD-3-Clause + + +// Runtime behaviour for the documentation version picker. +// +// Three jobs, all of which no-op on a doc set built without DOCS_VERSIONS (the +// _templates/components/nav-versions.html override renders nothing, so there is +// no picker to find): +// +// 1. Open and close the dropdown on click, rather than the theme's hover-only +// reveal — so it works from the keyboard and on touch. +// 2. Refresh the version list from the site's versions.json. The list is rendered +// into every page at build time, so an already-published doc set would +// otherwise never learn about releases cut after it. +// 3. Warn when the reader is not on the newest version, with a link to it. +// +// Written in the same plain-ES5-with-const style as the sibling _static scripts. + +// The site root is derived from this script's own URL, which is always +// `//_static/version-switcher.js` — so the root is three +// levels up, whatever the version is called and however deep the current page +// sits. Read at load time because document.currentScript is only set while the +// script is being evaluated, not later inside a callback. +// +// Deliberately not derived from DOCUMENTATION_OPTIONS.VERSION: that is Sphinx's +// `release` field, which exists to describe the project version. If anyone ever +// set it to a PEP 440 string (0.3.0) rather than the directory name (v0.3.0), +// matching against it would silently stop resolving. +const SITE_ROOT = document.currentScript + ? new URL("../../", document.currentScript.src).href + : null; + +// Label suffix conf.py's version_label() appends to the development version. +// Stripped for the banner text, where "main (development)" reads awkwardly. +const DEV_SUFFIX = " (development)"; + +// Upgrade each menu entry from "that version's front page" to "this page in that +// version", where it exists. Runs once, on first open, so a reader who never +// touches the picker pays nothing and the cost does not grow with each release. +function resolveMenuLinks(picker) { + const pending = picker.querySelectorAll("a[data-version-root]"); + pending.forEach(function (link) { + const root = link.dataset.versionRoot; + // Clear the marker first so a second open cannot re-probe the same entry. + delete link.dataset.versionRoot; + equivalentPageIn(root).then(function (href) { + link.href = href; + }); + }); +} + +function setupMenu(picker) { + const button = picker.querySelector(".js-version-menu"); + const menu = picker.querySelector(".nav-versions-choices"); + if (!button || !menu) { + return; + } + + function close() { + button.setAttribute("aria-expanded", "false"); + } + + button.addEventListener("click", function (event) { + event.stopPropagation(); + const open = button.getAttribute("aria-expanded") === "true"; + button.setAttribute("aria-expanded", open ? "false" : "true"); + if (!open) { + resolveMenuLinks(picker); + } + }); + + // Click anywhere else, or Escape, dismisses it — the behaviour a reader + // expects from a menu and what the hover-only original could not offer. + document.addEventListener("click", function (event) { + if (!picker.contains(event.target)) { + close(); + } + }); + document.addEventListener("keydown", function (event) { + if (event.key === "Escape") { + close(); + } + }); +} + +function renderVersions(picker, versions) { + const current = picker.querySelector(".js-version-menu span"); + const currentLabel = current ? current.textContent : null; + const list = document.createElement("ul"); + + versions.forEach(function (entry) { + const item = document.createElement("li"); + const link = document.createElement("a"); + link.setAttribute("role", "menuitem"); + // Assign via properties, not innerHTML, so a label from versions.json is + // inserted as text and cannot inject markup. + link.textContent = entry[0]; + link.href = entry[1]; + if (entry[0] === currentLabel) { + link.setAttribute("aria-current", "true"); + } else { + // Deep-link resolution is deferred to the first time the menu is opened + // (see resolveMenuLinks): probing every version here would cost one request + // per release on every page view, which grows with each release for readers + // who never touch the picker. + link.dataset.versionRoot = entry[1]; + } + item.appendChild(link); + list.appendChild(item); + }); + + const menu = picker.querySelector(".nav-versions-choices"); + // The override template puts nothing else inside the menu, so replacing its + // children swaps the list without disturbing anything. + menu.replaceChildren(list); +} + +// Map the current page onto another version, so a reader deep in the docs lands +// on the same topic rather than that version's front page. +// +// Falls back to the version root when the page does not exist there — pages get +// added, renamed and removed between releases, and a 404 is a worse outcome than +// the landing page. Verified with a HEAD request rather than assumed, since only +// the server knows what that version actually contains. +function equivalentPageIn(versionHref) { + if (!SITE_ROOT) { + return Promise.resolve(versionHref); + } + // The path of the current page relative to its own doc set, e.g. + // "quantization/config.html" from ".../v0.2.0/quantization/config.html". + const relative = window.location.href.slice(SITE_ROOT.length).split("/").slice(1).join("/"); + if (!relative) { + return Promise.resolve(versionHref); + } + const target = versionHref + relative; + return fetch(target, { method: "HEAD" }) + .then(function (response) { + return response.ok ? target : versionHref; + }) + .catch(function () { + return versionHref; + }); +} + +// Announce when the doc set being read is not the newest release, in one of two +// ways: `main` documents unreleased work, and an older release has been +// superseded. Both reuse the theme's own `.announcement` element, so they inherit +// its sticky positioning and the --sy-s-banner-height offset the header and +// sidebars already account for — worth far more than hand-rolling a bar and +// re-deriving those offsets. +function showVersionBanner(versions, currentLabel) { + if (document.querySelector(".announcement")) { + return; // a real announcement is configured; don't stack banners on it + } + + // versions[0] is `main`; the newest actual release is the first entry that + // isn't it. Ordering comes from assemble_versioned_site.sort_versions(). + const newest = versions.find(function (entry) { + return entry[0].indexOf(DEV_SUFFIX) === -1; + }); + // Nothing useful to point at until at least one release exists. + if (!newest || newest[0] === currentLabel) { + return; + } + + const development = currentLabel.indexOf(DEV_SUFFIX) !== -1; + const banner = document.createElement("div"); + // Development is a neutral heads-up; a superseded release is a caution. The + // two get different palettes in custom.css. + banner.className = development + ? "announcement version-banner version-banner-dev" + : "announcement version-banner version-banner-old"; + + const inner = document.createElement("div"); + inner.className = "announcement-inner"; + + const text = document.createElement("p"); + const emphasis = document.createElement("strong"); + if (development) { + emphasis.textContent = "in-development docs"; + text.append("These are ", emphasis, " and may describe unreleased features. "); + } else { + emphasis.textContent = "version " + currentLabel; + text.append("These docs are for ", emphasis, ", which is no longer the latest. "); + } + + const link = document.createElement("a"); + link.className = "version-banner-action"; + // Point at the version root immediately so the link is never dead, then + // upgrade it to this page's counterpart once the HEAD probe resolves. + link.href = newest[1]; + link.textContent = "Go to " + newest[0]; + equivalentPageIn(newest[1]).then(function (href) { + link.href = href; + }); + text.append(link); + + const close = document.createElement("button"); + close.className = "announcement-close"; + close.setAttribute("aria-label", "Close notification"); + close.innerHTML = ''; + + inner.append(text, close); + banner.appendChild(inner); + document.body.prepend(banner); + + // The theme's own script only wires the close button for a banner present at + // load, so this one manages its own teardown and height variable. + const style = document.createElement("style"); + const setHeight = function () { + style.textContent = ":root{--sy-s-banner-height:" + banner.clientHeight + "px}"; + }; + document.head.appendChild(style); + setHeight(); + window.addEventListener("resize", setHeight); + close.addEventListener("click", function () { + banner.remove(); + style.remove(); + }); +} + +document.addEventListener("DOMContentLoaded", function () { + const picker = document.querySelector("#version-picker"); + if (!picker) { + return; + } + setupMenu(picker); + + if (!SITE_ROOT) { + return; + } + fetch(SITE_ROOT + "versions.json", { cache: "no-cache" }) + .then(function (response) { + return response.ok ? response.json() : Promise.reject(response.status); + }) + .then(function (versions) { + if (!Array.isArray(versions) || versions.length === 0) { + return; + } + renderVersions(picker, versions); + const current = picker.querySelector(".js-version-menu span"); + if (current) { + showVersionBanner(versions, current.textContent); + } + }) + // Leave the build-time list in place when versions.json is missing or + // unreachable — a stale picker beats an empty one. The banner is skipped + // too, since without the manifest there is no way to know what is newest. + .catch(function () {}); +}); diff --git a/docs/src/_templates/components/nav-versions.html b/docs/src/_templates/components/nav-versions.html new file mode 100644 index 0000000..3d85e6c --- /dev/null +++ b/docs/src/_templates/components/nav-versions.html @@ -0,0 +1,59 @@ +{#- + Copyright 2026 Apple Inc. + + Use of this source code is governed by a BSD-3-Clause license that can + be found in the LICENSE file or at https://opensource.org/licenses/BSD-3-Clause +-#} + + +{#- + Override of shibuya/components/nav-versions.html. + + Same contract as the stock template — renders from the `versions` (list of + [label, href] pairs) and `current_version` values conf.py puts in html_context, + and renders nothing at all when `versions` is unset, so a local `make docs` + without DOCS_VERSIONS is unaffected. + + Differs in two ways: + + * Click to open, not hover. The stock template relies on `:hover` alone, which + gives no keyboard or touch affordance and pops open in passing. This uses an + aria-expanded button driven by _static/version-switcher.js, matching the + Copy page menu on the article. + * `aria-current` marks the entry the reader is on, so the active version is + identifiable without comparing it to the button text. + + The wrapper keeps the theme's `.nav-versions` / `.nav-versions-choices` class + names so shibuya's own dropdown styling still applies; custom.css adjusts what + the relocation into the brand cluster needs. +-#} + +{%- if versions -%} + +{%- endif -%} diff --git a/docs/src/_templates/partials/site-head.html b/docs/src/_templates/partials/site-head.html new file mode 100644 index 0000000..2b32edd --- /dev/null +++ b/docs/src/_templates/partials/site-head.html @@ -0,0 +1,74 @@ +{#- + Copyright 2026 Apple Inc. + + Use of this source code is governed by a BSD-3-Clause license that can + be found in the LICENSE file or at https://opensource.org/licenses/BSD-3-Clause +-#} + + +{#- + Override of shibuya/partials/site-head.html. + + Two changes to the stock header, both to do with versioned docs: + + 1. The version picker moves out of `.sy-head-extra` (where the theme puts it, + between the search box and the social links on the far right) and into the + brand cluster, immediately right of the project name. That is where readers + expect it — it labels the whole doc set, so it belongs with the title rather + than among the page-level controls. The stock include is dropped from + `.sy-head-extra` so the picker is not rendered twice. + + 2. `components/nav-versions.html` is replaced by our own markup so the trigger + is a real click-to-open menu button (see _static/version-switcher.js) rather + than the theme's hover-only reveal, matching the Copy page dropdown on the + article. Styling lives in _static/custom.css. + + Everything else is copied verbatim from the theme; keep it in sync when + upgrading shibuya. +-#} + +{%- macro render_logo(src, classname) -%} + {%- if src and src.startswith(('https://', 'http://')) -%} + {{ project }} + {%- elif src -%} + {{ project }} + {%- elif logo_url -%} + {{ project }} + {%- endif -%} +{%- endmacro -%} + +
+
+
+
+ + {{ render_logo(theme_light_logo, 'light-logo') }} + {{ render_logo(theme_dark_logo, 'dark-logo') }} + {{ project }} + + {%- include "components/nav-versions.html" -%} +
+
+ +
+ {%- include "components/searchbox.html" -%} + {%- include "components/nav-languages.html" -%} + {%- include "partials/nav-socials.html" -%} +
+
+
+ {%- include "components/theme-switch.html" -%} + +
+
+
diff --git a/docs/src/api/index.md b/docs/src/api/index.md new file mode 100644 index 0000000..9f0d1d4 --- /dev/null +++ b/docs/src/api/index.md @@ -0,0 +1,255 @@ +# API Reference + +## coreai_opt + +coreai_opt - A library for PyTorch model compression and optimizations. + +```{eval-rst} +.. autosummary:: + :toctree: generated + + coreai_opt.CoreMLExportError + coreai_opt.ExportBackend +``` + +## coreai_opt.casting + +Casting related utilities including FP32 -> FP16 and INT32 -> INT16 passes. + +```{eval-rst} +.. autosummary:: + :toctree: generated + + coreai_opt.casting.cast_fp32_to_fp16 + coreai_opt.casting.cast_int32_to_int16 + coreai_opt.casting.cast_to_16_bit_precision +``` + +## coreai_opt.config + +Configuration and specification modules for coreai_opt. + +```{eval-rst} +.. autosummary:: + :toctree: generated + + coreai_opt.config.CompressionConfig + coreai_opt.config.CompressionSpec + coreai_opt.config.CompressionType + coreai_opt.config.ModuleCompressionConfig + coreai_opt.config.OpCompressionConfig + coreai_opt.config.WeightOnlyModuleValidationMixin + coreai_opt.config.WeightOnlyOpValidationMixin +``` + +### coreai_opt.config.spec + +Base abstractions for compression specs, simulators, and component factories. + +```{eval-rst} +.. autosummary:: + :toctree: generated + + coreai_opt.config.spec.CompressionComponentFactoryBase + coreai_opt.config.spec.CompressionSimulatorBase + coreai_opt.config.spec.CompressionTargetTensor +``` + +## coreai_opt.coreai_utils + +Core AI MLIR-level compression transforms. + +```{eval-rst} +.. autosummary:: + :toctree: generated + + coreai_opt.coreai_utils.CompressionGranularity + coreai_opt.coreai_utils.DType + coreai_opt.coreai_utils.palettize_weights + coreai_opt.coreai_utils.quantize_weights + coreai_opt.coreai_utils.sparsify_weights +``` + +### coreai_opt.coreai_utils.common + +Common enums and constants for coreai_opt.coreai_utils. + +```{eval-rst} +.. autosummary:: + :toctree: generated + + coreai_opt.coreai_utils.common.QScheme +``` + +## coreai_opt.inspection + +Utilities for inspecting model operations and compression configuration. + +```{eval-rst} +.. autosummary:: + :toctree: generated + + coreai_opt.inspection.BoundaryEdge + coreai_opt.inspection.InputEdge + coreai_opt.inspection.ModelInspector + coreai_opt.inspection.ModelSummary + coreai_opt.inspection.ModuleContext + coreai_opt.inspection.ModuleInfo + coreai_opt.inspection.OpInfo + coreai_opt.inspection.SourceFrame +``` + +## coreai_opt.palettization + +Palettization specification and utilities for weight compression via lookup tables. + +```{eval-rst} +.. autosummary:: + :toctree: generated + + coreai_opt.palettization.KMeansPalettizer + coreai_opt.palettization.KMeansPalettizerConfig + coreai_opt.palettization.ModuleKMeansPalettizerConfig + coreai_opt.palettization.PalettizationSpec +``` + +### coreai_opt.palettization.config + +Palettization configuration classes. + +```{eval-rst} +.. autosummary:: + :toctree: generated + + coreai_opt.palettization.config.OpKMeansPalettizerConfig + coreai_opt.palettization.config.PATSchedule +``` + +### coreai_opt.palettization.spec + +Palettization specs, granularity classes, and factory functions. + +```{eval-rst} +.. autosummary:: + :toctree: generated + + coreai_opt.palettization.spec.DefaultTrainingConfig + coreai_opt.palettization.spec.PalettizationGranularity + coreai_opt.palettization.spec.PerGroupedChannelGranularity + coreai_opt.palettization.spec.PerTensorGranularity + coreai_opt.palettization.spec.TrainingStrategy + coreai_opt.palettization.spec.TrainingStrategyConfig + coreai_opt.palettization.spec.default_weight_palettization_spec +``` + +## coreai_opt.pruning + +Pruning infrastructure for coreai_opt. + +```{eval-rst} +.. autosummary:: + :toctree: generated + + coreai_opt.pruning.MagnitudePruner + coreai_opt.pruning.MagnitudePrunerConfig + coreai_opt.pruning.ModuleMagnitudePrunerConfig + coreai_opt.pruning.PruningSpec +``` + +### coreai_opt.pruning.config + +Pruning configuration exports. + +```{eval-rst} +.. autosummary:: + :toctree: generated + + coreai_opt.pruning.config.ConstantSparsitySchedule + coreai_opt.pruning.config.OpMagnitudePrunerConfig + coreai_opt.pruning.config.PolynomialDecaySchedule + coreai_opt.pruning.config.SparsityScheduleBase +``` + +### coreai_opt.pruning.spec + +Pruning spec components: specs, schemes, and parametrizations. + +```{eval-rst} +.. autosummary:: + :toctree: generated + + coreai_opt.pruning.spec.ChannelStructured + coreai_opt.pruning.spec.PruneImplBase + coreai_opt.pruning.spec.PruningScheme + coreai_opt.pruning.spec.Unstructured + coreai_opt.pruning.spec.default_weight_pruning_spec +``` + +## coreai_opt.quantization + +Quantization compressor, configuration, specs, and granularity classes. + +```{eval-rst} +.. autosummary:: + :toctree: generated + + coreai_opt.quantization.ExecutionMode + coreai_opt.quantization.InvalidExecutionModeError + coreai_opt.quantization.ModuleQuantizerConfig + coreai_opt.quantization.QuantizationSpec + coreai_opt.quantization.Quantizer + coreai_opt.quantization.QuantizerConfig +``` + +### coreai_opt.quantization.config + +Quantization configuration classes and execution mode. + +```{eval-rst} +.. autosummary:: + :toctree: generated + + coreai_opt.quantization.config.KVCacheQuantConfig + coreai_opt.quantization.config.OpQuantizerConfig + coreai_opt.quantization.config.QATSchedule +``` + +### coreai_opt.quantization.spec + +Quantization specs, schemes, granularity classes, and parameter calculators. + +```{eval-rst} +.. autosummary:: + :toctree: generated + + coreai_opt.quantization.spec.DynamicQParamsCalculator + coreai_opt.quantization.spec.GlobalMinMaxQParamsCalculator + coreai_opt.quantization.spec.MinMaxRangeCalculator + coreai_opt.quantization.spec.MovingAverageQParamsCalculator + coreai_opt.quantization.spec.PerBlockGranularity + coreai_opt.quantization.spec.PerChannelGranularity + coreai_opt.quantization.spec.PerTensorGranularity + coreai_opt.quantization.spec.QParamsCalculatorBase + coreai_opt.quantization.spec.QuantizationComponentFactory + coreai_opt.quantization.spec.QuantizationFormulation + coreai_opt.quantization.spec.QuantizationGranularity + coreai_opt.quantization.spec.QuantizationScheme + coreai_opt.quantization.spec.RangeCalculatorBase + coreai_opt.quantization.spec.RunningRangeMixin + coreai_opt.quantization.spec.StatefulQParamsCalculatorBase + coreai_opt.quantization.spec.StatelessQParamsCalculatorBase + coreai_opt.quantization.spec.StaticQParamsCalculator + coreai_opt.quantization.spec.default_activation_quantization_spec + coreai_opt.quantization.spec.default_weight_quantization_spec +``` + +### coreai_opt.quantization.spec.fake_quantize + +Fake quantization implementation base class and default implementation. + +```{eval-rst} +.. autosummary:: + :toctree: generated + + coreai_opt.quantization.spec.fake_quantize.FakeQuantizeImplBase +``` diff --git a/docs/src/conf.py b/docs/src/conf.py index ac707b2..5f54672 100644 --- a/docs/src/conf.py +++ b/docs/src/conf.py @@ -13,6 +13,8 @@ import enum import functools import inspect +import json +import os import pydoc import re import sys @@ -46,15 +48,22 @@ raise RuntimeError(msg) sys.path.insert(0, str(coreai_opt_root)) +from scripts.assemble_versioned_site import version_label # noqa: E402 from scripts.generate_api_index import generate_api_index # noqa: E402 # -- Project information ----------------------------------------------------- project = "CoreAI-Opt" -version = "main" project_copyright = "2026, Apple, Inc. All rights reserved" author = "Apple CoreAI-Opt Team" -release = "main" + +# Version label this build is published under. The docs workflow sets DOCS_VERSION +# to "main" (built from the main branch) or "vX.Y.Z" (built from a release tag); +# each lands in its own directory on the published site. A local `make docs` leaves +# it unset and reads as "main", which is what this file hardcoded previously. +docs_version = os.environ.get("DOCS_VERSION", "main") +version = docs_version +release = docs_version # -- General configuration --------------------------------------------------- @@ -168,6 +177,18 @@ mermaid_output_format = "svg" mermaid_cmd = "mmdc" +# mmdc drives headless Chrome through Puppeteer, and Chrome's own sandbox needs +# unprivileged user namespaces — which Ubuntu 23.10+ restricts via AppArmor, and +# containers commonly disallow. Without this, every diagram fails to render with +# "No usable sandbox!" and the build still *succeeds*, silently falling back to +# client-side rendering from a CDN. Disabling Chrome's sandbox is safe here +# because the only input is diagram source from this repo, and the whole thing +# already runs inside the CI sandbox. +# +# `--disable-dev-shm-usage` covers the other common container failure: a small +# /dev/shm makes Chrome crash partway through rendering instead of at launch. +mermaid_params = ["--puppeteerConfigFile", str(_docs_dir / "scripts" / "puppeteer-config.json")] + myst_heading_anchors = 4 # -- nbsphinx configuration ------------------------------------------------- @@ -215,7 +236,10 @@ # mid-word. See _static/sidebar-wrap.js. # copy-page-button.js is the runtime companion for the copy-page-button # template override at _templates/components/copy-page-button.html. -html_js_files = ["sidebar-wrap.js", "copy-page-button.js"] +# version-switcher.js drives the version picker: opens the menu, refreshes it +# from versions.json so an already-published doc set lists releases cut after it, +# and shows a banner when the reader is on a superseded version. +html_js_files = ["sidebar-wrap.js", "copy-page-button.js", "version-switcher.js"] # Theme options html_theme_options = { @@ -235,6 +259,36 @@ ], } +# -- Version switcher -------------------------------------------------------- +# +# Shibuya ships a version dropdown at theme/shibuya/components/nav-versions.html +# keyed off two template variables the theme never sets itself: `versions` (a list +# of [label, href] pairs) and `current_version`. Sphinx copies html_context into +# the template globals, so populating it here is all the picker needs. +# +# Both that component and partials/site-head.html are overridden in _templates/: +# the picker sits next to the project name rather than out among the page controls, +# and opens on click instead of hover. Styling is in _static/custom.css and the +# behaviour in _static/version-switcher.js. +# +# The docs workflow passes DOCS_VERSIONS as JSON. The override keeps the theme's +# `{% if versions %}` guard, so a local `make docs` with the variable unset renders +# no picker at all rather than a broken or single-entry one. +# +# hrefs must be absolute from the site root (e.g. /coreai-optimization/main/): +# a relative "../main/" would resolve against the current page's directory, so it +# would break on any nested page. +# +# `current_version` comes from the same version_label() the workflow used to build +# the list, so the button text can't drift from the entry it corresponds to — +# shibuya renders the two from separate template variables and would not complain. +_docs_versions = os.environ.get("DOCS_VERSIONS", "") +if _docs_versions: + html_context = { + "versions": json.loads(_docs_versions), + "current_version": version_label(docs_version), + } + html_show_sourcelink = False # Pygments (syntax highlighting) style @@ -424,9 +478,13 @@ def setup(app: Sphinx) -> None: app.connect("autodoc-skip-member", _autodoc_skip_member) # Generate api/index.md from the package tree before Sphinx reads sources. + # _write_if_changed, not write_text: this file is committed (and guarded by the + # check-api-doc-coverage pre-commit hook), so an unconditional rewrite would + # touch its mtime on every build and leave the file looking modified to git's + # stat cache even when its contents are unchanged. api_index = Path(__file__).parent / "api" / "index.md" api_index.parent.mkdir(parents=True, exist_ok=True) - api_index.write_text(generate_api_index()) + _write_if_changed(api_index, generate_api_index()) # Auto-generate preset stubs at their user-facing usage paths. Must run # after api/index.md exists (discovery reads it) and before Sphinx reads diff --git a/docs/tests/conftest.py b/docs/tests/conftest.py new file mode 100644 index 0000000..b0be713 --- /dev/null +++ b/docs/tests/conftest.py @@ -0,0 +1,27 @@ +# Copyright 2026 Apple Inc. +# +# Use of this source code is governed by a BSD-3-Clause license that can +# be found in the LICENSE file or at https://opensource.org/licenses/BSD-3-Clause + +"""Shared pytest fixtures for docs tests.""" + +from __future__ import annotations + +import sys +from pathlib import Path + +import pytest + +from coreai_opt._utils.repo_utils import find_repo_root + +# Add docs scripts to path so tests can import them +_repo_root = find_repo_root(__file__) +_scripts_dir = _repo_root / "docs" / "scripts" +if str(_scripts_dir) not in sys.path: + sys.path.insert(0, str(_scripts_dir)) + + +@pytest.fixture(scope="session") +def repo_root() -> Path: + """Return the repository root path.""" + return _repo_root diff --git a/docs/tests/test_api_doc_coverage.py b/docs/tests/test_api_doc_coverage.py new file mode 100644 index 0000000..bd70942 --- /dev/null +++ b/docs/tests/test_api_doc_coverage.py @@ -0,0 +1,79 @@ +# Copyright 2026 Apple Inc. +# +# Use of this source code is governed by a BSD-3-Clause license that can +# be found in the LICENSE file or at https://opensource.org/licenses/BSD-3-Clause + +"""Tests for Sphinx API reference coverage and rendering. + +Complements ``tests/test_api_visibility.py`` (which enforces ``__all__`` +declarations) to ensure every public symbol is both declared AND documented. +""" + +from __future__ import annotations + +import importlib +import importlib.util +from pathlib import Path + +from coreai_opt._utils.api_visibility_utils import find_public_packages + +_ROOT_PACKAGE = "coreai_opt" + + +def _load_module_from_path(name: str, path: Path): # noqa: ANN202 + """Import a Python file as a module by filesystem path.""" + spec = importlib.util.spec_from_file_location(name, path) + assert spec is not None and spec.loader is not None + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +def test_api_doc_coverage(repo_root: Path) -> None: + """Committed api/index.md must match the auto-generated version.""" + gen = _load_module_from_path( + "generate_api_index", repo_root / "docs" / "scripts" / "generate_api_index.py" + ) + api_index = repo_root / "docs" / "src" / "api" / "index.md" + assert api_index.exists(), ( + f"API index not found at {api_index}.\nRun `make render-api-index` to generate it." + ) + + assert api_index.read_text() == gen.generate_api_index(), ( + "docs/src/api/index.md is out of sync with the package tree.\n" + "Run `make render-api-index` to regenerate it, then stage the result." + ) + + +def test_autodoc_skip_filters_external_methods(repo_root: Path) -> None: + """No inherited external methods (torch, pydantic, etc.) survive the autodoc filter. + + Calls the real ``_autodoc_skip_member`` from ``docs/src/conf.py`` against + every member of every public API class. Any member that passes the filter + must originate from coreai_opt. + """ + conf = _load_module_from_path("docs_conf", repo_root / "docs" / "src" / "conf.py") + skip_fn = conf._autodoc_skip_member + + leaks: list[str] = [] + for pkg_name in find_public_packages(_ROOT_PACKAGE): + mod = importlib.import_module(pkg_name) + for sym in getattr(mod, "__all__", []): + cls = getattr(mod, sym, None) + if not isinstance(cls, type): + continue + for name in dir(cls): + member = getattr(cls, name, None) + if member is None: + continue + if skip_fn(app=None, what="class", name=name, obj=member, skip=False, options=None): + continue + origin = getattr(member, "__module__", None) or "" + if origin and not origin.startswith(_ROOT_PACKAGE): + leaks.append(f"{pkg_name}.{sym}.{name} (from {origin})") + + assert not leaks, ( + "External methods leak through _autodoc_skip_member in docs/src/conf.py.\n" + "These would appear in the Sphinx API reference:\n" + + "\n".join(f" - {s}" for s in sorted(set(leaks))) + ) diff --git a/docs/tests/test_docs.py b/docs/tests/test_docs.py new file mode 100644 index 0000000..86cda01 --- /dev/null +++ b/docs/tests/test_docs.py @@ -0,0 +1,92 @@ +# Copyright 2026 Apple Inc. +# +# Use of this source code is governed by a BSD-3-Clause license that can +# be found in the LICENSE file or at https://opensource.org/licenses/BSD-3-Clause + +""" +Test that documentation builds correctly. +""" + +import subprocess +from pathlib import Path + +import pytest + +# Lines of build output to quote in the failure message. `make docs` emits a few +# hundred lines, almost all of it Sphinx progress; the cause of a failure is +# always in the last handful. +_FAILURE_CONTEXT_LINES = 40 + + +def _build_failure_message(returncode: int, output: str) -> str: + """Return an assertion message that carries the build's own diagnostics. + + pytest renders a failing fixture's ``CompletedProcess`` with the middle + elided, so the actual error is the part that gets cut. Quoting the tail of the + output inside the message puts it in every reported ERROR block, where it + survives both that truncation and a trimmed CI log. + """ + lines = output.splitlines() + excerpt = lines[-_FAILURE_CONTEXT_LINES:] + omitted = len(lines) - len(excerpt) + header = f"`make docs` failed with exit code {returncode}." + if omitted > 0: + header += f" Last {len(excerpt)} of {len(lines)} output lines ({omitted} omitted):" + else: + header += " Full output:" + # Blank line before the block so pytest's `E ` prefixes stay readable. + return "\n".join([header, ""] + excerpt) + + +@pytest.fixture(scope="session") +def built_docs(repo_root: Path) -> Path: + """Build the docs once with ``make docs`` and return the HTML output directory. + + Session-scoped so both docs tests share a single build and each remains runnable + in isolation, without an implicit ordering dependency on another test building first. + """ + # Combine stdout and stderr so errors appear in context. + result = subprocess.run( + ["make", "docs"], + cwd=repo_root, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + ) + + # Always print output for visibility. + print(f"\n=== make docs output ===\n{result.stdout}") + + assert result.returncode == 0, _build_failure_message(result.returncode, result.stdout) + return repo_root / "docs" / "build" / "html" + + +def test_docs_built_index_html(built_docs: Path) -> None: + """Verify the docs build emitted the HTML entry point (index.html).""" + assert (built_docs / "index.html").exists(), ( + f"Expected documentation output at {built_docs / 'index.html'} does not exist" + ) + + +def test_docs_built_llms_txt(built_docs: Path) -> None: + """Verify the docs build emitted the llms.txt / llms-full.txt summaries.""" + assert (built_docs / "llms.txt").exists(), ( + f"Expected llms.txt at {built_docs / 'llms.txt'} does not exist" + ) + assert (built_docs / "llms-full.txt").exists(), ( + f"Expected llms-full.txt at {built_docs / 'llms-full.txt'} does not exist" + ) + + +def test_mermaid_svgs_generated(built_docs: Path) -> None: + """Verify ``make docs`` rendered mermaid diagrams to real, non-empty SVGs.""" + images_dir = built_docs / "_images" + svgs = list(images_dir.glob("mermaid-*.svg")) + assert svgs, f"No mermaid SVGs were generated in {images_dir}" + + # A real mermaid diagram contains an ```` root; an empty/degenerate placeholder + # (the silent-failure mode) does not. + malformed = [p.name for p in svgs if b" root): {malformed}" + ) diff --git a/scripts/ensure_pandoc.sh b/scripts/ensure_pandoc.sh index 2427421..75ac17f 100755 --- a/scripts/ensure_pandoc.sh +++ b/scripts/ensure_pandoc.sh @@ -9,9 +9,14 @@ # Ensure the `pandoc` binary is available, installing it if needed. # # nbsphinx shells out to `pandoc` to convert notebook markdown cells during the -# docs build. On macOS it comes from Homebrew. On Linux the dnf/apt repos do -# not ship pandoc, so we download the upstream static binary into /usr/local. -# Override the version with the PANDOC_VERSION environment variable. +# docs build. On macOS it comes from Homebrew. On Linux the dnf/apt repos do not +# ship pandoc, so we download the upstream static binary into the repo's own +# `.local/bin/` (gitignored) rather than a system prefix — CI runners and any +# other unprivileged environment cannot write to /usr/local, and a docs build +# should not need root. Override the version with PANDOC_VERSION. +# +# Prints nothing but diagnostics on stdout, so callers can capture the install +# location from `pandoc_bin_dir` below if they need to extend PATH themselves. # # Usage: ensure_pandoc.sh @@ -20,6 +25,17 @@ set -euo pipefail # shellcheck source=utils.sh source "$(dirname -- "${BASH_SOURCE[0]}")/utils.sh" +# Repo-local install prefix. Resolved from this script's location rather than the +# caller's working directory, so it lands in the same place wherever it runs from. +_repo_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" +pandoc_prefix="${PANDOC_PREFIX:-$_repo_root/.local}" +pandoc_bin_dir="$pandoc_prefix/bin" + +# An install from a previous run is on disk but not necessarily on PATH. +if [[ -x "$pandoc_bin_dir/pandoc" ]]; then + export PATH="$pandoc_bin_dir:$PATH" +fi + if command -v pandoc &>/dev/null; then echo "pandoc already installed: $(pandoc --version | head -n1)" exit 0 @@ -81,14 +97,28 @@ if ! echo "${expected_sha256} ${tarball}" | (cd "$tmpdir" && sha256sum --check) exit 1 fi -echo "Extracting to /usr/local..." -if ! tar xz --strip-components=1 -C /usr/local -f "${tmpdir}/${tarball}"; then +echo "Extracting to ${pandoc_prefix}..." +mkdir -p "$pandoc_prefix" +# `--no-same-owner --no-same-permissions` because the upstream tarball records +# root-owned entries with fixed modes; replaying those as an unprivileged user is +# what made extraction into a system prefix fail. +# +# The whole archive is extracted, not just bin/. Selecting a subset would need a +# glob, and the two tars disagree about those: GNU tar ignores patterns unless +# given --wildcards, while BSD tar rejects that flag outright. Extracting +# everything sidesteps the incompatibility, and the extra man pages are harmless +# in a prefix we own. +if ! tar xz --strip-components=1 --no-same-owner --no-same-permissions \ + -C "$pandoc_prefix" -f "${tmpdir}/${tarball}"; then echo "Error: failed to extract ${tarball}." >&2 exit 1 fi +export PATH="$pandoc_bin_dir:$PATH" + if ! command -v pandoc &>/dev/null; then echo "Error: pandoc installation failed. Please install pandoc manually." >&2 exit 1 fi echo "pandoc installed successfully: $(pandoc --version | head -n1)" +echo "Installed to ${pandoc_bin_dir} — add it to PATH to use pandoc directly." diff --git a/tests/devtools/test_assemble_versioned_site.py b/tests/devtools/test_assemble_versioned_site.py new file mode 100644 index 0000000..83ee1d0 --- /dev/null +++ b/tests/devtools/test_assemble_versioned_site.py @@ -0,0 +1,197 @@ +# Copyright 2026 Apple Inc. +# +# Use of this source code is governed by a BSD-3-Clause license that can +# be found in the LICENSE file or at https://opensource.org/licenses/BSD-3-Clause + +"""Tests for docs/scripts/assemble_versioned_site.py. + +The site assembler decides what the published documentation site contains, and +``actions/deploy-pages`` replaces the whole site with whatever it produces. A bug +here does not fail loudly — it takes previously published release docs offline. So +these tests focus on the properties that protect against that: version ordering +(which a lexicographic sort gets wrong), which directory names count as a doc set, +and the refusal to publish a tree that lost a version. +""" + +from __future__ import annotations + +import json +import shutil +from pathlib import Path + +import pytest +from _test_helpers import load_script + +_REPO_ROOT = Path(__file__).resolve().parents[2] +_SCRIPT = _REPO_ROOT / "docs" / "scripts" / "assemble_versioned_site.py" +_URL_PREFIX = "/coreai-optimization" + +asm = load_script(_SCRIPT) + + +@pytest.fixture +def site(tmp_path: Path) -> Path: + """A site tree holding several published versions plus a site-shell directory.""" + root = tmp_path / "site" + for name in ("main", "v0.2.1", "v0.9.0", "v0.10.0", ".git"): + (root / name).mkdir(parents=True) + return root + + +@pytest.fixture +def html(tmp_path: Path) -> Path: + """A minimal Sphinx HTML output directory.""" + out = tmp_path / "html" + out.mkdir() + (out / "index.html").write_text("docs") + return out + + +def run_plan(site: Path, version: str) -> int: + """Invoke the ``plan`` subcommand the way the workflow does.""" + argv = ["--site", str(site), "--version", version, "--url-prefix", _URL_PREFIX] + return asm.main([*argv, "plan"]) + + +def run_assemble(site: Path, version: str, html: Path) -> int: + """Invoke the ``assemble`` subcommand the way the workflow does.""" + argv = ["--site", str(site), "--version", version, "--url-prefix", _URL_PREFIX] + return asm.main([*argv, "assemble", "--html", str(html)]) + + +def test_discover_versions_orders_main_first_then_by_number(site: Path) -> None: + """`main` leads, then releases descend numerically — not lexicographically.""" + # v0.10.0 must outrank v0.9.0; sorted() on the strings would invert them. + assert asm.discover_versions(site) == ["main", "v0.10.0", "v0.9.0", "v0.2.1"] + + +def test_discover_versions_ignores_non_version_directories(site: Path) -> None: + """Site-shell directories must not be mistaken for published doc sets.""" + assert ".git" not in asm.discover_versions(site) + + +def test_discover_versions_accepts_a_four_segment_version(site: Path, html: Path) -> None: + """A version carrying the RELEASE.md extension segment is a real doc set. + + A downstream repo can add a 4th number via COREAI_OPT_VERSION_EXTENSION, so + `v0.2.1.1` must appear in the dropdown rather than being silently skipped. + """ + (site / "v0.2.1.1").mkdir() + + assert asm.discover_versions(site) == ["main", "v0.10.0", "v0.9.0", "v0.2.1.1", "v0.2.1"] + + assert run_assemble(site, "main", html) == 0 + manifest = json.loads((site / "versions.json").read_text()) + assert ["v0.2.1.1", "/coreai-optimization/v0.2.1.1/"] in manifest + + +def test_build_manifest_uses_root_absolute_hrefs() -> None: + """Hrefs must be absolute, or the dropdown breaks on nested pages.""" + manifest = asm.build_manifest(["main", "v0.3.0"], _URL_PREFIX) + assert manifest == [ + ["main (development)", "/coreai-optimization/main/"], + ["v0.3.0", "/coreai-optimization/v0.3.0/"], + ] + + +def test_build_manifest_without_url_prefix() -> None: + """A site served from the domain root still yields absolute hrefs.""" + assert asm.build_manifest(["main"], "") == [["main (development)", "/main/"]] + + +def test_version_label_marks_main_as_development() -> None: + """conf.py imports this, so the dropdown button matches its own menu entry.""" + assert asm.version_label("main") == "main (development)" + assert asm.version_label("v0.3.0") == "v0.3.0" + + +def test_assemble_publishes_only_the_named_version(site: Path, html: Path) -> None: + """Publishing one version must leave every other version untouched.""" + (site / "v0.2.1" / "index.html").write_text("old release") + + assert run_assemble(site, "v0.3.0", html) == 0 + + assert (site / "v0.3.0" / "index.html").read_text() == "docs" + # The frozen release is byte-for-byte as it was. + assert (site / "v0.2.1" / "index.html").read_text() == "old release" + + +def test_assemble_writes_the_site_shell(site: Path, html: Path) -> None: + """The root redirect, .nojekyll and manifest are all refreshed.""" + assert run_assemble(site, "main", html) == 0 + + assert (site / ".nojekyll").exists() + assert "url=main/" in (site / "index.html").read_text() + manifest = json.loads((site / "versions.json").read_text()) + assert manifest[0] == ["main (development)", "/coreai-optimization/main/"] + assert ["v0.2.1", "/coreai-optimization/v0.2.1/"] in manifest + + +def test_assemble_replaces_rather_than_merges(site: Path, html: Path) -> None: + """A page removed since the last build must not survive in the new one.""" + (site / "main" / "removed.html").write_text("stale") + + assert run_assemble(site, "main", html) == 0 + + assert not (site / "main" / "removed.html").exists() + + +def test_assemble_rejects_an_empty_build(site: Path, tmp_path: Path) -> None: + """A build that produced no index.html must not replace a live doc set.""" + empty = tmp_path / "empty" + empty.mkdir() + + assert run_assemble(site, "main", empty) == 1 + + +def test_assemble_fails_closed_when_a_published_version_is_missing(site: Path, html: Path) -> None: + """A tree missing a version its own manifest lists aborts the publish. + + This is the real hazard — a truncated or wrong-ref gh-pages checkout. Because + deploy-pages replaces the whole site, publishing such a tree would take the + missing release offline even though the branch still holds it. + """ + # A manifest from a previous publish that shipped a version now absent on disk. + assert run_assemble(site, "main", html) == 0 + shutil.rmtree(site / "v0.2.1") + + assert run_assemble(site, "main", html) == 1 + + +def test_assemble_fails_closed_on_an_unrecognized_directory(site: Path, html: Path) -> None: + """A directory that can't appear in the dropdown must not be silently ignored. + + Pages would still serve it, but no reader could navigate to it — so treat it as + a mistake to fix rather than skipping past it. + """ + (site / "latest").mkdir() + + assert run_assemble(site, "main", html) == 1 + + +def test_plan_includes_the_version_being_published( + site: Path, capsys: pytest.CaptureFixture[str] +) -> None: + """A brand-new version appears in the dropdown of its own first build.""" + assert run_plan(site, "v0.3.0") == 0 + + manifest = json.loads(capsys.readouterr().out) + assert ["v0.3.0", "/coreai-optimization/v0.3.0/"] in manifest + + +@pytest.mark.parametrize("bad", ["1.2.3", "latest", "v1.2.3rc1", "main-dev", "v1", "v1.2."]) +def test_rejects_version_names_that_are_not_doc_sets(site: Path, bad: str) -> None: + """Only `main` and a `v` + dotted-numbers tag may name a directory on the site.""" + with pytest.raises(SystemExit) as excinfo: + run_plan(site, bad) + assert excinfo.value.code != 0 + + +@pytest.mark.parametrize("good", ["main", "v1.2", "v1.2.3", "v1.2.3.4"]) +def test_accepts_version_names_with_any_segment_count(site: Path, good: str) -> None: + """Two or more numeric segments are all valid doc-set names. + + The release scheme is three segments today and four when a downstream repo + applies COREAI_OPT_VERSION_EXTENSION, so the count is not fixed. + """ + assert run_plan(site, good) == 0