Move the documentation from the wiki to a docs site

.copier-answers.yaml

@@ -1,14 +1,14 @@
 # Changes here will be overwritten by Copier
 _commit: 162463c
 _src_path: https://github.com/python-project-templates/base.git
-add_docs: false
+add_docs: true
 add_extension: cpp
 add_pyodide: false
 add_vcpkg: false
-add_wiki: true
+add_wiki: false
 email: dev@dau.dev
 github: dau-dev
-project_description: PCIe Driver for dau
+project_description: PCIe driver for dau projects
 project_name: dau-driver
 python_version_primary: '3.12'
 team: the dau-driver authors

.github/workflows/docs.yaml

@@ -0,0 +1,61 @@
+name: Publish Docs
+
+on:
+  workflow_run:
+    workflows: ["Build Status"]
+    branches: [main]
+    types: [completed]
+  workflow_dispatch:
+
+permissions:
+  actions: read
+  contents: write
+
+jobs:
+  docs:
+    runs-on: ubuntu-24.04
+    if: github.event.workflow_run.conclusion == 'success' || github.event_name == 'workflow_dispatch'
+    steps:
+      - name: Checkout
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1  # v7.0.1
+        with:
+          persist-credentials: false
+
+      - name: Setup Python
+        uses: actions-ext/python/setup@c8228d827a81b9f8c808825328af361c45461ba3
+        with:
+          version: "3.12"
+
+      - name: Download distributions
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c  # v8
+        with:
+          pattern: dist-ubuntu-24.04-3.12
+          merge-multiple: true
+          path: dist
+          run-id: ${{ github.event.workflow_run.id }}
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          repository: ${{ github.repository }}
+        if: github.event_name == 'workflow_run'
+
+      - name: Install from wheel
+        run: |
+          uv pip install dist/*.whl
+          uv pip install dist/*.whl --target . --no-deps
+          uv pip install "yardang[themes]"
+        if: github.event_name == 'workflow_run'
+
+      - name: Install from source (manual trigger)
+        run: |
+          uv pip install .[develop]
+          uv pip install "yardang[themes]"
+        if: github.event_name == 'workflow_dispatch'
+
+      - name: Build documentation
+        run: yardang build
+
+      - name: Publish documentation
+        uses: peaceiris/actions-gh-pages@84c30a85c19949d7eee79c4ff27748b70285e453  # v4.1.0
+        with:
+          publish_branch: gh-pages
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          publish_dir: docs/html

Makefile

@@ -30,8 +30,8 @@ lint-cpp:  ## run cpp linter
 	clang-format --dry-run -Werror -i -style=file `find ./cpp -name "*.*pp"`
 
 lint-docs:  ## lint docs with mdformat and codespell
-	python -m mdformat --check README.md $(wildcard docs/src) docs/wiki/
-	python -m codespell_lib README.md $(wildcard docs/src) docs/wiki/
+	python -m mdformat --check README.md $(wildcard docs/src) 
+	python -m codespell_lib README.md $(wildcard docs/src) 
 
 lint: lint-cpp lint-py  ## run project linters
 
@@ -47,8 +47,8 @@ fix-cpp:  ## fix cpp formatting
 	clang-format -i -style=file `find ./cpp -name "*.*pp"`
 
 fix-docs:  ## autoformat docs with mdformat and codespell
-	python -m mdformat README.md $(wildcard docs/src) docs/wiki/
-	python -m codespell_lib --write README.md $(wildcard docs/src) docs/wiki/
+	python -m mdformat README.md $(wildcard docs/src) 
+	python -m codespell_lib --write README.md $(wildcard docs/src) 
 
 fix: fix-cpp fix-py fix-docs  ## run project autoformatters
 

README.md

@@ -5,13 +5,14 @@ Host I/O for DAU devices: discovery, registers, DMA, and job execution over XDMA
 [![Build Status](https://github.com/dau-dev/dau-driver/actions/workflows/build.yaml/badge.svg?branch=main&event=push)](https://github.com/dau-dev/dau-driver/actions/workflows/build.yaml)
 [![codecov](https://codecov.io/gh/dau-dev/dau-driver/branch/main/graph/badge.svg)](https://codecov.io/gh/dau-dev/dau-driver)
 [![License](https://img.shields.io/github/license/dau-dev/dau-driver)](https://github.com/dau-dev/dau-driver)
+[![PyPI](https://img.shields.io/pypi/v/dau-driver.svg)](https://pypi.python.org/pypi/dau-driver)
 
 `dau-driver` is the host side of a DAU device. It discovers DAU-compatible XDMA
 endpoints, reads the canonical register block, decodes the device's capability
 advertisement, moves data over H2C/C2H DMA, allocates device storage, and runs
-the register-contract jobs the shells expose. All device semantics — register
-offsets, stream framing, capability words — come from `dau-core`; this package
-owns transport and execution only.
+the register-contract jobs the shells expose. All device semantics (register
+offsets, stream framing, capability words, record layouts) come from `dau-core`;
+this package owns transport and execution only.
 
 - **Linux** hosts use the Xilinx XDMA character devices under `/dev`.
 - **macOS** hosts use a DriverKit-based XDMA system extension through IOKit
@@ -29,7 +30,7 @@ device.check_bar_health()                          # all-ones reads fail here, n
 print(device.name, device.backend)                 # xdma0 linux-xdma
 print(device.capabilities.dma)                     # alignment / max-transfer constraints
 print(device.read_register_block_snapshot())       # the whole identity block
-print(device.read_status())                        # link, MIG, operator ready bits
+print(device.read_status())                        # the status word and last error
 ```
 
 The Python execution paths (discovery, registers, transfers, storage, the scan,
@@ -39,45 +40,49 @@ extension and the native extension are tested on hardware.
 
 ## Documentation
 
-The documentation lives in [the wiki](https://github.com/dau-dev/dau-driver/wiki),
-authored under [`docs/wiki`](docs/wiki) and published from there. It is in four
-parts, each answering a different need.
-
-**[Tutorial](https://github.com/dau-dev/dau-driver/wiki/First-Device)** — start here if dau-driver is new to
-you. Two guided lessons drive a device end to end against the fake transport, no
-board required:
-[Talk to a device without any hardware](https://github.com/dau-dev/dau-driver/wiki/First-Device) and
-[Run a scan over many batches](https://github.com/dau-dev/dau-driver/wiki/First-Scan-Run).
-
-**[How-to guides](https://github.com/dau-dev/dau-driver/wiki/Find-a-Device)** — directions for the jobs you
-will actually have. How to
-[find and check a device](https://github.com/dau-dev/dau-driver/wiki/Find-a-Device),
-[move data to and from it](https://github.com/dau-dev/dau-driver/wiki/Move-Data),
-[run scan jobs](https://github.com/dau-dev/dau-driver/wiki/Run-Scan-Jobs) across batches and cards,
-[allocate device storage](https://github.com/dau-dev/dau-driver/wiki/Allocate-Device-Storage), and
-[fence a PCIe operation with the deadman](https://github.com/dau-dev/dau-driver/wiki/Arm-the-Deadman)
-before touching the bus.
-
-**[Reference](https://github.com/dau-dev/dau-driver/wiki/Device-and-Discovery)** — the technical
-description, laid out to mirror the code: the
-[device and discovery](https://github.com/dau-dev/dau-driver/wiki/Device-and-Discovery) surface including
-transfers, storage and errors; the
-[execution contracts](https://github.com/dau-dev/dau-driver/wiki/Execution-Contracts) for stream, scan,
-pipeline and fleet jobs; and the
-[stream codec](https://github.com/dau-dev/dau-driver/wiki/Stream-Codec).
-
-**[Explanation](https://github.com/dau-dev/dau-driver/wiki/Transport-Boundary)** — the reasoning behind
-the design. Why [the driver owns almost
-nothing](https://github.com/dau-dev/dau-driver/wiki/Transport-Boundary) and where the `dau-core`
-boundary sits, why [staging overlaps compute and when the runner
-refuses](https://github.com/dau-dev/dau-driver/wiki/Overlapped-Staging), and why storage is handed out as
-[generation-checked handles rather than
-pointers](https://github.com/dau-dev/dau-driver/wiki/Storage-Handles).
-
-The [examples/](examples/) directory holds end-to-end host flows — record-batch
-round trip, stream aggregation, the OHLCV pipeline job — that run against real
+The full documentation is at
+[dau-dev.github.io/dau-driver](https://dau-dev.github.io/dau-driver/).
+
+- **Tutorial**: [Talk to a device without any hardware](docs/src/tutorial/first-device.md) ·
+  [Run a scan over many batches](docs/src/tutorial/first-scan-run.md). Both drive a device end
+  to end against the fake transport, no board required.
+- **How-to guides**: [Find and check a device](docs/src/how-to/find-a-device.md) ·
+  [Move data to and from it](docs/src/how-to/move-data.md) ·
+  [Run scan jobs](docs/src/how-to/run-scan-jobs.md) across batches and cards ·
+  [Allocate device storage](docs/src/how-to/allocate-device-storage.md) ·
+  [Fence a PCIe operation with the deadman](docs/src/how-to/arm-the-deadman.md) ·
+  [Develop and contribute](docs/src/how-to/develop.md).
+- **Reference**: [Device and discovery](docs/src/reference/device-and-discovery.md), including
+  transfers, storage and errors · [Execution contracts](docs/src/reference/execution-contracts.md)
+  for stream, scan, pipeline and fleet jobs · [Stream codec](docs/src/reference/stream-codec.md).
+- **Explanation**: [Why the driver owns almost nothing](docs/src/explanation/transport-boundary.md)
+  and where the `dau-core` boundary sits ·
+  [Why staging overlaps compute, and when the runner stops](docs/src/explanation/overlapped-staging.md) ·
+  [Why storage is handed out as generation-checked handles](docs/src/explanation/storage-handles.md).
+
+The [examples](https://github.com/dau-dev/dau-driver/tree/main/examples) directory holds end-to-end host flows (record-batch
+round trip, stream aggregation, the OHLCV pipeline job) that run against real
 hardware.
 
+## Installation
+
+Requires Python 3.12 or newer.
+
+```bash
+pip install dau-driver
+```
+
+Wheels are published for CPython 3.12 and 3.13 on Linux (x86_64 and aarch64)
+and macOS (arm64). Elsewhere pip builds the small native extension from source,
+which needs a C++ toolchain.
+
+On a host with a board, load the Linux XDMA module from `kernel/linux-xdma/` or
+the macOS DriverKit extension from `kernel/macos-xdma/`; each has its own build
+and load instructions. Without one, the package still imports and runs against
+its fake transport. Whoever can open the character devices (Linux) or the user
+client (macOS) can read and write the card's memory and registers, so the
+operating system's device permissions are the access control.
+
 ## License
 
 Apache-2.0, except `kernel/linux-xdma/`, which is Xilinx's XDMA reference

dau_driver/scan_execution.py

@@ -30,9 +30,10 @@
 
 @dataclass(frozen=True)
 class ScanBatch:
-    """One staged scan batch: the packed row payload (``row_count *
-    row_bytes`` bytes; the runner pads the DMA to the 64-byte grid while
-    INPUT_LENGTH stays exact) and its row count."""
+    """One staged scan batch: the packed row payload and its row count.
+
+    The payload is ``row_count * row_bytes`` bytes; the runner pads the DMA
+    to the 64-byte grid while ``INPUT_LENGTH`` stays exact."""
 
     payload: bytes
     row_count: int

dau_driver/tests/test_docs.py

@@ -0,0 +1,63 @@
+"""Every python block in the tutorials runs as written, in order.
+
+The tutorials are the pages that promise to work without a board, against the
+fake transport, so they are the ones run here; the how-to and reference pages
+show fragments for a real device. A page is one session: its blocks share a
+namespace, the way a reader typing them into one prompt would, and it runs in
+a scratch directory. Where a page shows what a block prints (an untagged or
+``text`` fence right after it, with at most a lead-in line such as
+"**Output:**" between), the output must match. A block marked with an
+``<!-- excerpt -->`` comment is skipped.
+"""
+
+from __future__ import annotations
+
+import contextlib
+import io
+import re
+from pathlib import Path
+
+import pytest
+
+DOCS = Path(__file__).resolve().parents[2] / "docs" / "src"
+PAGES = sorted((DOCS / "tutorial").glob("*.md"))
+FENCE = re.compile(r"(<!-- excerpt -->\s*)?```(\w*)\n(.*?)```", re.DOTALL)
+LEAD_IN = re.compile(r"\s*(?:[^\n]*:[^\n]*\s*)?")
+
+
+def _runs(page: Path) -> list[tuple[str, str | None]]:
+    """The page's runnable python blocks, each with the output shown after it."""
+    text = page.read_text(encoding="utf-8")
+    fences = list(FENCE.finditer(text))
+    runs = []
+    for index, match in enumerate(fences):
+        if match.group(2) != "python" or match.group(1):
+            continue
+        following = fences[index + 1] if index + 1 < len(fences) else None
+        shown = None
+        if (
+            following is not None
+            and following.group(2) in ("", "text")
+            and not following.group(1)
+            and LEAD_IN.fullmatch(text[match.end() : following.start()])
+        ):
+            shown = following.group(3)
+        runs.append((match.group(3), shown))
+    return runs
+
+
+def test_the_tutorials_are_found_and_check_output() -> None:
+    assert len(PAGES) == 2, PAGES
+    assert all(any(shown is not None for _body, shown in _runs(page)) for page in PAGES)
+
+
+@pytest.mark.parametrize("page", PAGES, ids=[f"{page.parent.name}/{page.stem}" for page in PAGES])
+def test_the_page_runs_as_written(page: Path, tmp_path, monkeypatch) -> None:
+    monkeypatch.chdir(tmp_path)
+    namespace: dict = {}
+    for number, (body, shown) in enumerate(_runs(page), start=1):
+        out = io.StringIO()
+        with contextlib.redirect_stdout(out):
+            exec(compile(body, f"{page.name} block {number}", "exec"), namespace)  # noqa: S102 (the docs are ours)
+        if shown is not None:
+            assert out.getvalue().strip() == shown.strip(), f"{page.name} block {number} printed something other than the page shows"