From 064b708bbc15e5bd11a6c377b1018462ce658b9c Mon Sep 17 00:00:00 2001 From: Tim Paine <3105306+timkpaine@users.noreply.github.com> Date: Fri, 23 Oct 2026 15:46:12 -0400 Subject: [PATCH 1/2] Move the documentation from the wiki to a docs site Signed-off-by: Tim Paine <3105306+timkpaine@users.noreply.github.com> --- .copier-answers.yaml | 4 +- .github/workflows/docs.yaml | 61 +++++++++++ .github/workflows/wiki.yaml | 34 ------ Makefile | 8 +- README.md | 87 ++++++++------- dau_driver/scan_execution.py | 7 +- dau_driver/tests/test_docs.py | 63 +++++++++++ .../explanation/overlapped-staging.md} | 71 ++++++------ .../explanation/storage-handles.md} | 76 +++++++------ .../explanation/transport-boundary.md} | 87 +++++++-------- .../how-to/allocate-device-storage.md} | 30 ++--- .../how-to/arm-the-deadman.md} | 49 ++++----- docs/src/how-to/develop.md | 87 +++++++++++++++ .../how-to/find-a-device.md} | 49 ++++----- .../Move-Data.md => src/how-to/move-data.md} | 32 +++--- .../how-to/run-scan-jobs.md} | 40 +++---- .../reference/device-and-discovery.md} | 79 +++++++------- .../reference/execution-contracts.md} | 23 ++-- .../reference/stream-codec.md} | 17 +-- .../tutorial/first-device.md} | 40 ++++--- .../tutorial/first-scan-run.md} | 40 +++---- docs/wiki/Installation.md | 34 ------ docs/wiki/_Footer.md | 1 - docs/wiki/_Sidebar.md | 41 ------- docs/wiki/contribute/Build-from-Source.md | 103 ------------------ docs/wiki/contribute/Contribute.md | 15 --- .../contribute/Local-Development-Setup.md | 55 ---------- pyproject.toml | 21 ++++ 28 files changed, 599 insertions(+), 655 deletions(-) create mode 100644 .github/workflows/docs.yaml delete mode 100644 .github/workflows/wiki.yaml create mode 100644 dau_driver/tests/test_docs.py rename docs/{wiki/explanation/Overlapped-Staging.md => src/explanation/overlapped-staging.md} (70%) rename docs/{wiki/explanation/Storage-Handles.md => src/explanation/storage-handles.md} (56%) rename docs/{wiki/explanation/Transport-Boundary.md => src/explanation/transport-boundary.md} (53%) rename docs/{wiki/how-to/Allocate-Device-Storage.md => src/how-to/allocate-device-storage.md} (78%) rename docs/{wiki/how-to/Arm-the-Deadman.md => src/how-to/arm-the-deadman.md} (59%) create mode 100644 docs/src/how-to/develop.md rename docs/{wiki/how-to/Find-a-Device.md => src/how-to/find-a-device.md} (79%) rename docs/{wiki/how-to/Move-Data.md => src/how-to/move-data.md} (85%) rename docs/{wiki/how-to/Run-Scan-Jobs.md => src/how-to/run-scan-jobs.md} (87%) rename docs/{wiki/reference/Device-and-Discovery.md => src/reference/device-and-discovery.md} (91%) rename docs/{wiki/reference/Execution-Contracts.md => src/reference/execution-contracts.md} (95%) rename docs/{wiki/reference/Stream-Codec.md => src/reference/stream-codec.md} (88%) rename docs/{wiki/tutorial/First-Device.md => src/tutorial/first-device.md} (87%) rename docs/{wiki/tutorial/First-Scan-Run.md => src/tutorial/first-scan-run.md} (84%) delete mode 100644 docs/wiki/Installation.md delete mode 100644 docs/wiki/_Footer.md delete mode 100644 docs/wiki/_Sidebar.md delete mode 100644 docs/wiki/contribute/Build-from-Source.md delete mode 100644 docs/wiki/contribute/Contribute.md delete mode 100644 docs/wiki/contribute/Local-Development-Setup.md diff --git a/.copier-answers.yaml b/.copier-answers.yaml index 8a78bb9..a01bcfb 100644 --- a/.copier-answers.yaml +++ b/.copier-answers.yaml @@ -1,11 +1,11 @@ # Changes here will be overwritten by Copier _commit: 162463c _src_path: https://github.com/python-project-templates/base.git -add_docs: false +add_docs: true add_extension: cpp add_pyodide: false add_vcpkg: false -add_wiki: true +add_wiki: false email: dev@dau.dev github: dau-dev project_description: PCIe Driver for dau diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml new file mode 100644 index 0000000..69e5208 --- /dev/null +++ b/.github/workflows/docs.yaml @@ -0,0 +1,61 @@ +name: Publish Docs + +on: + workflow_run: + workflows: ["Build Status"] + branches: [main] + types: [completed] + workflow_dispatch: + +permissions: + actions: read + contents: write + +jobs: + docs: + runs-on: ubuntu-24.04 + if: github.event.workflow_run.conclusion == 'success' || github.event_name == 'workflow_dispatch' + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Setup Python + uses: actions-ext/python/setup@c8228d827a81b9f8c808825328af361c45461ba3 + with: + version: "3.12" + + - name: Download distributions + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + pattern: dist-ubuntu-24.04-3.12 + merge-multiple: true + path: dist + run-id: ${{ github.event.workflow_run.id }} + github-token: ${{ secrets.GITHUB_TOKEN }} + repository: ${{ github.repository }} + if: github.event_name == 'workflow_run' + + - name: Install from wheel + run: | + uv pip install dist/*.whl + uv pip install dist/*.whl --target . --no-deps + uv pip install "yardang[themes]" + if: github.event_name == 'workflow_run' + + - name: Install from source (manual trigger) + run: | + uv pip install .[develop] + uv pip install "yardang[themes]" + if: github.event_name == 'workflow_dispatch' + + - name: Build documentation + run: yardang build + + - name: Publish documentation + uses: peaceiris/actions-gh-pages@84c30a85c19949d7eee79c4ff27748b70285e453 # v4.1.0 + with: + publish_branch: gh-pages + github_token: ${{ secrets.GITHUB_TOKEN }} + publish_dir: docs/html diff --git a/.github/workflows/wiki.yaml b/.github/workflows/wiki.yaml deleted file mode 100644 index a263c0d..0000000 --- a/.github/workflows/wiki.yaml +++ /dev/null @@ -1,34 +0,0 @@ -name: Publish Wiki - -on: - push: - branches: - - main - paths: - - "docs/**" - - "README.md" - workflow_dispatch: - -concurrency: - group: docs - cancel-in-progress: true - -permissions: - contents: write - -jobs: - deploy: - runs-on: ubuntu-24.04 - steps: - - name: Checkout - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - - name: Prepare wiki - run: cp README.md docs/wiki/Home.md - - - name: Publish wiki - uses: Andrew-Chen-Wang/github-wiki-action@1bbb4280446f9630e8e21a18012cbacf3b0f992e # v5.0.6 - with: - path: docs/wiki diff --git a/Makefile b/Makefile index 0ac9fb2..64f93aa 100644 --- a/Makefile +++ b/Makefile @@ -30,8 +30,8 @@ lint-cpp: ## run cpp linter clang-format --dry-run -Werror -i -style=file `find ./cpp -name "*.*pp"` lint-docs: ## lint docs with mdformat and codespell - python -m mdformat --check README.md $(wildcard docs/src) docs/wiki/ - python -m codespell_lib README.md $(wildcard docs/src) docs/wiki/ + python -m mdformat --check README.md $(wildcard docs/src) + python -m codespell_lib README.md $(wildcard docs/src) lint: lint-cpp lint-py ## run project linters @@ -47,8 +47,8 @@ fix-cpp: ## fix cpp formatting clang-format -i -style=file `find ./cpp -name "*.*pp"` fix-docs: ## autoformat docs with mdformat and codespell - python -m mdformat README.md $(wildcard docs/src) docs/wiki/ - python -m codespell_lib --write README.md $(wildcard docs/src) docs/wiki/ + python -m mdformat README.md $(wildcard docs/src) + python -m codespell_lib --write README.md $(wildcard docs/src) fix: fix-cpp fix-py fix-docs ## run project autoformatters diff --git a/README.md b/README.md index f2a466b..0cea8bc 100644 --- a/README.md +++ b/README.md @@ -5,13 +5,14 @@ Host I/O for DAU devices: discovery, registers, DMA, and job execution over XDMA [![Build Status](https://github.com/dau-dev/dau-driver/actions/workflows/build.yaml/badge.svg?branch=main&event=push)](https://github.com/dau-dev/dau-driver/actions/workflows/build.yaml) [![codecov](https://codecov.io/gh/dau-dev/dau-driver/branch/main/graph/badge.svg)](https://codecov.io/gh/dau-dev/dau-driver) [![License](https://img.shields.io/github/license/dau-dev/dau-driver)](https://github.com/dau-dev/dau-driver) +[![PyPI](https://img.shields.io/pypi/v/dau-driver.svg)](https://pypi.python.org/pypi/dau-driver) `dau-driver` is the host side of a DAU device. It discovers DAU-compatible XDMA endpoints, reads the canonical register block, decodes the device's capability advertisement, moves data over H2C/C2H DMA, allocates device storage, and runs -the register-contract jobs the shells expose. All device semantics — register -offsets, stream framing, capability words — come from `dau-core`; this package -owns transport and execution only. +the register-contract jobs the shells expose. All device semantics (register +offsets, stream framing, capability words, record layouts) come from `dau-core`; +this package owns transport and execution only. - **Linux** hosts use the Xilinx XDMA character devices under `/dev`. - **macOS** hosts use a DriverKit-based XDMA system extension through IOKit @@ -29,7 +30,7 @@ device.check_bar_health() # all-ones reads fail here, n print(device.name, device.backend) # xdma0 linux-xdma print(device.capabilities.dma) # alignment / max-transfer constraints print(device.read_register_block_snapshot()) # the whole identity block -print(device.read_status()) # link, MIG, operator ready bits +print(device.read_status()) # the status word and last error ``` The Python execution paths (discovery, registers, transfers, storage, the scan, @@ -39,45 +40,49 @@ extension and the native extension are tested on hardware. ## Documentation -The documentation lives in [the wiki](https://github.com/dau-dev/dau-driver/wiki), -authored under [`docs/wiki`](docs/wiki) and published from there. It is in four -parts, each answering a different need. - -**[Tutorial](https://github.com/dau-dev/dau-driver/wiki/First-Device)** — start here if dau-driver is new to -you. Two guided lessons drive a device end to end against the fake transport, no -board required: -[Talk to a device without any hardware](https://github.com/dau-dev/dau-driver/wiki/First-Device) and -[Run a scan over many batches](https://github.com/dau-dev/dau-driver/wiki/First-Scan-Run). - -**[How-to guides](https://github.com/dau-dev/dau-driver/wiki/Find-a-Device)** — directions for the jobs you -will actually have. How to -[find and check a device](https://github.com/dau-dev/dau-driver/wiki/Find-a-Device), -[move data to and from it](https://github.com/dau-dev/dau-driver/wiki/Move-Data), -[run scan jobs](https://github.com/dau-dev/dau-driver/wiki/Run-Scan-Jobs) across batches and cards, -[allocate device storage](https://github.com/dau-dev/dau-driver/wiki/Allocate-Device-Storage), and -[fence a PCIe operation with the deadman](https://github.com/dau-dev/dau-driver/wiki/Arm-the-Deadman) -before touching the bus. - -**[Reference](https://github.com/dau-dev/dau-driver/wiki/Device-and-Discovery)** — the technical -description, laid out to mirror the code: the -[device and discovery](https://github.com/dau-dev/dau-driver/wiki/Device-and-Discovery) surface including -transfers, storage and errors; the -[execution contracts](https://github.com/dau-dev/dau-driver/wiki/Execution-Contracts) for stream, scan, -pipeline and fleet jobs; and the -[stream codec](https://github.com/dau-dev/dau-driver/wiki/Stream-Codec). - -**[Explanation](https://github.com/dau-dev/dau-driver/wiki/Transport-Boundary)** — the reasoning behind -the design. Why [the driver owns almost -nothing](https://github.com/dau-dev/dau-driver/wiki/Transport-Boundary) and where the `dau-core` -boundary sits, why [staging overlaps compute and when the runner -refuses](https://github.com/dau-dev/dau-driver/wiki/Overlapped-Staging), and why storage is handed out as -[generation-checked handles rather than -pointers](https://github.com/dau-dev/dau-driver/wiki/Storage-Handles). - -The [examples/](examples/) directory holds end-to-end host flows — record-batch -round trip, stream aggregation, the OHLCV pipeline job — that run against real +The full documentation is at +[dau-dev.github.io/dau-driver](https://dau-dev.github.io/dau-driver/). + +- **Tutorial**: [Talk to a device without any hardware](docs/src/tutorial/first-device.md) · + [Run a scan over many batches](docs/src/tutorial/first-scan-run.md). Both drive a device end + to end against the fake transport, no board required. +- **How-to guides**: [Find and check a device](docs/src/how-to/find-a-device.md) · + [Move data to and from it](docs/src/how-to/move-data.md) · + [Run scan jobs](docs/src/how-to/run-scan-jobs.md) across batches and cards · + [Allocate device storage](docs/src/how-to/allocate-device-storage.md) · + [Fence a PCIe operation with the deadman](docs/src/how-to/arm-the-deadman.md) · + [Develop and contribute](docs/src/how-to/develop.md). +- **Reference**: [Device and discovery](docs/src/reference/device-and-discovery.md), including + transfers, storage and errors · [Execution contracts](docs/src/reference/execution-contracts.md) + for stream, scan, pipeline and fleet jobs · [Stream codec](docs/src/reference/stream-codec.md). +- **Explanation**: [Why the driver owns almost nothing](docs/src/explanation/transport-boundary.md) + and where the `dau-core` boundary sits · + [Why staging overlaps compute, and when the runner stops](docs/src/explanation/overlapped-staging.md) · + [Why storage is handed out as generation-checked handles](docs/src/explanation/storage-handles.md). + +The [examples](https://github.com/dau-dev/dau-driver/tree/main/examples) directory holds end-to-end host flows (record-batch +round trip, stream aggregation, the OHLCV pipeline job) that run against real hardware. +## Installation + +Requires Python 3.12 or newer. + +```bash +pip install dau-driver +``` + +Wheels are published for CPython 3.12 and 3.13 on Linux (x86_64 and aarch64) +and macOS (arm64). Elsewhere pip builds the small native extension from source, +which needs a C++ toolchain. + +On a host with a board, load the Linux XDMA module from `kernel/linux-xdma/` or +the macOS DriverKit extension from `kernel/macos-xdma/`; each has its own build +and load instructions. Without one, the package still imports and runs against +its fake transport. Whoever can open the character devices (Linux) or the user +client (macOS) can read and write the card's memory and registers, so the +operating system's device permissions are the access control. + ## License Apache-2.0, except `kernel/linux-xdma/`, which is Xilinx's XDMA reference diff --git a/dau_driver/scan_execution.py b/dau_driver/scan_execution.py index 2da14ad..26e083f 100644 --- a/dau_driver/scan_execution.py +++ b/dau_driver/scan_execution.py @@ -30,9 +30,10 @@ @dataclass(frozen=True) class ScanBatch: - """One staged scan batch: the packed row payload (``row_count * - row_bytes`` bytes; the runner pads the DMA to the 64-byte grid while - INPUT_LENGTH stays exact) and its row count.""" + """One staged scan batch: the packed row payload and its row count. + + The payload is ``row_count * row_bytes`` bytes; the runner pads the DMA + to the 64-byte grid while ``INPUT_LENGTH`` stays exact.""" payload: bytes row_count: int diff --git a/dau_driver/tests/test_docs.py b/dau_driver/tests/test_docs.py new file mode 100644 index 0000000..e5612e8 --- /dev/null +++ b/dau_driver/tests/test_docs.py @@ -0,0 +1,63 @@ +"""Every python block in the tutorials runs as written, in order. + +The tutorials are the pages that promise to work without a board, against the +fake transport, so they are the ones run here; the how-to and reference pages +show fragments for a real device. A page is one session: its blocks share a +namespace, the way a reader typing them into one prompt would, and it runs in +a scratch directory. Where a page shows what a block prints (an untagged or +``text`` fence right after it, with at most a lead-in line such as +"**Output:**" between), the output must match. A block marked with an +```` comment is skipped. +""" + +from __future__ import annotations + +import contextlib +import io +import re +from pathlib import Path + +import pytest + +DOCS = Path(__file__).resolve().parents[2] / "docs" / "src" +PAGES = sorted((DOCS / "tutorial").glob("*.md")) +FENCE = re.compile(r"(\s*)?```(\w*)\n(.*?)```", re.DOTALL) +LEAD_IN = re.compile(r"\s*(?:[^\n]*:[^\n]*\s*)?") + + +def _runs(page: Path) -> list[tuple[str, str | None]]: + """The page's runnable python blocks, each with the output shown after it.""" + text = page.read_text(encoding="utf-8") + fences = list(FENCE.finditer(text)) + runs = [] + for index, match in enumerate(fences): + if match.group(2) != "python" or match.group(1): + continue + following = fences[index + 1] if index + 1 < len(fences) else None + shown = None + if ( + following is not None + and following.group(2) in ("", "text") + and not following.group(1) + and LEAD_IN.fullmatch(text[match.end() : following.start()]) + ): + shown = following.group(3) + runs.append((match.group(3), shown)) + return runs + + +def test_the_tutorials_are_found_and_check_output() -> None: + assert len(PAGES) == 2, PAGES + assert all(any(shown is not None for _body, shown in _runs(page)) for page in PAGES) + + +@pytest.mark.parametrize("page", PAGES, ids=[f"{page.parent.name}/{page.stem}" for page in PAGES]) +def test_the_page_runs_as_written(page: Path, tmp_path, monkeypatch) -> None: + monkeypatch.chdir(tmp_path) + namespace: dict = {} + for number, (body, shown) in enumerate(_runs(page), start=1): + out = io.StringIO() + with contextlib.redirect_stdout(out): + exec(compile(body, f"{page.name} block {number}", "exec"), namespace) # noqa: S102 (the docs are ours) + if shown is not None: + assert out.getvalue().strip() == shown.strip(), f"{page.name} block {number} printed something other than the page shows" diff --git a/docs/wiki/explanation/Overlapped-Staging.md b/docs/src/explanation/overlapped-staging.md similarity index 70% rename from docs/wiki/explanation/Overlapped-Staging.md rename to docs/src/explanation/overlapped-staging.md index d0931e6..16a897e 100644 --- a/docs/wiki/explanation/Overlapped-Staging.md +++ b/docs/src/explanation/overlapped-staging.md @@ -1,9 +1,9 @@ -# Overlapped staging: why the runner ping-pongs, and why it refuses +# Overlapped staging: why the runner ping-pongs, and when it stops A scan over a large table is not one job. It is many, and how the host feeds them decides how much of the device's time is spent computing rather than waiting. -This page explains the shape `run_scan_batches` takes and the two refusals built -into it. +This page explains the shape `run_scan_batches` takes and the two cases it +refuses to run. ## The serial version, and what it wastes @@ -14,19 +14,19 @@ idle datapath, then an idle link with a busy datapath. The fix is a ping-pong: two input regions in device memory, alternating. While job *i* computes out of region A, batch *i + 1*'s DMA fills region B. That is all -`input_regions` is — two distinct addresses — and it is why passing the same -address twice raises rather than silently serializing. +`input_regions` is, two distinct addresses, and it is why passing the same +address twice raises rather than serializing without telling you. The overlap extends further than the DMA. `batches` may be a lazy iterable, and the runner pulls the next batch on a worker thread while the current job runs, so -whatever work the iterator does — encoding rows, reading a file — overlaps compute +whatever work the iterator does (encoding rows, reading a file) overlaps compute too. And because output regions are disjoint from input staging, the `collect` callback reading one job's results also overlaps the next batch's DMA. ## The overlap is a claim, so it is tested as one -An overlap that quietly degraded into serial execution would be invisible: the -results would still be correct, only slower. So the test suite pins it with a +An overlap that degraded into serial execution would go unnoticed: the results +would still be correct, only slower. So the test suite pins it with a transport that reports DONE only once the *next* batch's H2C write has landed. A serial stage-after-finish runner deadlocks into the timeout against that fake; a green run is proof the staging overlapped. @@ -35,12 +35,12 @@ This is the general pattern for testing a concurrency property. Asserting on timings is flaky; making the fake refuse to complete unless the property holds is not. -## How much it actually buys +## How much it saves Less than the arithmetic suggests, and the module says so rather than implying otherwise. Host writes and the job's own burst reads share one memory controller -on the reference board, so the win is bounded by the bandwidth headroom left over -— not by the ratio of DMA time to job time. +on the reference board, so the gain is bounded by the bandwidth left over, not by +the ratio of DMA time to job time. That is why the batch count is not a constant in the driver. Splitting a table into more batches hides more of the tail but adds per-job overhead; the crossover @@ -49,18 +49,17 @@ batches you hand it and the decision stays with the caller. ## The `per_key_state` guard -Some scan compositions carry state across the rows of a batch that is keyed — -a grouped aggregation holding a running total per key. Slice such a workload by +Some scan compositions keep state per key across the rows of a batch, such as a +grouped aggregation holding a running total per key. Slice such a workload by row position and a key's rows land in two different jobs, each producing a partial -the host has no way to know it must merge. The results are not obviously wrong; -they are quietly incomplete. +the host has no way to know it must merge. The results look right and are +incomplete. So `per_key_state` is a required keyword, and when it is True the runner checks -that `batches` is the object `key_partitioned_scan_batches` returns. Not a flag -on the batches, not a docstring — the type of the iterable. +that `batches` is the object `key_partitioned_scan_batches` returns. The check +is on the type of the iterable, not on a flag or a docstring. -This is worth dwelling on because it is a design choice with a cost. It makes the -runner aware of one specific batch producer, which is a coupling you would +This choice has a cost. It makes the runner aware of one specific batch producer, which is a coupling you would normally avoid. The alternative is a boolean the caller sets correctly and one day does not, on a failure mode that produces plausible numbers. Coupling one function to one other function is the cheaper mistake. @@ -68,13 +67,13 @@ function to one other function is the cheaper mistake. `key_partitioned_scan_batches` assigns longest-processing-time-first: the largest key group goes into the currently smallest batch. That needs every group's size before it can assign anything, so the first iteration consumes a full grouping -pass — the one place the lazy pipeline is not lazy. Key groups are indivisible, so +pass, the one place the lazy pipeline is not lazy. Key groups are indivisible, so one hot key can still dominate a batch. LPT bounds the imbalance by the largest group; it cannot beat it. -## Two batching contracts, not one +## Two batching contracts -`key_range_scan_batches` looks like the same idea and is not. It bounds the number +`key_range_scan_batches` looks like the same idea but is a different contract. It bounds the number of *distinct keys* in a batch, because the hash-mode grouped tile's table is a fixed-size hardware fact: a batch presenting more keys than it holds closes out `GROUP_CAPACITY_EXCEEDED` and is invalidated. @@ -86,8 +85,8 @@ batch per range. The interesting difference is what each function *does not* require. `key_partitioned_scan_batches` needs records ordered by key, because it groups them with a single pass. `key_range_scan_batches` does not, and preserves arrival -order within each batch — the hash table needs no key ordering, and imposing one -would give back exactly the staging freedom that tile exists to provide. +order within each batch. The hash table needs no key ordering, and imposing one +would give up the staging freedom that tile exists to provide. Both give the same strong property: each key lands in one batch, so merging the per-batch group records is a concatenation rather than an additive merge. The @@ -102,13 +101,13 @@ with each batch pulled exactly once by whichever pipeline demands it. That is wh makes it self-balancing: a faster card asks for more batches, with no pre-partitioning and no throughput model. -It also makes `per_key_state=True` impossible, and the function refuses it rather -than degrading. A key's rows would go to whichever cards happen to be idle, and +It also makes `per_key_state=True` impossible, so the function raises on it +rather than running with split state. A key's rows would go to whichever cards happen to be idle, and per-key state would split across devices that never see each other's rows. -Key-partitioning the input does not rescue it — the affinity would have to be -device-wide, which is precisely the pre-partitioning that demand-driven scheduling -exists to avoid. The refusal is the same guard `run_scan_batches` applies to -position-sliced batches, one level up. +Key-partitioning the input does not help: the affinity would have to be +device-wide, which is the pre-partitioning that demand-driven scheduling exists +to avoid. It is the same guard `run_scan_batches` applies to position-sliced +batches, one level up. `weights` biases the shared pull toward requested relative shares by weighted round-robin. It is a bias on *pull order*, not a partition: batches stay lazy and @@ -118,8 +117,8 @@ slow. ## The GIL detail that only shows up in tests `poll_yield_seconds` defaults to `None`, meaning the status poll spins. That is -right for the real transports, which release the GIL in their register read — -`os.pread` on Linux, `ctypes` on macOS — so concurrent pipelines interleave freely +right for the real transports, which release the GIL in their register read +(`os.pread` on Linux, `ctypes` on macOS), so concurrent pipelines interleave freely and a sleep would only add latency per poll. It is wrong for a pure-Python transport, which is what a concurrent test uses. @@ -132,7 +131,7 @@ The parameter exists for that case only. Cancellation is incomplete, and the docstring says so rather than leaving you to discover it. A device failure wakes pipelines blocked pulling a batch, but one -already inside its status wait runs to its own completion or timeout first — so an +already inside its status wait runs to its own completion or timeout first, so an early failure can surface only after its peers finish. Naming a limitation is cheaper than a caller building a timeout budget on an @@ -140,8 +139,8 @@ assumption that does not hold. ## Related -- [Run a scan over many batches](First-Scan-Run) — the runner and +- [Run a scan over many batches](../tutorial/first-scan-run.md): the runner and its guards, hands on. -- [Run scan jobs](Run-Scan-Jobs) — choosing a strategy for real work. -- [Execution contracts reference](Execution-Contracts) — every +- [Run scan jobs](../how-to/run-scan-jobs.md): choosing a strategy for real work. +- [Execution contracts reference](../reference/execution-contracts.md): every signature and constant. diff --git a/docs/wiki/explanation/Storage-Handles.md b/docs/src/explanation/storage-handles.md similarity index 56% rename from docs/wiki/explanation/Storage-Handles.md rename to docs/src/explanation/storage-handles.md index 9f7aa9e..600ba97 100644 --- a/docs/wiki/explanation/Storage-Handles.md +++ b/docs/src/explanation/storage-handles.md @@ -8,7 +8,7 @@ about why a host-side allocator is built this way. ## The bug this design does not have A conventional allocator hands back an address. Free it, allocate again, and the -new allocation may sit at the same address — so a stale reference to the old one +new allocation may sit at the same address, so a stale reference to the old one now reads and writes storage that belongs to something else. On a host that is a use-after-free. On a device it is worse, because there is no page table to fault: the DMA succeeds, the data is wrong, and nothing anywhere reports a problem. @@ -24,7 +24,7 @@ handle 0 generation 1 is stale, forged, or from another allocator ``` That sentence is the invariant: **freed storage is never readdressed through an -old name.** It comes for free in a few consequences. A double free faults, because +old name.** A few other checks follow from it. A double free faults, because `free` resolves first. A handle from another allocator faults, because the allocator token does not match. And a handle whose fields were tampered with faults, because the comparison is against the live record, not the caller's copy. @@ -35,28 +35,26 @@ Every operation reads the allocator's record rather than the handle's own `base` and `capacity`. `stage` checks the payload length against the *live* grant. `free` returns the *live* span to the free list. -This is not paranoia about malicious callers. It is that a frozen handle passed -around a program is a copy of facts that were true when it was issued, and the -allocator is the only thing that knows whether they still are. Trusting the copy -would make the generation check decorative — you would be validating identity and -then acting on unvalidated data. +The reason is not malicious callers. A frozen handle passed around a program is a +copy of facts that were true when it was issued, and the allocator is the only +thing that knows whether they still are. Trusting the copy would make the +generation check pointless: you would validate the identity and then act on +unvalidated data. ## Why ids are recycled -The allocator could simply never reuse an id: a monotonic counter, no recycling, -and the generation becomes unnecessary. That would be simpler, and it is the wrong -shape. +The allocator could instead never reuse an id: a monotonic counter, no recycling, +and no need for a generation. That would be simpler, but it would not match the +hardware. -The reason is what this allocator was a rehearsal for, and the rehearsal is -over. The shell's handle table has a fixed number of slots, because that is what -fits in BRAM. Slots get reused; the generation is exactly what makes reuse safe. -Keeping the software allocator on the same discipline is what let the API -survive the table arriving in hardware — `HandleTable` programs a slot straight -from the allocator's live record, and nothing above it changed. +The shell's handle table has a fixed number of slots, because that is what fits +in BRAM. Slots get reused, and the generation is what makes reuse safe. Because +the software allocator already followed the same rule, the API did not change +when the table arrived in hardware: `HandleTable` programs a slot straight from +the allocator's live record, and nothing above it changed. -The general principle: the dumb half of a lifecycle should be shaped like the half -that will eventually replace it, not like whatever is easiest in the language you -are currently writing. +The general rule: shape the simple software half of a lifecycle like the +hardware half that will replace it, not like whatever is easiest in Python. ## The host allocates; tiles consume @@ -68,54 +66,54 @@ to be one later. Reserved regions are how the fixed parts of the map stay out of the way. Lane output windows and fixed staging areas are carved out of the free list at construction, so nothing can be allocated on top of them. A reserved region -outside the tier, or with a non-positive size, raises immediately — a wrong +outside the tier, or with a non-positive size, raises immediately, so a wrong memory map fails at construction rather than by handing out an address inside a lane's output window and corrupting a result three jobs later. -## Why the device has to be the one that refuses +## Why the device has to make the check A host-side check is enough right up until the device reads on its own. A job over rows already resident on the device names the region it should read, and a -raw address survives its allocation: free the region, hand the space to someone -else, and the old address still reads perfectly. It just reads the new tenant's -bytes, and no layer above can tell that from an answer. +raw address outlasts its allocation: free the region, hand the space to someone +else, and the old address still reads without error. It reads the new owner's +bytes, and no layer above can tell that from a real answer. `HandleTable` closes that by programming the grant into a slot and letting the -job name `(id, generation)` instead of an address. A resolution the table -disagrees with refuses the job — the reader never issues a beat — and the -refusal arrives on the host as `HandleFaultError`, never as results, so a caller -cannot mistake it for a job that ran and summed nothing. +job name `(id, generation)` instead of an address. When the table disagrees with +a resolution, the shell does not start the job (the reader never issues a beat), +and the host gets a `HandleFaultError`, never results, so a caller cannot mistake +it for a job that ran and summed nothing. It is also why `arm` validates nothing. The stale handle has to reach the -device, or the table is never what refused it, and the refusal is a host-side -assertion wearing hardware's name. +device, or the table never checks it, and the fault would be a host-side +assertion presented as a hardware one. -## What it deliberately is not +## What it is not This is a first-fit allocator with a sorted, coalesced free list, and it makes no claim beyond that. It is not concurrent, has no size classes, and does not compact. Allocation is linear in the number of free spans. -That is proportional to the problem. Device tiers hold a handful of long-lived -regions — input staging, output windows, resident tables — not millions of small -transient objects. Exhaustion raises before anything is staged, naming the tier +That fits the problem. Device tiers hold a handful of long-lived regions (input +staging, output windows, resident tables), not millions of small transient +objects. Exhaustion raises before anything is staged, naming the tier and the request: ```text StorageExhaustedError: tier 'ddr' cannot hold 1073741824 bytes ``` -The one thing worth watching is that `stage` and `resolve` take any object with a +One thing to watch: `stage` and `resolve` take any object with a `write_dma` method rather than a typed device. That is the duck-typed device convention used throughout the package, and it is what lets a fake stand in for a -board — the same reason [the transport boundary](Transport-Boundary) is a +board, for the same reason [the transport boundary](transport-boundary.md) is a protocol rather than a base class. ## Related -- [Allocate device storage](Allocate-Device-Storage) — the allocator +- [Allocate device storage](../how-to/allocate-device-storage.md): the allocator in practice. -- [Device and discovery reference](Device-and-Discovery) — every +- [Device and discovery reference](../reference/device-and-discovery.md): every method and error. -- [The transport boundary](Transport-Boundary) — the same duck-typing choice, +- [The transport boundary](transport-boundary.md): the same duck-typing choice, one layer down. diff --git a/docs/wiki/explanation/Transport-Boundary.md b/docs/src/explanation/transport-boundary.md similarity index 53% rename from docs/wiki/explanation/Transport-Boundary.md rename to docs/src/explanation/transport-boundary.md index 6ff8453..b2ea715 100644 --- a/docs/wiki/explanation/Transport-Boundary.md +++ b/docs/src/explanation/transport-boundary.md @@ -1,7 +1,7 @@ # The transport boundary: why a driver owns almost nothing dau-driver moves bytes and runs jobs. It does not know what a register means, -what a batch header contains, or what an opcode computes — all of that comes from +what a batch header contains, or what an opcode computes. All of that comes from `dau-core`. This page is about where that line sits, why it is drawn there, and what it makes possible. @@ -14,67 +14,62 @@ capability advertisement. Transport code changes when the *host* changes: a new OS, a new devnode layout, a DMA quirk on one bitstream. Putting both in one package means every hardware-contract change touches the -driver, and every host quirk risks perturbing the contract. Worse, the contract -would then have exactly one consumer — this driver — and a contract with one -consumer is indistinguishable from an implementation detail. The compiler, the +driver, and every host quirk risks changing the contract. The contract would +also have one consumer, this driver, and a contract with one consumer is in +practice an implementation detail. The compiler, the simulator, and the golden models all need the same offsets. -So `dau-core` owns the semantics and dau-driver imports them. The concrete -consequence is that a hardware quirk gets fixed in the backend layer and stays -there. When one bitstream's AXI path corrupted whole-transfer C2H reads above a +So `dau-core` owns the semantics and dau-driver imports them. In practice, a +hardware quirk gets fixed in the backend layer and stays there. When one bitstream's AXI path corrupted whole-transfer C2H reads above a threshold, the fix was a chunking field on the device handle, applied inside `read_dma`. It did not become a rule in the protocol contract, because it was -never true of the protocol — only of that image. That is also why the field -could simply be DELETED once no such image remained: a quirk confined to the -layer it belonged to leaves nothing behind when it stops being true. +never true of the protocol, only of that image. It is also why the field could +be deleted once no such image remained, with nothing else to change. ## `DeviceTransport` is four methods The entire device-touching surface is a `Protocol` with `read_u32`, `write_u32`, -`read_dma`, and `write_dma`. Everything above it — register decoding, status bits, -transfer validation, job runners, the stream codec — is written against those four +`read_dma`, and `write_dma`. Everything above it (register decoding, transfer +validation, the job runners, the stream codec) is written against those four calls. -That is a deliberately austere interface, and its narrowness is the point. Two -real transports implement it: Linux XDMA character devices via `os.pread` and +The interface is kept that narrow on purpose. Two real transports implement it: Linux XDMA character devices via `os.pread` and `os.pwrite`, and a macOS DriverKit system extension via IOKit user-client calls. They share nothing but the protocol, and the layers above cannot tell them apart. -The narrowness has a second payoff that matters more day to day. A fake transport -is fifty lines. `FakeTransport` records DMA traffic and register writes in -dictionaries, and every Python execution path in the package — the scan runner's -ping-pong staging, the fleet's demand-driven scheduling, the bar shell's DDR -layout, capability refusals — is exercised against it in CI, on machines with no -FPGA. What sits below the boundary (the kernel module, the DriverKit extension, +It also makes testing cheap: a fake transport is fifty lines. `FakeTransport` +records DMA traffic and register writes in dictionaries, and every Python +execution path in the package (the scan runner's ping-pong staging, the fleet's +demand-driven scheduling, the bar shell's DDR layout, capability checks) runs +against it in CI, on machines with no FPGA. What sits below the boundary (the kernel module, the DriverKit extension, the native extension) is tested on hardware. This is the package's testing policy stated as an architectural constraint: **every Python execution path must run against a fake, and hardware-only -behaviour hides behind `DeviceTransport`.** If a code path cannot be reached without a board, it +behaviour sits behind `DeviceTransport`.** If a code path cannot be reached without a board, it is in the wrong layer. The fake lives in `dau_driver.testing`, shipped rather than kept in the test directory, because the driver's consumers test against it too. One definition -means the fake device's semantics cannot drift between packages — which they -would, immediately, if each package wrote its own. +means the fake device's semantics cannot drift between packages, as they would if +each package wrote its own. -## Discovery refuses more than it accepts +## Discovery skips anything that is not a DAU shell `discover_devices` will not return an XDMA endpoint just because it is an XDMA endpoint. It reads the register window, looks for the DAU magic word, and decodes a capability advertisement. Anything else is skipped. -The asymmetry between `discover_devices` and `identify_device` is deliberate. +`discover_devices` and `identify_device` treat a mismatch differently on purpose. Discovery scans a machine that may hold other people's cards, so a non-DAU -endpoint is *not an error* — it is simply not a match, and raising would make a -mixed machine undiscoverable. `identify_device` takes a path you named, so a bad +endpoint is not an error, just not a match, and raising would make a mixed +machine undiscoverable. `identify_device` takes a path you named, so a bad answer there is an error, and it raises `WrongBitstreamError` or `UnhealthyBarError` by name. -The all-ones case earns its own exception because it is the failure that does not -look like one. A dead AXI-lite path returns `0xFFFFFFFF` for every read. That is a -valid 32-bit number. It will decode as a version, a bitmap, a counter — plausible -values all the way down. `UnhealthyBarError` exists so that a dead path is a +The all-ones case has its own exception because it does not look like a failure. +A dead AXI-lite path returns `0xFFFFFFFF` for every read, which is a valid 32-bit +number. It decodes as a version, a bitmap or a counter, all of them plausible. `UnhealthyBarError` exists so that a dead path is a named failure rather than a strange number four layers later, and `check_bar_health()` exists so you can assert it before and after anything you care about. @@ -86,31 +81,31 @@ is the kind of fact that only transport code should know. ## Validate the shape before opening the channel -Every DMA transfer is checked against the device's *own advertised* constraints — -alignment, maximum size, channel availability — before a channel is opened. The +Every DMA transfer is checked against the device's *own advertised* constraints +(alignment, maximum size, channel availability) before a channel is opened. The errors are typed and say which rule was broken. -Checking against the advertisement rather than a constant is what makes the -driver survive a new board. `dma.max_aligned_transfer_bytes` is the value to use +Checking against the advertisement rather than a constant is what lets the +driver work on a new board. `dma.max_aligned_transfer_bytes` is the value to use for a maximum-sized loopback, not a hard-coded descriptor limit, because the limit is a property of the device in front of you. -Two related guards follow the same instinct of refusing rather than proceeding. +Two related guards also stop rather than proceed. `write_address` always writes both halves of an address pair, including a zero high half, because a design whose master is wider than 32 bits keeps the high half -across a LOW write — so writing only LOW inherits whatever the previous job left -and quietly addresses the wrong region. And when the handle knows the device's +across a LOW write, so writing only LOW reuses whatever the previous job left there +and addresses the wrong region. And when the handle knows the device's memory size, an address past it raises rather than wrapping to the bottom of memory, which is what the hardware would otherwise do. -The `memory_bytes` field defaults to `None`, and that is the honest choice: a -guard is only as good as the fact behind it. Inventing a default size would either +The `memory_bytes` field defaults to `None`, because a guard is only as good as +the fact behind it. Inventing a default size would either refuse valid addresses on a larger board or give false confidence on a smaller one. ## Where the model bends -Two places do not fit the clean picture, and both are worth naming. +Two places do not fit this picture. `write_address` is a module-level function rather than a method on `DauDevice`, because devices in this codebase are duck-typed. Test fakes and alternative @@ -119,16 +114,16 @@ this a method broke every one of them. The runners take `device` untyped for the same reason. `KeyedPipelineConfig` is a plain dataclass while the rest of the DAU stack uses -frozen pydantic models. It mirrors the `PIPELINE_*` register encodings — it is -wire packing, not configuration — and the rule the codebase follows is that +frozen pydantic models. It mirrors the `PIPELINE_*` register encodings (it is +wire packing, not configuration), and the rule the codebase follows is that register mirrors and DMA structs stay dataclasses while anything authored or composed follows the model patterns. ## Related -- [Find and check a device](Find-a-Device) — discovery and health in +- [Find and check a device](../how-to/find-a-device.md): discovery and health in practice. -- [Talk to a device without any hardware](First-Device) — the fake +- [Talk to a device without any hardware](../tutorial/first-device.md): the fake transport as a working tool. -- [Device and discovery reference](Device-and-Discovery) — the +- [Device and discovery reference](../reference/device-and-discovery.md): the protocol and every implementation of it. diff --git a/docs/wiki/how-to/Allocate-Device-Storage.md b/docs/src/how-to/allocate-device-storage.md similarity index 78% rename from docs/wiki/how-to/Allocate-Device-Storage.md rename to docs/src/how-to/allocate-device-storage.md index 190d3e8..439e30a 100644 --- a/docs/wiki/how-to/Allocate-Device-Storage.md +++ b/docs/src/how-to/allocate-device-storage.md @@ -22,13 +22,13 @@ allocator = StorageAllocator( ) ``` -| Parameter | Meaning | -| ---------------- | ---------------------------------------------------------------------------------------------------------- | -| `tier` | The tier's name; stamped onto every handle it issues. | -| `capacity_bytes` | The tier's size. Must be positive. | -| `base` | The tier's first address. Defaults to 0. | -| `reserved` | `(base, size)` regions carved out before anything is allocated — lane output windows, fixed staging areas. | -| `align` | Allocation alignment. Must be a positive power of two; defaults to 64, the DMA grid. | +| Parameter | Meaning | +| ---------------- | --------------------------------------------------------------------------------------------------------- | +| `tier` | The tier's name; stamped onto every handle it issues. | +| `capacity_bytes` | The tier's size. Must be positive. | +| `base` | The tier's first address. Defaults to 0. | +| `reserved` | `(base, size)` regions carved out before anything is allocated: lane output windows, fixed staging areas. | +| `align` | Allocation alignment. Must be a positive power of two; defaults to 64, the DMA grid. | A reserved region outside `[base, base + capacity_bytes)` or with a non-positive size raises `ValueError` at construction, so a wrong map fails @@ -54,7 +54,7 @@ print(hex(allocator.resolve(inputs))) The first allocation starts at `0x1000` because the first 4 KiB were reserved. Allocation is first-fit over the free list. `resolve(handle)` returns the device -address, generation-checked — use it wherever a register or a transfer needs an +address, generation-checked. Use it wherever a register or a transfer needs an address, rather than caching `handle.base`. `StorageExhaustedError` is raised when no free span can hold the request, before @@ -70,8 +70,8 @@ StorageExhaustedError: tier 'ddr' cannot hold 1073741824 bytes allocator.stage(device, inputs, payload) ``` -Resolves the handle, checks the payload against the *live* grant — never the -handle's own copy of the capacity — and writes it with `device.write_dma`. A +Resolves the handle, checks the payload against the *live* grant (never the +handle's own copy of the capacity) and writes it with `device.write_dma`. A payload larger than the grant raises `ValueError` rather than overrunning into the next allocation. @@ -89,8 +89,8 @@ allocator.resolve(inputs) ``` Freeing bumps the handle's generation and returns the span to the free list, -coalesced with its neighbours. The id itself is recycled — a fixed-slot hardware -handle table reuses slots, and the generation is what makes that safe. +coalesced with its neighbours. The id itself is recycled, because a fixed-slot +hardware handle table reuses slots, and the generation is what makes that safe. The consequence you want: freed storage is never readdressed through an old name. A stale handle raises instead of resolving to memory that now belongs to someone @@ -111,9 +111,9 @@ Resolve at the point of use. ## Next -- [Run scan jobs](Run-Scan-Jobs) — the input regions and output windows an +- [Run scan jobs](run-scan-jobs.md): the input regions and output windows an allocator hands out. -- [Move data to and from a device](Move-Data) — the transfers that consume +- [Move data to and from a device](move-data.md): the transfers that consume those addresses. -- [Storage handles](Storage-Handles) — why generations rather +- [Storage handles](../explanation/storage-handles.md): why generations rather than plain pointers. diff --git a/docs/wiki/how-to/Arm-the-Deadman.md b/docs/src/how-to/arm-the-deadman.md similarity index 59% rename from docs/wiki/how-to/Arm-the-Deadman.md rename to docs/src/how-to/arm-the-deadman.md index 0dd1405..a8605e6 100644 --- a/docs/wiki/how-to/Arm-the-Deadman.md +++ b/docs/src/how-to/arm-the-deadman.md @@ -3,25 +3,25 @@ This guide shows you how to run a sequence that touches the PCIe bus without risking a host you cannot get back. It applies to anything that removes an endpoint, resets a bridge, rescans, holds runtime power management, or -reprograms the fabric — not only to the obviously destructive steps. +reprograms the fabric, not only to the obviously destructive steps. The rule is short: **arm a cancellable auto-reboot guard before any PCIe-touching sequence, and disarm it only after the health ladder passes.** ## Why it is not optional -A bus operation that wedges the host does not fail; it stops answering. The -machine is up, the session is gone, and the only remedy is physical. A -guard armed beforehand turns that from an unrecoverable state into a wait: -if nothing disarms it, the host reboots itself and comes back. +A bus operation that hangs the host does not fail; it stops answering. The +machine is up, the session is gone, and the only fix is a power cycle. A +guard armed beforehand turns that into a wait: if nothing disarms it, the host +reboots itself and comes back. -This is the failure the rule exists for, and it has happened often enough -that the sequences worth running by hand are none of them. +This has happened often enough that none of these sequences should be run by +hand. ## Use a composed plan, not a script The `dau-build` hardware plans arm the guard around the destructive span, -take the host lock, carry the board's access facts from its platform +take the host lock, read the board's access facts from its platform definition, and support a dry run. A bare script does none of that. The plans this package's smoke command is injected into cover the two common @@ -53,14 +53,14 @@ sudo python -m dau_utils.deadman status sudo python -m dau_utils.deadman disarm ``` -`dau-driver` neither implements nor depends on it — `dau_utils` is an -out-of-repo prerequisite on the bench host, and nothing here will tell you it -is missing. Pair it with the host's own hardware watchdog so that a host -wedged below the point where the timer can fire still recovers. +`dau-driver` neither implements nor depends on it. `dau_utils` is a separate +prerequisite on the bench host, and nothing here will tell you it is missing. +Pair it with the host's own hardware watchdog, so that a host hung too far for +the timer to fire still recovers. -Choose the timeout by what the span actually takes, with margin. Too short -reboots a healthy host mid-programming; too long is a long wait for a -wedge you already know happened. +Choose the timeout from how long the span takes, with margin. Too short reboots +a healthy host mid-programming; too long means a long wait after a hang you +already know about. ## Disarm only after the device answers @@ -70,11 +70,10 @@ produced identified itself afterwards. So the ladder is: arm, do the destructive span, discover the device, read the magic word, read back the identity block, compare it against what was -built — and only then disarm. +built, and only then disarm. -Verification steps are deliberately placed **before** the disarm, which has a -consequence worth stating: an uncaught exception in a verify step reboots -the host. That is the intended behaviour. The alternative is the incident +Verification steps come **before** the disarm on purpose, so an uncaught +exception in a verify step reboots the host. That is the intended behaviour. The alternative is the incident this ordering exists to prevent, where a plan saw an endpoint, disarmed, and reported success while the board kept running the previous design. @@ -84,14 +83,14 @@ Verify the advertisement against the manifest that produced the bitstream: dau-driver-verify-advertisement --manifest /shell-build.artifacts.yaml ``` -Add `--require-build-id` to refuse a device whose build id does not match the -manifest's spec hash, rather than merely warning. +Add `--require-build-id` to fail, rather than warn, on a device whose build id +does not match the manifest's spec hash. ## Next -- [Find and check a device](Find-a-Device) — the health checks the ladder +- [Find and check a device](find-a-device.md): the health checks the ladder ends with. -- [Device and discovery](Device-and-Discovery) — `check_bar_health`, the +- [Device and discovery](../reference/device-and-discovery.md): `check_bar_health`, the register window, and the identity snapshot. -- [The transport boundary](Transport-Boundary) — why discovery refuses more - than it accepts. +- [The transport boundary](../explanation/transport-boundary.md): why discovery skips anything + that is not a DAU shell. diff --git a/docs/src/how-to/develop.md b/docs/src/how-to/develop.md new file mode 100644 index 0000000..d21a1d3 --- /dev/null +++ b/docs/src/how-to/develop.md @@ -0,0 +1,87 @@ +# Develop and contribute + +Contributions are welcome under the +[Apache 2.0 license](https://github.com/dau-dev/dau-driver/blob/main/LICENSE). +`kernel/linux-xdma/` is Xilinx's XDMA reference driver and keeps its own Dual +BSD/GPL terms. Every commit needs a Developer Certificate of Origin sign-off +(`git commit -s`), which a GitHub app checks on each pull request. + +## Set up + +```bash +git clone https://github.com/dau-dev/dau-driver.git +cd dau-driver +make develop +``` + +`make develop` installs the package in editable mode with its development +dependencies. The install builds a small native extension with nanobind, so it +needs a C++ toolchain. `make` with no arguments lists the targets. + +Nothing here needs a board. The Python paths run against the in-memory fake +transport in `dau_driver.testing`, which is how the suite exercises them. + +## Lint, type-check, test + +```bash +make lint # ruff for Python, clang-format for the extension under cpp/ +make checks # distribution check and ty +make test # pytest +``` + +Run all three before opening a pull request. The Python blocks in the README +and the tutorials run as tests, and the tutorials' printed output is checked, +so an example that stops working fails the suite. + +The hardware tests under `dau_driver/tests/integration/` skip unless they are +pointed at a board: `DAU_XDMA_MACOS=1` on a macOS host with the DriverKit +extension active, or `DAU_XDMA_HARDWARE=1` on a Linux XDMA host. Fence +any run that touches the bus with the deadman first; see +[how to arm the deadman](arm-the-deadman.md). + +## Kernel drivers + +The host drivers live under `kernel/`, each with its own build and load +instructions in its README. The Linux XDMA module is built against the running +kernel. The macOS DriverKit extension is built with Xcode; to check that it +compiles without signing it: + +```bash +cd kernel/macos-xdma +make compile-unsigned +``` + +Whoever can open the character devices (Linux) or the user client (macOS) can +read and write the card's memory and registers. The operating system's device +permissions are the access control. + +## Wheels + +Release wheels are built with cibuildwheel in CI, from `[tool.cibuildwheel]` in +`pyproject.toml`. To build one locally: + +```bash +make dist-py-wheel +``` + +## Documentation + +The pages under `docs/src` and the README make up the documentation site. It +is built with [yardang](https://github.com/python-project-templates/yardang) +from `[tool.yardang]` in `pyproject.toml`, which lists the pages in order, and +the docs workflow publishes it to GitHub Pages after each green build on +`main`. To build it locally: + +```bash +uv pip install "yardang[themes]" +yardang build +``` + +The site lands in `docs/html`. + +## Pull requests + +Work on a fork and open a pull request against `main`. A draft is fine while +the change is in progress. Squash into logical commits, all signed. Open an +issue first for anything larger than a fix, and use the discussions page for +questions. diff --git a/docs/wiki/how-to/Find-a-Device.md b/docs/src/how-to/find-a-device.md similarity index 79% rename from docs/wiki/how-to/Find-a-Device.md rename to docs/src/how-to/find-a-device.md index 189eac7..1dabb6e 100644 --- a/docs/wiki/how-to/Find-a-Device.md +++ b/docs/src/how-to/find-a-device.md @@ -4,9 +4,9 @@ This guide shows you how to get from "there is a card in the machine" to a validated `DauDevice` handle, and how to read the failure when you do not get one. -Discovery is deliberately strict. A DAU device is not "an XDMA endpoint"; it is -an XDMA endpoint whose register window reports the DAU magic word and decodes a -capability advertisement. Anything else is skipped or refused by name. +Discovery is strict on purpose. A DAU device is an XDMA endpoint whose register +window reports the DAU magic word and decodes a capability advertisement. +Anything else is skipped, or raises a named error when you asked for it by path. ## Discover every device on the host @@ -29,8 +29,8 @@ devices = discover_devices("/some/other/dev") ``` An endpoint whose BAR reads all ones, or whose magic word is not DAU's, is -skipped rather than returned — broad discovery should not fail because one card -in the box is running someone else's bitstream. +skipped rather than returned: broad discovery should not fail because one card in +the box is running someone else's bitstream. ## Identify one device by path @@ -42,14 +42,14 @@ from dau_driver import identify_device device = identify_device("/dev/xdma0_user") ``` -This does raise, which is the point: you named this device, so a bad answer is -an error rather than an omission. +This one raises: you named this device, so a bad answer is an error rather than +an omission. -| Exception | Meaning | -| --------------------- | ---------------------------------------------------------------------------------------------------- | -| `UnhealthyBarError` | The magic word read all ones. The endpoint or its AXI-lite BAR is not responding. | -| `WrongBitstreamError` | The window responded, but with something other than the DAU magic word. | -| `RegisterAccessError` | A register read failed after the window was located — a short read, or the node could not be opened. | +| Exception | Meaning | +| --------------------- | --------------------------------------------------------------------------------------------------- | +| `UnhealthyBarError` | The magic word read all ones. The endpoint or its AXI-lite BAR is not responding. | +| `WrongBitstreamError` | The window responded, but with something other than the DAU magic word. | +| `RegisterAccessError` | A register read failed after the window was located: a short read, or the node could not be opened. | A path that does not exist reads back as zero rather than the magic word, so it surfaces as `WrongBitstreamError`, not as a missing-file error. @@ -65,16 +65,15 @@ wrong bitstream. ## Check the register path before you trust it -A dead AXI-lite path returns `0xFFFFFFFF`, which is a perfectly plausible -number. Check for it explicitly: +A dead AXI-lite path returns `0xFFFFFFFF`, which looks like a valid number. Check for it explicitly: ```python device.check_bar_health() ``` Raises `UnhealthyBarError` if `MAGIC` or `STREAM_PROTOCOL_VERSION` reads all -ones. Call it before a sequence you care about and again afterwards — a -successful DMA that leaves the BAR dead is worth knowing about immediately. +ones. Call it before a sequence you care about and again afterwards, because a +successful DMA that leaves the BAR dead is something to know about at once. To check other offsets, pass them: @@ -95,8 +94,8 @@ print(device.read_counters()) `read_register_block_snapshot()` reads the whole identity block in one pass: versions, platform id, shell version, build id, status, DMA constraints, the -operator bitmap, and the counters. The composed-capability words — lane count, -host opcode bitmap, sort capacity — are only read on register map 0.2 and above, +operator bitmap, and the counters. The composed-capability words (lane count, +host opcode bitmap, sort capacity) are only read on register map 0.2 and above, because older shells do not decode those offsets. `device.capabilities.operators` lists what the device executes. @@ -107,7 +106,7 @@ never be submitted to the device. ## Read the concrete capability contract The register bitmap is a coarse advertisement. A design's shell-build manifest -carries the exact composed contract — per-tile opcodes and input types, with no +holds the exact composed contract, per-tile opcodes and input types, with no bitmap-times-types expansion: ```python @@ -158,13 +157,13 @@ NativeInfo(backend='nanobind', abi_version=1, package_version='0.1.0', extension ``` `backend='python'` with `extension_available=False` means the compiled extension -is not present and the pure-Python fallback is in use. Everything still works; -hot paths are slower. +is not present. Everything still works: the extension only reports its own +backend and ABI version today, and every path runs in Python either way. ## Next -- [Move data to and from a device](Move-Data) — DMA and the stream codec. -- [Device and discovery reference](Device-and-Discovery) — every +- [Move data to and from a device](move-data.md): DMA and the stream codec. +- [Device and discovery reference](../reference/device-and-discovery.md): every field and signature. -- [The transport boundary](Transport-Boundary) — why discovery - refuses so much. +- [The transport boundary](../explanation/transport-boundary.md): why discovery + is strict. diff --git a/docs/wiki/how-to/Move-Data.md b/docs/src/how-to/move-data.md similarity index 85% rename from docs/wiki/how-to/Move-Data.md rename to docs/src/how-to/move-data.md index 5f79cc5..7fa96ad 100644 --- a/docs/wiki/how-to/Move-Data.md +++ b/docs/src/how-to/move-data.md @@ -50,8 +50,8 @@ computing the descriptor limit yourself. ## Guard against addressing past the end of memory -An address beyond the device's memory does not fail — it wraps to the bottom, and -you silently read or overwrite the wrong region. Tell the handle how big its +An address beyond the device's memory does not fail. It wraps to the bottom, and +you read or overwrite the wrong region without any error. Tell the handle how big its memory is, then program addresses through the method form: ```python @@ -64,13 +64,13 @@ device.write_address_register(RegisterOffset.INPUT_ADDRESS_LOW, RegisterOffset.I ``` `RegisterAccessError` is then raised for an address at or past the limit. Left as -`None`, the caller is trusted — the guard is only as good as the fact, and -inventing a default would either refuse valid addresses or give false confidence. +`None`, the caller is trusted: the guard is only as good as the fact, and an +invented default would either reject valid addresses or give false confidence. The module-level `write_address` function reads `memory_bytes` off the device by default. Pass an integer to override the limit, or `None` to disable the check -entirely — but disable it deliberately, because the default is what makes the -guard apply to callers who never thought about it. +entirely, but only on purpose: the default is what makes the guard apply to +callers who never thought about it. ## Program a 64-bit address into a register pair @@ -86,11 +86,11 @@ write_address(device, RegisterOffset.INPUT_ADDRESS_LOW, RegisterOffset.INPUT_ADD Always write both halves, which is what `write_address` does even when the high half is zero. A design whose master is wider than 32 bits keeps the high half -across a LOW write, so writing only LOW inherits whatever the previous job left -and quietly addresses the wrong region. +across a LOW write, so writing only LOW reuses whatever the previous job left there +and addresses the wrong region. Pass the offsets explicitly rather than deriving `low + 4`. The job pair is -adjacent, but a NoC lane's is not — its high half is at `0x14`, because `0x04` is +adjacent, but a NoC lane's is not: its high half is at `0x14`, because `0x04` is the result length. `write_address` is a function, not a method, because devices here are duck-typed: @@ -166,10 +166,10 @@ For a single-column batch use `decode_column`; for the one-row result of a scalar aggregation use `decode_scalar_result`, which returns the typed `value` directly. -Decoding validates as it goes — version, header length, alignment, payload -windows, per-column widths and row counts, and the operation descriptors — and +Decoding validates as it goes (version, header length, alignment, payload +windows, per-column widths and row counts, and the operation descriptors) and raises `InvalidStreamError` on any mismatch. A truncated or misaligned read -surfaces there rather than as a plausible wrong number. +fails there rather than coming back as a plausible wrong number. ## Run one aggregation as a single call @@ -191,7 +191,7 @@ print(result.value) ``` The operation is checked against the device's advertised capabilities *before* -any DMA — an unsupported opcode or a mismatched output type raises +any DMA: an unsupported opcode or a mismatched output type raises `StreamExecutionError` with nothing staged. `trigger` is the hook that starts the job between the H2C write and the C2H read; supply the register sequence your shell expects, or use `dau_driver.testing.golden_aggregation_trigger` against a @@ -199,9 +199,9 @@ fake. ## Next -- [Run scan jobs](Run-Scan-Jobs) — many batches, with staging overlapped +- [Run scan jobs](run-scan-jobs.md): many batches, with staging overlapped against compute. -- [Stream codec reference](Stream-Codec) — every encoder and +- [Stream codec reference](../reference/stream-codec.md): every encoder and decoder entry point. -- [Allocate device storage](Allocate-Device-Storage) — handing out device +- [Allocate device storage](allocate-device-storage.md): handing out device addresses safely. diff --git a/docs/wiki/how-to/Run-Scan-Jobs.md b/docs/src/how-to/run-scan-jobs.md similarity index 87% rename from docs/wiki/how-to/Run-Scan-Jobs.md rename to docs/src/how-to/run-scan-jobs.md index 383d967..9a568d3 100644 --- a/docs/wiki/how-to/Run-Scan-Jobs.md +++ b/docs/src/how-to/run-scan-jobs.md @@ -34,7 +34,7 @@ Each batch's `payload` must be exactly `row_count * row_bytes`. The runner pads the DMA out to the 64-byte grid while `INPUT_LENGTH` stays exact. Your `collect` reads lane registers and output regions, which are disjoint from -the input staging — so collection overlaps the next batch's DMA too. +the input staging, so collection overlaps the next batch's DMA too. ## Overlap the encoding as well @@ -75,7 +75,7 @@ batches = key_partitioned_scan_batches( ``` Assignment is longest-processing-time-first: the largest key group goes to the -currently smallest batch. **Records must arrive ordered by key** — a key +currently smallest batch. **Records must arrive ordered by key**: a key appearing in two runs raises `ValueError`. Record order within each key is preserved. @@ -85,9 +85,9 @@ when that batch is requested. Key groups are indivisible, so one hot key can dominate a batch. Pass `per_key_state=True` when running these. The runner requires it for a -stateful composition and checks that the batches actually came from this function -— a position-sliced iterable raises `ValueError` rather than silently splitting a -key's state. +stateful composition and checks that the batches came from this function. A +position-sliced iterable raises `ValueError` rather than splitting a key's state +without telling you. ### Capacity-bounded batches @@ -107,8 +107,8 @@ device's `GROUP_CAPACITY_EXCEEDED` close-out. The key space is partitioned into ranges of `key_capacity` keys and one batch is emitted per range. Records need **not** arrive in key order here, and arrival order is preserved -within each batch — the hash table needs no key ordering, and imposing one would -give back the staging freedom the tile exists to provide. The trade is a counting +within each batch. The hash table needs no key ordering, and imposing one would +give up the staging freedom the tile exists to provide. The trade is a counting pass over every key before the first batch, and row counts that may be uneven: key ranges are equal in keys, not in rows. @@ -118,8 +118,8 @@ is a plain concatenation. If your batches came from somewhere else, use ## Query rows that are already on the device -When the device already holds the data — a resident table queried many times — -re-staging every batch is the wrong shape. The feed is the larger term of the +When the device already holds the data, such as a resident table queried many +times, re-staging every batch is wasted work. The feed is the larger term of the device path, so lift the DMA out of the loop: ```python @@ -137,7 +137,7 @@ results = run_resident_scan( Nothing is staged. Each repeat programs the address and length, starts the job, waits, and calls `collect`, so you get one result per repeat rather than one -merged answer — enough to assert that a rerun is bit-identical instead of +merged answer, so you can check that a rerun returns identical results instead of assuming it. The address must be on the 64-byte DMA grid, and it is raw: it outlives the @@ -158,9 +158,9 @@ except HandleFaultError as exc: `install` programs the grant from the allocator's live record, and `capacity` is the number of slots the shell was built with. The job then names -`(id, generation)`: a resolution the device disagrees with refuses the job -before the reader issues a beat, and that refusal reaches you as -`HandleFaultError` rather than as results. +`(id, generation)`. When the device disagrees with a resolution, the shell does +not start the job (the reader never issues a beat), and you get a +`HandleFaultError` rather than results. Call `table.free(handle)` when the region is released, so nothing resolves through the slot until it is installed again. @@ -192,7 +192,7 @@ exact device object that produced it. `input_regions` is reused as **device-local** addresses on every card, never shared memory. -Bias the split with `weights` — one positive, finite value per device — to +Bias the split with `weights`, one positive, finite value per device, to pre-weight each shared pull toward the requested relative shares. Batches stay lazy and are never queued ahead. @@ -203,7 +203,7 @@ never see each other's state. Run it on a single device with A failure is raised as `FleetScanError` annotated with the batch index and the device. Note that cancelling wakes pipelines blocked pulling a batch, but one -already inside its status wait runs to its own completion or timeout first — so +already inside its status wait runs to its own completion or timeout first, so an early failure can surface only after its peers finish. ## Report progress @@ -232,17 +232,17 @@ scan job 3 failed: status=0x3 last_error=2 Raise `timeout_seconds` for large batches; the default is 60 seconds. -If you are running concurrent pipelines against a pure-Python transport — a fake -in a test — set `poll_yield_seconds` to a small value. The real transports release +If you are running concurrent pipelines against a pure-Python transport, such as +a fake in a test, set `poll_yield_seconds` to a small value. The real transports release the GIL in their register read, so pipelines interleave without it and a sleep would only add latency per poll; a pure-Python transport spinning would hold the GIL and flatten self-balancing toward an even split regardless of device speed. ## Next -- [Allocate device storage](Allocate-Device-Storage) — where the input regions +- [Allocate device storage](allocate-device-storage.md): where the input regions and output windows come from. -- [Execution contracts reference](Execution-Contracts) — every +- [Execution contracts reference](../reference/execution-contracts.md): every runner's full signature. -- [Overlapped staging](Overlapped-Staging) — why the ping-pong +- [Overlapped staging](../explanation/overlapped-staging.md): why the ping-pong and the `per_key_state` guard are shaped this way. diff --git a/docs/wiki/reference/Device-and-Discovery.md b/docs/src/reference/device-and-discovery.md similarity index 91% rename from docs/wiki/reference/Device-and-Discovery.md rename to docs/src/reference/device-and-discovery.md index 6775205..5b0ce49 100644 --- a/docs/wiki/reference/Device-and-Discovery.md +++ b/docs/src/reference/device-and-discovery.md @@ -73,16 +73,16 @@ declared and never pre-biased. The module-level `read_u32(path, offset)` and and a caller using them composes the window itself. The window is discovered rather than configured. -`capability_decode.REGISTER_WINDOW_OFFSETS` is the candidate list — +`capability_decode.REGISTER_WINDOW_OFFSETS` is the candidate list, `(0x0, 0x1000)`, the shell base first and the stream-job contract window -second — and discovery returns the first offset whose `MAGIC` reads back as +second, and discovery returns the first offset whose `MAGIC` reads back as the DAU magic word, retrying a few times to let a freshly programmed fabric settle. A composition top decodes only the low twelve address bits and returns all ones for anything unmapped, so a health probe must stay on contract registers inside the 4 KiB window. -64-bit registers are read as two separate 32-bit accesses, deliberately: the -user BAR can short-read an eight-byte access. +64-bit registers are read as two separate 32-bit accesses on purpose: the user +BAR can short-read an eight-byte access. ### DMA @@ -97,8 +97,8 @@ DMA device addresses are device memory addresses, not AXI-lite register offsets. ## `DeviceTransport` -A `Protocol`. Implement it to swap the backend — the fakes in the test suite do -exactly this. +A `Protocol`. Implement it to swap the backend, as the fakes in the test suite +do. ```python class DeviceTransport(Protocol): @@ -124,11 +124,12 @@ a memory limit is known and the address is at or past it. `memory_bytes` defaults to a sentinel, not to `None`, and the sentinel means "read `device.memory_bytes`". Passing `None` explicitly disables the bound -check; passing an integer overrides it. A plain `None` default would have made -the guard silently inactive for every caller who did not know to opt in. +check; passing an integer overrides it. A plain `None` default would have turned +the guard off for every caller who did not know to opt in. Offsets are passed explicitly; register pairs are not always adjacent. The job -pair is, but a NoC lane's high half is at `0x14` — `0x04` is its result length. +pair is, but a NoC lane's high half is at `0x14`, because `0x04` is its result +length. Module-level helpers `read_u32(path, offset)` and `read_u64(path, offset)` read registers directly from a node path. @@ -174,20 +175,20 @@ is not decoded into named flags. ### `DeviceCounters` -`h2c_bytes`, `c2h_bytes`, `rows`, `batches`, `stalls` — all `int`. +`h2c_bytes`, `c2h_bytes`, `rows`, `batches`, `stalls`, all `int`. ## Capability decode Shared register-block decode semantics for every transport backend. -| Function | Returns | -| --------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------- | -| `decode_platform_id(value)` | The ASCII platform id, or `""` when the word is not printable ASCII. | -| `decode_register_map_version(value)` | The `RegisterMapVersion`. An all-ones word is a dead aperture, not a version, and decodes as `NOT_ADVERTISED_REGISTER_MAP_VERSION` (0.0). | -| `read_register_block_snapshot(read_register)` | The discovery-time identity snapshot. `read_register` abstracts the backend's window access. | -| `operators_from_snapshot(snapshot)` | `(hardware operators, host-computed operators)` decoded by register map version. | -| `contract_from_manifest_metadata(metadata)` | The `CapabilityContract` from a shell-build manifest's `capability_contract` key, or `None`. | -| `operators_from_contract(contract)` | `(hardware operators, host-computed operators)` from the concrete composition contract — per-tile opcodes and input types, no bitmap expansion. | +| Function | Returns | +| --------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------- | +| `decode_platform_id(value)` | The ASCII platform id, or `""` when the word is not printable ASCII. | +| `decode_register_map_version(value)` | The `RegisterMapVersion`. An all-ones word is a dead aperture, not a version, and decodes as `NOT_ADVERTISED_REGISTER_MAP_VERSION` (0.0). | +| `read_register_block_snapshot(read_register)` | The discovery-time identity snapshot. `read_register` abstracts the backend's window access. | +| `operators_from_snapshot(snapshot)` | `(hardware operators, host-computed operators)` decoded by register map version. | +| `contract_from_manifest_metadata(metadata)` | The `CapabilityContract` from a shell-build manifest's `capability_contract` key, or `None`. | +| `operators_from_contract(contract)` | `(hardware operators, host-computed operators)` from the concrete composition contract: per-tile opcodes and input types, no bitmap expansion. | Register map 0.2 and above advertises the composed truth: `OPERATOR_BITMAP` carries only HDL-executed opcodes, typed INT32 at register level, and @@ -213,12 +214,12 @@ reserved regions. Raises `ValueError` for a non-positive capacity, an `align` that is not a positive power of two, or a reserved region with non-positive size or lying outside the tier. -| Method | Behaviour | -| ----------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `allocate(nbytes) -> StorageHandle` | First-fit, aligned. Raises `StorageExhaustedError` when no span fits, `ValueError` for a non-positive size. | -| `free(handle)` | Bump the handle's generation and return the live span to the free list, coalesced. Raises `StaleHandleError` on a stale, forged, or foreign handle — so a double free faults. | -| `resolve(handle) -> int` | The handle's device address, generation-checked. Raises `StaleHandleError`. | -| `stage(device, handle, data)` | Resolve, check `data` against the live grant, and `device.write_dma`. Raises `ValueError` when the payload exceeds the grant. | +| Method | Behaviour | +| ----------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `allocate(nbytes) -> StorageHandle` | First-fit, aligned. Raises `StorageExhaustedError` when no span fits, `ValueError` for a non-positive size. | +| `free(handle)` | Bump the handle's generation and return the live span to the free list, coalesced. Raises `StaleHandleError` on a stale, forged, or foreign handle, so a double free faults. | +| `resolve(handle) -> int` | The handle's device address, generation-checked. Raises `StaleHandleError`. | +| `stage(device, handle, data)` | Resolve, check `data` against the live grant, and `device.write_dma`. Raises `ValueError` when the payload exceeds the grant. | ### `StorageHandle` @@ -249,31 +250,31 @@ rather than discovered. A non-positive capacity raises `ValueError`. | Method | Behaviour | | ---------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `install(handle)` | Program the handle's grant into slot `handle.id` from the allocator's **live** record. Raises `StaleHandleError` when the allocator will not resolve it, `HandleTableError` for a slot past `capacity` or a base off the 64-byte grid. | -| `free(handle)` | Drop the slot out of liveness. Unconditional on the allocator — freeing can only make the table refuse more, so it is safe whether or not the host has released the span yet. | +| `free(handle)` | Drop the slot out of liveness. Unconditional on the allocator: freeing can only make the table reject more jobs, so it is safe whether or not the host has released the span yet. | | `arm(handle)` | Name the handle the next job resolves, in place of an input address. | | `run_resident_scan(handle, *, row_count, row_bytes, collect, repeats=1, timeout_seconds=60.0)` | `scan_execution.run_resident_scan` with the raw address replaced by the handle: arm once, then run the job `repeats` times. Returns one `collect(device)` per repeat. | -Two authorities, deliberately different. `install` trusts the host — the live -record is what a slot is programmed from, and a handle the host no longer -believes in must never become a grant the device does. `arm` trusts nothing: a -stale handle has to reach the device, or the table is never the thing that -refused it. +`install` and `arm` trust different things on purpose. `install` trusts the +host: the live record is what a slot is programmed from, and a handle the host +no longer considers valid must never become a grant the device accepts. `arm` +checks nothing: a stale handle has to reach the device, or the table never gets +to reject it. ### `HandleFaultError` -The shell refused to start the job on the handle it was armed with. A -`PipelineJobError`, so a caller catching every job failure still catches it, -with its own type because the whole value of the table is that this case is -distinguishable: the job did not run, nothing was read, nothing was written, and +The shell would not start the job on the handle it was armed with. It is a +`PipelineJobError`, so a caller catching every job failure still catches it, and +it has its own type because the point of the table is that this case can be told +apart: the job did not run, nothing was read, nothing was written, and no result register moved. Results for repeats that did run are dropped with it. `code` is a `dau_core.registers.HandleFaultCode`: -| Code | Meaning | -| --------------- | ------------------------------------------------------------------------------------------- | -| `BAD_ID` | The table has no such slot; capacity is planned at build time, not grown at run time. | -| `STALE` | The slot is freed, or live at another generation — that region belongs to someone else now. | -| `OUT_OF_BOUNDS` | The job asks for more bytes than the grant. | +| Code | Meaning | +| --------------- | ------------------------------------------------------------------------------------------ | +| `BAD_ID` | The table has no such slot; capacity is planned at build time, not grown at run time. | +| `STALE` | The slot is freed, or live at another generation; that region belongs to someone else now. | +| `OUT_OF_BOUNDS` | The job asks for more bytes than the grant. | A code outside that set came from the datapath, and the datapath ran; it is raised as a plain `PipelineJobError`. diff --git a/docs/wiki/reference/Execution-Contracts.md b/docs/src/reference/execution-contracts.md similarity index 95% rename from docs/wiki/reference/Execution-Contracts.md rename to docs/src/reference/execution-contracts.md index bff2899..76ee5b8 100644 --- a/docs/wiki/reference/Execution-Contracts.md +++ b/docs/src/reference/execution-contracts.md @@ -66,7 +66,7 @@ run_scan_batches( ``` Run one job per batch with ping-pong input staging: batch *i + 1* DMAs into the -alternate region — and, for a lazy iterable, is produced — while job *i* computes. +alternate region (and, for a lazy iterable, is produced) while job *i* computes. Returns the `collect(device)` values in batch order. Per batch the runner writes `INPUT_ADDRESS_LOW`/`HIGH` and `INPUT_LENGTH_LOW`, @@ -108,17 +108,16 @@ Run the job over rows already on the device, uploading nothing. Per repeat the runner writes `INPUT_ADDRESS_LOW`/`HIGH` and `INPUT_LENGTH_LOW`, sets `JOB_CONTROL` to `START`, waits for DONE, and calls `collect`. Returns one `collect(device)` per repeat rather than one merged answer, so a caller can -assert every run is bit-identical instead of trusting that a rerun of the same -bytes must be. +check that every run returns identical results instead of assuming it. `row_count`, `row_bytes`, and `repeats` must be positive, and `address` must be -on the `DMA_ALIGN_BYTES` grid — an unaligned region is a silent-wrong-bytes -failure on silicon. All four raise `ValueError`. +on the `DMA_ALIGN_BYTES` grid, because an unaligned region returns wrong bytes on +silicon without any error. All four raise `ValueError`. `address` is raw, and a raw address outlives the allocation it came from. `HandleTable.run_resident_scan` is this same run with a generation-checked handle in its place; see -[the handle table](Device-and-Discovery#handle-table). +the handle table in [device and discovery](device-and-discovery.md). ### `key_partitioned_scan_batches` @@ -134,7 +133,7 @@ key_partitioned_scan_batches( Build key-confined batches by longest-processing-time-first greedy assignment: the largest key group goes into the currently smallest batch. Records must be -ordered by key — a key appearing in more than one run raises `ValueError`. Record +ordered by key: a key appearing in more than one run raises `ValueError`. Record order within each key is preserved. Creating the iterable consumes nothing. First iteration consumes one complete @@ -156,7 +155,7 @@ key_range_scan_batches( ) -> Iterator[ScanBatch] ``` -Split records into batches each holding at most `key_capacity` distinct keys — the +Split records into batches each holding at most `key_capacity` distinct keys, the host half of the hash-mode grouped tile's overflow contract. The sorted key space is partitioned into ranges of `key_capacity` keys and one batch is emitted per range. @@ -244,7 +243,7 @@ A host-side mirror of the `PIPELINE_*` configuration registers, as a dataclass. | `map_on_key` | `bool` | `False` | `filter_op_word()`, `map_op_word()`, and `map_constant_word()` produce the packed -register values. A `None` `map_config` packs as ADD 0 on the value lane — +register values. A `None` `map_config` packs as ADD 0 on the value lane, the identity. `configure_pipeline(device, config)` writes all five `PIPELINE_*` registers. @@ -290,8 +289,8 @@ registry entry. `BAR_ROW_BYTES` (16) and `BAR_RECORD_BYTES` (48) come from the same module. -`staging` is the board's `DdrStaging` — its input regions, output region, sizes, -and alignment — passed in from the composed platform. It is a keyword argument +`staging` is the board's `DdrStaging` (its input regions, output region, sizes, +and alignment), passed in from the composed platform. It is a keyword argument rather than a module constant because addresses and region sizes are board geometry, and a package-level default is how one board's memory map ends up applied to another. @@ -318,4 +317,4 @@ host that assumed four lanes on a shell advertising its own count is the reason the constant is gone. `record_bytes` defaults to the fused bar shell's record; key-partitioned scan shells reuse the runner with their own width. Pass `validate_record_count=False` for a lane whose count register latches an *input* -row count and emits one fixed-width record — the `result_length == count * record_bytes` invariant does not hold there, and `RESULT_LENGTH` is trusted alone. +row count and emits one fixed-width record. The `result_length == count * record_bytes` invariant does not hold there, so only `RESULT_LENGTH` is used. diff --git a/docs/wiki/reference/Stream-Codec.md b/docs/src/reference/stream-codec.md similarity index 88% rename from docs/wiki/reference/Stream-Codec.md rename to docs/src/reference/stream-codec.md index 677b078..31bae72 100644 --- a/docs/wiki/reference/Stream-Codec.md +++ b/docs/src/reference/stream-codec.md @@ -1,9 +1,10 @@ # Stream codec reference `dau_driver.stream_codec` encodes typed columns as DAU stream batches and decodes -them back. The wire format — `BatchHeader`, `ColumnDescriptor`, +them back. The wire format (`BatchHeader`, `ColumnDescriptor`, `OperationDescriptor`, `LogicalType`, `StreamFlag`, `RecordBatchLimits`, -`InvalidStreamError` — is defined in `dau_core.stream`; this module marshals it. +`InvalidStreamError`) and the per-type widths (`LOGICAL_TYPE_BYTE_WIDTHS`) are +defined in `dau_core.stream`; this module marshals them. Batches are 64-byte aligned throughout: the descriptor table starts at `BatchHeader.SIZE_BYTES`, each payload begins on a 64-byte boundary, and the @@ -79,12 +80,12 @@ raises `InvalidStreamError` naming the column. Requires `pyarrow`. All methods are static. -| Method | Returns | -| --------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------- | -| `decode_column(data)` | `DecodedStreamColumn` — a single-column batch. Requires `column_count == 1`. | -| `decode_record_batch(data, *, limits=DEFAULT_RECORD_BATCH_LIMITS)` | `DecodedRecordBatch` — the N-column counterpart. | -| `decode_record_batch_to_arrow(data, *, names=None, limits=DEFAULT_RECORD_BATCH_LIMITS)` | A PyArrow `RecordBatch`. Column names default to `f{field_id}`. | -| `decode_scalar_result(data)` | `DecodedStreamResult` — a one-row batch with its `value` unpacked. | +| Method | Returns | +| --------------------------------------------------------------------------------------- | --------------------------------------------------------------------------- | +| `decode_column(data)` | `DecodedStreamColumn`: a single-column batch. Requires `column_count == 1`. | +| `decode_record_batch(data, *, limits=DEFAULT_RECORD_BATCH_LIMITS)` | `DecodedRecordBatch`: the N-column counterpart. | +| `decode_record_batch_to_arrow(data, *, names=None, limits=DEFAULT_RECORD_BATCH_LIMITS)` | A PyArrow `RecordBatch`. Column names default to `f{field_id}`. | +| `decode_scalar_result(data)` | `DecodedStreamResult`: a one-row batch with its `value` unpacked. | Decoding validates the stream version (0.1), the header length, 64-byte alignment of the descriptor and payload offsets, every payload window against the diff --git a/docs/wiki/tutorial/First-Device.md b/docs/src/tutorial/first-device.md similarity index 87% rename from docs/wiki/tutorial/First-Device.md rename to docs/src/tutorial/first-device.md index c34a3a0..6896e76 100644 --- a/docs/wiki/tutorial/First-Device.md +++ b/docs/src/tutorial/first-device.md @@ -1,15 +1,15 @@ # Talk to a device without any hardware -In this tutorial we will drive a DAU device end to end — read its identity +In this tutorial we will drive a DAU device end to end (read its identity registers, check the health of its register path, validate a DMA transfer, and -run one aggregation job — without a board, a bitstream, or a driver install. +run one aggregation job) without a board, a bitstream, or a driver install. Everything runs against the fake transport that ships with the package for exactly this purpose. Work in a Python session with `dau-driver` installed. Each step shows the code to run and an **Output** block with what it prints. -## Step 1 — Open a device +## Step 1: Open a device A `DauDevice` is a validated handle. On a real host, `discover_devices()` produces one; here we build one over an in-memory transport so nothing is @@ -35,11 +35,11 @@ backend: linux-xdma platform: DPV1 ``` -The paths passed to `fake_device` are never opened — the transport intercepts +The paths passed to `fake_device` are never opened; the transport intercepts every read and write. Anything you do below would work identically against a real device. -## Step 2 — Read what the device says it can do +## Step 2: Read what the device says it can do Capabilities are decoded from the device's own register block, so they describe the bitstream that is loaded, not the driver. Run: @@ -58,7 +58,6 @@ for operator in device.capabilities.operators: ```text alignment_bytes: 64 max_transfer_bytes: 4096 - SUM ['INT32'] -> INT64 MIN ['INT32'] -> INT64 MAX ['INT32'] -> INT64 @@ -70,7 +69,7 @@ respect in Step 4. And the operator list is typed: `SUM` over `INT32` produces `INT64`, and asking for any other output type is an error the driver catches before touching the bus. -## Step 3 — Read the identity and status registers +## Step 3: Read the identity and status registers Register access goes through the same transport. Preload a few values so the fake has something to report, then read them back. Run: @@ -99,11 +98,11 @@ status: 0b111 last error: 0 DeviceCounters(h2c_bytes=0, c2h_bytes=0, rows=1000000, batches=12, stalls=0) ``` -`0x44415531` is the DAU magic word — it is how discovery tells a DAU shell from +`0x44415531` is the DAU magic word, which is how discovery tells a DAU shell from any other XDMA bitstream. `DeviceStatus` holds the status word as read, and the counters are 64-bit values read as two 32-bit register accesses. -## Step 4 — Check the register path is healthy +## Step 4: Check the register path is healthy A dead AXI-lite path does not fail; it returns all ones, which looks like a perfectly good number. `check_bar_health()` is the check for that. Run: @@ -142,7 +141,7 @@ UnhealthyBarError: MAGIC read all ones from /tmp/xdma0_user; BAR/AXI-lite path m Call this before and after any sequence you care about. It is cheap and it turns a wrong number into a named failure. -## Step 5 — Validate a transfer before opening a channel +## Step 5: Validate a transfer before opening a channel Transfers are shape-checked against the device's own DMA constraints, before any channel is opened. Run: @@ -167,15 +166,15 @@ h2c 0x40 128 xdma0_h2c_0 ``` 100 bytes is not a multiple of the device's 64-byte alignment, so it never -reaches the bus. The valid transfer resolves to the H2C channel path — one +reaches the bus. The valid transfer resolves to the H2C channel path, as one `DmaTransfer` describing exactly what is about to happen. -## Step 6 — Run an aggregation job +## Step 6: Run an aggregation job Now the whole round trip: encode a column into a DAU stream batch, DMA it in, let the device compute, and decode the scalar result. The fake device needs something to play the part of the hardware, which is what -`golden_aggregation_trigger` does — it decodes the input batch and stages the +`golden_aggregation_trigger` does: it decodes the input batch and stages the golden answer where the C2H read will find it. Run: ```python @@ -237,7 +236,7 @@ Eight int32 values became a 256-byte stream batch: a header, a column descriptor an operation descriptor, and the payload, all padded onto the 64-byte grid the DMA engine requires. -## Step 7 — Watch a capability check refuse +## Step 7: Watch a capability check stop a job The driver checks the requested operation against the device's advertised capabilities *before* any DMA. Run: @@ -276,13 +275,12 @@ StreamExecutionError: device does not support average for int32 h2c writes since: 1 ``` -The write count did not go up. Nothing was staged, because the refusal happened -first. +The write count did not go up. Nothing was staged, because the check ran first. ## What you have done -You have exercised the Python host surface — discovery handle, capabilities, -registers, status, BAR health, transfer validation, and a full job round trip — +You have exercised the Python host surface (discovery handle, capabilities, +registers, status, BAR health, transfer validation, and a full job round trip) against a transport that needs no hardware. This is also how the package's own test suite runs: every Python execution path has to work against a fake, so hardware-only behaviour stays behind the `DeviceTransport` boundary and is @@ -291,8 +289,8 @@ tested on a board. From here: - To run many batches with staging overlapped against compute, see - [Run a scan over many batches](First-Scan-Run). + [Run a scan over many batches](first-scan-run.md). - To find real devices on a real host, see - [Find and check a device](Find-a-Device). + [Find and check a device](../how-to/find-a-device.md). - For why the fake exists at all, read - [The transport boundary](Transport-Boundary). + [The transport boundary](../explanation/transport-boundary.md). diff --git a/docs/wiki/tutorial/First-Scan-Run.md b/docs/src/tutorial/first-scan-run.md similarity index 84% rename from docs/wiki/tutorial/First-Scan-Run.md rename to docs/src/tutorial/first-scan-run.md index 84f29fa..c0f564a 100644 --- a/docs/wiki/tutorial/First-Scan-Run.md +++ b/docs/src/tutorial/first-scan-run.md @@ -5,14 +5,14 @@ the streaming runner, and watch it ping-pong between two device regions so each batch's DMA overlaps the previous batch's compute. Then we will meet the guard that stops a stateful composition from being fed the wrong kind of batch. -As in [Talk to a device without any hardware](First-Device), everything runs -against the fake transport — no board required. Work in a Python session with +As in [Talk to a device without any hardware](first-device.md), everything runs +against the fake transport, with no board required. Work in a Python session with `dau-driver` installed. -## Step 1 — Build some batches +## Step 1: Build some batches A `ScanBatch` is a packed row payload and its row count. Here each row is four -int32 fields — a symbol id and a price, with two fields unused. Run: +int32 fields: a symbol id and a price, with two fields unused. Run: ```python import struct @@ -49,9 +49,9 @@ for batch in batches: ``` Three rows at 16 bytes each is 48 bytes. Note that 48 is not a multiple of the -64-byte DMA grid — hold that thought for Step 2. +64-byte DMA grid; Step 2 comes back to that. -## Step 2 — Run them +## Step 2: Run them The runner needs two distinct device addresses to alternate between, the row width, and a `collect` callback that reads each job's results off the device once @@ -98,13 +98,13 @@ bytes: [64, 64] Three things happened here. -The two batches landed in *different* regions — `0x0` then `0x8000000`. That is +The two batches landed in *different* regions, `0x0` then `0x8000000`. That is the ping-pong: while job 0 computes out of region A, batch 1's DMA is already filling region B. Batch 2 would go back to region A, and so on. The payloads went out as 64 bytes, not 48. The runner pads the DMA up to the -64-byte grid, while `INPUT_LENGTH` — which is what the device actually reads — -stays exact at 48. That is why `collect` returned 48 for each batch. +64-byte grid, while `INPUT_LENGTH`, which is what the device reads, stays exact +at 48. That is why `collect` returned 48 for each batch. `collect` was called once per batch, after that batch's job reported DONE, and the results came back in batch order. @@ -113,9 +113,9 @@ Because `batches` here is a list, all the encoding happened before the run. Pass generator instead and the runner will pull each batch just before it stages it, so the encoding work overlaps compute too. -## Step 3 — Confine each key to one batch +## Step 3: Confine each key to one batch -Some scan compositions carry per-key state across a batch — a grouped aggregation, +Some scan compositions keep per-key state across a batch, a grouped aggregation for instance. Slicing rows by position would then split a key's state across two jobs. `key_partitioned_scan_batches` builds batches where each key lands in exactly one. Run: @@ -157,7 +157,7 @@ assignment is longest-processing-time-first: the largest key group goes into the currently smallest batch. Records must arrive ordered by key, and record order within each key is preserved. -## Step 4 — Meet the guard +## Step 4: Meet the guard `per_key_state` is not advisory. Try running the plain list of batches from Step 1 with it set: @@ -187,7 +187,7 @@ A stateful composition only accepts batches that came from on a flag you could get wrong, so a position-sliced batch cannot reach a stateful job by accident. -## Step 5 — Bound the distinct keys per batch +## Step 5: Bound the distinct keys per batch A different constraint applies to the hash-mode grouped tile: its table holds a fixed number of *distinct keys*, and a batch presenting more is invalidated by the @@ -212,22 +212,22 @@ Two batches: the first holds keys `AAA` and `BBB` (four rows), the second holds counts differ. Because each key lands in exactly one batch, combining the per-batch group records -is a plain concatenation. And unlike Step 3, records need not arrive in key order -— the hash table needs no key ordering, so arrival order is preserved within each -batch. +is a plain concatenation. And unlike Step 3, records need not arrive in key +order: the hash table needs no key ordering, so arrival order is preserved within +each batch. ## What you have done You have run a multi-batch scan with overlapped staging, built key-confined batches two different ways, and seen the guard that keeps a stateful composition -from silently reading half its state. +from reading half its state. From here: - For choosing between these batching strategies on real work, and for spreading batches across several cards, see - [Run scan jobs](Run-Scan-Jobs). + [Run scan jobs](../how-to/run-scan-jobs.md). - For the full signature of every runner, see the - [execution contracts reference](Execution-Contracts). + [execution contracts reference](../reference/execution-contracts.md). - For why the overlap is shaped this way, read - [Overlapped staging](Overlapped-Staging). + [Overlapped staging](../explanation/overlapped-staging.md). diff --git a/docs/wiki/Installation.md b/docs/wiki/Installation.md deleted file mode 100644 index 665616b..0000000 --- a/docs/wiki/Installation.md +++ /dev/null @@ -1,34 +0,0 @@ -## Prerequisites - -Python 3.12 or later. On a host with a board, the Linux XDMA module from -`kernel/linux-xdma/` or the macOS DriverKit extension from `kernel/macos-xdma/` -must be loaded; without one, the package still imports and runs against its fake -transport. - -## Install from the repository - -There is no package index release yet. Install from a clone, or straight from -GitHub: - -```bash -pip install git+https://github.com/dau-dev/dau-driver.git -``` - -For development, clone and install editable with the development extras: - -```bash -git clone https://github.com/dau-dev/dau-driver.git -cd dau-driver -make develop -``` - -The install builds a small native extension with nanobind, so a C++ toolchain is -required on the installing machine. The transports themselves are Python; the -extension only reports its own backend and ABI version today. - -## Kernel drivers - -The host-side drivers live under [`kernel/`](https://github.com/dau-dev/dau-driver/tree/main/kernel) -with their own build and load instructions. Whoever can open the character -devices (Linux) or the user client (macOS) can read and write the card's memory -and registers; the operating system's device permissions are the access control. diff --git a/docs/wiki/_Footer.md b/docs/wiki/_Footer.md deleted file mode 100644 index ff46823..0000000 --- a/docs/wiki/_Footer.md +++ /dev/null @@ -1 +0,0 @@ -_This wiki is autogenerated. To made updates, open a PR against the original source file in [`docs/wiki`](https://github.com/dau-dev/dau-driver/tree/main/docs/wiki)._ diff --git a/docs/wiki/_Sidebar.md b/docs/wiki/_Sidebar.md deleted file mode 100644 index 03cc0e6..0000000 --- a/docs/wiki/_Sidebar.md +++ /dev/null @@ -1,41 +0,0 @@ - - -**[Home](Home)** - -**Get Started** - -- [Installation](Installation) -- [Contributing](Contribute) -- [Development Setup](Local-Development-Setup) -- [Build from Source](Build-from-Source) - -**Tutorial** - -- [First Device](First-Device) -- [First Scan Run](First-Scan-Run) - -**How-to guides** - -- [Find a Device](Find-a-Device) -- [Move Data](Move-Data) -- [Run Scan Jobs](Run-Scan-Jobs) -- [Allocate Device Storage](Allocate-Device-Storage) -- [Arm the Deadman](Arm-the-Deadman) - -**Reference** - -- [Device and Discovery](Device-and-Discovery) -- [Execution Contracts](Execution-Contracts) -- [Stream Codec](Stream-Codec) - -**Explanation** - -- [The Transport Boundary](Transport-Boundary) -- [Overlapped Staging](Overlapped-Staging) -- [Storage Handles](Storage-Handles) diff --git a/docs/wiki/contribute/Build-from-Source.md b/docs/wiki/contribute/Build-from-Source.md deleted file mode 100644 index 5cbf11b..0000000 --- a/docs/wiki/contribute/Build-from-Source.md +++ /dev/null @@ -1,103 +0,0 @@ -`dau-driver` is written in Python. While prebuilt wheels are provided for end users, it is also straightforward to build `dau-driver` from either the Python [source distribution](https://packaging.python.org/en/latest/specifications/source-distribution-format/) or the GitHub repository. - -- [Make commands](#make-commands) -- [Prerequisites](#prerequisites) -- [Clone](#clone) -- [Install Python dependencies](#install-python-dependencies) -- [Build](#build) -- [Lint and Autoformat](#lint-and-autoformat) -- [Testing](#testing) - -## Make commands - -As a convenience, `dau-driver` uses a `Makefile` for commonly used commands. You can print the main available commands by running `make` with no arguments - -```bash -> make - -build build the library -clean clean the repository -fix run autofixers -install install library -lint run lints -test run the tests -``` - -## Prerequisites - -`dau-driver` has a few system-level dependencies which you can install from your machine package manager. Other package managers like `conda`, `nix`, etc, should also work fine. - -## Clone - -Clone the repo with: - -```bash -git clone https://github.com/dau-dev/dau-driver.git -cd dau-driver -``` - -## Install Python dependencies - -Python build and develop dependencies are specified in the `pyproject.toml`, but you can manually install them: - -```bash -make requirements -``` - -Note that these dependencies would otherwise be installed normally as part of [PEP517](https://peps.python.org/pep-0517/) / [PEP518](https://peps.python.org/pep-0518/). - -## Build - -Build the python project in the usual manner: - -```bash -make build -``` - -## Lint and Autoformat - -`dau-driver` has linting and auto formatting. - -| Language | Linter | Autoformatter | Description | -| :------- | :---------- | :------------ | :---------- | -| Python | `ruff` | `ruff` | Style | -| Markdown | `mdformat` | `mdformat` | Style | -| Markdown | `codespell` | | Spelling | - -**Python Linting** - -```bash -make lint-py -``` - -**Python Autoformatting** - -```bash -make fix-py -``` - -**Documentation Linting** - -```bash -make lint-docs -``` - -**Documentation Autoformatting** - -```bash -make fix-docs -``` - -## Testing - -`dau-driver` has extensive Python tests. The tests can be run via `pytest`. First, install the Python development dependencies with - -```bash -make develop -``` - -**Python** - -```bash -make test -``` diff --git a/docs/wiki/contribute/Contribute.md b/docs/wiki/contribute/Contribute.md deleted file mode 100644 index b9d18a3..0000000 --- a/docs/wiki/contribute/Contribute.md +++ /dev/null @@ -1,15 +0,0 @@ -Contributions are welcome on this project. We distribute under the terms of the [Apache 2.0 license](https://github.com/dau-dev/dau-driver/blob/main/LICENSE). - -> [!NOTE] -> -> `dau-driver` requires [Developer Certificate of Origin](https://en.wikipedia.org/wiki/Developer_Certificate_of_Origin) for all contributions. -> This is enforced by a [Probot GitHub App](https://probot.github.io/apps/dco/), which checks that commits are "signed". -> Read [instructions to configure commit signing](Local-Development-Setup#configure-commit-signing). - -For **bug reports** or **small feature requests**, please open an issue on our [issues page](https://github.com/dau-dev/dau-driver/issues). - -For **questions** or to discuss **larger changes or features**, please use our [discussions page](https://github.com/dau-dev/dau-driver/discussions). - -For **contributions**, please see our [developer documentation](Local-Development-Setup). We have `help wanted` and `good first issue` tags on our issues page, so these are a great place to start. - -For **documentation updates**, make PRs that update the pages in `/docs/wiki`. The documentation is pushed to the GitHub wiki automatically through a GitHub workflow. Note that direct updates to this wiki will be overwritten. diff --git a/docs/wiki/contribute/Local-Development-Setup.md b/docs/wiki/contribute/Local-Development-Setup.md deleted file mode 100644 index d2771cb..0000000 --- a/docs/wiki/contribute/Local-Development-Setup.md +++ /dev/null @@ -1,55 +0,0 @@ -## Table of Contents - -- [Table of Contents](#table-of-contents) -- [Step 1: Build from Source](#step-1-build-from-source) -- [Step 2: Configuring Git and GitHub for Development](#step-2-configuring-git-and-github-for-development) - - [Create your fork](#create-your-fork) - - [Configure remotes](#configure-remotes) - - [Authenticating with GitHub](#authenticating-with-github) -- [Guidelines](#guidelines) - -## Step 1: Build from Source - -To work on `dau-driver`, you are going to need to build it from source. See -[Build from Source](Build-from-Source) for -detailed build instructions. - -Once you've built `dau-driver` from a `git` clone, you will also need to -configure `git` and your GitHub account for `dau-driver` development. - -## Step 2: Configuring Git and GitHub for Development - -### Create your fork - -The first step is to create a personal fork of `dau-driver`. To do so, click -the "fork" button at https://github.com/dau-dev/dau-driver, or just navigate -[here](https://github.com/dau-dev/dau-driver/fork) in your browser. Set the -owner of the repository to your personal GitHub account if it is not -already set that way and click "Create fork". - -### Configure remotes - -Next, you should set some names for the `git` remotes corresponding to -main dau-dev repository and your fork. See the [GitHub Docs](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/working-with-forks/configuring-a-remote-repository-for-a-fork) for more information. - -### Authenticating with GitHub - -If you have not already configured `ssh` access to GitHub, you can find -instructions to do so -[here](https://docs.github.com/en/authentication/connecting-to-github-with-ssh), -including instructions to create an SSH key if you have not done -so. Authenticating with SSH is usually the easiest route. If you are working in -an environment that does not allow SSH connections to GitHub, you can look into -[configuring a hardware -passkey](https://docs.github.com/en/authentication/authenticating-with-a-passkey/about-passkeys) -or adding a [personal access -token](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens) -to avoid the need to type in your password every time you push to your fork. - -## Guidelines - -After developing a change locally, ensure that both [lints](Build-from-Source#lint-and-autoformat) and [tests](Build-from-Source#testing) pass. Commits should be squashed into logical units, and all commits must be signed (e.g. with the `-s` git flag). We require [Developer Certificate of Origin](https://en.wikipedia.org/wiki/Developer_Certificate_of_Origin) for all contributions. - -If your work is still in-progress, open a [draft pull request](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/about-pull-requests#draft-pull-requests). Otherwise, open a normal pull request. It might take a few days for a maintainer to review and provide feedback, so please be patient. If a maintainer asks for changes, please make said changes and squash your commits if necessary. If everything looks good to go, a maintainer will approve and merge your changes for inclusion in the next release. - -Please note that non substantive changes, large changes without prior discussion, etc, are not accepted and pull requests may be closed. diff --git a/pyproject.toml b/pyproject.toml index 867e94e..c2884e4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -200,3 +200,24 @@ section-order = [ "F401", "F403", ] + +[tool.yardang] +title = "dau-driver" +root = "README.md" +pages = [ + "docs/src/tutorial/first-device.md", + "docs/src/tutorial/first-scan-run.md", + "docs/src/how-to/find-a-device.md", + "docs/src/how-to/move-data.md", + "docs/src/how-to/run-scan-jobs.md", + "docs/src/how-to/allocate-device-storage.md", + "docs/src/how-to/arm-the-deadman.md", + "docs/src/how-to/develop.md", + "docs/src/reference/device-and-discovery.md", + "docs/src/reference/execution-contracts.md", + "docs/src/reference/stream-codec.md", + "docs/src/explanation/transport-boundary.md", + "docs/src/explanation/overlapped-staging.md", + "docs/src/explanation/storage-handles.md", +] +use-autoapi = true From 887970c1f822abc923226998b2955c69bea6072e Mon Sep 17 00:00:00 2001 From: Tim Paine <3105306+timkpaine@users.noreply.github.com> Date: Fri, 23 Oct 2026 15:46:12 -0400 Subject: [PATCH 2/2] Describe the package as the PCIe driver for dau projects Signed-off-by: Tim Paine <3105306+timkpaine@users.noreply.github.com> --- .copier-answers.yaml | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.copier-answers.yaml b/.copier-answers.yaml index a01bcfb..3010480 100644 --- a/.copier-answers.yaml +++ b/.copier-answers.yaml @@ -8,7 +8,7 @@ add_vcpkg: false add_wiki: false email: dev@dau.dev github: dau-dev -project_description: PCIe Driver for dau +project_description: PCIe driver for dau projects project_name: dau-driver python_version_primary: '3.12' team: the dau-driver authors diff --git a/pyproject.toml b/pyproject.toml index c2884e4..f7ecaff 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,7 +11,7 @@ name = "dau-driver" authors = [ {name = "the dau-driver authors", email = "dev@dau.dev"}, ] -description = "PCIe Driver for dau" +description = "PCIe driver for dau projects" readme = "README.md" license = { text = "Apache-2.0" } version = "0.1.0"