diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 8410595..66d2518 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -8,7 +8,6 @@ on: - v* paths-ignore: - LICENSE - - README.md pull_request: branches: - main diff --git a/Makefile b/Makefile index a0551ed..3866fd0 100644 --- a/Makefile +++ b/Makefile @@ -52,7 +52,7 @@ check-dist: ## check python sdist and wheel with check-dist check-types: ## check python types with ty ty check --python $$(which python) -checks: check-dist +checks: check-dist check-types # Alias check: checks @@ -101,7 +101,7 @@ dist-check: ## run python dist checker with twine dist: clean dist-build dist-check ## build all dists -publish: dist ## publish python assets +publish: dist ## build the distributions a release would publish (publishing is a separate step) ######### # CLEAN # diff --git a/README.md b/README.md index ea05443..c1b0598 100644 --- a/README.md +++ b/README.md @@ -5,14 +5,37 @@ Host utilities for FPGA bench machines [![Build Status](https://github.com/dau-dev/dau-utils/actions/workflows/build.yaml/badge.svg?branch=main&event=push)](https://github.com/dau-dev/dau-utils/actions/workflows/build.yaml) [![codecov](https://codecov.io/gh/dau-dev/dau-utils/branch/main/graph/badge.svg)](https://codecov.io/gh/dau-dev/dau-utils) [![License](https://img.shields.io/github/license/dau-dev/dau-utils)](https://github.com/dau-dev/dau-utils) -[![PyPI](https://img.shields.io/pypi/v/dau-utils.svg)](https://pypi.python.org/pypi/dau-utils) ## Overview -Two small command-line tools for the Linux machine an FPGA card is plugged -into. Each one does a single thing and leaves the decisions (which devices, -when to arm) to the caller. Higher layers such as `dau-build` run them -from their hardware command plans; you can also run them by hand. +Small command-line tools for the Linux machine an FPGA card is plugged into. +Each one does a single thing and leaves the decisions (which devices, when to +arm) to the caller. Higher layers such as `dau-build` run them from their +hardware command plans; you can also run them by hand. Both tools print what +they would do with `--dry-run`, which needs no privilege, and refuse an +ambiguous request rather than guessing. + +## Requirements + +- Linux with systemd (the deadman schedules a transient timer) and the + `sysrq` trigger enabled for `b` (`kernel.sysrq` must allow it). +- `sudo` without a password prompt for the invoking user, for `systemd-run` + and `systemctl`: the deadman runs them with `sudo -n` and treats a prompt + as a failure. The runtime-PM tool writes sysfs directly and is run under + `sudo` itself. +- `pciutils` (`lspci`) for pattern-based device discovery. +- Python 3.11 or later. + +## Installation + +There is no package index release yet. Install from a clone, or straight from +GitHub: + +```bash +pip install git+https://github.com/dau-dev/dau-utils.git +``` + +For development, clone and run `make develop`. ## Deadman: recover from a hung PCIe device @@ -29,12 +52,26 @@ dau-utils-deadman disarm # operation returned: cancel the reboot dau-utils-deadman status ``` +The timeout is a policy choice: long enough for the operation to finish, +short enough that a wedged host comes back without anyone present. The +default is 180 seconds. + If the host hangs before `disarm`, the timer fires `sysrq-b` (`emergency_restart()`), which resets the machine without going through the -stuck driver. The same operations are available from Python: -`dau_utils.deadman.arm`, `disarm`, `is_armed`, and an `armed()` context -manager. Failures raise `DeadmanError`; if you see one, assume the host is -not protected. +stuck driver. `arm` confirms the timer is active before it reports success, +and `disarm` confirms it is gone; a missing command, a failed `systemd-run`, +or a call that does not answer within fifteen seconds raises `DeadmanError`. +If you see one, assume the host is not protected, or not yet safe. + +The unit name is the tool's own (`dau-deadman`, or `dau-deadman-` to +run more than one); it will not stop or reset any other unit. + +The same operations are available from Python: `dau_utils.deadman.arm`, +`disarm`, `is_armed`, and an `armed()` context manager. The context manager +disarms on the way out whatever the block did, including on an exception, +which is right for Python work. A hardware plan that fails a step must leave +the timer armed, so that the host reboots if the step wedged it; the plans in +`dau-build` call `arm` and `disarm` directly for that reason. Arm it before every PCIe rescan, flash or register probe against a device that has hung before. Synthesis and other work that does not touch PCIe does @@ -43,23 +80,29 @@ not need it. ## Runtime PM: hold and release PCIe power management `dau-utils-pci-runtime-pm` holds or releases Linux PCI runtime power -management for devices matched against `lspci -Dnn`. You choose the patterns; -the tool does the sysfs writes: +management for devices named by PCI address or matched against `lspci -Dnn` +by text pattern. You choose the devices; the tool writes `power/control` and +`d3cold_allowed` under the sysfs device root: ```bash -dau-utils-pci-runtime-pm hold --pattern Thunderbolt --pattern 10ee:7011 -dau-utils-pci-runtime-pm release --pattern Thunderbolt --pattern 10ee:7011 +sudo dau-utils-pci-runtime-pm hold --device 0000:04:00.0 +sudo dau-utils-pci-runtime-pm hold --pattern "Downstream Bridge" --pattern abcd:ef01 +sudo dau-utils-pci-runtime-pm release --pattern "Downstream Bridge" --pattern abcd:ef01 ``` -`--dry-run` prints the writes it would make without touching sysfs. +A device must be a PCI address (`dddd:bb:dd.f`), which is the only form that +stays inside the sysfs root. `--device` and `--pattern` are alternatives, not +a combination. Every attribute in the plan is checked before the first write, +so a device is never left half held. A pattern that matches nothing is +tolerated, and said on stderr, because the device being recovered may not +enumerate. + +`--dry-run` prints the writes without touching sysfs and needs no privilege. ## Development ```bash -pip install -e .[develop] +make develop python -m pytest dau_utils/tests python -m ruff check dau_utils ``` - -> [!NOTE] -> This library was generated using [copier](https://copier.readthedocs.io/en/stable/) from the [Base Python Project Template repository](https://github.com/python-project-templates/base). diff --git a/dau_utils/deadman.py b/dau_utils/deadman.py index b2e7c31..973b32c 100644 --- a/dau_utils/deadman.py +++ b/dau_utils/deadman.py @@ -1,9 +1,9 @@ """Prime a forced reboot before a risky PCIe operation, cancel it on success. -The dpv1 wedge class that needs a power cycle is a driver/PCIe hang where the -kernel and systemd stay alive -- they keep petting the hardware watchdog, so -systemd's ``RuntimeWatchdogSec`` never fires, yet ``systemctl reboot`` hangs on -the stuck device. The recovery is ``sysrq-b`` (``emergency_restart()``), which +The hang this recovers from is a driver or PCIe wedge where the kernel and +systemd stay alive: they keep petting the hardware watchdog, so systemd's +``RuntimeWatchdogSec`` never fires, yet ``systemctl reboot`` blocks on the +stuck device. The recovery is ``sysrq-b`` (``emergency_restart()``), which resets immediately without touching the wedged driver. ``arm`` schedules that reset as a transient systemd timer so it survives the @@ -11,18 +11,34 @@ flash, or register probe and ``disarm`` once it returns cleanly. If the box wedges before ``disarm``, the timer fires and reboots it. A full kernel lock (systemd itself dead) is still caught by systemd's own hardware watchdog. + +Every call into systemd is bounded and non-interactive (``sudo -n``), and a +call that fails, hangs or is missing surfaces as :class:`DeadmanError`: the +caller must not treat the host as protected, or as safe, on anything less +than a confirmed answer. """ from __future__ import annotations import argparse +import re +import shlex import subprocess import sys -from collections.abc import Iterator, Sequence +from collections.abc import Generator, Sequence from contextlib import contextmanager DEFAULT_UNIT = "dau-deadman" +# policy: long enough for a reprogram and rescan to complete, short enough +# that a wedged host comes back without anyone present DEFAULT_TIMEOUT_S = 180 +# systemd answers in well under a second; a call that takes this long is +# the hang this tool exists to survive, and must not hang the tool itself +COMMAND_TIMEOUT_S = 15 + +# the only units this tool may create, stop or reset: its own namespace. An +# arbitrary --unit would let `disarm` stop any service on the host. +_UNIT = re.compile(r"dau-deadman(-[a-z0-9]+)*") # sysrq 'b' = emergency_restart(): reboot past a wedged driver, where a clean # `systemctl reboot` would block on it. Best-effort sync first so a live @@ -39,12 +55,21 @@ class DeadmanError(RuntimeError): """A deadman operation could not be confirmed -- treat the host as unsafe.""" +def check_unit(unit: str) -> str: + """The unit name, if it is in this tool's namespace.""" + if not _UNIT.fullmatch(unit): + raise ValueError(f"deadman unit must be dau-deadman or dau-deadman-, got {unit!r}") + return unit + + def arm_command(timeout_s: int = DEFAULT_TIMEOUT_S, *, unit: str = DEFAULT_UNIT) -> tuple[str, ...]: """The ``systemd-run`` invocation that schedules the reset ``timeout_s`` from now.""" + check_unit(unit) if timeout_s < 1: raise ValueError(f"deadman timeout must be at least 1 second, got {timeout_s}") return ( "sudo", + "-n", "systemd-run", f"--unit={unit}", f"--on-active={timeout_s}", @@ -58,53 +83,92 @@ def arm_command(timeout_s: int = DEFAULT_TIMEOUT_S, *, unit: str = DEFAULT_UNIT) def disarm_commands(*, unit: str = DEFAULT_UNIT) -> tuple[tuple[str, ...], ...]: """Stop the pending timer/service and clear any failed state, idempotently.""" + check_unit(unit) return ( - ("sudo", "systemctl", "stop", f"{unit}.timer", f"{unit}.service"), - ("sudo", "systemctl", "reset-failed", f"{unit}.timer", f"{unit}.service"), + ("sudo", "-n", "systemctl", "stop", f"{unit}.timer", f"{unit}.service"), + ("sudo", "-n", "systemctl", "reset-failed", f"{unit}.timer", f"{unit}.service"), ) def status_command(*, unit: str = DEFAULT_UNIT) -> tuple[str, ...]: """List the pending deadman timer, if armed.""" + check_unit(unit) return ("systemctl", "list-timers", "--all", f"{unit}.timer") +def _run(command: Sequence[str], *, check: bool) -> subprocess.CompletedProcess: + """One bounded, captured call; every failure mode is a DeadmanError.""" + try: + return subprocess.run(tuple(command), check=check, capture_output=True, text=True, timeout=COMMAND_TIMEOUT_S) + except FileNotFoundError as error: + raise DeadmanError(f"{command[0]} is not installed: {error}") from error + except subprocess.TimeoutExpired as error: + raise DeadmanError(f"{shlex.join(command)} did not answer within {COMMAND_TIMEOUT_S}s") from error + except subprocess.CalledProcessError as error: + detail = (error.stderr or error.stdout or "").strip() + raise DeadmanError(f"{shlex.join(command)} failed with exit {error.returncode}: {detail}") from error + except OSError as error: + raise DeadmanError(f"{shlex.join(command)} could not run: {error}") from error + + def _is_active(name: str) -> bool: - """True only if systemctl positively reports ``name`` running. A failed - query (D-Bus down, sudo denied) is treated as active -- we cannot claim a - unit is stopped unless systemctl confirms it.""" - result = subprocess.run(("systemctl", "is-active", name), check=False, capture_output=True, text=True) + """True only if systemctl positively reports ``name`` not running. A failed + query (D-Bus down, sudo denied, a hang) is treated as active -- we cannot + claim a unit is stopped unless systemctl confirms it.""" + try: + result = _run(("systemctl", "is-active", name), check=False) + except DeadmanError: + return True return result.stdout.strip() not in _INACTIVE_STATES def is_armed(*, unit: str = DEFAULT_UNIT) -> bool: """True if a deadman timer or its service is still live for ``unit``.""" + check_unit(unit) return _is_active(f"{unit}.timer") or _is_active(f"{unit}.service") def arm(timeout_s: int = DEFAULT_TIMEOUT_S, *, unit: str = DEFAULT_UNIT) -> None: - """Schedule the forced reset. Refuses to stomp an already-armed timer so - concurrent callers cannot silently cancel each other's protection; clears - only inactive stale state before scheduling.""" + """Schedule the forced reset and confirm the timer is live. Refuses to + stomp an already-armed timer so concurrent callers cannot silently cancel + each other's protection; clears only inactive stale state before + scheduling.""" + command = arm_command(timeout_s, unit=unit) if is_armed(unit=unit): raise DeadmanError(f"{unit} is already armed; disarm it before arming again") - subprocess.run(("sudo", "systemctl", "reset-failed", f"{unit}.timer", f"{unit}.service"), check=False, capture_output=True) - subprocess.run(arm_command(timeout_s, unit=unit), check=True) + _run(disarm_commands(unit=unit)[1], check=False) + _run(command, check=True) + if not is_armed(unit=unit): + raise DeadmanError(f"systemd-run returned but {unit}.timer is not active; the host is NOT protected") def disarm(*, unit: str = DEFAULT_UNIT) -> None: """Cancel the pending reset and confirm it is gone. Raises ``DeadmanError`` if the timer cannot be verified inactive -- the caller must not treat the host as safe until this returns cleanly.""" - subprocess.run(("sudo", "systemctl", "stop", f"{unit}.timer", f"{unit}.service"), check=False, capture_output=True) + stop, reset_failed = disarm_commands(unit=unit) + _run(stop, check=False) if is_armed(unit=unit): raise DeadmanError(f"{unit} still armed after stop; reset may still fire -- intervene before trusting the host") - subprocess.run(("sudo", "systemctl", "reset-failed", f"{unit}.timer", f"{unit}.service"), check=False, capture_output=True) + _run(reset_failed, check=False) + + +def status(*, unit: str = DEFAULT_UNIT) -> int: + """Print the timer listing; return systemctl's exit code.""" + result = _run(status_command(unit=unit), check=False) + sys.stdout.write(result.stdout) + sys.stderr.write(result.stderr) + return result.returncode @contextmanager -def armed(timeout_s: int = DEFAULT_TIMEOUT_S, *, unit: str = DEFAULT_UNIT) -> Iterator[None]: - """Arm around a risky block; disarm on the way out, success or exception.""" +def armed(timeout_s: int = DEFAULT_TIMEOUT_S, *, unit: str = DEFAULT_UNIT) -> Generator[None, None, None]: + """Arm around a risky block; disarm on the way out, success or exception. + + This is for Python callers whose failure is a Python exception. A hardware + plan that fails a step must leave the timer ARMED so the host reboots if + the step wedged it; use ``arm``/``disarm`` directly there. + """ arm(timeout_s, unit=unit) try: yield @@ -112,17 +176,27 @@ def armed(timeout_s: int = DEFAULT_TIMEOUT_S, *, unit: str = DEFAULT_UNIT) -> It disarm(unit=unit) -def main(argv: Sequence[str] | None = None) -> int: +def _parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser(description="Prime a forced reboot before a risky PCIe op; cancel it on success") - parser.add_argument("action", choices=("arm", "disarm", "status"), help="Deadman action") - parser.add_argument("--timeout", type=int, default=DEFAULT_TIMEOUT_S, help="Seconds before the reset fires when arming") - parser.add_argument("--unit", default=DEFAULT_UNIT, help="Transient systemd unit name") - parser.add_argument("--dry-run", action="store_true", help="Print the command(s) without running them") - args = parser.parse_args(argv) + actions = parser.add_subparsers(dest="action", required=True) + arm_parser = actions.add_parser("arm", help="schedule the reset") + arm_parser.add_argument("--timeout", type=int, default=DEFAULT_TIMEOUT_S, help="seconds before the reset fires") + arm_parser.add_argument("--unit", default=DEFAULT_UNIT, type=check_unit, help="transient systemd unit, dau-deadman or dau-deadman-") + arm_parser.add_argument("--dry-run", action="store_true", help="print the command without running it") + disarm_parser = actions.add_parser("disarm", help="cancel the pending reset") + disarm_parser.add_argument("--unit", default=DEFAULT_UNIT, type=check_unit, help="transient systemd unit, dau-deadman or dau-deadman-") + disarm_parser.add_argument("--dry-run", action="store_true", help="print the commands without running them") + status_parser = actions.add_parser("status", help="list the pending timer") + status_parser.add_argument("--unit", default=DEFAULT_UNIT, type=check_unit, help="transient systemd unit, dau-deadman or dau-deadman-") + return parser + + +def main(argv: Sequence[str] | None = None) -> int: + args = _parser().parse_args(argv) if args.action == "arm": if args.dry_run: - print(" ".join(arm_command(args.timeout, unit=args.unit))) + print(shlex.join(arm_command(args.timeout, unit=args.unit))) return 0 try: arm(args.timeout, unit=args.unit) @@ -135,7 +209,7 @@ def main(argv: Sequence[str] | None = None) -> int: if args.action == "disarm": if args.dry_run: for command in disarm_commands(unit=args.unit): - print(" ".join(command)) + print(shlex.join(command)) return 0 try: disarm(unit=args.unit) @@ -145,8 +219,11 @@ def main(argv: Sequence[str] | None = None) -> int: print(f"deadman disarmed (unit {args.unit})") return 0 - subprocess.run(status_command(unit=args.unit), check=False) - return 0 + try: + return status(unit=args.unit) + except DeadmanError as error: + print(f"deadman status unavailable: {error}", file=sys.stderr) + return 1 if __name__ == "__main__": diff --git a/dau_utils/pci_runtime_pm.py b/dau_utils/pci_runtime_pm.py index 83ad4df..a2a3aa0 100644 --- a/dau_utils/pci_runtime_pm.py +++ b/dau_utils/pci_runtime_pm.py @@ -1,6 +1,18 @@ +"""Hold or release Linux PCI runtime power management for named devices. + +A device is named by its PCI address (``dddd:bb:dd.f``) or discovered from +``lspci -Dnn`` by text patterns, never both at once. The writes go to the +device's ``power/control`` and ``d3cold_allowed`` attributes under the sysfs +device root; because every device name is a PCI address, no name can leave +that root, and the whole plan is checked before the first write so a missing +attribute leaves nothing half applied. +""" + from __future__ import annotations import argparse +import re +import shlex import subprocess import sys from collections.abc import Sequence @@ -8,6 +20,25 @@ from pathlib import Path DEFAULT_SYSFS_ROOT = Path("/sys/bus/pci/devices") +# lspci answers at once; a longer wait is a wedged bus, which is what the +# caller is about to protect against, not something to hang on here +LSPCI_TIMEOUT_S = 15 + +# the sysfs names are PCI addresses; the attribute values are the two states +# of each knob. Nothing else is ever written. +_BDF = re.compile(r"[0-9a-fA-F]{4}:[0-9a-fA-F]{2}:[0-9a-fA-F]{2}\.[0-7]") +_VALUES = frozenset({"on", "auto", "0", "1"}) + + +class RuntimePmError(RuntimeError): + """The plan could not be made or applied as asked.""" + + +def check_device(device: str) -> str: + """The device, if it is a PCI address; the only form that stays inside the sysfs root.""" + if not _BDF.fullmatch(device): + raise RuntimePmError(f"device must be a PCI address (dddd:bb:dd.f), got {device!r}") + return device @dataclass(frozen=True) @@ -15,6 +46,10 @@ class RuntimePmWrite: path: Path value: str + def __post_init__(self) -> None: + if self.value not in _VALUES: + raise RuntimePmError(f"runtime PM write value must be one of {sorted(_VALUES)}, got {self.value!r}") + def discover_pci_devices(lspci_output: str, *, patterns: Sequence[str] = ()) -> tuple[str, ...]: matches: list[str] = [] @@ -22,7 +57,7 @@ def discover_pci_devices(lspci_output: str, *, patterns: Sequence[str] = ()) -> if any(pattern in line for pattern in patterns): parts = line.split(maxsplit=1) if parts: - matches.append(parts[0]) + matches.append(check_device(parts[0])) return tuple(dict.fromkeys(matches)) @@ -30,7 +65,7 @@ def plan_runtime_pm_writes(mode: str, devices: Sequence[str], *, sysfs_root: Pat control, d3cold_allowed = _mode_values(mode) writes: list[RuntimePmWrite] = [] for device in devices: - device_root = sysfs_root / device + device_root = sysfs_root / check_device(device) writes.extend( ( RuntimePmWrite(device_root / "power" / "control", control), @@ -40,49 +75,68 @@ def plan_runtime_pm_writes(mode: str, devices: Sequence[str], *, sysfs_root: Pat return tuple(writes) +def missing_targets(writes: Sequence[RuntimePmWrite]) -> tuple[RuntimePmWrite, ...]: + """The writes whose attribute does not exist; checked before anything is written.""" + return tuple(write for write in writes if not write.path.exists()) + + def apply_runtime_pm_writes(writes: Sequence[RuntimePmWrite]) -> tuple[RuntimePmWrite, ...]: - applied: list[RuntimePmWrite] = [] + """Apply every write, or none: a missing attribute refuses the whole plan + so a device is never left half held or half released.""" + missing = missing_targets(writes) + if missing: + raise RuntimePmError("missing sysfs attributes: " + ", ".join(str(write.path) for write in missing)) for write in writes: - if write.path.exists(): - write.path.write_text(f"{write.value}\n") - applied.append(write) - return tuple(applied) + write.path.write_text(f"{write.value}\n") + return tuple(writes) -def main(argv: Sequence[str] | None = None) -> int: +def _parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser(description="Hold or release Linux PCI runtime PM for matching devices") - parser.add_argument("mode", choices=("hold", "release"), help="Runtime PM mode to apply") - parser.add_argument("--device", action="append", default=[], help="PCI BDF to update; may be provided more than once") - parser.add_argument("--pattern", action="append", default=[], help="lspci text pattern to discover; may be provided more than once") + parser.add_argument("mode", choices=("hold", "release"), help="runtime PM mode to apply") + selector = parser.add_mutually_exclusive_group(required=True) + selector.add_argument("--device", action="append", default=[], help="PCI address to update (dddd:bb:dd.f); may be repeated") + selector.add_argument("--pattern", action="append", default=[], help="lspci text pattern to discover devices by; may be repeated") parser.add_argument("--sysfs-root", type=Path, default=DEFAULT_SYSFS_ROOT, help="PCI sysfs device root") - parser.add_argument("--lspci-output", help="Use provided lspci output instead of running lspci -Dnn") - parser.add_argument("--dry-run", action="store_true", help="Print writes without applying them") - args = parser.parse_args(argv) + parser.add_argument("--lspci-output", help="lspci -Dnn text to discover from instead of running lspci (needs --pattern)") + parser.add_argument("--dry-run", action="store_true", help="print the writes without applying them") + return parser - patterns = tuple(args.pattern) - if args.device: - devices: tuple[str, ...] = tuple(args.device) - elif patterns: - lspci_output = args.lspci_output if args.lspci_output is not None else _lspci_output() - devices = discover_pci_devices(lspci_output, patterns=patterns) - else: - devices = () - writes = plan_runtime_pm_writes(args.mode, devices, sysfs_root=args.sysfs_root) + +def main(argv: Sequence[str] | None = None) -> int: + parser = _parser() + args = parser.parse_args(argv) + if args.lspci_output is not None and not args.pattern: + parser.error("--lspci-output only makes sense with --pattern") + + try: + if args.device: + devices: tuple[str, ...] = tuple(check_device(device) for device in args.device) + else: + lspci_output = args.lspci_output if args.lspci_output is not None else _lspci_output() + devices = discover_pci_devices(lspci_output, patterns=tuple(args.pattern)) + if not devices: + # tolerated on purpose: a device that no longer enumerates is the + # one being recovered, and there is nothing to hold for it + print(f"no device matched {shlex.join(args.pattern)}; nothing to write", file=sys.stderr) + writes = plan_runtime_pm_writes(args.mode, devices, sysfs_root=args.sysfs_root) + except RuntimePmError as error: + print(f"runtime PM: {error}", file=sys.stderr) + return 1 if args.dry_run: for write in writes: print(f"write {write.path} {write.value}") return 0 - applied = apply_runtime_pm_writes(writes) - for write in applied: - print(f"wrote {write.path} {write.value}") - skipped = tuple(write for write in writes if write not in applied) - for write in skipped: - print(f"skipped {write.path} (missing)", file=sys.stderr) - if skipped: - print(f"applied {len(applied)} of {len(writes)} runtime PM writes", file=sys.stderr) + missing = missing_targets(writes) + if missing: + for write in missing: + print(f"missing {write.path}", file=sys.stderr) + print(f"applied 0 of {len(writes)} runtime PM writes: the plan is refused whole", file=sys.stderr) return 1 + for write in apply_runtime_pm_writes(writes): + print(f"wrote {write.path} {write.value}") return 0 @@ -91,11 +145,18 @@ def _mode_values(mode: str) -> tuple[str, str]: return "on", "0" if mode == "release": return "auto", "1" - raise ValueError(f"unknown runtime PM mode: {mode}") + raise RuntimePmError(f"unknown runtime PM mode: {mode}") def _lspci_output() -> str: - return subprocess.run(("lspci", "-Dnn"), check=True, text=True, stdout=subprocess.PIPE).stdout + try: + return subprocess.run(("lspci", "-Dnn"), check=True, text=True, stdout=subprocess.PIPE, timeout=LSPCI_TIMEOUT_S).stdout + except FileNotFoundError as error: + raise RuntimePmError("lspci is not installed (pciutils)") from error + except subprocess.TimeoutExpired as error: + raise RuntimePmError(f"lspci did not answer within {LSPCI_TIMEOUT_S}s") from error + except subprocess.CalledProcessError as error: + raise RuntimePmError(f"lspci failed with exit {error.returncode}") from error if __name__ == "__main__": diff --git a/dau_utils/tests/test_all.py b/dau_utils/tests/test_all.py deleted file mode 100644 index bc121a7..0000000 --- a/dau_utils/tests/test_all.py +++ /dev/null @@ -1,5 +0,0 @@ -from dau_utils import * - - -def test_all(): - assert True diff --git a/dau_utils/tests/test_deadman.py b/dau_utils/tests/test_deadman.py index 0dab4fc..a0a8259 100644 --- a/dau_utils/tests/test_deadman.py +++ b/dau_utils/tests/test_deadman.py @@ -1,6 +1,7 @@ from __future__ import annotations import runpy +import shlex import subprocess import sys @@ -8,11 +9,13 @@ from dau_utils import deadman from dau_utils.deadman import ( + COMMAND_TIMEOUT_S, DEFAULT_TIMEOUT_S, DEFAULT_UNIT, DeadmanError, arm, arm_command, + check_unit, disarm, disarm_commands, is_armed, @@ -24,45 +27,57 @@ def test_arm_command_schedules_a_transient_sysrq_reset_timer() -> None: command = arm_command(120, unit="dau-deadman") - assert command[:6] == ( + assert command[:7] == ( "sudo", + "-n", "systemd-run", "--unit=dau-deadman", "--on-active=120", "--timer-property=AccuracySec=1s", "--collect", ) - assert command[6:8] == ("/bin/sh", "-c") - assert "/proc/sysrq-trigger" in command[8] - assert command[8].strip().endswith("echo b > /proc/sysrq-trigger") + assert command[7:9] == ("/bin/sh", "-c") + assert "/proc/sysrq-trigger" in command[9] + assert command[9].strip().endswith("echo b > /proc/sysrq-trigger") def test_arm_command_rejects_a_nonpositive_timeout() -> None: for bad in (0, -5): - try: + with pytest.raises(ValueError, match="at least 1 second"): arm_command(bad) - except ValueError: - continue - raise AssertionError(f"expected ValueError for timeout {bad}") + + +def test_units_outside_the_deadman_namespace_are_refused() -> None: + """`disarm --unit sshd` would stop sshd; the tool may only touch its own units.""" + assert check_unit("dau-deadman") == "dau-deadman" + assert check_unit("dau-deadman-bench2") == "dau-deadman-bench2" + for bad in ("sshd", "dau-deadman.service", "dau-deadman-*", "", "dau_deadman"): + with pytest.raises(ValueError, match="dau-deadman"): + check_unit(bad) + with pytest.raises(ValueError): + disarm_commands(unit="sshd") + with pytest.raises(SystemExit): + main(["disarm", "--unit", "sshd", "--dry-run"]) def test_disarm_stops_the_timer_and_service_then_clears_failed_state() -> None: stop, reset = disarm_commands(unit="dau-deadman") - assert stop == ("sudo", "systemctl", "stop", "dau-deadman.timer", "dau-deadman.service") - assert reset == ("sudo", "systemctl", "reset-failed", "dau-deadman.timer", "dau-deadman.service") + assert stop == ("sudo", "-n", "systemctl", "stop", "dau-deadman.timer", "dau-deadman.service") + assert reset == ("sudo", "-n", "systemctl", "reset-failed", "dau-deadman.timer", "dau-deadman.service") def test_status_command_lists_the_named_timer() -> None: assert status_command(unit="dau-deadman") == ("systemctl", "list-timers", "--all", "dau-deadman.timer") -def test_cli_arm_dry_run_prints_the_scheduled_reset_command(capsys) -> None: +def test_cli_arm_dry_run_prints_the_scheduled_reset_command_shell_quoted(capsys) -> None: exit_code = main(["arm", "--timeout", "90", "--dry-run"]) assert exit_code == 0 printed = capsys.readouterr().out.strip() - assert printed == " ".join(arm_command(90, unit=DEFAULT_UNIT)) + assert printed == shlex.join(arm_command(90, unit=DEFAULT_UNIT)) + assert shlex.split(printed) == list(arm_command(90, unit=DEFAULT_UNIT)) # the reset script survives the quoting def test_cli_disarm_dry_run_prints_both_teardown_commands(capsys) -> None: @@ -70,7 +85,7 @@ def test_cli_disarm_dry_run_prints_both_teardown_commands(capsys) -> None: assert exit_code == 0 lines = capsys.readouterr().out.splitlines() - assert lines == [" ".join(command) for command in disarm_commands(unit=DEFAULT_UNIT)] + assert lines == [shlex.join(command) for command in disarm_commands(unit=DEFAULT_UNIT)] def test_cli_arm_dry_run_defaults_to_the_module_timeout(capsys) -> None: @@ -79,20 +94,34 @@ def test_cli_arm_dry_run_defaults_to_the_module_timeout(capsys) -> None: assert f"--on-active={DEFAULT_TIMEOUT_S}" in capsys.readouterr().out +def test_cli_refuses_flags_that_do_not_belong_to_the_action() -> None: + with pytest.raises(SystemExit): + main(["disarm", "--timeout", "999"]) + with pytest.raises(SystemExit): + main(["status", "--dry-run"]) + + class _FakeSystemctl: - """Stands in for subprocess.run: is-active returns a scripted state, and - every mutating call (sudo/systemd-run/stop) is recorded.""" + """Stands in for subprocess.run: is-active returns a scripted state, a + systemd-run marks the timer active, and every call is recorded with the + timeout it was given.""" - def __init__(self, active_states: dict[str, str]) -> None: - self.active_states = active_states + def __init__(self, active_states: dict[str, str], *, arm_takes: bool = True) -> None: + self.active_states = dict(active_states) + self.arm_takes = arm_takes self.calls: list[tuple[str, ...]] = [] + self.timeouts: list[float | None] = [] - def __call__(self, command, check=False, capture_output=False, text=False): + def __call__(self, command, check=False, capture_output=False, text=False, timeout=None): command = tuple(command) self.calls.append(command) + self.timeouts.append(timeout) if command[:2] == ("systemctl", "is-active"): state = self.active_states.get(command[2], "inactive") return subprocess.CompletedProcess(command, 0, stdout=f"{state}\n", stderr="") + if command[:3] == ("sudo", "-n", "systemd-run") and self.arm_takes: + unit = command[3].removeprefix("--unit=") + self.active_states[f"{unit}.timer"] = "active" return subprocess.CompletedProcess(command, 0, stdout="", stderr="") @@ -103,16 +132,44 @@ def test_arm_refuses_to_replace_an_already_armed_timer(monkeypatch) -> None: with pytest.raises(DeadmanError, match="already armed"): arm(120) - assert not any(call[:2] == ("sudo", "systemd-run") for call in fake.calls) + assert not any(call[:3] == ("sudo", "-n", "systemd-run") for call in fake.calls) -def test_arm_schedules_when_no_timer_is_live(monkeypatch) -> None: - fake = _FakeSystemctl({}) # everything inactive +def test_arm_schedules_when_no_timer_is_live_and_confirms_it(monkeypatch) -> None: + fake = _FakeSystemctl({}) # everything inactive until systemd-run monkeypatch.setattr(deadman.subprocess, "run", fake) arm(120) - assert any(call[:2] == ("sudo", "systemd-run") for call in fake.calls) + assert any(call[:3] == ("sudo", "-n", "systemd-run") for call in fake.calls) + assert fake.calls[-1][:2] == ("systemctl", "is-active") # confirmed after scheduling + assert all(timeout == COMMAND_TIMEOUT_S for timeout in fake.timeouts) # every call is bounded + + +def test_arm_reports_an_unprotected_host_when_the_timer_does_not_come_up(monkeypatch) -> None: + fake = _FakeSystemctl({}, arm_takes=False) + monkeypatch.setattr(deadman.subprocess, "run", fake) + + with pytest.raises(DeadmanError, match="NOT protected"): + arm(120) + + +def test_a_missing_or_hanging_systemd_is_a_deadman_error(monkeypatch) -> None: + def missing(command, **kwargs): + raise FileNotFoundError(command[0]) + + monkeypatch.setattr(deadman.subprocess, "run", missing) + with pytest.raises(DeadmanError, match="not installed"): + deadman._run(("systemd-run",), check=True) + + def hangs(command, **kwargs): + raise subprocess.TimeoutExpired(command, float(kwargs.get("timeout") or 0)) + + monkeypatch.setattr(deadman.subprocess, "run", hangs) + with pytest.raises(DeadmanError, match="did not answer"): + deadman._run(("systemctl", "is-active", "x"), check=False) + # and a query that cannot be answered counts as armed + assert is_armed() is True def test_disarm_raises_when_the_timer_survives_the_stop(monkeypatch) -> None: @@ -129,7 +186,8 @@ def test_disarm_succeeds_only_once_the_timer_is_confirmed_inactive(monkeypatch) disarm() # no raise - assert ("sudo", "systemctl", "stop", f"{DEFAULT_UNIT}.timer", f"{DEFAULT_UNIT}.service") in fake.calls + stop, reset = disarm_commands(unit=DEFAULT_UNIT) + assert stop in fake.calls and reset in fake.calls # execution runs the commands the dry run prints def test_is_armed_treats_a_failed_query_as_still_armed(monkeypatch) -> None: @@ -149,6 +207,15 @@ def test_cli_disarm_reports_failure_when_timer_cannot_be_confirmed_gone(monkeypa assert "DISARM FAILED" in capsys.readouterr().err +def test_cli_status_returns_systemctls_exit_code(monkeypatch, capsys) -> None: + def failing(command, **kwargs): + return subprocess.CompletedProcess(tuple(command), 3, stdout="", stderr="Failed to connect to bus\n") + + monkeypatch.setattr(deadman.subprocess, "run", failing) + assert main(["status"]) == 3 + assert "Failed to connect" in capsys.readouterr().err + + def test_module_entrypoint_runs_cli_for_uninstalled_checkout(capsys, monkeypatch) -> None: monkeypatch.setattr(sys, "argv", ["deadman", "arm", "--timeout", "42", "--dry-run"]) monkeypatch.delitem(sys.modules, "dau_utils.deadman", raising=False) diff --git a/dau_utils/tests/test_pci_runtime_pm.py b/dau_utils/tests/test_pci_runtime_pm.py index a5c3387..00cb767 100644 --- a/dau_utils/tests/test_pci_runtime_pm.py +++ b/dau_utils/tests/test_pci_runtime_pm.py @@ -1,21 +1,26 @@ from __future__ import annotations import runpy +import subprocess import sys from pathlib import Path -from dau_utils.pci_runtime_pm import discover_pci_devices, main, plan_runtime_pm_writes +import pytest +from dau_utils import pci_runtime_pm +from dau_utils.pci_runtime_pm import RuntimePmError, RuntimePmWrite, apply_runtime_pm_writes, discover_pci_devices, main, plan_runtime_pm_writes + +# synthetic lspci -Dnn text: a root port, a bridge behind it, and an accelerator behind that LSPCI_OUTPUT = """ -0000:00:07.0 PCI bridge [0604]: Intel Corporation Raptor Lake-P Thunderbolt 4 PCI Express Root Port #0 [8086:a76e] -0000:00:0d.2 USB controller [0c03]: Intel Corporation Raptor Lake-P Thunderbolt 4 NHI #0 [8086:a73e] -0000:02:00.0 PCI bridge [0604]: Intel Corporation JHL6240 Thunderbolt 3 Bridge (Low Power) [8086:15c0] (rev 01) -0000:04:00.0 Processing accelerators [1200]: Xilinx Corporation 7-Series FPGA Hard PCIe block (AXI/debug) [10ee:7011] +0000:00:07.0 PCI bridge [0604]: Example Silicon Root Port [1234:0001] +0000:00:0d.2 USB controller [0c03]: Example Silicon Host Controller [1234:0002] +0000:02:00.0 PCI bridge [0604]: Example Silicon Downstream Bridge [1234:0003] (rev 01) +0000:04:00.0 Processing accelerators [1200]: Example Accelerator Co. FPGA card [abcd:ef01] """ def test_discovers_devices_matching_explicit_patterns() -> None: - assert discover_pci_devices(LSPCI_OUTPUT, patterns=("Thunderbolt", "JHL", "10ee:7011")) == ( + assert discover_pci_devices(LSPCI_OUTPUT, patterns=("Root Port", "Host Controller", "Bridge", "abcd:ef01")) == ( "0000:00:07.0", "0000:00:0d.2", "0000:02:00.0", @@ -44,6 +49,18 @@ def test_runtime_pm_write_plan_maps_hold_and_release_to_sysfs_knobs() -> None: ] +def test_a_device_that_is_not_a_pci_address_cannot_leave_the_sysfs_root() -> None: + """The only confinement sysfs allows (its device entries are symlinks, so a + resolved-path check would fail on a real host) is the name itself.""" + for bad in ("../../../etc", "/etc/passwd", "0000:04:00", "0000:04:00.9", "xdma0", ""): + with pytest.raises(RuntimePmError, match="PCI address"): + plan_runtime_pm_writes("hold", (bad,)) + with pytest.raises(RuntimePmError, match="PCI address"): + discover_pci_devices("../escape PCI bridge [0604]: thing", patterns=("thing",)) + with pytest.raises(RuntimePmError, match="value must be one of"): + RuntimePmWrite(Path("/sys/x"), "rm -rf") + + def test_cli_dry_run_prints_hold_writes_for_explicit_device(capsys) -> None: exit_code = main(["hold", "--device", "0000:04:00.0", "--dry-run"]) @@ -56,18 +73,7 @@ def test_cli_dry_run_prints_hold_writes_for_explicit_device(capsys) -> None: def test_cli_dry_run_can_discover_devices_from_lspci_fixture(capsys) -> None: exit_code = main( - [ - "release", - "--dry-run", - "--pattern", - "Thunderbolt", - "--pattern", - "JHL", - "--pattern", - "10ee:7011", - "--lspci-output", - LSPCI_OUTPUT, - ] + ["release", "--dry-run", "--pattern", "Root Port", "--pattern", "Bridge", "--pattern", "abcd:ef01", "--lspci-output", LSPCI_OUTPUT] ) assert exit_code == 0 @@ -76,28 +82,47 @@ def test_cli_dry_run_can_discover_devices_from_lspci_fixture(capsys) -> None: assert lines[-1] == "write /sys/bus/pci/devices/0000:04:00.0/d3cold_allowed 1" -def test_cli_without_patterns_matches_no_devices(capsys) -> None: - exit_code = main(["hold"]) +def test_cli_refuses_an_ambiguous_or_empty_selection() -> None: + with pytest.raises(SystemExit): + main(["hold"]) # nothing selected + with pytest.raises(SystemExit): + main(["hold", "--device", "0000:04:00.0", "--pattern", "Bridge", "--dry-run"]) # both selectors + with pytest.raises(SystemExit): + main(["hold", "--device", "0000:04:00.0", "--lspci-output", LSPCI_OUTPUT, "--dry-run"]) # lspci text without a pattern + + +def test_cli_rejects_a_device_that_is_not_a_pci_address(capsys) -> None: + assert main(["hold", "--device", "../../etc", "--dry-run"]) == 1 + assert "PCI address" in capsys.readouterr().err + + +def test_cli_says_so_when_no_device_matches(capsys) -> None: + """Zero matches are tolerated (the device being recovered may not + enumerate), and said out loud rather than passed silently.""" + exit_code = main(["hold", "--pattern", "Nonesuch", "--lspci-output", LSPCI_OUTPUT]) assert exit_code == 0 - assert capsys.readouterr().out == "" + captured = capsys.readouterr() + assert captured.out == "" + assert "no device matched Nonesuch; nothing to write" in captured.err -def test_cli_missing_sysfs_paths_surface_skips_and_fail(tmp_path, capsys) -> None: +def test_cli_missing_sysfs_paths_refuse_the_whole_plan(tmp_path, capsys) -> None: exit_code = main(["hold", "--device", "0000:04:00.0", "--sysfs-root", str(tmp_path)]) assert exit_code == 1 captured = capsys.readouterr() assert captured.out == "" - err_lines = captured.err.splitlines() - assert err_lines == [ - f"skipped {tmp_path}/0000:04:00.0/power/control (missing)", - f"skipped {tmp_path}/0000:04:00.0/d3cold_allowed (missing)", - "applied 0 of 2 runtime PM writes", + assert captured.err.splitlines() == [ + f"missing {tmp_path}/0000:04:00.0/power/control", + f"missing {tmp_path}/0000:04:00.0/d3cold_allowed", + "applied 0 of 2 runtime PM writes: the plan is refused whole", ] -def test_cli_partial_apply_reports_skip_and_fails(tmp_path, capsys) -> None: +def test_cli_writes_nothing_when_one_attribute_is_missing(tmp_path, capsys) -> None: + """A device half held is the state the tool exists to avoid; the plan is + checked before the first write.""" device_root = tmp_path / "0000:04:00.0" (device_root / "power").mkdir(parents=True) control = device_root / "power" / "control" @@ -107,13 +132,15 @@ def test_cli_partial_apply_reports_skip_and_fails(tmp_path, capsys) -> None: exit_code = main(["hold", "--device", "0000:04:00.0", "--sysfs-root", str(tmp_path)]) assert exit_code == 1 - assert control.read_text() == "on\n" + assert control.read_text() == "auto\n" # untouched captured = capsys.readouterr() - assert captured.out.splitlines() == [f"wrote {control} on"] + assert captured.out == "" assert captured.err.splitlines() == [ - f"skipped {device_root}/d3cold_allowed (missing)", - "applied 1 of 2 runtime PM writes", + f"missing {device_root}/d3cold_allowed", + "applied 0 of 2 runtime PM writes: the plan is refused whole", ] + with pytest.raises(RuntimePmError, match="missing sysfs attributes"): + apply_runtime_pm_writes(plan_runtime_pm_writes("hold", ("0000:04:00.0",), sysfs_root=tmp_path)) def test_cli_applies_present_sysfs_paths(tmp_path, capsys) -> None: @@ -132,6 +159,25 @@ def test_cli_applies_present_sysfs_paths(tmp_path, capsys) -> None: assert capsys.readouterr().err == "" +def test_lspci_is_bounded_and_its_absence_is_reported(monkeypatch, capsys) -> None: + seen = {} + + def fake_run(command, **kwargs): + seen["timeout"] = kwargs.get("timeout") + return subprocess.CompletedProcess(tuple(command), 0, stdout=LSPCI_OUTPUT) + + monkeypatch.setattr(pci_runtime_pm.subprocess, "run", fake_run) + assert main(["hold", "--pattern", "abcd:ef01", "--dry-run"]) == 0 + assert seen["timeout"] == pci_runtime_pm.LSPCI_TIMEOUT_S + + def missing(command, **kwargs): + raise FileNotFoundError("lspci") + + monkeypatch.setattr(pci_runtime_pm.subprocess, "run", missing) + assert main(["hold", "--pattern", "abcd:ef01", "--dry-run"]) == 1 + assert "pciutils" in capsys.readouterr().err + + def test_module_entrypoint_runs_cli_for_uninstalled_checkout(capsys, monkeypatch) -> None: monkeypatch.setattr(sys, "argv", ["pci_runtime_pm", "hold", "--device", "0000:04:00.0", "--dry-run"]) monkeypatch.delitem(sys.modules, "dau_utils.pci_runtime_pm", raising=False) diff --git a/dau_utils/tests/test_readme.py b/dau_utils/tests/test_readme.py new file mode 100644 index 0000000..cc0e9a4 --- /dev/null +++ b/dau_utils/tests/test_readme.py @@ -0,0 +1,48 @@ +"""Every command the README shows runs as written, in its unprivileged dry-run +form. The README is the first thing a bench operator reads; a command that +does not run there is a claim the repository cannot back.""" + +from __future__ import annotations + +import re +import shlex +from pathlib import Path + +import pytest + +from dau_utils import deadman, pci_runtime_pm + +README = Path(__file__).resolve().parents[2] / "README.md" +BLOCK = re.compile(r"```bash\n(.*?)```", re.DOTALL) +TOOLS = {"dau-utils-deadman": deadman.main, "dau-utils-pci-runtime-pm": pci_runtime_pm.main} + + +def _documented_commands() -> list[str]: + found = [] + for block in BLOCK.findall(README.read_text(encoding="utf-8")): + for line in block.splitlines(): + argv = shlex.split(line.split("#", 1)[0]) + if argv and argv[0] == "sudo": + argv = argv[1:] + if argv and argv[0] in TOOLS: + found.append(shlex.join(argv)) + return found + + +@pytest.mark.parametrize("command", _documented_commands()) +def test_a_documented_command_runs_as_a_dry_run(command: str, monkeypatch, capsys) -> None: + argv = shlex.split(command) + tool, args = TOOLS[argv[0]], argv[1:] + if args[0] == "status": + # status queries systemd; prove it composes and exits with systemctl's code + import subprocess + + monkeypatch.setattr(deadman.subprocess, "run", lambda c, **k: subprocess.CompletedProcess(tuple(c), 0, stdout="", stderr="")) + elif "--pattern" in args: + monkeypatch.setattr( + pci_runtime_pm, "_lspci_output", lambda: "0000:04:00.0 Processing accelerators [1200]: Example Downstream Bridge FPGA card [abcd:ef01]\n" + ) + args = [*args, "--dry-run"] + else: + args = [*args, "--dry-run"] + assert tool(args) == 0, command diff --git a/pyproject.toml b/pyproject.toml index 9cd6f6e..87225f1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,12 +20,9 @@ classifiers = [ "Development Status :: 3 - Alpha", "Programming Language :: Python", "Programming Language :: Python :: Implementation :: CPython", - "Programming Language :: Python :: Implementation :: PyPy", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", - "Programming Language :: Python :: 3.13", - "Programming Language :: Python :: 3.14", ] dependencies = [] @@ -74,9 +71,7 @@ replace = 'version = "{new_version}"' [tool.coverage.run] branch = true -omit = [ - "dau_utils/tests/integration/", -] +omit = [] [tool.coverage.report] exclude_also = [