diff --git a/.github/scripts/depgraph.js b/.github/scripts/depgraph.js new file mode 100644 index 00000000..4faa4572 --- /dev/null +++ b/.github/scripts/depgraph.js @@ -0,0 +1,261 @@ +/*! + * Copyright (c) 2026-present, The Dash Core developers + * SPDX-License-Identifier: MIT + * See the accompanying file LICENSE or https://opensource.org/license/MIT + */ + +// @ts-check + +// Submits `uv.lock` to the dependency graph. GitHub currently natively parses +// `Cargo.lock` but cannot parse `uv.lock`, this script parses it for submission +// to the dependency graph. + +const fs = require("node:fs"); + +// Submission tag, keyed to overwrite autogenerated results from `pyproject.toml`. +const PY_MANIFEST_KEY = "pyproject.toml"; + +// Identification of this script. +const DETECTOR_PROFILE = { + name: "depgraph.js", + version: "1.0.0", + url: "https://github.com/dashpay/base-sdk", +}; + +// Matches `name[extras]==version`, capturing name in 1 and version in 2, ends at whitespace, marker or backslash. +const RE_PIN = /^([A-Za-z0-9][A-Za-z0-9._-]*)(?:\[[^\]]*\])?==([^\s;\\]+)/; + +// Matches a distribution name, an extras suffix allowed, and nothing else. +const RE_NAME = /^[A-Za-z0-9][A-Za-z0-9._-]*(?:\[[^\]]*\])?$/; + +// Matches an unindented comment. +const RE_HEADER = /^#/; + +// Matches an indented comment. +const RE_OWNED = /^\s+#/; + +// Matches an indented `# via`, capturing what trails it, which may be empty. +const RE_VIA = /^\s+#\s+via\b(.*)$/; + +// Matches an indented comment holding one token, captured. +const RE_VIA_ITEM = /^\s+#\s+(\S.*)$/; + +/** + * PEP 503-style text normalisation. + * + * @param {string} name + * @returns {string} + */ +function normalise(name) { + return name.toLowerCase().replace(/[-_.]+/g, "-"); +} + +/** + * The package URL for a pinned distribution, local version encoded. + * + * @param {string} name normalised name + * @param {string} version + * @returns {string} + */ +function purlFor(name, version) { + return `pkg:pypi/${name}@${version.replace(/\+/g, "%2B")}`; +} + +/** + * Parse a `via` entry. + * + * @param {string} entry + * @param {string} line the line it was read from, named in the error + * @returns {string} normalised name, extras dropped + */ +function viaName(entry, line) { + if (!RE_NAME.test(entry)) { + throw new Error(`unsupported \`via\` entry: ${line.trim()}`); + } + return normalise(entry.replace(/\[.*$/, "")); +} + +/** + * Parse `uv export --format requirements-txt --no-hashes` output. + * + * Two shapes are read, a pin and the `# via` beneath it holding a name or list. + * + * @param {string} text + * @returns {Map} + */ +function parseExport(text) { + /** @type {Map} */ + const packages = new Map(); + /** @type {{ version: string, via: string[] } | null} */ + let current = null; + let listing = false; + + for (const raw of text.split("\n")) { + const line = raw.replace(/\r$/, ""); + + if (line.trim() === "" || RE_HEADER.test(line)) { + current = null; + listing = false; + continue; + } + + if (current !== null && RE_OWNED.test(line)) { + const via = RE_VIA.exec(line); + if (via) { + const rest = via[1].trim(); + listing = rest === ""; + if (!listing) { + current.via.push(viaName(rest, line)); + } + continue; + } + + const listed = RE_VIA_ITEM.exec(line); + if (listed && listing) { + current.via.push(viaName(listed[1].trim(), line)); + } + continue; + } + + // Extras are matched so they cannot hide a pin. + const pin = RE_PIN.exec(line); + if (pin === null) { + throw new Error(`unsupported requirement: ${line.trim()}`); + } + + const name = normalise(pin[1]); + const held = packages.get(name); + if (held !== undefined && held.version !== pin[2]) { + throw new Error( + `${name} is pinned at both ${held.version} and ${pin[2]}`, + ); + } + + current = { version: pin[2], via: [] }; + packages.set(name, current); + listing = false; + } + + return packages; +} + +/** + * Build the `resolved` map a snapshot carries, keyed and cross-referenced + * by the package URL. + * + * All entries are scoped `development`, since they make up the devshell. + * + * @param {Map} packages + * @param {string} project normalised name of the workspace project + * @returns {Record} + */ +function resolveGraph(packages, project) { + /** @type {Record} */ + const resolved = {}; + + for (const [name, pkg] of packages) { + if (pkg.via.length === 0) { + throw new Error(`${name} has no \`via\`; export --no-emit-project`); + } + for (const parent of pkg.via) { + if (parent !== project && !packages.has(parent)) { + throw new Error(`${name} names ${parent}, not a pin nor ${project}`); + } + } + + const purl = purlFor(name, pkg.version); + resolved[purl] = { + package_url: purl, + relationship: pkg.via.includes(project) ? "direct" : "indirect", + scope: "development", + dependencies: [], + }; + } + + // `via` names parents, a snapshot states children, so invert the edges. + for (const [name, pkg] of packages) { + const child = purlFor(name, pkg.version); + for (const parent of pkg.via) { + const owner = packages.get(parent); + if (owner === undefined) { + continue; + } + resolved[purlFor(parent, owner.version)].dependencies.push(child); + } + } + + return resolved; +} + +/** + * @param {{ sha: string, ref: string, resolved: Record }} params + * @returns {object} + */ +function buildSnapshot({ sha, ref, resolved }) { + return { + version: 0, + job: { + id: process.env.GITHUB_RUN_ID, + correlator: `${process.env.GITHUB_WORKFLOW}-${process.env.GITHUB_JOB}`, + }, + sha, + ref, + detector: DETECTOR_PROFILE, + scanned: new Date().toISOString(), + manifests: { + [PY_MANIFEST_KEY]: { + name: PY_MANIFEST_KEY, + file: { source_location: PY_MANIFEST_KEY }, + resolved, + }, + }, + }; +} + +/** + * @param {{ github: import("@actions/github").getOctokit, context: import("@actions/github").context, core: any }} params + */ +module.exports = async ({ github, context, core }) => { + const source = process.env.REQUIREMENTS; + if (source === undefined) { + throw new Error("REQUIREMENTS names the export to submit; it is unset"); + } + + const project = process.env.PROJECT; + if (project === undefined) { + throw new Error("PROJECT names the workspace project; it is unset"); + } + + const packages = parseExport(fs.readFileSync(source, "utf8")); + if (packages.size === 0) { + throw new Error(`${source} states no pinned versions`); + } + + const resolved = resolveGraph(packages, normalise(project)); + const snapshot = buildSnapshot({ + sha: context.sha, + ref: context.ref, + resolved, + }); + + const entries = Object.values(resolved); + const direct = entries.filter((e) => e.relationship === "direct").length; + core.info(`submitting ${entries.length} packages, ${direct} direct`); + + const { data } = await github.request( + "POST /repos/{owner}/{repo}/dependency-graph/snapshots", + { + owner: context.repo.owner, + repo: context.repo.repo, + ...snapshot, + }, + ); + if (data.result === "INVALID") { + throw new Error(`snapshot refused: ${data.message}`); + } + core.info(`snapshot ${data.id}: ${data.message}`); +}; + +module.exports.parseExport = parseExport; +module.exports.resolveGraph = resolveGraph; +module.exports.buildSnapshot = buildSnapshot; diff --git a/.github/workflows/build_msrv.yml b/.github/workflows/build_msrv.yml index 009f4aa8..c09c35f5 100644 --- a/.github/workflows/build_msrv.yml +++ b/.github/workflows/build_msrv.yml @@ -45,12 +45,22 @@ jobs: node-version: 24 - name: Set up Python + id: python uses: actions/setup-python@v6 with: python-version-file: pyproject.toml + - name: Set up uv + uses: astral-sh/setup-uv@v10.0.1 + with: + version: 0.12.9 + enable-cache: true + cache-dependency-glob: uv.lock + - name: Install Python dependencies - run: pip install ".[dev]" + run: | + uv sync --locked --extra dev --python '${{ steps.python.outputs.python-path }}' + echo "${PWD}/.venv/bin" >> "${GITHUB_PATH}" - name: Install CodeQL id: setup-codeql @@ -82,22 +92,24 @@ jobs: uses: actions/cache@v5 with: path: ~/.codeql - key: codeql-packs-${{ hashFiles('contrib/codeql/codeql-pack.lock.yml') }} + key: codeql-packs-${{ hashFiles('maint/codeql/*/codeql-pack.lock.yml') }} - name: Run linters - run: python3 contrib/lint_all.py --exclude lint_codeql + run: | + python3 maint/lint_all.py --exclude lint_codeql + python3 maint/lint/lint_codeql.py check env: RUSTUP_TOOLCHAIN: 1.85.0 - name: Run CodeQL - run: python3 contrib/lint/lint_codeql.py --with-suite=rust-security-and-quality + run: python3 maint/lint/lint_codeql.py run --lang=rust --with-suite=rust-security-and-quality env: RUSTUP_TOOLCHAIN: 1.85.0 - name: Check PR commit messages if: github.event_name == 'pull_request' run: > - python3 contrib/lint/lint_unconv.py + python3 maint/lint/lint_unconv.py -r "${{ github.event.pull_request.base.sha }}..${{ github.event.pull_request.head.sha }}" build: diff --git a/.github/workflows/build_nightly.yml b/.github/workflows/build_nightly.yml index 0d529d4a..c93e9a93 100644 --- a/.github/workflows/build_nightly.yml +++ b/.github/workflows/build_nightly.yml @@ -88,7 +88,7 @@ jobs: - name: Check formatting if: matrix.config.name == 'full' - run: python contrib/lint/lint_rust.py + run: python maint/lint/lint_rust.py - name: Test package (with coverage) if: matrix.config.name == 'full' diff --git a/.github/workflows/pages.yml b/.github/workflows/pages.yml index 7f681835..9be1cde0 100644 --- a/.github/workflows/pages.yml +++ b/.github/workflows/pages.yml @@ -39,12 +39,22 @@ jobs: run: cargo install wasm-pack@0.15.0 - name: Set up Python + id: python uses: actions/setup-python@v6 with: python-version-file: pyproject.toml + - name: Set up uv + uses: astral-sh/setup-uv@v10.0.1 + with: + version: 0.12.9 + enable-cache: true + cache-dependency-glob: uv.lock + - name: Install Python dependencies - run: pip install ".[dev]" + run: | + uv sync --locked --extra dev --python '${{ steps.python.outputs.python-path }}' + echo "${PWD}/.venv/bin" >> "${GITHUB_PATH}" - name: Test documentation tooling run: pytest diff --git a/.github/workflows/repo_depgraph.yml b/.github/workflows/repo_depgraph.yml new file mode 100644 index 00000000..ca243a0e --- /dev/null +++ b/.github/workflows/repo_depgraph.yml @@ -0,0 +1,49 @@ +name: Push dependency graph + +on: + push: + branches: [develop] + paths: + - uv.lock + - .github/scripts/depgraph.js + - .github/workflows/repo_depgraph.yml + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }} + cancel-in-progress: true + +permissions: + contents: write + +jobs: + submit: + name: Submit uv lockfile + runs-on: ubuntu-24.04-arm + + steps: + - name: Checkout + uses: actions/checkout@v6 + with: + fetch-depth: 1 + persist-credentials: false + + - name: Set up uv + uses: astral-sh/setup-uv@v10.0.1 + with: + version: 0.12.9 + enable-cache: true + cache-dependency-glob: uv.lock + + - name: Export resolved dependencies + run: uv export --locked --format requirements-txt --no-emit-project --all-extras --no-hashes -o "${RUNNER_TEMP}/requirements.txt" + + - name: Submit snapshot + uses: actions/github-script@v8 + env: + REQUIREMENTS: ${{ runner.temp }}/requirements.txt + PROJECT: dash-base-sdk + with: + script: | + const script = require("./.github/scripts/depgraph.js"); + await script({ github, context, core }); diff --git a/README.md b/README.md index c1fe4675..b38d447f 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,7 @@ > [!NOTE] > Solid lines are build dependencies. Dotted lines are test dependencies. - + ```mermaid %%{init: { "flowchart": { "curve": "basis" } } }%% @@ -64,7 +64,7 @@ graph LR params --> p2p_core ``` - + ## Features diff --git a/contrib/README.md b/contrib/README.md index 8da1b40a..627ac8f0 100644 --- a/contrib/README.md +++ b/contrib/README.md @@ -3,7 +3,7 @@ Python 3.x and thus, assume a host capable of running Python. For guidance on installing Python on your host, visit https://www.python.org/downloads/ - + ## Preparing the virtual environment @@ -15,14 +15,11 @@ recommended to create a fresh virtual environment. > your program of choice's documentation if using a different manager. ```bash -# Create a new venv -uv venv .venv +# Create .venv and install the versions uv.lock pins +uv sync --locked --extra dev # Enter venv source .venv/bin/activate - -# Install dependencies -uv pip install -e ".[dev]" ``` > [!WARNING] @@ -90,24 +87,11 @@ sudo apt install git nodejs -y sudo dnf install -y git nodejs24 ``` -## Running linters - -All linters available in [`contrib/lint/`](../contrib/lint) are listed below. The first verb is implied if no verb is -specified at runtime. Verbs may accept arguments of their own, for more information, run an individual lint script with -`--help`. To run all scripts, use [`lint_all.py`](./lint_all.py). + -| Name | Purpose | Verbs | Depends on | -| ---- | ------- | ---------- | ---------- | -| [`lint_cargo.py`](./lint/lint_cargo.py) | Enforce MSRV across Rust build dependency graph, check/format TOML files against [`.taplo.toml`](../.taplo.toml) | `check` , `apply`, `apply-all` | (MSRV enforcement) `cargo` (TOML formatting) `taplo` | -| [`lint_codeql.py`](./lint/lint_codeql.py) | Query Rust sources against [`contrib/codeql/*.ql`](./codeql) | `run` | `codeql`, `rustc` | -| [`lint_javascript.py`](./lint/lint_javascript.py) | Lint Javascript sources against [`eslint.config.mjs`](js/eslint.config.mjs) | *None* | `npx` (part of Node.js), `eslint` (auto-retrieved by script) | -| [`lint_markdown.py`](./lint/lint_markdown.py) | Lint Markdown [documentation](../docs/dev/about_docs.md) | *None* | `pymarkdownlnt` | -| [`lint_python.py`](./lint/lint_python.py) | Lint Python sources against `[tool.ruff]` options in [`pyproject.toml`](../pyproject.toml) | *None* | `ruff` | -| [`lint_rust.py`](./lint/lint_rust.py) | Lint Rust sources against [`rustfmt.toml`](../rustfmt.toml) | *None* | `cargo`, `rustfmt` | -| [`lint_semgrep.py`](./lint/lint_semgrep.py) | Lint Rust sources against [`contrib/semgrep/*.yml`](./semgrep) | *None* | `semgrep` | -| [`lint_unconv.py`](./lint/lint_unconv.py) | Lint commit names in ranges specified against [`unconv.toml`](../unconv.toml) | *None* | `git` | + -### Verifying bisectability +## Verifying bisectability As a general rule of thumb, each commit must individually compile and pass linters. To help with this, we have a helper script, [`git_filter.py`](./git_filter.py) that creates a temporary worktree and executes supplied commands for each @@ -118,7 +102,7 @@ commit in a specified range so the worktree isn't blocked by the validation run. ./contrib/git_filter.py --fast-fail develop branch_name -- bash -c 'cargo clippy --all-targets --no-default-features -- -D warnings && cargo clippy --all-targets --features full -- -D warnings && cargo test --all-targets --features full && -./contrib/lint_all.py' +./maint/lint_all.py' ``` - + diff --git a/contrib/common.py b/contrib/common.py deleted file mode 100644 index 07f9a6ae..00000000 --- a/contrib/common.py +++ /dev/null @@ -1,325 +0,0 @@ -#!/usr/bin/env python3 -# coding: latin-1 - -# -# Copyright (c) 2026-present, The Dash Core developers -# SPDX-License-Identifier: MIT -# See the accompanying file LICENSE or https://opensource.org/license/MIT -# - -"""Shared constants and helpers for lint scripts.""" - -from __future__ import annotations - -import argparse -import os -import re -import shutil -import subprocess -import sys -from functools import cache -from pathlib import Path -from typing import TYPE_CHECKING, NoReturn - -if TYPE_CHECKING: - from collections.abc import Callable, Mapping - from typing import TextIO - -# ANSI escape codes for terminal output. -ANSI_BOLD = "\033[1m" -ANSI_DIM = "\033[2m" -ANSI_GREEN = "\033[32m" -ANSI_RED = "\033[31m" -ANSI_RESET = "\033[0m" - -# Cargo workspace roots, relative to the repository root. -CARGO_WORKSPACES: tuple[str, ...] = (".", "docs/samples") - -# Rust source roots the analysers scan, relative to the repository root. -SOURCE_DIRS: tuple[str, ...] = ("pkgs", "docs/samples") - -# Assumed base branch for codebase. -DEFAULT_BASE = "develop" - -# Return codes. -RETCODE_ERR = 1 -RETCODE_PASS = 0 -RETCODE_SKIP = 77 - -# Matches an address that names something other than a path on disk. -_OFF_DISK_RE = re.compile(r"^(?:[A-Za-z][A-Za-z0-9+.\-]*:|//|#)") - - -class _VerbParser(argparse.ArgumentParser): - """Parser spelling a usage fault in the harness' return codes.""" - - def exit(self, status: int = 0, message: str | None = None) -> NoReturn: - if message: - self._print_message(message, sys.stderr) - sys.exit(RETCODE_ERR if status else RETCODE_PASS) - - -def declare_verbs( - description: str, - verbs: Mapping[str, str], -) -> argparse.ArgumentParser: - """Return a parser taking one of *verbs*. - - *verbs* maps each verb to what it does, and insertion order picks the - default, so the first entry must avoid mutating effects. - """ - if not verbs: - raise ValueError("no verbs declared") - default = next(iter(verbs)) - parser = _VerbParser( - description=description, - formatter_class=argparse.RawTextHelpFormatter, - ) - parser.add_argument( - "verb", - choices=tuple(verbs), - default=default, - nargs="?", - help="\n".join( - f"{name}: {what}" + (" (default)" if name == default else "") - for name, what in verbs.items() - ), - ) - return parser - - -def off_disk(target: str) -> bool: - """Whether *target* addresses something other than a file on disk.""" - return _OFF_DISK_RE.match(target) is not None - - -@cache -def _entries(where: Path) -> frozenset[str]: - """Return the names *where* holds, as the filesystem spells them.""" - return frozenset(entry.name for entry in where.iterdir()) - - -def spelt_as_stored(root: Path, target: Path) -> bool: - """Whether *target* is spelt as the filesystem under *root* holds it. - - A case-insensitive filesystem resolves a misspelt path, so a wrong-case - link passes `exists()` on macOS and Windows and then serves a 404 from a - case-sensitive host. Each component is matched against its directory. - """ - # Normalising first drops the `..` a caller may have left in the path, - # which names no directory entry and so would fail the walk outright. - target = target.resolve() - if not target.is_relative_to(root): - return True - probe = root - for part in target.relative_to(root).parts: - if part not in _entries(probe): - return False - probe = probe / part - return True - - -def is_plain_file(root: Path, name: str) -> bool: - """Whether *name* is a regular file inside *root*, reached without links.""" - path = root / name - if not path.is_file(): - return False - try: - relative = path.relative_to(root) - except ValueError: - return False - probe = root - for part in relative.parts: - probe = probe / part - if probe.is_symlink(): - return False - return path.resolve().is_relative_to(root.resolve()) - - -def git_run(cwd: Path | str, *args: str) -> subprocess.CompletedProcess[str]: - """Run a git command in *cwd* and return the result.""" - return subprocess.run( # noqa: S603 - [require_bin("git"), *args], - capture_output=True, - check=False, - cwd=str(cwd), - encoding="utf-8", - errors="replace", - ) - - -def git_out(cwd: Path | str, *args: str) -> str: - """Run a git command in *cwd*, raise on failure, return its output.""" - result = git_run(cwd, *args) - if result.returncode != 0: - fault = result.stderr.strip() or result.stdout.strip() - raise RuntimeError(f"git {args[0]}: {fault or result.returncode}") - return result.stdout.strip() - - -def relay( - text: str, - repo_root: Path, - *, - stream: TextIO | None = None, - drop: Callable[[str], bool] | None = None, -) -> None: - """Print *text* with paths shortened against *repo_root*.""" - prefix = str(repo_root) + "/" - for line in text.splitlines(): - if drop is not None and drop(line): - continue - print(line.replace(prefix, ""), file=stream or sys.stdout) - - -def touched(repo_root: Path, suffixes: tuple[str, ...]) -> list[str]: - """Return the files matching *suffixes* that this branch has changed.""" - base = git_out(repo_root, "merge-base", DEFAULT_BASE, "HEAD") - return [ - name - for name in git_out(repo_root, "diff", "--name-only", base).splitlines() - if name.endswith(suffixes) and is_plain_file(repo_root, name) - ] - - -def format_table( - headers: tuple[str, ...], - rows: list[tuple[str, ...]], - status_colors: dict[str, str] | None = None, -) -> str: - """Render a markdown table with optional color on the last column.""" - colors = status_colors or {} - widths = [ - max(len(h), *(len(r[i]) for r in rows), 0) for i, h in enumerate(headers) - ] - - def fmt(cells: tuple[str, ...], *, color: bool = False) -> str: - parts: list[str] = [] - for i, cell in enumerate(cells): - pre = post = "" - if color and i == len(cells) - 1 and colors: - pre = colors.get(cell, ANSI_DIM) - post = ANSI_RESET - pad = widths[i] - len(cell) - parts.append(f" {pre}{cell}{post}{' ' * pad} ") - return f"|{'|'.join(parts)}|" - - sep = "|" + "|".join("-" * (w + 2) for w in widths) + "|" - return "\n".join( - [ - fmt(headers), - sep, - *(fmt(r, color=True) for r in rows), - ] - ) - - -def find_up( - start: Path, - predicate: Callable[[Path], bool], - label: str = "matching directory", -) -> Path: - """Walk upward from *start*, returning the first matching directory.""" - for directory in (start, *start.parents): - if predicate(directory): - return directory - raise FileNotFoundError(f"{label} not found above {start}") - - -def find_up_file(start: Path, name: str) -> Path | None: - """Walk upward from *start*, returning the first *name* found.""" - for directory in (start, *start.parents): - candidate = directory / name - if candidate.is_file(): - return candidate - return None - - -def is_workspace_root(d: Path) -> bool: - """Return True if *d* looks like a Cargo workspace root.""" - cargo = d / "Cargo.toml" - return ( - cargo.is_file() - and "[workspace]" in cargo.read_text(encoding="utf-8") - and (d / "pkgs").is_dir() - ) - - -def require_bin(name: str, path: str | None = None) -> str: - """Return the path to *name* or raise FileNotFoundError.""" - result = shutil.which(name, path=path) - if result is None and os.name == "nt": - result = shutil.which(f"{name}.exe", path=path) - if result is None: - where = "in expected path" if path else "in PATH" - raise FileNotFoundError(f"error: {name} binary not found {where}") - return result - - -@cache -def root_dir() -> Path: - """Return the workspace root (directory containing Cargo.toml).""" - return find_up( - Path(__file__).resolve().parent, - is_workspace_root, - "workspace Cargo.toml", - ) - - -def usable_threads() -> int: - """Return a conservative thread count (total CPUs minus one).""" - return max(1, (os.cpu_count() or 2) - 1) - - -def usable_mem() -> int: - """Return half the physical RAM in MiB. - - Raises RuntimeError when physical RAM cannot be determined. - """ - total = _physical_ram_bytes() - return total // (2 * 1024 * 1024) - - -def _physical_ram_bytes() -> int: - """Return total physical RAM in bytes.""" - if sys.platform.startswith("linux"): - for line in Path("/proc/meminfo").read_text(encoding="utf-8").splitlines(): - if line.startswith("MemTotal:"): - return int(line.split()[1]) * 1024 - raise RuntimeError("MemTotal not found in /proc/meminfo") - if sys.platform == "darwin": - try: - out = subprocess.check_output( - ["sysctl", "-n", "hw.memsize"], # noqa: S607 - ) - return int(out.strip()) - except ( - FileNotFoundError, subprocess.CalledProcessError, ValueError, - ) as exc: - raise RuntimeError( - "could not determine physical RAM on macOS", - ) from exc - if sys.platform == "win32": - try: - out = subprocess.check_output( - ["powershell", "-NoProfile", "-Command", # noqa: S607 - "(Get-CimInstance Win32_ComputerSystem)" - ".TotalPhysicalMemory"], - ).decode() - value = out.strip() - if value.isdigit(): - return int(value) - except (FileNotFoundError, subprocess.CalledProcessError): - pass - try: - out = subprocess.check_output( - ["wmic", "computersystem", "get", # noqa: S607 - "TotalPhysicalMemory", "/value"], - ).decode() - for line in out.splitlines(): - if line.startswith("TotalPhysicalMemory="): - return int(line.split("=", 1)[1].strip()) - except (FileNotFoundError, subprocess.CalledProcessError, ValueError): - pass - raise RuntimeError("could not determine physical RAM on Windows") - raise RuntimeError(f"unsupported platform: {sys.platform}") diff --git a/contrib/common.py b/contrib/common.py new file mode 120000 index 00000000..304e73b7 --- /dev/null +++ b/contrib/common.py @@ -0,0 +1 @@ +../maint/common.py \ No newline at end of file diff --git a/contrib/lint/lint_semgrep.py b/contrib/lint/lint_semgrep.py deleted file mode 100755 index b25b5bc6..00000000 --- a/contrib/lint/lint_semgrep.py +++ /dev/null @@ -1,60 +0,0 @@ -#!/usr/bin/env python3 -# coding: latin-1 - -# -# Copyright (c) 2026-present, The Dash Core developers -# SPDX-License-Identifier: MIT -# See the accompanying file LICENSE or https://opensource.org/license/MIT -# - -"""Runs semgrep rules against the workspace.""" - -from __future__ import annotations - -import subprocess -import sys - -from common import ( - RETCODE_ERR, - RETCODE_PASS, - SOURCE_DIRS, - require_bin, - root_dir, -) - - -def main() -> int: - semgrep_bin = require_bin("semgrep") - - repo_root = root_dir() - config_dir = repo_root / "contrib" / "semgrep" - target_dirs = [repo_root / where for where in SOURCE_DIRS] - - configs: list[str] = [] - for cfg in sorted(config_dir.glob("*.yml")): - configs.extend(["--config", str(cfg)]) - - if not configs: - raise FileNotFoundError( - "no semgrep configs found in contrib/semgrep/", - ) - - result = subprocess.run( # noqa: S603 - [ - semgrep_bin, - "scan", - *configs, - "--error", - *[str(d) for d in target_dirs], - ], - check=False, - ) - return RETCODE_PASS if result.returncode == 0 else RETCODE_ERR - - -if __name__ == "__main__": - try: - sys.exit(main()) - except Exception as exc: # noqa: BLE001 - print(exc, file=sys.stderr) - sys.exit(RETCODE_ERR) diff --git a/docs/README.md b/docs/README.md index a4380544..765886ec 100644 --- a/docs/README.md +++ b/docs/README.md @@ -15,7 +15,7 @@ utilised by your packages. * Base packages. These packages implement specific algorithms but without chain-distinguishing consensus logic. * Protocol packages. These packages define the Dash protocol as deployed, blocks, transactions, chain parameters. ---8<-- "README.md:crate-graph" + *Note: Solid lines are build dependencies, dotted lines are test dependencies.* diff --git a/docs/dev/about_docs.md b/docs/dev/about_docs.md index 28f4e541..d2151c5e 100644 --- a/docs/dev/about_docs.md +++ b/docs/dev/about_docs.md @@ -79,6 +79,37 @@ outside `docs/` resolve to the forge instead. > Zensical treats on-disk `.md` links as documentation and will fail to build if they are located outside `docs/`. > This does not affect non-Markdown files and directories. +### Splicing + +To keep material that would otherwise be duplicated from falling out of sync, a document can be split into reusable +segments that other documents splice in, or to assemble a document outright. This lets the on-disk layout keep +documentation close to the material it describes, while keeping the prose cohesive when it is read as a webpage. + +Splicing happens when the site is built, so a document read on GitHub is the file as stored on-disk, without the +segments spliced in. We use a custom comment syntax to ensure that splice markers are invisible, so they don't interfere +with GitHub and WYSIWYG editors. + +This makes Zensical the primary authority on the _shape_ of a document and it is recommended to [preview](#preview) your +edits and the pages that could be affected by your edits to ensure it remains pleasant to read. + +To splice in a whole document, or a spliced segment of it (like `setup`), the syntax is as below. + +```markdown + + +``` + +To create a spliceable segment, wrap the desired text in `start` and `end` markers carrying its label (like `setup`). + +```markdown + +Carried into the splice. + +``` + +Links in spliced material are resolved against the file that defines them, not the page splicing it in (see +[link processing](#link-processing)). Splices may nest, and a directive inside a code fence is inert. + ## Postprocessing > [!WARNING] diff --git a/docs/dev/getting_started.md b/docs/dev/getting_started.md index 9a9406a2..ca260d6f 100644 --- a/docs/dev/getting_started.md +++ b/docs/dev/getting_started.md @@ -25,4 +25,4 @@ components at the supported versions without further intervention. Should you wa version, please consult the vendor documentation for `RUSTUP_TOOLCHAIN` ([source](https://rust-lang.github.io/rustup/environment-variables.html)). ---8<-- "contrib/README.md:setup" + diff --git a/docs/dev/maintenance.md b/docs/dev/maintenance.md new file mode 100644 index 00000000..7e650eae --- /dev/null +++ b/docs/dev/maintenance.md @@ -0,0 +1,4 @@ +# Maintenance + + + diff --git a/docs/preprocess.py b/docs/preprocess.py index c4cd5c02..0da24d90 100644 --- a/docs/preprocess.py +++ b/docs/preprocess.py @@ -95,7 +95,13 @@ def run(self, lines: list[str]) -> list[str]: _FENCE_RE = re.compile(r"^\s*(`{3,}|~{3,})(.*)$") # Matches a splice from another file, or one named section from it. -_INCLUDE_RE = re.compile(r'^\s*--8<--\s+"([^"]+)"\s*$') +_INCLUDE_RE = re.compile(r"^\s*\s*$") + +# Matches any directive-shaped comment, whatever it names. +_DIRECTIVE_RE = re.compile(r"^\s*\s*$") + +# The directives this module answers for. Anything else is a misspelling. +_DIRECTIVES = frozenset({"include", "start", "end"}) # Matches the target of an inline link, and any title trailing it. _LINK_RE = re.compile( @@ -113,6 +119,13 @@ def run(self, lines: list[str]) -> list[str]: _DEPTH_LIMIT = 8 +def _named_directive(line: str) -> None: + """Raise when *line* holds a directive this module does not answer for.""" + found = _DIRECTIVE_RE.match(line) + if found is not None and found.group(1) not in _DIRECTIVES: + raise ValueError(f"{found.group(1)}: no such directive") + + class _Fences: """Running fence state over a sequence of lines.""" @@ -173,8 +186,11 @@ def _expand( for line in lines: # A directive inside a fence is the syntax being shown, not used. - match = None if fences.covers(line) else _INCLUDE_RE.match(line) + fenced = fences.covers(line) + match = None if fenced else _INCLUDE_RE.match(line) if match is None: + if not fenced: + _named_directive(line) output.append(line) continue if budget <= 0: @@ -328,7 +344,7 @@ def test_include_splices_a_section(self) -> None: with self._scratch( whole="above\n\ninside\n\nbelow\n", ) as home: - out = self._render(f'--8<-- "{home}/whole.md:mid"\n') + out = self._render(f'\n') assert "inside" in out assert "above" not in out assert "below" not in out @@ -338,66 +354,80 @@ def test_include_refuses_an_unknown_section(self) -> None: with self._scratch(whole="nothing marked\n") as home: with pytest.raises(ValueError, match="no such section"): - self._render(f'--8<-- "{home}/whole.md:mid"\n') + self._render(f'\n') def test_include_rejects_an_absolute_path(self) -> None: import pytest with pytest.raises(ValueError, match="outside the repository"): - self._render('--8<-- "/etc/hosts"\n') + self._render('\n') def test_include_rejects_a_traversal(self) -> None: import pytest with pytest.raises(ValueError, match="outside the repository"): - self._render('--8<-- "../../../../etc/hosts"\n') + self._render('\n') def test_include_inside_a_fence_is_left_alone(self) -> None: - out = self._render('```\n--8<-- "unconv.toml"\n```\n') - assert "8<--" in out + out = self._render('```\n\n```\n') + assert "[include:maint/unconv.toml]" in out assert "[global]" not in out + def test_a_misspelt_directive_is_refused(self) -> None: + import pytest + + with pytest.raises(ValueError, match="no such directive"): + self._render("\n") + + def test_a_section_marker_is_a_known_directive(self) -> None: + out = self._render("\nkept\n\n") + assert "kept" in out + + def test_a_misspelt_directive_in_a_fence_is_left_alone(self) -> None: + out = self._render("```\n\n```\n") + assert "[inclde:README.md]" in out + def test_include_nests(self) -> None: with self._scratch( - outer='--8<-- "unconv.toml"\n', + outer='\n', ) as home: - out = self._render(f'--8<-- "{home}/outer.md"\n') - assert "8<--" not in out + out = self._render(f'\n') + assert "[include:" not in out assert "global" in out def test_include_refuses_a_cycle(self) -> None: import pytest with self._scratch(loop="") as home: - spec = f'--8<-- "{home}/loop.md"\n' + spec = f'\n' (root_dir() / home / "loop.md").write_text(spec, encoding="utf-8") with pytest.raises(ValueError, match="nested past the limit"): self._render(spec) def test_titled_link_is_rebased(self) -> None: with self._scratch(page='[a](../README.md "root")\n') as home: - out = self._render(f'--8<-- "{home}/page.md"\n') + out = self._render(f'\n') assert f'href="{_REPO}/blob/{_BRANCH}/README.md"' in out assert 'title="root"' in out def test_caged_link_is_rebased(self) -> None: with self._scratch(page="[a](<../README.md>)\n") as home: - out = self._render(f'--8<-- "{home}/page.md"\n') + out = self._render(f'\n') assert f'href="{_REPO}/blob/{_BRANCH}/README.md"' in out def test_bare_link_is_rebased(self) -> None: - with self._scratch(page="[a](../unconv.toml)\n") as home: - out = self._render(f'--8<-- "{home}/page.md"\n') - assert f'href="{_REPO}/blob/{_BRANCH}/unconv.toml"' in out + with self._scratch(page="[a](../maint/unconv.toml)\n") as home: + out = self._render(f'\n') + assert f'href="{_REPO}/blob/{_BRANCH}/maint/unconv.toml"' in out def test_page_under_docs_is_addressed_from_the_site(self) -> None: with self._scratch(page="[a](../docs/dev/guide_rust.md)\n") as home: - out = self._render(f'--8<-- "{home}/page.md"\n') + out = self._render(f'\n') assert 'href="/dev/guide_rust/"' in out def test_off_disk_link_is_left_alone(self) -> None: with self._scratch(page="[a](tel:+15551212) [b](irc://x/y)\n") as home: - out = self._render(f'--8<-- "{home}/page.md"\n') + out = self._render(f'\n') assert 'href="tel:+15551212"' in out assert 'href="irc://x/y"' in out @@ -406,7 +436,7 @@ def test_missing_link_target_is_refused(self) -> None: with self._scratch(page="[a](./nope.md)\n") as home: with pytest.raises(ValueError, match="no such file"): - self._render(f'--8<-- "{home}/page.md"\n') + self._render(f'\n') @staticmethod def _pointer() -> IncludePreprocessor: @@ -437,11 +467,13 @@ def test_wrong_case_link_is_refused(self) -> None: # Refused either as missing or as misspelt, by the host's case rules. with self._scratch(page="[a](../README.MD)\n") as home: with pytest.raises(ValueError, match=r"no such file|not spelt"): - self._render(f'--8<-- "{home}/page.md"\n') + self._render(f'\n') def test_include_survives_a_fenced_info_string(self) -> None: - out = self._render('```\n```text\n--8<-- "unconv.toml"\n```\n') - assert "8<--" in out + out = self._render( + '```\n```text\n\n```\n' + ) + assert "[include:maint/unconv.toml]" in out assert "[global]" not in out def test_alert_survives_a_fenced_info_string(self) -> None: @@ -451,21 +483,21 @@ def test_alert_survives_a_fenced_info_string(self) -> None: def test_site_root_link_is_left_for_postprocessing(self) -> None: with self._scratch(page="[a](/dev/about_docs/)\n") as home: - out = self._render(f'--8<-- "{home}/page.md"\n') + out = self._render(f'\n') assert 'href="/dev/about_docs/"' in out def test_a_fence_an_include_opens_holds_over_the_parent(self) -> None: with self._scratch(opener="```\n", body="spliced text\n") as home: out = self._pointer().run([ - f'--8<-- "{home}/opener.md"', - f'--8<-- "{home}/body.md"', + f'', + f'', "```", - f'--8<-- "{home}/body.md"', + f'', ]) # Held back while the fence the first splice opened is still open, # then spliced once the parent's own marker closes that fence. assert out[0] == "```" - assert out[1].startswith("--8<--") + assert out[1].startswith("