diff --git a/.github/dependabot.yml b/.github/dependabot.yml index e18ae10ccc..b65cdf26b3 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -4,6 +4,11 @@ updates: directory: "/" schedule: interval: "weekly" + cooldown: + # default-days also covers patch releases and actions not using semver. + default-days: 14 + semver-major-days: 90 + semver-minor-days: 30 groups: actions: patterns: diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml index deab27e150..7f06c99e9c 100644 --- a/.github/workflows/backport.yml +++ b/.github/workflows/backport.yml @@ -31,8 +31,8 @@ jobs: startsWith(github.event.comment.body, '/backport') ) steps: - - uses: actions/checkout@v7 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Create backport pull requests - uses: korthout/backport-action@v4 + uses: korthout/backport-action@2e830a1d0b8269505846ddd407a70876913ad1f8 # v4.6.0 with: label_pattern: '^port/(.+)$' diff --git a/.github/workflows/bigendian.yml b/.github/workflows/bigendian.yml new file mode 100644 index 0000000000..be8d18d0b1 --- /dev/null +++ b/.github/workflows/bigendian.yml @@ -0,0 +1,200 @@ +# Big-endian test: build and test borg on s390x (big-endian) under qemu +# user-mode emulation, and check that a repository written on a big-endian +# machine can be read on a little-endian one and the other way round. +# +# Why: borg's repository format is architecture independent and the native code +# has explicit big-endian code paths (e.g. the __builtin_bswap64 calls in the +# chunker kernels), but every other machine we test on is little-endian, so +# those code paths are never executed and a bug in them would only be found by +# the users of s390x, some ppc64 and some mips machines. +# +# Everything inside the container is emulated instruction by instruction, so +# this is slow. Therefore it does not run for every pull request, but only when +# native code or format relevant code was touched (plus weekly and on demand). + +name: Big-endian + +on: + pull_request: + branches: [ master ] + paths: + - '**.pyx' + - '**.pxd' + - 'src/borg/**/*.c' + - 'src/borg/**/*.h' + - 'src/borg/chunkers/**' + - 'src/borg/crypto/**' + - 'src/borg/repository.py' + - 'src/borg/repoobj.py' + - 'src/borg/archive.py' + - 'scripts/endian_interop_test.py' + - '.github/workflows/bigendian.yml' + schedule: + - cron: '43 5 * * 3' # Wednesdays at 05:43 UTC + workflow_dispatch: # Allow manual trigger + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +permissions: + contents: read + +env: + PY_COLORS: "1" + +jobs: + s390x: + name: s390x (big-endian, emulated) + runs-on: ubuntu-24.04 + timeout-minutes: 120 + + env: + # Debian trixie has python 3.13, and all our dependencies that ship binary + # wheels (blake3, backports-zstd, PyYAML, cffi) have s390x wheels for it - + # only the small C/Cython extensions are compiled (emulated) from source. + IMAGE: docker.io/s390x/debian:trixie-slim + CONTAINER: borg-s390x + # both sides of the interoperability test share this directory: + INTEROP: ${{ github.workspace }}/.interop + # a hung emulated test must fail instead of eating the job timeout: + PYTEST_TIMEOUT: "600" + # The endianness sensitive parts: the native code (chunkers, crypto, + # compression, hashindex, item) and the code that reads/writes the + # repository format. Extend this if it turns out to be too narrow. + PYTEST_TARGETS: >- + borg.testsuite.chunkers + borg.testsuite.crypto + borg.testsuite.compress_test + borg.testsuite.hashindex_test + borg.testsuite.item_test + borg.testsuite.repoobj_test + borg.testsuite.repository_test + borg.testsuite.archive_test + borg.testsuite.helpers.msgpack_test + + steps: + - uses: actions/checkout@v7 + with: + # Just fetching one commit is not enough for setuptools-scm, so we fetch all. + fetch-depth: 0 + fetch-tags: true + + - name: Set up Python + uses: actions/setup-python@v7 + with: + # same python version as in the container, so both sides are comparable + python-version: '3.13' + + # The wheels pip has to build from source inside the emulated container + # (msgpack, argon2-cffi-bindings, borghash, borgstore) are the expensive + # part of the container setup, so keep them from run to run. + - name: Cache pip-built wheels (s390x) + uses: actions/cache@v6 + with: + path: .pip-cache-s390x + key: s390x-pip-${{ hashFiles('pyproject.toml') }} + restore-keys: | + s390x-pip- + + - name: Install Linux packages + run: | + sudo apt-get update + sudo apt-get install -y pkg-config build-essential + sudo apt-get install -y libssl-dev libacl1-dev liblz4-dev + + - name: Build borg (native, little-endian) + run: | + set -euxo pipefail + # Note: a non-editable install on purpose - the s390x build below uses + # the same source tree and an editable install would put the extension + # modules of one architecture in there for the other one to pick up. + python -m venv "$RUNNER_TEMP/venv-native" + "$RUNNER_TEMP/venv-native/bin/pip" install --upgrade pip wheel + # Note: no pytest and no other development dependencies in this venv on + # purpose - this is what a normal borg installation looks like. + "$RUNNER_TEMP/venv-native/bin/pip" install . + "$RUNNER_TEMP/venv-native/bin/borg" --version + "$RUNNER_TEMP/venv-native/bin/python" -c 'import sys; assert sys.byteorder == "little", sys.byteorder' + # the container has no git, so tell setuptools-scm the version we got here: + echo "BORG_VERSION=$("$RUNNER_TEMP/venv-native/bin/borg" --version | cut -d' ' -f2)" >> $GITHUB_ENV + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + with: + platforms: s390x + + - name: Start the s390x container + run: | + set -euxo pipefail + mkdir -p .pip-cache-s390x + docker run -d --name "$CONTAINER" --platform linux/s390x \ + -v "${{ github.workspace }}:/borg" -w /borg \ + -e PIP_CACHE_DIR=/borg/.pip-cache-s390x \ + -e SETUPTOOLS_SCM_PRETEND_VERSION_FOR_BORGBACKUP="$BORG_VERSION" \ + -e HOST_UID="$(id -u)" -e HOST_GID="$(id -g)" \ + "$IMAGE" sleep infinity + # if this says anything but s390x, the emulation did not kick in: + test "$(docker exec "$CONTAINER" uname -m)" = "s390x" + + - name: Build borg (s390x, big-endian) + run: | + docker exec -i "$CONTAINER" bash -s <<'EOF' + set -euxo pipefail + export DEBIAN_FRONTEND=noninteractive + apt-get update + apt-get install -y --no-install-recommends \ + python3 python3-dev python3-venv \ + build-essential pkg-config libssl-dev libacl1-dev liblz4-dev + python3 -c 'import sys; assert sys.byteorder == "big", sys.byteorder; print("byteorder:", sys.byteorder)' + python3 -m venv /venv + /venv/bin/pip install --upgrade pip wheel + /venv/bin/pip install pytest pytest-xdist pytest-benchmark pytest-timeout + /venv/bin/pip install /borg + /venv/bin/borg --version + chown -R "$HOST_UID:$HOST_GID" /borg/.pip-cache-s390x + EOF + + - name: Run the endianness sensitive tests on s390x + run: | + docker exec -i \ + -e PYTEST_TARGETS="$PYTEST_TARGETS" \ + -e PYTEST_TIMEOUT="$PYTEST_TIMEOUT" \ + -e PY_COLORS="$PY_COLORS" \ + "$CONTAINER" bash -s <<'EOF' + set -euxo pipefail + # run against the installed borg, not against the source tree: + cd /tmp + /venv/bin/python -m pytest -v -n4 -rs --benchmark-skip --pyargs $PYTEST_TARGETS + EOF + + - name: Write test data and a repository on s390x + run: | + set -euxo pipefail + "$RUNNER_TEMP/venv-native/bin/python" scripts/endian_interop_test.py testdata "$INTEROP" + docker exec -i -e INTEROP=/borg/.interop "$CONTAINER" bash -s <<'EOF' + set -euxo pipefail + BORG=/venv/bin/borg /venv/bin/python /borg/scripts/endian_interop_test.py write "$INTEROP" be + chown -R "$HOST_UID:$HOST_GID" "$INTEROP" + EOF + + - name: Read the s390x repository on x86_64 (and write to it) + run: | + set -euxo pipefail + export BORG="$RUNNER_TEMP/venv-native/bin/borg" + PY="$RUNNER_TEMP/venv-native/bin/python" + $PY scripts/endian_interop_test.py verify "$INTEROP" be --as le + $PY scripts/endian_interop_test.py write "$INTEROP" le + $PY scripts/endian_interop_test.py compare "$INTEROP" be le + + - name: Read the x86_64 archives on s390x + run: | + docker exec -i -e INTEROP=/borg/.interop "$CONTAINER" bash -s <<'EOF' + set -euxo pipefail + BORG=/venv/bin/borg /venv/bin/python /borg/scripts/endian_interop_test.py verify "$INTEROP" le --as be + chown -R "$HOST_UID:$HOST_GID" "$INTEROP" + EOF + + - name: Stop the s390x container + if: ${{ always() }} + run: docker rm -f "$CONTAINER" || true diff --git a/.github/workflows/black.yaml b/.github/workflows/black.yaml index 33ef1f60f1..146c3beba3 100644 --- a/.github/workflows/black.yaml +++ b/.github/workflows/black.yaml @@ -19,12 +19,15 @@ concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.ref }} cancel-in-progress: ${{ github.event_name == 'pull_request' }} +permissions: + contents: read + jobs: lint: runs-on: ubuntu-24.04 timeout-minutes: 5 steps: - - uses: actions/checkout@v7 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - uses: psf/black@87928e6d6761a4a6d22250e1fee5601b3998086e # 26.5.1 with: version: "~= 24.0" diff --git a/.github/workflows/canary-py312-ubuntu2604.yml b/.github/workflows/canary-py312-ubuntu2604.yml new file mode 100644 index 0000000000..c4658898da --- /dev/null +++ b/.github/workflows/canary-py312-ubuntu2604.yml @@ -0,0 +1,49 @@ +# Inverted canary: watch for Python 3.12 becoming available on ubuntu-26.04. +# +# ubuntu-26.04 runners currently only work with the image's default Python +# (3.14) - actions/setup-python has no cached builds of older Pythons for +# this image yet, so requesting 3.12 fails. This probe tries daily. +# +# The logic is inverted on purpose: while 3.12 is still unavailable, the +# run stays GREEN (expected state, no noise). The day setup-python 3.12 +# succeeds, the job FAILS loudly - GitHub emails scheduled-workflow +# failures to the user who last modified the cron line, so a red run here +# is the "3.12 works now" notification. When it fires: add ubuntu-26.04 +# / 3.12 jobs to ci.yml and delete this workflow. + +name: Canary (Python 3.12 on ubuntu-26.04) + +on: + schedule: + - cron: '0 6 * * *' # Run at 06:00 UTC + workflow_dispatch: # Allow manual trigger + +permissions: {} + +jobs: + probe: + name: Probe setup-python 3.12 + runs-on: ubuntu-26.04 + timeout-minutes: 30 + + steps: + - name: Try to set up Python 3.12 + id: setup + continue-on-error: true + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: '3.12' + + - name: Still unsupported - stay green + if: steps.setup.outcome == 'failure' + run: | + echo "::notice::Python 3.12 is still unavailable on ubuntu-26.04 (expected - nothing to do)." + + - name: Python 3.12 works now - fail to trigger the notification email + if: steps.setup.outcome == 'success' + run: | + python --version + python -c 'import ssl, sqlite3, zlib, lzma' + python -m pip --version + echo "::error::Python 3.12 is NOW AVAILABLE on ubuntu-26.04! Add it to ci.yml and delete this canary workflow. (This failure is the notification - see the comment at the top of the workflow file.)" + exit 1 diff --git a/.github/workflows/canary.yml b/.github/workflows/canary.yml index c36a6d2737..818cee44b7 100644 --- a/.github/workflows/canary.yml +++ b/.github/workflows/canary.yml @@ -8,6 +8,13 @@ on: permissions: contents: read +env: + # Force colored tox and pytest output even without a tty - the GitHub + # Actions log viewer renders ANSI colors. tox passes both vars through + # to pytest (pass_env = ["*"]). + PY_COLORS: "1" # pytest + TOX_COLORED: "yes" # tox's own output + jobs: canary_tests: name: Canary (${{ matrix.os }}, ${{ matrix.python-version }}, ${{ matrix.toxenv }}) @@ -37,13 +44,13 @@ jobs: toxenv: py314-none steps: - - uses: actions/checkout@v7 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: fetch-depth: 0 fetch-tags: true - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v7 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: ${{ matrix.python-version }} @@ -107,20 +114,34 @@ jobs: timeout-minutes: 180 env: - PY_COLORS: 1 MSYS2_ARG_CONV_EXCL: "*" MSYS2_ENV_CONV_EXCL: "*" + # see the "Cache pip-built wheels" step. MSYS2_ENV_CONV_EXCL above keeps + # this a Windows path when it enters the msys2 shell. + PIP_CACHE_DIR: ${{ github.workspace }}\.pip-cache defaults: run: shell: msys2 {0} steps: - - uses: actions/checkout@v7 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: fetch-depth: 0 - - uses: msys2/setup-msys2@v2 + # Same as the "Cache pip-built wheels" step in ci.yml (MSYS2's mingw + # Python cannot use PyPI's win_amd64 wheels). Own key, because the + # unlocked requirements may resolve to different versions; falls back + # to (and gets picked up by) the ci.yml cache via the shared prefix. + - name: Cache pip-built wheels + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: .pip-cache + key: windows-msys2-pip-canary-${{ hashFiles('pyproject.toml', 'requirements.d/pyinstaller.txt') }} + restore-keys: | + windows-msys2-pip- + + - uses: msys2/setup-msys2@66cd2cce69caa17b53920067426061ca1de3a884 # v2.32.0 with: msystem: UCRT64 update: true diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7f20f77cc7..edbec097ce 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -28,6 +28,13 @@ concurrency: permissions: contents: read +env: + # Force colored tox and pytest output even without a tty - the GitHub + # Actions log viewer renders ANSI colors. tox passes both vars through + # to pytest (pass_env = ["*"]). + PY_COLORS: "1" # pytest + TOX_COLORED: "yes" # tox's own output + jobs: lint: @@ -35,8 +42,8 @@ jobs: timeout-minutes: 5 steps: - - uses: actions/checkout@v7 - - uses: astral-sh/ruff-action@v4.1.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: astral-sh/ruff-action@278981a28ce3188b1e39527901f38254bf3aac89 # v4.1.0 security: @@ -44,9 +51,9 @@ jobs: timeout-minutes: 5 steps: - - uses: actions/checkout@v7 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Set up Python - uses: actions/setup-python@v7 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: '3.11' - name: Install dependencies @@ -64,14 +71,14 @@ jobs: needs: [lint] steps: - - uses: actions/checkout@v7 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: # Just fetching one commit is not enough for setuptools-scm, so we fetch all. fetch-depth: 0 fetch-tags: true - name: Set up Python - uses: actions/setup-python@v7 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: '3.12' @@ -175,19 +182,19 @@ jobs: timeout-minutes: 360 steps: - - uses: actions/checkout@v7 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: # Just fetching one commit is not enough for setuptools-scm, so we fetch all. fetch-depth: 0 fetch-tags: true - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v7 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: ${{ matrix.python-version }} - name: Cache pip - uses: actions/cache@v6 + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ~/.cache/pip key: ${{ runner.os }}-${{ runner.arch }}-pip-${{ hashFiles('requirements.d/development.lock.txt') }} @@ -196,7 +203,7 @@ jobs: ${{ runner.os }}-${{ runner.arch }}- - name: Cache tox environments - uses: actions/cache@v6 + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: .tox key: ${{ runner.os }}-${{ runner.arch }}-tox-${{ matrix.toxenv }}-${{ hashFiles('requirements.d/development.lock.txt', 'pyproject.toml') }} @@ -345,13 +352,13 @@ jobs: - name: Attest binaries provenance (${{ matrix.binary }}) if: ${{ matrix.binary && startsWith(github.ref, 'refs/tags/') }} - uses: actions/attest-build-provenance@v4 + uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8 # v4.2.2 with: subject-path: 'artifacts/*' - name: Upload binaries (${{ matrix.binary }}) if: ${{ matrix.binary && startsWith(github.ref, 'refs/tags/') }} - uses: actions/upload-artifact@v7 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: ${{ matrix.binary }} path: artifacts/* @@ -367,7 +374,7 @@ jobs: - name: Upload test results to Codecov if: ${{ !cancelled() && !contains(matrix.toxenv, 'mypy') && !contains(matrix.toxenv, 'docs') }} - uses: codecov/codecov-action@v7 + uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 env: OS: ${{ runner.os }} python: ${{ matrix.python-version }} @@ -379,7 +386,7 @@ jobs: - name: Upload coverage to Codecov if: ${{ !cancelled() && !contains(matrix.toxenv, 'mypy') && !contains(matrix.toxenv, 'docs') }} - uses: codecov/codecov-action@v7 + uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 env: OS: ${{ runner.os }} python: ${{ matrix.python-version }} @@ -411,14 +418,16 @@ jobs: artifact_prefix: borg-freebsd-15-x86_64-gh - os: netbsd - version: '10.1' + version: '11.0' display_name: NetBSD do_binaries: false - os: openbsd - version: '7.8' + version: '7.9' display_name: OpenBSD do_binaries: false + # extra RAM for the mfs-backed TMPDIR, see the openbsd test step + memory: 12G - os: omnios version: 'r151056' @@ -427,18 +436,35 @@ jobs: steps: - name: Check out repository - uses: actions/checkout@v7 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: fetch-depth: 0 fetch-tags: true + # .pip-cache lives inside the workspace, which cross-platform-actions + # rsyncs into the VM and back, so wheels built from sdists in one run + # (most of the VM setup time) are reused by the next one. + - name: Cache pip-built wheels + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: .pip-cache + key: ${{ matrix.os }}-${{ matrix.version }}-pip-${{ hashFiles('requirements.d/development.lock.txt') }} + restore-keys: | + ${{ matrix.os }}-${{ matrix.version }}-pip- + - name: Start VM for ${{ matrix.display_name }} id: cross_os - uses: cross-platform-actions/action@v1.3.0 + # a normal boot takes < 1 minute; since 1.4.0 the action bounds + # waiting for boot itself, this is just an additional safety net. + timeout-minutes: 15 + uses: cross-platform-actions/action@24ef01df165c76df1ed2b9f9e9212e78dc2fc963 # v1.4.0 with: operating_system: ${{ matrix.os }} version: ${{ matrix.version }} shell: bash + # the default VM size (2 cpus) leaves half of the runner's 4 vcpus idle + cpu_count: 4 + memory: ${{ matrix.memory || '8G' }} - name: Test on ${{ matrix.display_name }} shell: cpa.sh {0} @@ -447,6 +473,18 @@ jobs: run: | set -euxo pipefail + # a hung test must fail instead of stalling the suite until the + # job timeout (tox passes all env vars through to pytest). + export PYTEST_TIMEOUT=300 + + # colored tox/pytest output - the workflow-level env vars do not + # reach into the VM, so export them here again. + export PY_COLORS=1 + export TOX_COLORED=yes + + # see the "Cache pip-built wheels" step + export PIP_CACHE_DIR="$PWD/.pip-cache" + case "${{ matrix.os }}" in freebsd) export IGNORE_OSVERSION=yes @@ -502,50 +540,49 @@ jobs: netbsd) arch="$(uname -m)" sudo -E mkdir -p /usr/pkg/etc/pkgin - echo "https://ftp.NetBSD.org/pub/pkgsrc/packages/NetBSD/${arch}/10.1/All" | sudo tee /usr/pkg/etc/pkgin/repositories.conf > /dev/null + echo "https://ftp.NetBSD.org/pub/pkgsrc/packages/NetBSD/${arch}/11.0/All" | sudo tee /usr/pkg/etc/pkgin/repositories.conf > /dev/null sudo -E pkgin update sudo -E pkgin -y upgrade sudo -E pkgin -y install lz4 git sudo -E pkgin -y install rust sudo -E pkgin -y install pkg-config - sudo -E pkgin -y install py311-pip py311-virtualenv py311-tox - sudo -E ln -sf /usr/pkg/bin/python3.11 /usr/pkg/bin/python3 - sudo -E ln -sf /usr/pkg/bin/pip3.11 /usr/pkg/bin/pip3 - sudo -E ln -sf /usr/pkg/bin/virtualenv-3.11 /usr/pkg/bin/virtualenv3 - sudo -E ln -sf /usr/pkg/bin/tox-3.11 /usr/pkg/bin/tox3 + # rust already depends on python313, so use that + sudo -E pkgin -y install py313-pip py313-virtualenv py313-tox + sudo -E ln -sf /usr/pkg/bin/python3.13 /usr/pkg/bin/python3 + sudo -E ln -sf /usr/pkg/bin/pip3.13 /usr/pkg/bin/pip3 + sudo -E ln -sf /usr/pkg/bin/virtualenv-3.13 /usr/pkg/bin/virtualenv3 + sudo -E ln -sf /usr/pkg/bin/tox-3.13 /usr/pkg/bin/tox3 # Ensure base system admin tools are on PATH for the non-root shell export PATH="/sbin:/usr/sbin:$PATH" - echo "--- Preparing an extattr-enabled filesystem ---" - # On many NetBSD setups /tmp is tmpfs without extended attributes. - # Create a FFS image with extended attributes enabled and use it for TMPDIR. - VNDDEV="vnd0" - IMGFILE="/tmp/fs.img" - sudo -E dd if=/dev/zero of=${IMGFILE} bs=1m count=1024 - sudo -E vndconfig -c "${VNDDEV}" "${IMGFILE}" - sudo -E newfs -O 2ea /dev/r${VNDDEV}a - MNT="/mnt/eafs" - sudo -E mkdir -p ${MNT} - sudo -E mount -t ffs -o extattr /dev/${VNDDEV}a $MNT - export TMPDIR="${MNT}/tmp" + # On the netbsd 11 VM, / is a fs with extended attributes. + export TMPDIR="/tmp_eafs" sudo -E mkdir -p ${TMPDIR} sudo -E chmod 1777 ${TMPDIR} touch ${TMPDIR}/testfile lsextattr user ${TMPDIR}/testfile && echo "[xattr] *** xattrs SUPPORTED on ${TMPDIR}! ***" - tox3 -e py311-none + tox3 -e py313-none ;; openbsd) - sudo -E pkg_add lz4 git - sudo -E pkg_add rust - sudo -E pkg_add openssl%3.5 - sudo -E pkg_add py3-pip py3-virtualenv py3-tox + # Put the temp tree (pip/cc build temps, pytest tmp dirs and the + # borg test repos in them) on a memory-backed filesystem instead + # of the slow FFS disk. Sized generously (the pytest tmp tree is + # only cleaned up after the run); the VM gets 12G RAM for this. + sudo mkdir -p /mfs + # -O2: FFS2 format - mfs defaults to FFS1, whose 32 bit + # timestamps silently wrap (breaks test_extract_y2261). + sudo mount_mfs -O2 -s 6g swap /mfs + sudo chmod 1777 /mfs + export TMPDIR=/mfs + + sudo -E pkg_add lz4 git rust openssl%3.5 py3-pip py3-virtualenv py3-tox export BORG_OPENSSL_NAME=eopenssl35 - tox -e py312-none + tox -e py313-none ;; omnios) @@ -561,6 +598,12 @@ jobs: export TMPDIR=/var/tmp/borg-ci mkdir -p "$TMPDIR" + # show whether xattrs work on the ZFS-backed TMPDIR (they are files in a + # hidden per-file attribute directory there, listed via runat(1)) + touch "$TMPDIR/testfile" + /usr/bin/runat "$TMPDIR/testfile" ls -a && echo "*** xattrs supported on $TMPDIR ***" + rm "$TMPDIR/testfile" + python3 -m venv .venv . .venv/bin/activate python -V @@ -601,7 +644,7 @@ jobs: - name: Upload artifacts if: startsWith(github.ref, 'refs/tags/') && matrix.do_binaries - uses: actions/upload-artifact@v7 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: ${{ matrix.artifact_prefix }} path: artifacts/* @@ -609,13 +652,13 @@ jobs: - name: Attest provenance if: startsWith(github.ref, 'refs/tags/') && matrix.do_binaries - uses: actions/attest-build-provenance@v4 + uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8 # v4.2.2 with: subject-path: 'artifacts/*' - name: Upload test results to Codecov if: ${{ !cancelled() }} - uses: codecov/codecov-action@v7 + uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 env: OS: ${{ matrix.os }} with: @@ -626,7 +669,7 @@ jobs: - name: Upload coverage to Codecov if: ${{ !cancelled() }} - uses: codecov/codecov-action@v7 + uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 env: OS: ${{ matrix.os }} with: @@ -642,21 +685,39 @@ jobs: timeout-minutes: 90 needs: [lint] + permissions: + contents: read + id-token: write + attestations: write + env: - PY_COLORS: 1 MSYS2_ARG_CONV_EXCL: "*" MSYS2_ENV_CONV_EXCL: "*" + # see the "Cache pip-built wheels" step. MSYS2_ENV_CONV_EXCL above keeps + # this a Windows path when it enters the msys2 shell. + PIP_CACHE_DIR: ${{ github.workspace }}\.pip-cache defaults: run: shell: msys2 {0} steps: - - uses: actions/checkout@v7 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: fetch-depth: 0 - - uses: msys2/setup-msys2@v2 + # MSYS2's mingw Python cannot use PyPI's win_amd64 wheels, so pip builds + # all compiled deps from source - incl. building maturin via cargo, just + # to build the blake3 wheel. Persist the wheels pip builds. + - name: Cache pip-built wheels + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: .pip-cache + key: windows-msys2-pip-${{ hashFiles('pyproject.toml', 'requirements.d/pyinstaller.txt') }} + restore-keys: | + windows-msys2-pip- + + - uses: msys2/setup-msys2@66cd2cce69caa17b53920067426061ca1de3a884 # v2.32.0 with: msystem: UCRT64 update: true @@ -682,10 +743,36 @@ jobs: # build sdist and wheel in dist/... python -m build - - uses: actions/upload-artifact@v7 + # Same layout and naming as the binaries of the other platforms, so that + # the release job picks this up as a release asset, too. The single-file + # binary keeps its .exe extension - Windows needs it to run the file. + - name: Prepare binaries (borg-windows-x86_64-gh) + run: | + pushd dist/binary + echo "single-file binary" + ./borg.exe -V + echo "single-directory binary" + ./borg-dir/borg.exe -V + tar czf borg.tgz borg-dir + popd + mkdir -p artifacts + cp dist/binary/borg.exe artifacts/borg-windows-x86_64-gh.exe + cp dist/binary/borg.tgz artifacts/borg-windows-x86_64-gh.tgz + echo "binary files" + ls -l artifacts/ + + - name: Attest binaries provenance (borg-windows-x86_64-gh) + if: ${{ startsWith(github.ref, 'refs/tags/') }} + uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8 # v4.2.2 with: - name: borg-windows - path: dist/binary/borg.exe + subject-path: 'artifacts/*' + + - name: Upload binaries (borg-windows-x86_64-gh) + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: borg-windows-x86_64-gh + path: artifacts/* + if-no-files-found: error - name: Run tests run: | @@ -697,7 +784,7 @@ jobs: - name: Upload test results to Codecov if: ${{ !cancelled() }} - uses: codecov/codecov-action@v7 + uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 env: OS: ${{ runner.os }} python: '3.11' @@ -709,7 +796,7 @@ jobs: - name: Upload coverage to Codecov if: ${{ !cancelled() }} - uses: codecov/codecov-action@v7 + uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0 env: OS: ${{ runner.os }} python: '3.11' @@ -718,3 +805,61 @@ jobs: report_type: coverage env_vars: OS,python files: coverage.xml + + release: + # Build the sdist and draft the GitHub release with the binaries the jobs + # above built for this tag, see .github/workflows/release.yml. + # + # vm_tests is continue-on-error (a flaky BSD VM must not stop a release), so + # this waits for it, but only requires native_tests to have succeeded. A + # binary that did not get built is warned about while drafting the release. + if: ${{ !cancelled() && startsWith(github.ref, 'refs/tags/') && needs.native_tests.result == 'success' }} + needs: [native_tests, vm_tests] + + permissions: + contents: write + id-token: write + attestations: write + + uses: ./.github/workflows/release.yml + + pypi: + # Upload the sdist the release job built to PyPI. + # + # This job can not live in release.yml with the rest of the release code: + # PyPI trusted publishing does not work from a reusable workflow, see + # https://docs.pypi.org/trusted-publishers/troubleshooting/ + # + # One-time setup, so that no API token has to be stored anywhere: + # - on pypi.org, add a trusted publisher to the "borgbackup" project: + # owner "borgbackup", repository "borg", workflow "ci.yml", + # environment "pypi". + # - create the "pypi" environment in the repository settings. Configuring + # required reviewers for it makes this (irreversible) upload wait for an + # approval, which is the last chance to stop a release. + if: ${{ startsWith(github.ref, 'refs/tags/') && needs.release.result == 'success' }} + needs: [release] + + runs-on: ubuntu-24.04 + timeout-minutes: 30 + + environment: + name: pypi + url: https://pypi.org/project/borgbackup/ + + permissions: + contents: read + id-token: write # trusted publishing + + steps: + - name: Get the sdist built by the release job + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: sdist + path: dist + + - name: What we are about to upload + run: ls -l dist/ + + - name: Upload to PyPI + uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # v1.14.2 diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 869a598e0c..834bffadcc 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -46,16 +46,16 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v7 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: # Just fetching one commit is not enough for setuptools-scm, so we fetch all. fetch-depth: 0 - name: Set up Python - uses: actions/setup-python@v7 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: 3.11 - name: Cache pip - uses: actions/cache@v6 + uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ~/.cache/pip key: ${{ runner.os }}-pip-${{ hashFiles('requirements.d/development.txt') }} @@ -69,7 +69,7 @@ jobs: sudo apt-get install -y libssl-dev libacl1-dev liblz4-dev # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@v4.37.4 + uses: github/codeql-action/init@5595ccaf912efad79be6eef63a5619ff05969be3 # v4.37.6 with: languages: ${{ matrix.language }} # If you wish to specify custom queries, you can do so here or in a config file. @@ -83,4 +83,4 @@ jobs: pip3 install -r requirements.d/development.txt pip3 install -ve . - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v4.37.4 + uses: github/codeql-action/analyze@5595ccaf912efad79be6eef63a5619ff05969be3 # v4.37.6 diff --git a/.github/workflows/fame.yml b/.github/workflows/fame.yml index cb96c605b4..1baf34394c 100644 --- a/.github/workflows/fame.yml +++ b/.github/workflows/fame.yml @@ -42,13 +42,13 @@ jobs: pull-requests: write steps: - - uses: actions/checkout@v7 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: ref: master fetch-depth: 0 # git blame needs the whole history - name: Set up Python - uses: actions/setup-python@v7 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: '3.14' diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000000..4a9a29ebb8 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,171 @@ +# Release automation: build the source distribution, publish it to PyPI and +# create a GitHub release with the standalone binaries. +# +# This is called by ci.yml when a tag is pushed, and it deliberately runs as +# part of that same workflow run: that is where the binaries for the tag are +# built, and artifacts can only be downloaded within the run that created them. +# +# The GitHub release is created as a *draft* on purpose: +# - the release notes want a human, +# - the detached GPG signature of the sdist can only be made locally +# (scripts/sdist-sign), so it has to be added by hand, +# - and the binaries should be tried out before the release becomes visible. +# Publishing the draft is a single click in the GitHub UI. +# +# The upload to PyPI is *not* done here, but by the "pypi" job in ci.yml: PyPI +# trusted publishing can not be used from a reusable workflow, see +# https://docs.pypi.org/trusted-publishers/troubleshooting/ - so that job has to +# live in a workflow that is triggered by an event. This job hands the sdist +# over to it as a workflow artifact. + +name: Release + +on: + workflow_call: + +permissions: + contents: read + +env: + # The standalone binaries expected for a release: for every platform the + # single-file binary and, as a .tgz, the single-directory variant. See the + # "binary" entries of the native_tests matrix, the "artifact_prefix" entries + # of the vm_tests matrix and the windows_tests job in ci.yml - keep in sync. + EXPECTED_ASSETS: >- + borg-linux-glibc239-x86_64-gh + borg-linux-glibc239-x86_64-gh.tgz + borg-linux-glibc239-arm64-gh + borg-linux-glibc239-arm64-gh.tgz + borg-macos-15-arm64-gh + borg-macos-15-arm64-gh.tgz + borg-macos-15-x86_64-gh + borg-macos-15-x86_64-gh.tgz + borg-freebsd-15-x86_64-gh + borg-freebsd-15-x86_64-gh.tgz + borg-windows-x86_64-gh.exe + borg-windows-x86_64-gh.tgz + +jobs: + github_release: + name: Draft the GitHub release + runs-on: ubuntu-24.04 + timeout-minutes: 30 + + permissions: + contents: write # to create the release + id-token: write # to attest the sdist + attestations: write + + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + # Just fetching one commit is not enough for setuptools-scm, so we fetch all. + fetch-depth: 0 + fetch-tags: true + + - name: Set up Python + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: '3.13' + + - name: Install Linux packages + run: | + sudo apt-get update + sudo apt-get install -y pkg-config build-essential + sudo apt-get install -y libssl-dev libacl1-dev liblz4-dev + + - name: Build the sdist + run: | + set -euxo pipefail + python -m pip install --upgrade pip build twine + # only a sdist: we do not publish wheels, they would be platform specific. + python -m build --sdist + twine check dist/* + ls -l dist/ + + - name: Check that the sdist is complete and installable + # A release that cannot be installed from PyPI is the worst kind of + # release, and nothing else in CI ever installs borg from a sdist or + # without the development requirements. + run: | + set -euxo pipefail + python -m venv "$RUNNER_TEMP/venv-sdist" + "$RUNNER_TEMP/venv-sdist/bin/pip" install --upgrade pip + "$RUNNER_TEMP/venv-sdist/bin/pip" install dist/borgbackup-*.tar.gz + "$RUNNER_TEMP/venv-sdist/bin/borg" --version + "$RUNNER_TEMP/venv-sdist/bin/borg" --help > /dev/null + + - name: Attest the sdist provenance + uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8 # v4.2.2 + with: + subject-path: 'dist/*.tar.gz' + + - name: Download the binaries built for this tag + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + path: assets + # the binary artifacts are all named like the binaries they contain + pattern: 'borg-*-gh' + merge-multiple: true + + - name: Check that no binary is missing + run: | + set -uo pipefail + ls -l assets/ || true + missing="" + for asset in $EXPECTED_ASSETS; do + test -f "assets/$asset" || missing="$missing $asset" + done + if [ -n "$missing" ]; then + # not an error: vm_tests is continue-on-error, so e.g. a flaky FreeBSD + # VM should not stop the release - but do not lose it silently either. + echo "::warning::binaries missing from this release:$missing" + fi + + - name: Create the draft release + env: + GH_TOKEN: ${{ github.token }} + TAG: ${{ github.ref_name }} + run: | + set -euxo pipefail + # 2.0.0b23 and friends are pre-releases, 2.1.0 is not. + prerelease="" + case "$TAG" in *a*|*b*|*rc*) prerelease="--prerelease" ;; esac + cat > release-notes.md <\`. + EOF + mkdir -p assets # there may not have been any artifact to download + if gh release view "$TAG" > /dev/null 2>&1; then + # a re-run of this job: keep the (possibly already edited) release and + # just replace its assets. + gh release upload "$TAG" --clobber dist/*.tar.gz $(find assets -type f | sort) + else + gh release create "$TAG" \ + --draft $prerelease \ + --title "borg $TAG" \ + --notes-file release-notes.md \ + dist/*.tar.gz $(find assets -type f | sort) + fi + gh release view "$TAG" --json isDraft,isPrerelease,assets + + - name: Keep the sdist for the PyPI upload + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: sdist + path: dist/*.tar.gz + if-no-files-found: error diff --git a/FAME.md b/FAME.md index 54fc3ffa93..4a45d90cf0 100644 --- a/FAME.md +++ b/FAME.md @@ -1,11 +1,11 @@ # Contributors -372 people have contributed to Borg, with 10,607 commits in total. +372 people have contributed to Borg, with 10,733 commits in total. Thanks to everyone who helped! ![Contributors by commit count](FAME.svg) -Generated on 2026-08-03 by `scripts/fame.py`, which computes the statistics with +Generated on 2026-08-10 by `scripts/fame.py`, which computes the statistics with [git-fame](https://github.com/casperdcl/git-fame), from the `master` branch only - commits that exist solely on other branches or in unmerged pull requests are not counted. @@ -21,30 +21,30 @@ show up there, so the number understates early contributions. Generated files | Contributor | Commits | Lines | Files | |:---|---:|---:|---:| -| Thomas Waldmann | 7,044 | 76,773 | 461 | -| Marian Beermann | 1,140 | 8,400 | 157 | -| Jonas Borgström | 560 | 1,028 | 38 | +| Thomas Waldmann | 7,148 | 82,508 | 490 | +| Marian Beermann | 1,140 | 8,158 | 156 | +| Jonas Borgström | 560 | 986 | 38 | | Antoine Beaupré | 285 | 441 | 33 | -| Mrityunjay Raj | 198 | 7,917 | 102 | +| Mrityunjay Raj | 217 | 8,493 | 102 | | Andrey Bienkowski | 72 | 210 | 16 | | Ted Lawson | 71 | 4,606 | 39 | -| Thalian | 63 | 594 | 32 | +| Thalian | 63 | 584 | 31 | | Martin Hostettler | 46 | 2,559 | 9 | -| Milkey Mouse | 42 | 248 | 14 | +| Milkey Mouse | 42 | 245 | 13 | | Dan Christensen | 40 | 5 | 2 | -| dependabot[bot] | 37 | 46 | 7 | +| dependabot[bot] | 39 | 46 | 7 | | Rayyan Ansari | 34 | 91 | 12 | | Abdel-Rahman | 30 | 3 | 3 | | Manuel Riel | 28 | 94 | 6 | | Nehalenniæ Oudin | 26 | 48 | 11 | | anarcat | 24 | 34 | 4 | -| Hugo Wallenburg | 23 | 1,189 | 13 | +| Hugo Wallenburg | 23 | 1,183 | 12 | | Michael Hanselmann | 22 | 71 | 3 | -| SanskritFritz | 17 | 258 | 1 | | Björn Ketelaars | 17 | 33 | 6 | +| SanskritFritz | 17 | 0 | 0 | | Lee Bousfield | 14 | 160 | 9 | -| Alan Jenkins | 14 | 38 | 2 | -| Daniel Rudolf | 13 | 230 | 14 | +| Alan Jenkins | 14 | 35 | 2 | +| Daniel Rudolf | 13 | 220 | 14 | | Robin Schneider | 13 | 19 | 8 | | Elmar Hoffmann | 12 | 90 | 6 | | Jürg Rast | 12 | 73 | 11 | @@ -71,7 +71,7 @@ show up there, so the number understates early contributions. Generated files | Felix Schwarz | 6 | 4 | 2 | | Alexander-N | 6 | 0 | 0 | | Paul D | 5 | 185 | 46 | -| Simon Frei | 5 | 174 | 3 | +| Simon Frei | 5 | 132 | 3 | | Dominik Stadler | 5 | 77 | 1 | | elandorr | 5 | 40 | 3 | | remyabel | 5 | 25 | 3 | @@ -86,7 +86,6 @@ show up there, so the number understates early contributions. Generated files | Franco Ayala | 4 | 39 | 6 | | Suryansh Pal | 4 | 36 | 9 | | 8bit | 4 | 30 | 5 | -| oxiedi | 4 | 26 | 1 | | Steve Groesz | 4 | 22 | 1 | | Ryan Polley | 4 | 12 | 4 | | Artem Sheremet | 4 | 10 | 1 | @@ -94,6 +93,7 @@ show up there, so the number understates early contributions. Generated files | Josh Holland | 4 | 3 | 2 | | Narendra Vardi | 4 | 3 | 1 | | Alexander Pyhalov | 4 | 0 | 0 | +| oxiedi | 4 | 0 | 0 | | Stephan Herbers | 3 | 49 | 2 | | Charmi Kadi | 3 | 46 | 2 | | Martin Richtarsky | 3 | 34 | 4 | @@ -103,7 +103,7 @@ show up there, so the number understates early contributions. Generated files | Michael Gajda | 3 | 23 | 2 | | eoli3n | 3 | 22 | 1 | | Will | 3 | 21 | 4 | -| Andrea Gelmini | 3 | 18 | 13 | +| Andrea Gelmini | 3 | 17 | 12 | | Oleg Drokin | 3 | 16 | 1 | | kmq | 3 | 15 | 1 | | jungle-boogie | 3 | 13 | 1 | @@ -125,12 +125,13 @@ show up there, so the number understates early contributions. Generated files | jhemmje | 3 | 0 | 0 | | Thomas Portmann | 2 | 206 | 5 | | Ken Kundert | 2 | 149 | 4 | +| ThomasWaldmann | 2 | 145 | 1 | | trxvorr | 2 | 103 | 7 | | Syed Ali Ghazi Ejaz | 2 | 90 | 4 | | David Rambo | 2 | 87 | 3 | -| Soumik Dutta | 2 | 85 | 3 | -| Eric Wolf | 2 | 51 | 4 | +| Soumik Dutta | 2 | 80 | 3 | | Divyansh Agrawal | 2 | 43 | 4 | +| Eric Wolf | 2 | 38 | 3 | | Ioannis Cherouvim | 2 | 32 | 2 | | zDEFz | 2 | 27 | 1 | | Lapinot | 2 | 24 | 4 | @@ -200,7 +201,6 @@ show up there, so the number understates early contributions. Generated files | Mike Mason | 1 | 38 | 3 | | borkd | 1 | 37 | 1 | | James Vasile | 1 | 36 | 1 | -| lexa-a | 1 | 31 | 1 | | Uriel | 1 | 30 | 1 | | vancheese | 1 | 28 | 8 | | Nic Donaldson | 1 | 26 | 1 | @@ -208,7 +208,6 @@ show up there, so the number understates early contributions. Generated files | Mike | 1 | 23 | 1 | | Sitaram Chamarty | 1 | 23 | 1 | | Jonathan Zacsh | 1 | 21 | 1 | -| edvatar | 1 | 16 | 2 | | Félix Sipma | 1 | 13 | 1 | | remyabel2 | 1 | 13 | 1 | | Antonio Larrosa | 1 | 12 | 1 | @@ -222,7 +221,7 @@ show up there, so the number understates early contributions. Generated files | Jubjub | 1 | 6 | 1 | | Mher Kazandjian | 1 | 6 | 1 | | TawfeeqShaik | 1 | 6 | 1 | -| ThomasWaldmann | 1 | 6 | 1 | +| edvatar | 1 | 6 | 2 | | Florent Hemmi | 1 | 5 | 1 | | Leo Antunes | 1 | 5 | 1 | | dataprolet | 1 | 5 | 1 | @@ -381,6 +380,7 @@ show up there, so the number understates early contributions. Generated files | jeroen tiebout | 1 | 0 | 0 | | kannes | 1 | 0 | 0 | | klemens | 1 | 0 | 0 | +| lexa-a | 1 | 0 | 0 | | lumbric | 1 | 0 | 0 | | mirobertod | 1 | 0 | 0 | | nain-F49FF806 | 1 | 0 | 0 | diff --git a/FAME.svg b/FAME.svg index 7a695be0c9..6822437656 100644 --- a/FAME.svg +++ b/FAME.svg @@ -19,19 +19,19 @@ top 20 of 371 by commits on the master branch Thomas Waldmann -7,044 +7,148 Marian Beermann - + 1,140 Jonas Borgström - + 560 Antoine Beaupré - + 285 Mrityunjay Raj - -198 + +217 Andrey Bienkowski 72 @@ -71,10 +71,10 @@ Michael Hanselmann 22 -SanskritFritz +Björn Ketelaars 17 -Björn Ketelaars +SanskritFritz 17 generated by scripts/fame.py using git-fame diff --git a/docs/changes.rst b/docs/changes.rst index 629f8315f9..f88a6a6513 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -104,7 +104,7 @@ Compatibility notes: - removed --numeric-owner (use --numeric-ids) - removed --nobsdflags (use --noflags) - removed --noatime (default now, see also --atime) - - removed --save-space option (does not change behaviour) + - removed --save-space option (does not change behavior) - removed --bypass-lock option - removed --remote-path option (use the BORG_REMOTE_PATH environment variable) - removed --rsh option (use the BORG_RSH environment variable) @@ -167,67 +167,148 @@ Version 2.0.0b23 (not released yet) New features: -- faster: +- crypto: protect metadata and object header in the modes that do not encrypt, #9104. + + The "none" and "authenticated" modes had a no-op repo object envelope: only the chunk + id over the plaintext was checked, so a repo object's metadata (which selects the + decompressor!) and its object header were not verified at all. They now use a tagged + envelope: every object slot carries a 32 byte tag over the payload, the object header, + the chunk id and the meta/data slot, verified before the payload is used. + + - "authenticated-sha256" / "authenticated-blake3": the tag is a MAC (HMAC-SHA256 resp. + keyed BLAKE3, key derived from crypt_key), thus these modes now detect tampering with + metadata and object header, not just with the chunk content. + - "none-sha256" / "none-blake3": the tag is an unkeyed checksum, which detects accidental + corruption. It is no authentication - these modes have no key and make no such claim. + - The tag is deterministic (no nonce/session), so repositories with the same key material + store byte-identical objects for identical input. + - The unencrypted modes are named "-" now, because the hash *is* what protects + the data there: "none-sha256", "none-blake3", "authenticated-sha256", + "authenticated-blake3". Bare "--encryption none" / "--encryption authenticated" are + rejected, and "--id-hash" only applies to the encrypted modes now. + - "none-blake3" is new: the unencrypted mode was limited to sha256 before. + + Breaking: borg 2 beta repositories using the old "none" or "authenticated" formats are not + supported any more - create a new repository, or transfer the archives with borg 2.0.0b23 + or older first. Reading borg 1.x "none"/"authenticated" repositories for + "borg transfer --from-borg1" is not affected. +- faster create: - use multi-threaded zstd compression for big chunks, #9961 - use multi-threaded blake3 hashing for big chunks, #9958 - - extract: avoid refetching/reparsing repeated chunks, #1678 - - fetch_many: serve all-zero chunks without repository access; also cache - recently parsed chunks, #1678 - - do not verify the chunk id on every read from an encrypted repo (the AEAD - authentication covers reads), see BORG_ASSERT_ID, #9994, #7362 -- Chunkers: - - - fastcdc is the new default chunker, for file content data as well as for the - item metadata stream, #9957. Compared to the previous default "buzhash", it is - faster and its Gear table is derived from secret key material (instead of only - XORing a 32bit seed into the table), so chunk cut points are much harder to - predict without the key (better resistance against fingerprinting attacks). - - rabin-aes: new chunker with cryptographically sound resistance against - chunk-size fingerprinting (UHF-then-PRF: secret Rabin polynomial + AES-128, - following eprint 2025/558); uses AES hw acceleration or OpenSSL, ~700 MB/s - - goldilocks-aes: new chunker, like rabin-aes but with the reference universal - hash of eprint 2025/558 (Goldilocks prime-field polynomial hash); about half - the rabin-aes speed, mainly a comparison baseline - - toeplitz-aes: new chunker, like rabin-aes but with a tabulated LFSR/Toeplitz - hash as the universal hash (secret 2 KiB table, fixed public polynomial); - optimal 2^-64 collision bound, fastest of the three AES chunkers - - fastcdc: SIMD-accelerated scan kernel (NEON on aarch64, AVX2 on x86-64, - blocked scalar elsewhere), ~1.4x faster on Apple Silicon; cut points stay - bit-identical (1-byte granularity, same golden chunk points) - - buzhash64: SIMD scan kernel (same technique and dispatch), ~1.5x faster - on Apple Silicon; cut points stay bit-identical - - zero-copy fill: the file reader writes data (and zeros for sparse holes) - directly into the chunker's scan buffer, replacing several per-byte copy - chains; fastcdc ~+40%, buzhash64 ~+20%, buzhash ~+15%, - rabin-aes/toeplitz-aes ~+16%, goldilocks-aes ~+9% - - lazy buffer compaction: compact the scan buffer only when the free tail - gets too small for a full read block (instead of memmoving on every - refill), removing ~80% of the compaction copy traffic; cut points stay - bit-identical (all CDC chunkers) + - overlap pack hash/store and build of the next pack, #9988. + BORG_PACK_ASYNC=no disables the store-thread (debugging aid). + - chunkers, crypto: release the GIL in pure-C hot paths + - give each thread its own LZ4 scratch buffer, #10032 +- faster extract / mount: + + - avoid refetching/reparsing repeated chunks: serve all-zero chunks without + repository access, cache recently parsed chunks, #1678 + - do not verify the chunk id on every read from an encrypted repo (the + AEAD authentication covers reads), see BORG_ASSERT_ID, #9994, #7362 +- more, faster, and more secure chunkers: + + - fastcdc is the new and faster default chunker, #9957 + - fastcdc / buzhash64: SIMD-accelerated scan kernel, #10034, #10043: + + - AVX-512 / AVX2 on x86-64 (Intel / AMD), NEON on aarch64, plus a portable + blockwise one; all bit-identical to the sequential loop + - the sequential loop is what runs unless one of BORG_FASTCDC_KERNEL / + BORG_BUZHASH64_KERNEL / BORG_AES_CHUNKER_KERNEL selects another kernel; + which one is fastest depends on the CPU *and* the compiler, so nothing + is chosen automatically + - toeplitz-aes, rabin-aes, goldilocks-aes: fingerprinting-resistant chunkers + (UHF-then-PRF), with direct AES hardware acceleration (AES-NI or + VAES/AVX-512) or via OpenSSL, #9987, #10043 + - zero-copy fill and lazy buffer compaction optimizations + - log the chunker and its scan kernel at debug level +- compression: support zstd's negative ("fast") levels, ``zstd,-1`` .. ``zstd,-128``, #9950. + They trade compression ratio for speed. Compatible with existing repositories. +- check: + + - keep pack check results, add --max-age to reuse them, #9696, #9925 + - report missing chunks grouped as chunk -> files -> archives, #9218, #9965 + - calendar-aware --max-age, symmetric clock-skew window + - stream one line per missing chunk id, run report on abort, lower report caps, #9218 +- help environment: new help topic about environment variables, #10061 - webdav: serve archives via WebDAV / HTTP, including PAX tar downloads - this is a nice replacement for `borg mount` in some use cases, #9942 - mount: expose POSIX ACLs on Linux mounts (not enforced), #1042 - analyze: report deduplicated size of a set of archives, #5741 +- repo-compress: was temporarily gone, now re-added with pack support, #9663 +- version: add --json output, #10004 +- benchmark cpu: + + - add a throughput column (MB/s), #10049 + - measure hashes and compressors at several buffer sizes + - compress deterministic compressible data instead of random noise + - measure algorithms the way borg uses them (e.g. multithreading on/off + depending on data size) + - use --chunking / --hashing / --encrypting / --compressing / --msgpacking + to run only a subset of the benchmarks, #10050 +- completion: generate fish and tcsh completions, #9989, #9503 +- list: --sort-by=field[,field,...], #9009 - BORG_UNITS env var: si / iec / raw size formatting, replaces the --iec option, #5513 - BORG_PROGRESS_FPS env var: how often --progress output is updated, #8041 Fixes: +- create: do not use ctime for the files cache on Windows, #7193. + + ctime is the file *creation* time on Windows, so a ctime based files cache mode + did not notice content changes of files that kept their size and inode number. + The default is ``mtime,size,inode`` there now and an explicitly given ctime based + mode warns and uses the corresponding mtime based mode. +- extract: restore the timestamps using SetFileTime on Windows, #7269. + + os.utime can not set the birthtime (creation time) there, nor can it work on a file + descriptor or on a symlink itself. Thus, the birthtime was not restored at all and the + timestamps of a symlink were set on the symlink's target. Also, failing to set the + timestamps is not silently ignored anymore, but gives a warning. - re-add XXH64 to read borg 1.x integrity data, #9935 +- list: add {blake3} format key, #9984 - support date: archive patterns for --from-borg1, #9949 - fix calculate_relative_offset year offset for Feb 29, #9967 - bind pack object header into AEAD authentication - fix false repo relocation warning on macOS due to NFC/NFD path differences, #2913 - release chunk data memoryviews (fixes PyPy memory leak), #1755, #9978 +- fix DownloadPipeline.fetch_many() crashing on a missing chunk, #10024 +- lrucache: make it thread-safe +- crypto: start a new session after encrypting 2 TiB with one aes256-ocb session key, #6501 +- check: + + - flush pack writer in ArchiveChecker.finish() before dropping the index + - handle Ctrl-C at safe boundaries, #7893, #9966 + - report invalid pack names instead of crashing + - honest per-run interrupt count, drop redundant save, fix stale SIGINT docs Other changes: -- crypto: start a new session after encrypting 2TiB with one aes256-ocb session key, #6501 -- chunkers: refactor the shared machinery (buffering, min/max clamping, normalized - chunking, iterator protocol) into a common ChunkerBase class instead of keeping a - copy per chunker; fastcdc also shares the window-less scan loop with the AES - chunkers via a _scan() hook. Cut points stay bit-identical for all chunkers. +- support Python 3.15 +- borgstore: require 0.6.x, with blake3 support +- shtab: require >=1.9.3 +- list: validate --format keys, #9984 +- crypto: raise IntegrityError for truncated AEAD/AE envelopes +- chunkers: refactor the shared machinery into a common ChunkerBase class +- PackReader.read(): return a memoryview of the in-memory pack instead of copying +- remove avoidable per-chunk memory copies on the hot data path, #10059, #10060 + + - compress: lz4 decompresses directly into the result bytes object + - compress: do not copy chunk data to bytes, use the buffer protocol + - chunkers: read file data directly into the caller's buffer (if possible) + - crypto: AEAD encrypt/decrypt directly into the result bytes object +- write_chunkindex_to_repo: reduce memory needs, #9886 +- export-tar/import-tar: zstd (de)compression is in-process now, #10067 +- mount/webdav: unify the 3 archive-as-filesystem implementations, #10020. + Behavior changes that fell out of the unification: + + - webdav reads now go through DownloadPipeline.fetch_many(), so the all-zero + chunk shortcut and the parsed-chunk cache (#1678) apply to webdav as well. + - the mounts get webdav's Unicode NFC lookup fallback (macOS decomposes names). + - directories report st_nlink >= 2 (hlfuse behavior) in both mounts. + - a chunk that is read to its end is no longer put into the data cache, so a + full download does not evict the chunks that partial (range) reads need - this was + the FUSE behavior, now webdav shares it. - removed some global options (they were difficult to use and spammed the help output): - --remote-path -> BORG_REMOTE_PATH @@ -239,11 +320,29 @@ Other changes: - docs: - update README + - README: show the contributor chart in the "Helping" section - new borg2 demo screencast (see www.borgbackup.org), #6303 - fix/refactor return codes documentation, #9905 - fix chunks index / memory usage internals documentation, #9937 - update help for some commands, #9948 - fix grammar/typos, #9972 + - add chunker guide (user-level and cryptographic) + - FAME.md: update contributor statistics, #10022 + - crypto: misc. improvements to code and docs, #6501, ... + - GitHub issue #10000: "We Are Borg" joke collection +- CI / tests: + + - give the test VMs 4 CPUs / 8 GiB RAM + - cache pip-built wheels (Windows, BSDs, OmniOS, Haiku) + - upgrade cross-platform-actions to 1.4.0 + - upgrade to NetBSD 11.0 + - upgrade to OpenBSD 7.9 + - OpenBSD: put TMPDIR on an mfs + - fix VM job hangs, use all runner CPUs + - fail hung tests after 5 minutes on the test VMs + - time out the "Start VM" step after 15 minutes + - time-bound LRUCache.test_threaded_access + - benchmark crud json-lines: I/O throughput may round to 0 Version 2.0.0b22 (2026-07-22) @@ -1162,7 +1261,7 @@ New features: Bug fixes: -- fix Ctrl-C / SIGINT behaviour for pyinstaller-made binaries, #8155 +- fix Ctrl-C / SIGINT behavior for pyinstaller-made binaries, #8155 - delete: fix error handling with Ctrl-C - rcompress: fix error handling with Ctrl-C - delete: fix error handling when no archive is specified, #8256 @@ -1192,7 +1291,7 @@ New features: - BORG_EXIT_CODES=modern: optional more specific return codes (for errors and warnings). The default value of this new environment variable is "legacy", which should result in - a behaviour similar to borg 1.2 and older (only using rc 0, 1 and 2). + a behavior similar to borg 1.2 and older (only using rc 0, 1 and 2). "modern" exit codes are much more specific (see the internals/frontends docs). - implement "borg version" (shows client and server version), #7829 @@ -1441,7 +1540,7 @@ Other changes: - use local time / local timezone to output timestamps, #7283 - update development.lock.txt, including a setuptools security fix, #7227 -- remove --save-space option (does not change behaviour) +- remove --save-space option (does not change behavior) - remove part files from final archive - remove --consider-part-files, related stats code, update docs - transfer: drop part files diff --git a/docs/development.rst b/docs/development.rst index 4a07ff70f9..34910e56da 100644 --- a/docs/development.rst +++ b/docs/development.rst @@ -531,22 +531,29 @@ Checklist: - Optional: run tox and/or binary builds on all supported platforms via vagrant, check for test failures. This is now optional as we do platform testing and binary building on GitHub. -- Create sdist, sign it, upload release to (test) PyPi: +- When GitHub CI looks good on the release PR, merge it and push the release tag. - :: + Pushing the tag makes CI build the standalone binaries and then release: + the ``release`` job (``.github/workflows/release.yml``) builds the sdist, + checks that it can be installed, attests its provenance and drafts the GitHub + release with the sdist and all standalone binaries attached. The ``pypi`` job + then uploads the sdist to PyPI. + + If the ``pypi`` environment has required reviewers configured, the upload to + PyPI waits for an approval - that is the last chance to stop it. Also watch + out for a warning while the release is drafted: it means that a binary is + missing because its build did not succeed. +- Sign the sdist and add the signature to the drafted GitHub release:: scripts/sdist-sign X.Y.Z - scripts/upload-pypi X.Y.Z test - scripts/upload-pypi X.Y.Z Note: the signature is not uploaded to PyPi any more, but we upload it to github releases. -- When GitHub CI looks good on the release PR, merge it and then check "Actions": - GitHub will create binary assets after the release PR is merged within the - CI testing of the merge. Check the "Upload binaries" step on Ubuntu (AMD/Intel - and ARM64) and macOS (Intel and ARM64), fetch the ZIPs with the binaries. -- Unpack the ZIPs and test the binaries, upload the binaries to the GitHub - release page (borg-OS-SPEC-ARCH-gh and borg-OS-SPEC-ARCH-gh.tgz). +- Download the binaries from the drafted release and test them. For macOS + binaries **with** FUSE support, document the macFUSE version in the release + notes: macFUSE uses a kernel extension that needs to be compatible with the + code contained in the binary. +- Review the release notes and publish the drafted GitHub release. - Close the release milestone on GitHub. - `Update borgbackup.org @@ -557,13 +564,3 @@ Checklist: - Mailing list. - Mastodon / BlueSky / X (aka Twitter). - IRC channel (change ``/topic``). - -- Create a GitHub release, include: - - - pypi dist package and signature - - Standalone binaries (see above for how to create them). - - - For macOS binaries **with** FUSE support, document the macFUSE version - in the README of the binaries. macFUSE uses a kernel extension that needs - to be compatible with the code contained in the binary. - - A link to ``CHANGES.rst``. diff --git a/docs/faq.rst b/docs/faq.rst index e4fd87e59a..2c731ebc8e 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -1032,33 +1032,79 @@ code). Is there a way to limit bandwidth with Borg? -------------------------------------------- -Borg has no built-in bandwidth limiting - the ``--remote-ratelimit`` and -``--upload-ratelimit`` options were removed. +Borg has no bandwidth limiting option - ``--remote-ratelimit`` and +``--upload-ratelimit`` were removed. There are 2 ways to do it anyway: -For repositories accessed via ssh, bandwidth can be limited with pipeviewer_: +Using borgstore's bandwidth limit +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Create a wrapper script: /usr/local/bin/pv-wrapper +borgstore, which Borg uses for repository access, can limit the transfer rate:: -:: + # 16 Mbit/s == 2 MB/s, 0 (the default) means unlimited: + export BORGSTORE_BANDWIDTH=16000000 + +Pros: + +- works for all backends (``sftp://``, ``rest://``, ``rclone:``, ``s3://``, ...). +- limits both directions. +- needs no additional software. + +Cons: + +- the value is given in **bits** per second (easy to confuse with ``pv``, which + uses bytes per second). +- only the object payload is accounted for, protocol overhead (ssh, TLS, http, + ...) comes on top, so the real usage is a bit higher than the given rate. +- it does not shape the traffic: an object is transferred at full speed and + Borg then waits until the given rate is reached on average. As Borg transfers + rather big objects (pack files are up to 50MB), the connection will be busy + for a while and idle afterwards. + +There is also ``BORGSTORE_LATENCY`` (in microseconds), which adds a delay per +backend call. + +Using pv on the ssh connection +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - #!/bin/sh +For ``rest://`` repositories, Borg connects via ssh, so the transfer can be +limited with pipeviewer_. Put a ``pv`` on each side of the connection using an +ssh ``ProxyCommand`` (this needs ``nc``), e.g. in ``~/.ssh/config``:: + + Host borghost ## -q, --quiet do not output any transfer information at all ## -L, --rate-limit RATE limit transfer to RATE bytes per second - RATE=307200 - pv -q -L $RATE | "$@" + ProxyCommand pv -q -L 307200 | nc %h %p | pv -q -L 307200 -Add BORG_RSH environment variable to use pipeviewer wrapper script with ssh. +The first ``pv`` limits the upload, the second one the download, each to RATE +bytes per second. -:: +Pros: - export BORG_RSH='/usr/local/bin/pv-wrapper ssh' +- smoother, as ``pv`` limits the rate of the data stream itself. +- each direction has its own limit. +- the ssh protocol overhead is included in the limit. +- the rate can be changed on the fly:: -Now Borg will be bandwidth limited. The nice thing about ``pv`` is that you can -change rate-limit on the fly: + pv -R $(pidof pv) -L 102400 -:: + As the ``ProxyCommand`` above runs 2 ``pv`` processes, ``pidof`` will print 2 + pids - give ``pv -R`` the pid of the direction you want to change. + +Cons: + +- only works for ``rest://`` repositories. ``sftp://`` does not run an external + ssh command (borgstore uses paramiko and creates the connection itself) and + a ``ProxyCommand`` in ``~/.ssh/config`` is not used there. +- needs ``pv`` and ``nc``. + +Using rclone's bandwidth limit +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +For ``rclone:`` repositories, you can also use rclone's own bandwidth limiting, +see its ``--bwlimit`` option. rclone picks up options from the environment, so +you can use:: - pv -R $(pidof pv) -L 102400 + export RCLONE_BWLIMIT=300k .. _pipeviewer: https://www.ivarch.com/programs/pv.shtml diff --git a/docs/internals/data-structures.rst b/docs/internals/data-structures.rst index b7b0e1fb28..71ec05ae5e 100644 --- a/docs/internals/data-structures.rst +++ b/docs/internals/data-structures.rst @@ -36,9 +36,13 @@ config/ cache/ checked-packs - repository check progress (partial checks, full checks' checkpointing), - the set of packs checked so far this cycle (pack id -> timestamp, result), - as a hashtable with an appended integrity hash + repository check results (pack id -> timestamp, result), as a hashtable with an + appended integrity hash. Records are kept across checks: ``check --max-age`` + skips packs whose intact record is younger than the given age, and partial checks + (``--max-duration``) verify the least-recently-checked packs first so repeated + runs cover the whole repository. Records of corrupt packs are kept for repair and + always re-verified. Records of packs no longer listed in packs/ are pruned when a + check finishes. There is a list of pointers to archive objects in this directory: @@ -111,7 +115,8 @@ Repo object metadata Metadata is a MessagePack-encoded (and encrypted/authenticated) dict with: - ctype (compression type 0..255) -- clevel (compression level 0..255) +- clevel (compression level, one byte, interpreted depending on ctype - see + :ref:`data-compression`) - csize (overall compressed (and maybe obfuscated) data size) - psize (only when obfuscated: payload size without the obfuscation trailer) - size (uncompressed size of the data) @@ -420,6 +425,10 @@ Borg has these chunkers (the default is "fastcdc"): underlying paper); about half the rabin-aes speed, mainly a comparison baseline. +All chunkers support sparse file processing (``borg create --sparse``): hole +ranges in the input file are then detected (via ``SEEK_HOLE``/``SEEK_DATA``) +and seeked over instead of being read, processing their content as all-zero. + For some more general usage hints see also ``--chunker-params``. "fixed" chunker @@ -439,11 +448,6 @@ The default is not to have a differently sized header. bytes) recommended. E.g.: 4194304 would cut 4MiB sized chunks. - HEADER_SIZE: optional, defaults to 0 (no header). -The fixed chunker also supports processing sparse files (reading only the ranges -with data and seeking over the empty hole ranges). - -``borg create --sparse --chunker-params fixed,BLOCK_SIZE[,HEADER_SIZE]`` - "fastcdc" chunker +++++++++++++++++ @@ -915,6 +919,54 @@ need any special handling when reading, because the session id is part of every header. Because the advantages of the individual session keys just add up, frequent session key changes also keep the total advantage low over the lifetime of a borg key. +.. _tagged_envelope: + +Modes without encryption +~~~~~~~~~~~~~~~~~~~~~~~~ + +The ``authenticated-*`` and ``none-*`` modes do not encrypt: the payload of a repository +object slot (the compressed chunk data resp. the packed metadata, see `Repository objects`_) +is stored as-is. Every slot still carries a 32 byte tag:: + + TYPE(1) + reserved(1) + tag(32) + payload + +``TYPE`` is the key type byte (which identifies the mode, see ``KeyType``), ``reserved`` is +zero. The tag is computed over the envelope header, the AAD and the payload:: + + aad_full = aad + chunk_id + tag = MAC(tag_key, TYPE || reserved || len16_be(aad_full) || aad_full || payload) + +``aad`` is what ``RepoObj`` authenticates alongside the payload: the object header prefix +(magic, format version, chunk id) and the slot tag (``M`` for meta, ``D`` for data), see +:ref:`pack-format`. Consequently, the tag detects modification of the payload, of the +metadata, of the object header, a swap of the meta and the data slot, and an object slice +taken from a different object. The length prefix keeps the boundary between the AAD and the +payload unambiguous. + +There is no nonce, no session and no other state: the tag is deterministic. Two repositories +with the same key material therefore store byte-identical objects for identical input, which +allows deduplicating them on the filesystem level (e.g. with CoW/dedup tools). + +The modes differ in the tag algorithm and in whether they have a key at all: + +- ``authenticated-sha256`` / ``authenticated-blake3``: the tag is a **MAC** (HMAC-SHA256 resp. + keyed BLAKE3), so only somebody who has the borg key can compute it - this detects malicious + tampering, not just accidental corruption. The MAC key is derived from ``crypt_key``:: + + tag_key = sha256(crypt_key + b"borg-repoobj-mac-hmac-sha256")[:32] # authenticated-sha256 + tag_key = sha256(crypt_key + b"borg-repoobj-mac-blake3")[:32] # authenticated-blake3 + + It is deliberately not derived from ``id_key``: chunk ids are public, and related repositories + share the id key (see ``borg repo-create --other-repo``), which must not enable them to forge + each other's objects. ``--copy-crypt-key`` shares ``crypt_key`` and thus opts into producing + byte-identical objects across the related repositories. +- ``none-sha256`` / ``none-blake3``: there is no key at all, so the tag is an **unkeyed** hash + (plain SHA-256 resp. BLAKE3 over the same input), i.e. a checksum. It detects accidental + corruption and reads that returned the wrong bytes, but anybody who modifies an object can + recompute it - it is no protection against malicious tampering. For the same reason, the chunk + ids of these modes are unkeyed hashes of the plaintext, which makes all repositories of such a + mode dedup identically. + Legacy modes ~~~~~~~~~~~~ @@ -924,8 +976,12 @@ Old repositories (which used AES-CTR mode) are supported read-only to be able to AES-CTR mode is not supported for new repositories and the related code will be removed in a future release. -Both modes -~~~~~~~~~~ +The same applies to the borg 1.x ``none`` and ``authenticated`` modes: their envelope is just +the type byte followed by the payload, so nothing about an object is verified except the chunk +id over the plaintext. They were replaced by the tagged modes described above. + +All modes +~~~~~~~~~ Encryption keys (and other secrets) are kept either in the keys directory on the client ('keyfile' mode) or under the keys/ namespace in the repository @@ -1007,19 +1063,25 @@ Compression ----------- Borg supports the following compression methods, each identified by a ctype value -in the range between 0 and 255 (and augmented by a clevel 0..255 value for the +in the range between 0 and 255 (and augmented by a one-byte clevel value for the compression level): - none (no compression, pass through data 1:1), identified by 0x00 - lz4 (low compression, but super fast), identified by 0x01 -- zstd (level 1-22 offering a wide range: level 1 is lower compression and high - speed, level 22 is higher compression and lower speed) - identified by 0x03 +- zstd (level -128..22 offering a wide range: level 22 is higher compression and lower + speed, level 1 is lower compression and high speed, and the negative "fast" levels + trade still more compression for still more speed) - identified by 0x03 - zlib (level 0-9, level 0 is no compression [but still adding zlib overhead], level 1 is low, level 9 is high compression), identified by 0x05 - lzma (level 0-9, level 0 is low, level 9 is high compression), identified by 0x02. -The type byte is followed by a byte indicating the compression level. +The type byte is followed by a byte indicating the compression level. How that byte is +interpreted depends on the compression type: for zstd it is an ``int8_t``, so that the +negative levels fit (level -1 is stored as 255, -128 as 128). For all other types it is +an unsigned byte, with 255 meaning "no level applies" (as for none and lz4). Levels 1..22 +occupy the same byte values either way, so zstd data written by older borg versions keeps +its meaning. Speed: none > lz4 > zlib > lzma, lz4 > zstd Compression: lzma > zlib > lz4 > none, zstd > lz4 diff --git a/docs/internals/frontends.rst b/docs/internals/frontends.rst index 9b9bbe8fb3..fea84ca753 100644 --- a/docs/internals/frontends.rst +++ b/docs/internals/frontends.rst @@ -356,6 +356,10 @@ Archive formats array under the *archives* key, while :ref:`borg_create` returns a single archive object under the *archive* key. +:ref:`borg_create` with ``--dry-run`` does not create an archive, so there is no *archive* key. +Instead, it returns *dry_run* (true) and a reduced *stats* object with *nfiles* and *original_size*, +both computed from file system metadata without reading the file contents. + Both formats contain a *name* key with the archive name, the *id* key with the hexadecimal archive ID, and the *start* key with the start timestamp. @@ -561,6 +565,84 @@ Example (excerpt) of ``borg diff --json-lines``:: {"path": "file3", "changes": [{"type": "removed", "size": 0}]} +Archive Analysis +++++++++++++++++ + +:ref:`borg_analyze` ``--json`` emits the numbers of its text report as one object. All sizes are +byte values; the compression factor the text report shows is ``stored_size / source_size``. + +Without ``--by-name``, the *dedup_size* and *hotspots* keys are present. + +*dedup_size* describes the considered set of archives: + +considered_archives + Number of archives matching the archive filters +total_archives + Number of non-deleted archives in the repository +whole_repository + True if no archive was left over by the filters, so the considered set is the whole + repository. Every referenced chunk is then trivially exclusive to the set, and the + *exclusive* key is absent. +deduplicated + Object with *source_size* and *stored_size*: the summed size of the union of chunks the + considered archives reference, chunks shared within the set counted once +exclusive + Object with *source_size* and *stored_size*: the chunks referenced only by the considered + set, i.e. what deleting the whole set would free. Absent if *whole_repository* is true. +unreferenced + Object with *stored_size* and *chunks*: the chunks no non-deleted archive references, which + ``borg compact`` could free. Their source size is not known, as it is only recorded in the + archives referencing a chunk. +total_chunks + Number of chunks in the repository chunk index +missing_chunks + Number of chunks referenced by an archive but absent from the repository chunk index + +*hotspots* is a list of objects with *path* (directory path) and *size* (bytes of chunks added or +removed in that directory between consecutive archives), busiest directory first. It is ``null`` +if fewer than two archives matched, as hot spots need at least two archives to compare. + +With ``--by-name``, the *by_name* key is present instead, decomposing the whole repository: + +archives + Number of non-deleted archives in the repository +names + List of objects with *name*, *archives* (number of archives with that name), *source_size* + and *stored_size*. The sizes are what is exclusive to that name: no archive of another name + references those chunks. Biggest *stored_size* first. +shared + Object with *source_size* and *stored_size*: the chunks referenced by two or more names +unreferenced + As above +total + Object with *archives*, *source_size* and *stored_size*. Each chunk is counted in exactly one + of *names*, *shared* and *unreferenced*, so the *names* and *shared* sizes add up to *total*. +total_chunks, missing_chunks + As above + +Example of ``borg analyze -a 'sh:userA-*' --json``:: + + { + "dedup_size": { + "considered_archives": 2, + "deduplicated": {"source_size": 3000, "stored_size": 3536}, + "exclusive": {"source_size": 2000, "stored_size": 3338}, + "missing_chunks": 0, + "total_archives": 3, + "total_chunks": 13, + "unreferenced": {"chunks": 0, "stored_size": 0}, + "whole_repository": false + }, + "encryption": {"encryption": "aes256-ocb", "id_hash": "sha256"}, + "hotspots": [{"path": "home/user/src", "size": 1000}], + "repository": { + "id": "06e4027d32f8eae8333f8fe06b1c2c46bf12f22ad10bd4d04a0f30751a26d77b", + "last_modified": "2026-08-01T22:46:05.886533", + "location": "/home/user/repository" + } + } + + .. _msgid: Message IDs @@ -735,6 +817,10 @@ Warnings {}: {} BackupFileNotFoundError rc: 107 {}: {} + BackupTimeoutError rc: 111 + {}: {} + BackupBrokenSymlinkError rc: 112 + {}: {} Operations - cache.begin_transaction diff --git a/docs/internals/packs.rst b/docs/internals/packs.rst index 638f772ce4..0f8d6acec9 100644 --- a/docs/internals/packs.rst +++ b/docs/internals/packs.rst @@ -54,12 +54,14 @@ The fixed part of each blob header is 49 bytes (``REPOOBJ_HEADER_SIZE``): ``REPOOBJ_HEADER_SIZE = len(OBJ_MAGIC) + 1 + 32 + 4 + 4 = 49`` Format version ``0x02`` (``OBJ_VERSION_HEADER_AAD``) binds the header's first 41 bytes (``OBJ_MAGIC`` -+ version + ``chunk_id`` -- ``REPOOBJ_HEADER_AAD_SIZE``) into the AEAD authentication of ++ version + ``chunk_id`` -- ``REPOOBJ_HEADER_AAD_SIZE``) into the authentication of ``encrypted_meta`` and ``encrypted_data`` as additional authenticated data (AAD: data that is -authenticated together with the ciphertext, but not itself encrypted). This applies to the AEAD -encryption modes (AES-256-OCB, ChaCha20-Poly1305). ``meta_size`` and ``data_size`` are excluded from -the AAD; tampering with either still fails authentication, because it changes the length of the -ciphertext slice being decrypted. A forged ``chunk_id``, version, or magic byte therefore fails AEAD +authenticated together with the ciphertext, but not itself encrypted). This applies to all borg 2 +modes: the AEAD encryption modes (AES-256-OCB, ChaCha20-Poly1305) authenticate it with their AEAD +tag, the ``authenticated-*`` modes with their MAC and the ``none-*`` modes with their (unkeyed) +checksum, see :ref:`tagged_envelope`. ``meta_size`` and ``data_size`` are excluded from +the AAD; tampering with either still fails the check, because it changes the length of the +slice being read. A forged ``chunk_id``, version, or magic byte therefore fails authentication in ``RepoObj.parse()``/``parse_meta()``. ``encrypted_meta`` and ``encrypted_data`` each add a one-byte slot tag on top of the shared header @@ -82,7 +84,7 @@ decrypting, so it does not check header AAD authentication. The fixed 49-byte blob header. ``meta_size`` and ``data_size`` drive traversal; integrity comes from the content-addressed pack name and the - per-blob AEAD, which authenticates magic/version/chunk_id as additional + per-blob tag, which authenticates magic/version/chunk_id as additional authenticated data. A reader locates the next blob by advancing:: diff --git a/docs/internals/security.rst b/docs/internals/security.rst index 079ca2be72..ad09641c91 100644 --- a/docs/internals/security.rst +++ b/docs/internals/security.rst @@ -75,10 +75,21 @@ object's metadata (like e.g.: manifest vs. archive vs. user file content data). When loading data from the repo, borg verifies that the type of object it got matches the type it wanted. borg 2 does not use TAMs any more. -As both the object's metadata and data are AEAD encrypted and also bound to +As both the object's metadata and data are authenticated and also bound to the object ID (via giving the ID as AAD), there is no way an attacker (without access to the borg key) could change the type of the object or move content -to a different object ID. +to a different object ID. This holds for the AEAD encryption modes (where the +AEAD tag authenticates them) as well as for the ``authenticated-*`` modes +(where a MAC does, see :ref:`tagged_envelope`). + +It does **not** hold for the ``none-*`` modes: they have no key, so their objects +carry an unkeyed checksum rather than a MAC, and an attacker who modifies an +object can simply recompute it. What still constrains an attacker there is the +object ID being the (unkeyed) hash of the plaintext: the content of an existing +object can not be replaced without the ID no longer matching. But the object's +metadata, the archives list and the manifest are not anchored to anything secret, +so a ``none-*`` repository provides no tamper protection - only detection of +accidental corruption. This effectively 'anchors' each archive to the key, which is controlled by the client, thereby anchoring the DAG starting from the archives list entry, @@ -201,10 +212,50 @@ Notable: recommended periodic audit for this). +Authenticated modes +~~~~~~~~~~~~~~~~~~~ + +Modes: ``--encryption authenticated-(sha256|blake3)`` + +Supported: borg 2.0+ + +These modes do not encrypt: everything in the repository is readable by anybody who +can read the repository. They do authenticate, though - like an AEAD mode minus the +data encryption: + +- Every repository object slot (metadata and data) carries a MAC over the payload, the + object header and the slot, see :ref:`tagged_envelope`. Reading verifies it before the + payload is used for anything (in particular before decompressing it), so tampering with + any of them is detected, whether accidental or malicious. +- The chunk IDs are MACs over the plaintext, as in the encrypted modes. +- The MAC key lives in a borg key (repokey or keyfile, ``--key-location``), protected by a + passphrase like the keys of the encrypted modes. An attacker without that key can neither + forge objects nor chunk IDs. + +Different from the AEAD modes, the MAC is deterministic (a MAC needs no nonce): there is no +session key, no IV and thus no usage limit to observe, and identical input produces identical +objects. + +Unencrypted modes +~~~~~~~~~~~~~~~~~ + +Modes: ``--encryption none-(sha256|blake3)`` + +Supported: borg 2.0+ + +These modes have no key at all: they neither encrypt nor authenticate. Every repository +object slot carries an *unkeyed* checksum (see :ref:`tagged_envelope`), which detects +accidental corruption - bad storage hardware, a truncated write, a read that returned the +wrong bytes - before the data is used. It is not a protection against an attacker: whoever +modifies an object can recompute the checksum, and the chunk IDs are unkeyed hashes as well. + +You are advised not to use these modes. Use ``authenticated-*`` instead if you do not want +your data encrypted but do want to detect tampering; it is the same thing plus a key. + Legacy modes ~~~~~~~~~~~~ -Modes: ``--encryption (repokey|keyfile)[-blake2]`` +Modes: ``--encryption (repokey|keyfile)[-blake2]``, ``--encryption (none|authenticated)`` Supported: borg < 2.0 @@ -213,6 +264,11 @@ derived via HMAC-SHA256 or (in the ``-blake2`` variants) Blake2b. ``blake2b`` is only used by these legacy modes; new repositories use ``sha256`` or ``blake3`` (see above). +The borg 1.x ``none`` and ``authenticated`` modes belong here, too: their repository +objects have no tag at all, so nothing about an object is verified except the chunk ID +over the plaintext - not even the object's metadata. They were replaced by the modes +described above, which cover metadata and object header as well. + borg 2.0 does not support creating new repos using these modes, but ``borg transfer`` can still read such existing repos. diff --git a/docs/man/borg-create.1 b/docs/man/borg-create.1 index e302b853f1..d0356f3a07 100644 --- a/docs/man/borg-create.1 +++ b/docs/man/borg-create.1 @@ -28,7 +28,7 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] .in \\n[rst2man-indent\\n[rst2man-indent-level]]u .. -.TH "borg-create" "1" "2026-07-21" "" "borg backup tool" +.TH "borg-create" "1" "2026-08-17" "" "borg backup tool" .SH Name borg-create \- Creates a new archive. .SH SYNOPSIS @@ -46,6 +46,23 @@ The slashdot hack in paths (recursion roots) is triggered by using \fB/./\fP: strip the prefix on the left side of \fB\&./\fP from the archived items (in this case, \fBthis/gets/archived\fP will be the path in the archived item). .sp +If a recursion root (a path given on the command line or in a patterns file) is a +symlink, borg follows it and archives what it points to \- using the path you gave. +If \fBcurrent\fP is a symlink pointing to the directory \fB20260801\-2345\fP, +\fBborg create ARCHIVE current\fP thus archives \fBcurrent\fP as a directory (with the +metadata of \fB20260801\-2345\fP) and recurses into it, archiving the contained fs +objects as \fBcurrent/...\fP\&. As the archived paths do not change when the symlink +target changes, the files cache keeps working for such backups. +.sp +Note that the symlink itself is then not in the archive (and neither is its target +path), so restoring will create a real directory (or file) where the symlink was. +If you want the symlink archived as a symlink, do not give it as a recursion root, +but let borg find it while recursing (symlinks found that way are never followed). +A recursion root that is a symlink with a non\-existing target is skipped with a warning. +.sp +If you give both a symlink and its target as recursion roots, borg archives the fs +objects only once, under the path given first (like for any other root given twice). +.sp When specifying \(aq\-\(aq as a path, borg will read data from standard input and create a file named \(aqstdin\(aq in the created archive from that data. In some cases, it is more appropriate to use \-\-content\-from\-command. See the section \fIReading from stdin\fP @@ -71,9 +88,9 @@ the files cache. This comparison can operate in different modes as given by \fB\-\-files\-cache\fP: .INDENT 0.0 .IP \(bu 2 -ctime,size,inode (default) +ctime,size,inode (default on POSIX systems) .IP \(bu 2 -mtime,size,inode (default behaviour of borg versions older than 1.1.0rc4) +mtime,size,inode (default on Windows) .IP \(bu 2 ctime,size (ignore the inode number) .IP \(bu 2 @@ -109,6 +126,13 @@ can be arbitrarily set from userspace, e.g., to set mtime back to the same value it had before a content change happened. This can be used maliciously as well as well\-meant, but in both cases mtime\-based cache modes can be problematic. .UNINDENT +.sp +On Windows, ctime is the file \fIcreation\fP time, not the \(dqmetadata change time\(dq it is +on POSIX systems. A ctime based mode would therefore not notice content changes of a +file that keeps its size and inode number, so borg defaults to \fBmtime,size,inode\fP +there. If a ctime based mode is given explicitly on Windows, borg warns and uses the +corresponding mtime based mode instead: ctime,size,inode \-> mtime,size,inode, +ctime,size \-> mtime,size, rechunk,ctime \-> rechunk,mtime. .INDENT 0.0 .TP .B The \fB\-\-files\-changed\fP option controls how Borg detects if a file has changed during backup: @@ -134,11 +158,22 @@ The \fB\-\-progress\fP option shows (from left to right) Original and (uncompres deduplicated size (O and U respectively), then the Number of files (N) processed so far, followed by the currently processed path. .sp +Sizes of GB and above are shown with enough decimal places that even MB\-sized progress +stays visible. On a terminal, this needs a width of at least 110 columns \- on narrower +terminals, the compact format is used, so that the path stays readable. If the output +does not go to a terminal (e.g. into a logfile), the precise format is always used. +.sp When using \fB\-\-stats\fP, you will get some statistics about how much data was added \- the \(dqThis Archive\(dq deduplicated size there is most interesting as that is how much your repository will grow. Please note that the \(dqAll archives\(dq stats refer to -the state after creation. Also, the \fB\-\-stats\fP and \fB\-\-dry\-run\fP options are mutually -exclusive because the data is not actually compressed and deduplicated during a dry run. +the state after creation. +.sp +When \fB\-\-stats\fP is used together with \fB\-\-dry\-run\fP, only the number of files and the +original size are reported. They are computed from file system metadata, without reading +the file contents, so a dry run stays fast. As data is not actually read, chunked, and +deduplicated during a dry run, the deduplicated size is unknown. The sizes of data read +from standard input, from a command\(aqs output, or from special files (\fB\-\-read\-special\fP) +are also unknown in a dry run and counted as zero. .sp The \fB\-\-stats\fP output also reports the store statistics (lines prefixed with \(dqStore\(dq), taken from the storage layer after this run. These cover the backend and @@ -272,16 +307,19 @@ do not read and store ACLs into archive do not read and store xattrs into archive .TP .B \-\-sparse -detect sparse holes in input (supported only by fixed chunker) +detect sparse holes in input and seek over them instead of reading them .TP .BI \-\-files\-cache \ MODE -operate files cache in MODE. default: ctime,size,inode +operate files cache in MODE. default: ctime,size,inode (on Windows: mtime,size,inode, because ctime is file creation time there). .TP .BI \-\-files\-changed \ MODE specify how to detect if a file has changed during backup (ctime, mtime, disabled). default: ctime (on Windows: mtime, because ctime is file creation time there). .TP .B \-\-read\-special open and read block and char device files as well as FIFOs as if they were regular files. Also follows symlinks pointing to these kinds of files. +.TP +.BI \-\-read\-special\-timeout \ SECONDS +when reading from FIFOs or character devices (see \-\-read\-special): skip the file with an error if no data arrives for more than SECONDS (this includes waiting for a FIFO\(aqs writer to connect). Give 0 to wait forever. default: 1800 seconds. .UNINDENT .SS Archive options .INDENT 0.0 @@ -293,7 +331,7 @@ add a comment text to the archive manually specify the archive creation date/time (yyyy\-mm\-ddThh:mm:ss[(+|\-)HH:MM] format, (+|\-)HH:MM is the UTC offset, default: local time zone). Alternatively, give a reference file/directory. .TP .BI \-\-chunker\-params \ PARAMS -specify the chunker parameters (ALGO, CHUNK_MIN_EXP, CHUNK_MAX_EXP, HASH_MASK_BITS, HASH_WINDOW_SIZE). default: buzhash,19,23,21,4095 +specify the chunker parameters (ALGO, CHUNK_MIN_EXP, CHUNK_MAX_EXP, HASH_MASK_BITS, NC_LEVEL). default: fastcdc,19,23,21,2 .TP .BI \-C \ COMPRESSION\fR,\fB \ \-\-compression \ COMPRESSION select compression algorithm, see the output of the \(dqborg help compression\(dq command for details. @@ -358,7 +396,7 @@ $ fusermount \-u sshfs\-mount # Make a big effort in fine\-grained deduplication (big chunk management # overhead, needs a lot of RAM and disk space; see the formula in the internals docs): -$ borg create \-\-chunker\-params buzhash,10,23,16,4095 small /smallstuff +$ borg create \-\-chunker\-params fastcdc,10,23,16,2 small /smallstuff # Backup a raw device (must not be active/in use/mounted at that time) $ borg create \-\-read\-special \-\-chunker\-params fixed,4194304 my\-sdx /dev/sdX @@ -556,6 +594,9 @@ to borg (maybe implementing your own recursion or your own rules), you can use .sp Borg supports paths with the slashdot hack to strip path prefixes here also. So, be careful not to unintentionally trigger that. +.sp +Symlinks given this way are never followed (unlike recursion roots are), they are +archived as symlinks. .SH SEE ALSO .sp \fIborg\-common(1)\fP, \fIborg\-delete(1)\fP, \fIborg\-prune(1)\fP, \fIborg\-check(1)\fP, \fIborg\-patterns(1)\fP, \fIborg\-placeholders(1)\fP, \fIborg\-compression(1)\fP, \fIborg\-repo\-create(1)\fP diff --git a/docs/man/borg-environment.1 b/docs/man/borg-environment.1 new file mode 100644 index 0000000000..c7f45c7db5 --- /dev/null +++ b/docs/man/borg-environment.1 @@ -0,0 +1,612 @@ +.\" Man page generated from reStructuredText +.\" by the Docutils 0.22.4 manpage writer. +. +. +.nr rst2man-indent-level 0 +. +.de1 rstReportMargin +\\$1 \\n[an-margin] +level \\n[rst2man-indent-level] +level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] +- +\\n[rst2man-indent0] +\\n[rst2man-indent1] +\\n[rst2man-indent2] +.. +.de1 INDENT +.\" .rstReportMargin pre: +. RS \\$1 +. nr rst2man-indent\\n[rst2man-indent-level] \\n[an-margin] +. nr rst2man-indent-level +1 +.\" .rstReportMargin post: +.. +.de UNINDENT +. RE +.\" indent \\n[an-margin] +.\" old: \\n[rst2man-indent\\n[rst2man-indent-level]] +.nr rst2man-indent-level -1 +.\" new: \\n[rst2man-indent\\n[rst2man-indent-level]] +.in \\n[rst2man-indent\\n[rst2man-indent-level]]u +.. +.TH "borg-environment" "1" "2026-08-14" "" "borg backup tool" +.SH Name +borg-environment \- Details regarding environment +.SH DESCRIPTION +.sp +Borg uses some environment variables for automation: +.INDENT 0.0 +.TP +.B General: +.INDENT 7.0 +.TP +.B BORG_REPO +When set, use the value to give the default repository location. +Use this so you do not need to type \fB\-\-repo /path/to/my/repo\fP all the time. +.TP +.B BORG_OTHER_REPO +Similar to BORG_REPO, but gives the default for \fB\-\-other\-repo\fP\&. +.TP +.B BORG_PASSPHRASE (and BORG_OTHER_PASSPHRASE) +When set, use the value to answer the passphrase question for encrypted repositories. +It is used when a passphrase is needed to access an encrypted repo as well as when a new +passphrase should be initially set when initializing an encrypted repo. +See also BORG_NEW_PASSPHRASE. +.TP +.B BORG_PASSCOMMAND (and BORG_OTHER_PASSCOMMAND) +When set, use the standard output of the command (trailing newlines are stripped) to answer the +passphrase question for encrypted repositories. +It is used when a passphrase is needed to access an encrypted repo as well as when a new +passphrase should be initially set when initializing an encrypted repo. Note that the command +is executed without a shell. So variables, like \fB$HOME\fP will work, but \fB~\fP won\(aqt. +If BORG_PASSPHRASE is also set, it takes precedence. +See also BORG_NEW_PASSPHRASE. +.TP +.B BORG_PASSPHRASE_FD (and BORG_OTHER_PASSPHRASE_FD) +When set, specifies a file descriptor to read a passphrase +from. Programs starting borg may choose to open an anonymous pipe +and use it to pass a passphrase. This is safer than passing via +BORG_PASSPHRASE, because on some systems (e.g. Linux) environment +can be examined by other processes. +If BORG_PASSPHRASE or BORG_PASSCOMMAND are also set, they take precedence. +.TP +.B BORG_NEW_PASSPHRASE +When set, use the value to answer the passphrase question when a \fBnew\fP passphrase is asked for. +This variable is checked first. If it is not set, BORG_PASSPHRASE and BORG_PASSCOMMAND will also +be checked. +Main use case for this is to fully automate \fBborg key change\-passphrase\fP\&. +.TP +.B BORG_DISPLAY_PASSPHRASE +When set, use the value to answer the \(dqdisplay the passphrase for verification\(dq question when defining a new passphrase for encrypted repositories. +.TP +.B BORG_DEBUG_PASSPHRASE +When set to YES, display debugging information that includes passphrases used and passphrase related env vars set. +.TP +.B BORG_EXIT_CODES +When set to \(dqmodern\(dq, the borg process will return more specific exit codes (rc). +When set to \(dqlegacy\(dq, the borg process will return rc 2 for all errors, 1 for all warnings, 0 for success. +Default is \(dqmodern\(dq. +.TP +.B BORG_HOST_ID +Borg usually computes a host id from the FQDN plus the results of \fBuuid.getnode()\fP (which usually returns +a unique id based on the MAC address of the network interface. Except if that MAC happens to be all\-zero \- in +that case it returns a random value, which is not what we want (because it kills automatic stale lock removal). +So, if you have an all\-zero MAC address or other reasons to better control the host id externally, just set this +environment variable to a unique value. If all your FQDNs are unique, you can just use the FQDN. If not, +use \%\&. +.TP +.B BORG_HOSTNAME +When set, use this value as the hostname (instead of the auto\-detected one), e.g. to run borg +on one host, but impersonate another host. This affects the hostname stored in newly created +archives as well as the \fB{hostname}\fP placeholder. +.TP +.B BORG_USERNAME +When set, use this value as the username (instead of the auto\-detected one), e.g. to run borg +as one user, but impersonate another user. This affects the username stored in newly created +archives as well as the \fB{user}\fP placeholder. +.TP +.B BORG_LOCK_WAIT +You can set the default value for the \fB\-\-lock\-wait\fP option with this, so +you do not need to give it as a command line option. +.TP +.B BORG_LOGGING_CONF +When set, use the given filename as INI\-style logging configuration (see +\%). +A basic example conf can be found at \fBdocs/misc/logging.conf\fP\&. +.TP +.B BORG_RSH +When set, use this command instead of \fBssh\fP\&. This can be used to specify ssh options, such as +a custom identity file \fBssh \-i /path/to/private/key\fP\&. See \fBman ssh\fP for other options. +This is the replacement for the removed \fB\-\-rsh CMD\fP command line option. +borg also gives this to borgstore as \fBBORGSTORE_RSH\fP, except if that is already set. +.TP +.B BORG_REMOTE_PATH +When set, use the given path as borg executable on the remote (defaults to \(dqborg\(dq if unset). +This is the replacement for the removed \fB\-\-remote\-path PATH\fP command line option. +.TP +.B BORG_UNITS +Determines how borg formats sizes in its human\-readable output: +.INDENT 7.0 +.IP \(bu 2 +\fBsi\fP (default): decimal units, e.g. \fB1.23 MB\fP (1kB = 1000B) +.IP \(bu 2 +\fBiec\fP: binary units, e.g. \fB1.18 MiB\fP (1KiB = 1024B) +.IP \(bu 2 +\fBraw\fP: exact byte counts, e.g. \fB1234567 B\fP +.UNINDENT +.sp +Use \fBraw\fP if you want to parse sizes with scripts (e.g. for monitoring), +so you do not have to deal with scaled values and different units. +Alternatively, use a command\(aqs \fB\-\-json\fP output or, for the commands +supporting \fB\-\-format\fP, the size related format keys \- sizes are given +as byte counts there anyway. +.sp +\fBBORG_UNITS=iec\fP is the replacement for the removed \fBBORG_IEC\fP environment +variable (and for the \fB\-\-iec\fP command line option removed before that). +.TP +.B BORG_PROGRESS_FPS +How often the \fB\-\-progress\fP output is updated at most, in updates per +second (default: 5). Fractional values are allowed, e.g. +\fBBORG_PROGRESS_FPS=0.1\fP limits it to one update every 10 seconds. +Lower values are useful when the output goes into a logfile rather than +to an interactive terminal. +.TP +.B BORG_SPINNER +Controls the spinner borg animates on a terminal while doing work of unknown +duration: +.INDENT 7.0 +.IP \(bu 2 +unset (default): animate, using Unicode frames if the terminal can display them +.IP \(bu 2 +\fBascii\fP: animate, but only use ASCII frames (\fB|/\-\e\fP) +.IP \(bu 2 +\fBoff\fP: do not animate, only output the messages next to the spinner +.UNINDENT +.sp +The spinner is animated only on an interactive terminal anyway (and never +with \fB\-\-log\-json\fP), and its colour follows the usual \fBNO_COLOR\fP and +\fBCOLORTERM\fP conventions. See also \fBBORG_PROGRESS_FPS\fP: it also gives +the spinner its frame rate. +.TP +.B BORG_DEBUG_PROFILE +When set to a filename, write an execution profile in Borg format into that file +(see \fIdebugging\fP). If the filename ends with \fB\&.pyprof\fP, a Python\-compatible +profile is written instead. +This is the replacement for the removed \fB\-\-debug\-profile\fP command line option. +Note: every borg invocation writes the profile, so unset it again when you are done. +.TP +.B BORG_REPO_PERMISSIONS +Set repository permissions, see also: \fIborg_serve\fP +.TP +.B BORG_FILES_CACHE_SUFFIX +When set to a value at least one character long, instructs borg to use a specifically named +(based on the suffix) alternative files cache. This can be used to avoid loading and saving +cache entries for backup sources other than the current sources. +.TP +.B BORG_FILES_CACHE_TTL +When set to a numeric value, this determines the maximum \(dqtime to live\(dq for the files cache +entries (default: 2). The files cache is used to determine quickly whether a file is unchanged. +.TP +.B BORG_ASSERT_ID +Comma\-separated list of the places where borg shall verify that a chunk\(aqs content matches +its chunk id (\fBchunkid == id_hash(content)\fP) after decrypting and decompressing it. +Verifying costs a full hash pass over everything that is read at such a place. +.sp +Default (variable not set): +.INDENT 7.0 +.INDENT 3.5 +.sp +.EX +BORG_ASSERT_ID=repair,transfer,rechunk +.EE +.UNINDENT +.UNINDENT +.sp +These are the place names that can be listed: +.INDENT 7.0 +.TP +.B read +Every read that decompresses a chunk: \fBborg extract\fP, \fBborg mount\fP, +\fBborg export\-tar\fP, \fBborg diff\fP, ... This is by far the most data borg reads, so +this place is \fBnot\fP in the default, see the explanation below. +.TP +.B repair +\fBborg check \-\-repair\fP\&. It rebuilds archives from the item metadata stream it reads, +re\-packing it into new chunks with freshly computed ids, and it recreates manifest and +archives directory entries from what it reads. +.TP +.B transfer +\fBborg transfer\fP, for everything it reads from the source repository. Transferring +re\-anchors the content in another repository, which is a trust boundary. +.TP +.B rechunk +\fBborg recreate \-\-chunker\-params ...\fP, i.e. re\-chunking reads. Re\-chunking computes +new chunk ids from the content it reads, so a violation would not be noticeable any +more afterwards. (Re\-chunking in \fBborg transfer\fP is covered by \fBtransfer\fP\&.) +.UNINDENT +.sp +An unknown place name is an error. An empty value (\fBBORG_ASSERT_ID=\fP) verifies at none of +these places, but still where borg always verifies (see below). +.sp +Why \fBread\fP is not in the default: for encrypted repositories (all the AEAD ciphersuites), +the chunk id is part of the AEAD additional authenticated data, so a successful decryption +already proves that a holder of the repository key deliberately stored exactly this +ciphertext for exactly this chunk id. A malicious or buggy \fBrepository\fP can therefore not +swap, splice or substitute objects, whether the id is verified on read or not. What the id +check adds is the detection of chunks whose content does not match their id, which only a +malicious or compromised \fBborg client that had your borg key\fP could have written (e.g. to +poison future deduplication). If that is in your threat model \- e.g. because some machines +writing into the repository are not fully trusted \- add \fBread\fP to the list: +.INDENT 7.0 +.INDENT 3.5 +.sp +.EX +BORG_ASSERT_ID=read,repair,transfer,rechunk +.EE +.UNINDENT +.UNINDENT +.sp +Otherwise, running \fBborg check \-\-verify\-data\fP periodically is recommended: it is the +audit that re\-certifies the invariant for all chunks in the background, instead of on +every read. +.sp +Independent of this variable, borg always verifies the chunk id: +.INDENT 7.0 +.IP \(bu 2 +in \fBborg check \-\-verify\-data\fP\&. That audit is what makes not verifying elsewhere +defensible, so it is not configurable (there is no \fBverify_data\fP place name). +.IP \(bu 2 +for \fBauthenticated\fP and \fBnone\fP mode repositories: there is no AEAD there, so the id +check \fIis\fP the read path\(aqs integrity check and switching it off would remove it +completely. Same for reading borg 1.x repositories (\fBborg transfer\fP). +.UNINDENT +.TP +.B BORG_BLAKE3_MT_THRESHOLD +When set to a numeric value, chunks of at least that many KiB get their id computed by +multi\-threaded BLAKE3, smaller ones single\-threaded (default: 256, i.e. 256KiB). +Only relevant for repositories using \fB\-\-id\-hash blake3\fP\&. +Multi\-threading only pays off for big enough chunks and the break\-even point depends on +the machine\(aqs core count, so the default is deliberately conservative. +Run \fBscripts/blake3\-optimize\-mt\-threshold.py\fP to measure the best value for your +machine \- it sweeps input sizes, prints the recommended threshold and the command to +set it, and can optionally show a chart of the measurements in your browser +(\fB\-\-html \-\-open\fP). +0 means \(dqalways multi\-threaded\(dq, a very large value effectively disables multi\-threading. +.TP +.B BORG_ZSTD_MT_WORKERS +When set to a numeric value, use that many threads to zstd\-compress a single chunk +(default: the cpu count). 0 or 1 means single\-threaded compression. +Only relevant when compressing with \fBzstd\fP\&. +Chunks below 768KiB are always compressed single\-threaded: libzstd will not use a +compression job smaller than 512KiB, so a small chunk gets split very unevenly and +multi\-threading it would be slower than not doing it at all. +Multi\-threading trades a little compression ratio for speed (measured at \fBzstd,3\fP: ++0.05% archive size for 1MiB chunks, +0.64% for 8MiB ones, more at higher levels), and +it uses more cpu time in total to reduce the wallclock time. Set it to 1 if you would +rather have the smaller archive, or if borg has to share the cpu with other work. +.TP +.B BORG_FASTCDC_KERNEL / BORG_BUZHASH64_KERNEL +Select the scan kernel the \fBfastcdc\fP / \fBbuzhash64\fP chunker uses (default: +\fBscalar\fP, the plain sequential loop). Accepted values are \fBavx512\fP, \fBavx2\fP, +\fBneon\fP, \fBblockwise\fP and \fBscalar\fP\&. +All kernels chunk identically \- same cut points, same chunk ids \- and differ only in +speed, so this is safe to change at any time, also for an existing repository. +Which kernel is fastest is not predictable from the instruction set: it depends on the +cpu and on the compiler that built borg, and the sequential loop wins on some machines. +Nothing is selected automatically, so measure on your own hardware with +\fBborg benchmark cpu \-\-chunking\fP before setting these. +\fBavx512\fP and \fBavx2\fP exist only on x86\-64, \fBneon\fP only on aarch64, and only if the +compiler that built borg supported them; \fBscalar\fP and \fBblockwise\fP are portable C +and always available. +Requesting a kernel that this build or this cpu cannot run is an error rather than a +silent fallback, so a benchmark can not accidentally measure a different kernel. +\fBborg create \-\-debug\fP logs the chunker and the kernel it was created with. +.TP +.B BORG_AES_CHUNKER_KERNEL +Select the scan kernel used by the AES based chunkers \- one variable for all three of +\fBtoeplitz\-aes\fP, \fBrabin\-aes\fP and \fBgoldilocks\-aes\fP (default: \fBevp\fP, the portable +OpenSSL path). Accepted values are \fBvaes\fP, \fBaes\-ni\fP, \fBaes\-arm64\fP and \fBevp\fP\&. +As with the chunker kernels above, all of them chunk identically and differ only in +speed, nothing is selected automatically, and a kernel that can not run here is an +error rather than a silent fallback. +\fBvaes\fP and \fBaes\-ni\fP exist only on x86\-64, \fBaes\-arm64\fP only on aarch64. +\fBvaes\fP additionally needs a compiler that knows it (gcc >= 11 / clang >= 14), so a +cpu supporting VAES is not by itself enough to have that kernel available. +.TP +.B BORG_SHOW_SYSINFO +When set to no (default: yes), system information (like OS, Python version, ...) in +exceptions is not shown. +Please only use for good reasons as it makes issues harder to analyze. +.TP +.B BORG_MSGPACK_VERSION_CHECK +Controls whether Borg checks the \fBmsgpack\fP version. +The default is \fByes\fP (strict check). Set to \fBno\fP to disable the version check and +allow any installed \fBmsgpack\fP version. Use this at your own risk; malfunctioning or +incompatible \fBmsgpack\fP versions may cause subtle bugs or repository data corruption. +.TP +.B BORG_FUSE_IMPL +Choose the low\-level FUSE implementation borg shall use for \fBborg mount\fP\&. +This is a comma\-separated list of implementation names, they are tried in the +given order, e.g.: +.INDENT 7.0 +.IP \(bu 2 +\fBmfusepy,pyfuse3,llfuse\fP: default, first try to load mfusepy, then pyfuse3, then llfuse. +.IP \(bu 2 +\fBllfuse,pyfuse3\fP: first try to load llfuse, then try to load pyfuse3. +.IP \(bu 2 +\fBmfusepy\fP: only try to load mfusepy +.IP \(bu 2 +\fBpyfuse3\fP: only try to load pyfuse3 +.IP \(bu 2 +\fBllfuse\fP: only try to load llfuse +.IP \(bu 2 +\fBnone\fP: do not try to load an implementation +.UNINDENT +.TP +.B BORG_SELFTEST +This can be used to influence borg\(aqs built\-in self\-tests. The default is to execute the tests +at the beginning of each borg command invocation. +.sp +BORG_SELFTEST=disabled can be used to switch off the tests and rather save some time. +Disabling is not recommended for normal borg users, but large scale borg storage providers can +use this to optimize production servers after at least doing a one\-time test borg (with +self\-tests not disabled) when installing or upgrading machines/OS/Borg. +.TP +.B BORG_WORKAROUNDS +A list of comma\-separated strings that trigger workarounds in borg, +e.g. to work around bugs in other software. +.sp +Currently known strings are: +.INDENT 7.0 +.TP +.B basesyncfile +Use the more simple BaseSyncFile code to avoid issues with sync_file_range. +You might need this to run borg on WSL (Windows Subsystem for Linux) or +in systemd.nspawn containers on some architectures (e.g. ARM). +Using this does not affect data safety, but might result in a more bursty +write\-to\-disk behavior (not continuously streaming to disk). +.TP +.B retry_erofs +Retry opening a file without O_NOATIME if opening a file with O_NOATIME +caused EROFS. You will need this to make archives from volume shadow copies +in WSL1 (Windows Subsystem for Linux 1). +.TP +.B authenticated_no_key +Work around a lost passphrase or key for an \fBauthenticated\-*\fP mode repository +(these are only authenticated, but not encrypted). +If the key is missing in the repository config, add \fBkey = anything\fP there. +.sp +Without the key, borg can not verify anything that needs it: neither the +authentication tag of the repository objects nor the chunk ids. It therefore +reads the repository \fBunverified\fP \- a corrupted or tampered repository will +not be detected. (This only concerns the \fBauthenticated\-*\fP modes; the +\fBnone\-*\fP modes need no key and keep verifying their checksums.) +.sp +This workaround is \fBonly\fP for emergencies and \fBonly\fP to extract data +from an affected repository (read\-only access): +.INDENT 7.0 +.INDENT 3.5 +.sp +.EX +BORG_WORKAROUNDS=authenticated_no_key borg extract \-\-repo repo archive +.EE +.UNINDENT +.UNINDENT +.sp +After you have extracted all data you need, you MUST delete the repository: +.INDENT 7.0 +.INDENT 3.5 +.sp +.EX +BORG_WORKAROUNDS=authenticated_no_key borg delete repo +.EE +.UNINDENT +.UNINDENT +.sp +Now you can init a fresh repo. Make sure you do not use the workaround any more. +.UNINDENT +.UNINDENT +.TP +.B Output formatting: +.INDENT 7.0 +.TP +.B BORG_CHECK_FORMAT +Giving the default value for \fBborg check \-\-format=X\fP\&. +.TP +.B BORG_LIST_FORMAT +Giving the default value for \fBborg list \-\-format=X\fP\&. +.TP +.B BORG_REPO_LIST_FORMAT +Giving the default value for \fBborg repo\-list \-\-format=X\fP\&. +.TP +.B BORG_PRUNE_FORMAT +Giving the default value for \fBborg prune \-\-format=X\fP\&. +.UNINDENT +.TP +.B Some automatic \(dqanswerers\(dq (if set, they automatically answer confirmation questions): +.INDENT 7.0 +.TP +.B BORG_UNKNOWN_UNENCRYPTED_REPO_ACCESS_IS_OK=no (or =yes) +For \(dqWarning: Attempting to access a previously unknown unencrypted repository\(dq +.TP +.B BORG_RELOCATED_REPO_ACCESS_IS_OK=no (or =yes) +For \(dqWarning: The repository at location ... was previously located at ...\(dq +.TP +.B BORG_CHECK_I_KNOW_WHAT_I_AM_DOING=NO (or =YES) +For \(dqThis is a potentially dangerous function...\(dq (check \-\-repair) +.TP +.B BORG_DELETE_I_KNOW_WHAT_I_AM_DOING=NO (or =YES) +For \(dqYou requested to DELETE the repository completely \fIincluding\fP all archives it contains:\(dq +.UNINDENT +.sp +Note: answers are case sensitive. setting an invalid answer value might either give the default +answer or ask you interactively, depending on whether retries are allowed (they by default are +allowed). So please test your scripts interactively before making them a non\-interactive script. +.TP +.B Directories and files: +Borg 2 uses the platformdirs library (\%) to determine +default directory locations. This means that default paths are \fBplatform\-specific\fP: +.INDENT 7.0 +.IP \(bu 2 +Linux: XDG Base Directory Specification paths are used (e.g. \fB~/.config/borg\fP, +\fB~/.cache/borg\fP, \fB~/.local/share/borg\fP). \fBXDG_*\fP environment variables are +honoured (see \%). +.IP \(bu 2 +macOS: native macOS directories are used by default (e.g. \fB~/Library/Application Support/borg\fP, +\fB~/Library/Caches/borg\fP). \fBXDG_*\fP environment variables are honoured if set. +.IP \(bu 2 +Windows: Windows AppData directories are used (e.g. \fBC:\eUsers\e\eAppData\eRoaming\eborg\fP, +\fBC:\eUsers\e\eAppData\eLocal\eborg\fP). \fBXDG_*\fP environment variables are \fBnot\fP honoured. +.UNINDENT +.sp +On all platforms, you can override each directory individually using the specific environment +variables described below. You can also set \fBBORG_BASE_DIR\fP to force borg to use +\fBBORG_BASE_DIR/.config/borg\fP, \fBBORG_BASE_DIR/.cache/borg\fP, etc., regardless of the platform. +.sp +Default directory locations by platform (when no \fBBORG_*\fP environment variables are set): +.INDENT 7.0 +.INDENT 3.5 +.sp +.EX +Directory Linux macOS Windows +Config ~/.config/borg ~/Library/Application Support/borg %APPDATA%\eborg +Cache ~/.cache/borg ~/Library/Caches/borg %LOCALAPPDATA%\eborg\eCache +Data ~/.local/share/borg ~/Library/Application Support/borg %LOCALAPPDATA%\eborg +Runtime /run/user//borg ~/Library/Caches/TemporaryItems/borg %TEMP%\eborg +Keys /keys /keys \ekeys +Security /security /security \esecurity +.EE +.UNINDENT +.UNINDENT +.INDENT 7.0 +.TP +.B BORG_BASE_DIR +Defaults to \fB$HOME\fP or \fB~$USER\fP or \fB~\fP (in that order). +If you want to move all borg\-specific folders to a custom path at once, all you need to do is +to modify \fBBORG_BASE_DIR\fP: the other paths for cache, config etc. will adapt accordingly +(assuming you didn\(aqt set them to a different custom value). +.TP +.B BORG_CACHE_DIR +Defaults to the platform\-specific cache directory (see table above). +If \fBBORG_BASE_DIR\fP is set, defaults to \fB$BORG_BASE_DIR/.cache/borg\fP\&. +On Linux and macOS, \fBXDG_CACHE_HOME\fP is also honoured if \fBBORG_BASE_DIR\fP is not set. +This directory contains the local cache and might need a lot +of space for dealing with big repositories. Make sure you\(aqre aware of the associated +security aspects of the cache location: \fIcache_security\fP +.TP +.B BORG_CONFIG_DIR +Defaults to the platform\-specific config directory (see table above). +If \fBBORG_BASE_DIR\fP is set, defaults to \fB$BORG_BASE_DIR/.config/borg\fP\&. +On Linux and macOS, \fBXDG_CONFIG_HOME\fP is also honoured if \fBBORG_BASE_DIR\fP is not set. +This directory contains all borg configuration directories, see the FAQ +for a security advisory about the data in this directory: \fIhome_config_borg\fP +.TP +.B BORG_DATA_DIR +Defaults to the platform\-specific data directory (see table above). +If \fBBORG_BASE_DIR\fP is set, defaults to \fB$BORG_BASE_DIR/.local/share/borg\fP\&. +On Linux and macOS, \fBXDG_DATA_HOME\fP is also honoured if \fBBORG_BASE_DIR\fP is not set. +This directory contains all borg data directories, see the FAQ +for a security advisory about the data in this directory: \fIhome_data_borg\fP +.TP +.B BORG_RUNTIME_DIR +Defaults to the platform\-specific runtime directory (see table above). +If \fBBORG_BASE_DIR\fP is set, defaults to \fB$BORG_BASE_DIR/.cache/borg\fP\&. +On Linux and macOS, \fBXDG_RUNTIME_DIR\fP is also honoured if \fBBORG_BASE_DIR\fP is not set. +This directory contains borg runtime files, like e.g. the socket file. +.TP +.B BORG_SECURITY_DIR +Defaults to \fB$BORG_DATA_DIR/security\fP\&. +This directory contains security relevant data. +.TP +.B BORG_KEYS_DIR +Defaults to \fB$BORG_CONFIG_DIR/keys\fP\&. +This directory contains keys for encrypted repositories. +.TP +.B BORG_KEY_FILE +When set, use the given path as repository key file. Please note that this is only +for rather special applications that externally fully manage the key files: +.INDENT 7.0 +.IP \(bu 2 +this setting only applies to the keyfile modes (not to the repokey modes). +.IP \(bu 2 +using a full, absolute path to the key file is recommended. +.IP \(bu 2 +all directories in the given path must exist. +.IP \(bu 2 +this setting forces borg to use the key file at the given location. +.IP \(bu 2 +the key file must either exist (for most commands) or will be created (\fBborg repo\-create\fP). +.IP \(bu 2 +you need to give a different path for different repositories. +.IP \(bu 2 +you need to point to the correct key file matching the repository the command will operate on. +.UNINDENT +.TP +.B TMPDIR +This is where temporary files are stored (might need a lot of temporary space for some +operations), see \% +for details. +.UNINDENT +.TP +.B Building: +.INDENT 7.0 +.TP +.B BORG_OPENSSL_NAME +Defines the subdirectory name for OpenSSL (setup.py). +.TP +.B BORG_OPENSSL_PREFIX +Adds given OpenSSL header file directory to the default locations (setup.py). +.TP +.B BORG_LIBACL_PREFIX +Adds given prefix directory to the default locations. If an \(aqinclude/acl/libacl.h\(aq is found +Borg will be linked against the system libacl instead of a bundled implementation. (setup.py) +.TP +.B BORG_LIBLZ4_PREFIX +Adds given prefix directory to the default locations. If a \(aqinclude/lz4.h\(aq is found Borg +will be linked against the system liblz4 instead of a bundled implementation. (setup.py) +.UNINDENT +.TP +.B Automatic option environment variables: +Borg uses jsonargparse (\%) with \fBdefault_env=True\fP, +which means that every command\-line option can also be set via an environment variable. +.sp +The environment variable name is derived from the program name (\fBborg\fP), +the subcommand (if any), and the option name, all converted to uppercase +with dashes replaced by underscores. +.sp +For \fBtop\-level options\fP (not specific to a subcommand), the pattern is: +.INDENT 7.0 +.INDENT 3.5 +.sp +.EX +BORG_